From 2e05f50ba471522d519adfa3a41ff8eebcd0d890 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Mon, 7 Sep 2026 11:18:08 +1000 Subject: [PATCH 1/2] Manifest-driven validate(): the schema block is the spec, shared by the builders and a PR check (#119) builders/_validate.py enforces a manifest's `schema` block on a raw-shaped frame -- columns and `pattern` runs (#120), dtype families (#122), exact `known_nulls` and the `nulls:` placement rule (#121), row_count_floor, date_range -- and MEASURES the overlap window against the previous vintage (no series or period may disappear, no populated cell may go empty; revisions are returned for the builder to bound, never asserted equal). Two callers use it unchanged: - builders/business_cycle.py and builders/business_cycle_fred.py call validate(frame.reset_index(), manifest, previous) and keep only what a schema cannot say: the year grid, the economy set, value bands, recency, each FRED series' first observation, USREC 0/1, and the per-series revision bound. The FRED builder's MONTHLY_FROM / LAST_OBS / KNOWN_HOLES constants are gone -- the manifest's nulls block is the single source. _template.py shows the two-layer shape. - scripts/validate_datasets.py, run by .github/workflows/validate-datasets.yml on every PR: a conformance pass over all 44 manifests (canonical dtypes, name-xor-pattern with one capture group, exact known_nulls, a nulls block on every dynamic snapshot, known_nulls_total only inside header-less sheet reads) and the byte-level validate() over every CSV. Non-CSV formats are conformance-only and the log says so. Two contract fixes the validator surfaced, both in manifests: - us_business_cycle_monthly.csv.yml: `recent: 1` and no exact counts. The live frame on 2026-09-07 ran to 2026-08 with UMCSENT, CPILFESL and INDPRO unpublished for that month (they release mid-month); the old hand-written check asserted zero trailing nulls and would have failed the first monthly refresh on this lag. Every count in that file moves with the newest month, so placement is the contract. - countries.csv.yml: `Country code: 1` -- Namibia's ISO alpha-2 code is the string "NA", which a default read_csv parses as missing. Declared with the reason; a lecture that needs it must pass keep_default_na=False. Verified: 44/44 manifests pass under pandas 2.3.3 (CI) and 3.0.5 (the lectures' pin); both dynamic builders dry-run green against the live sources and reproduce the committed vintages (0 cells revised); 16 mutations of the committed frames are caught, and the four that are not are the builder layer's (grid, first observation) or permitted by design (a lagging newest month, a backfilled hole). Co-Authored-By: Claude Fable 5.1 --- .github/workflows/consumed-file-check.yml | 6 +- .github/workflows/validate-datasets.yml | 42 +++ AGENTS.md | 2 +- PLAN.md | 2 +- builders/README.md | 11 +- builders/_template.py | 41 ++- builders/_validate.py | 367 +++++++++++++++++++++ builders/business_cycle.py | 84 ++--- builders/business_cycle_fred.py | 73 ++-- lectures/countries.csv.yml | 6 + lectures/us_business_cycle_monthly.csv.yml | 23 +- manifest-schema.yml | 6 +- scripts/validate_datasets.py | 134 ++++++++ 13 files changed, 673 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/validate-datasets.yml create mode 100644 builders/_validate.py create mode 100644 scripts/validate_datasets.py diff --git a/.github/workflows/consumed-file-check.yml b/.github/workflows/consumed-file-check.yml index 6d6a713..977babd 100644 --- a/.github/workflows/consumed-file-check.yml +++ b/.github/workflows/consumed-file-check.yml @@ -1,6 +1,8 @@ # Go-live guardrail (PLAN Phase 5): a PR must not break a file a live lecture -# consumes. The narrowest possible check — full PR validation (manifest schema, -# dtypes, invariant tests) comes later in Phase 5 and will subsume this job. +# consumes. This job is the byte-integrity (sha256) and catalog-freshness gate; +# its sibling validate-datasets.yml (2026-09-07, #119) is the schema gate — +# manifest conformance and per-dataset invariants. Together they are the "PR +# validation" Phase 5 promised; neither subsumes the other. name: consumed-file-check diff --git a/.github/workflows/validate-datasets.yml b/.github/workflows/validate-datasets.yml new file mode 100644 index 0000000..747bfda --- /dev/null +++ b/.github/workflows/validate-datasets.yml @@ -0,0 +1,42 @@ +# PR validation (PLAN Phase 5; QuantEcon/data-lectures#119): the manifest's +# schema block is an executable contract, and this is where it executes on +# every PR. Two passes, both in scripts/validate_datasets.py: +# +# conformance every manifest obeys manifest-schema.yml's rules -- canonical +# dtype names (#122), name-xor-pattern with one capture group +# (#120), exact known_nulls and a `nulls:` placement block on +# every dynamic snapshot (#121), known_nulls_total only inside +# header-less sheet reads +# bytes every CSV's committed bytes satisfy its own schema block, +# through the same builders/_validate.py the dynamic builders +# run -- so a manifest that drifts from its file fails here +# before it can fail a refresh +# +# Non-CSV formats get the conformance pass only, and the log says so. +# consumed-file-check.yml stays beside this as the byte-integrity (sha256) and +# catalog-freshness gate; this job is the schema half PLAN Phase 5 promised. +# +# pandas is pinned to requirements.txt's version (2.3.3), deliberately one +# major behind the lectures' anaconda=2026.07 (pandas 3): the validator +# compares dtypes by family precisely so it is green on both, and running the +# older one here is the standing check that it stays so. + +name: validate-datasets + +on: + pull_request: + push: + branches: [main] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: false # lectures/ is plain git; a pointer here should fail (see consumed-file-check.yml) + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pandas==2.3.3 PyYAML==6.0.3 + - run: python scripts/validate_datasets.py diff --git a/AGENTS.md b/AGENTS.md index 13483c3..bc93f17 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,7 +122,7 @@ Where one builder produces a **set** of files, name it for the set and let each **Where a builder reads its input from.** The normal case is the third-party upstream, fetched at run time: eight of the nine `committed` builders here do that, and it is the fetch stage of the contract below. A builder reads from `sources/` **only when the input cannot be re-fetched** — the upstream is gone, unlocatable, or was inherited with no recoverable source. `sources/` is that exception layer, not a general input tree, and it is emphatically not "the big-file directory": the defining property is un-refetchability, not size. What it must never be is a network read from another QuantEcon repo — that is how a retired repo becomes load-bearing again. -Builders follow four stages — **fetch → pre-process → validate → write** — and only write on validation pass (expected columns/dtypes, row-count floor, recency of date range, no all-NaN columns, and a **bounded** overlap window against the previous vintage — a tracking snapshot is revised by its source, so the test is a tolerance plus a printed summary, never equality). Lectures always read the last-good snapshot: an upstream outage may fail a refresh, it must never break a lecture build. +Builders follow four stages — **fetch → pre-process → validate → write** — and only write on validation pass. **The validate stage is shared**: `builders/_validate.py` reads the manifest's `schema` block as its spec (columns and `pattern` runs, dtype families, exact `known_nulls`, the `nulls:` placement rule, `row_count_floor`, `date_range`) and measures the overlap window against the previous vintage; a builder calls `validate(frame.reset_index(), manifest, previous)` and layers on only what the schema cannot say — value bands, a grid check, the per-series revision **bound** (a tracking snapshot is revised by its source, so the test is a tolerance plus a printed summary, never equality). The same function runs over every committed CSV on every PR (`scripts/validate_datasets.py`, `validate-datasets.yml`), so a manifest that drifts from its bytes fails the PR, not the next refresh. Lectures always read the last-good snapshot: an upstream outage may fail a refresh, it must never break a lecture build. **One builder per source, composite files where a lecture reads series together, shared fetch libraries** (decided 2026-09-01, #26): `builders/_fred.py` is the `Fred` class for FRED reads — no `pandas_datareader` in a builder — and a lecture's FRED data is one file with the lecture's own variable names as columns, not six. A builder that writes a set validates every file before writing any. diff --git a/PLAN.md b/PLAN.md index f6aa33d..2a0a190 100644 --- a/PLAN.md +++ b/PLAN.md @@ -296,7 +296,7 @@ Only one file genuinely forces LFS, and it is not a dataset: Full automation: - [x] Audit dashboard workflow ([#20](https://github.com/QuantEcon/data-lectures/issues/20), added 2026-07-17): `.github/workflows/audit-dashboard.yml` rebuilds the full-universe data audit + migration tracker from the 8 lecture repos' `main` (push to main / weekly / dispatch) and deploys it with the published tree to Pages. Strict mode fails the build on an unannotated data reference or a `migration.yml` status the scan contradicts -- [ ] PR validation: manifest schema check + per-dataset invariant tests (expected columns/dtypes, row-count floor, date-range recency, no all-NaN columns, overlap-window agreement with the previous vintage) on every PR touching data. The schema decisions these tests force — column patterns for wide files, `known_nulls` exact-vs-ceiling, a canonical dtype vocabulary — have their own issues under the [#14](https://github.com/QuantEcon/data-lectures/issues/14) tracker: [#120](https://github.com/QuantEcon/data-lectures/issues/120) column patterns, [#121](https://github.com/QuantEcon/data-lectures/issues/121) `known_nulls`, [#122](https://github.com/QuantEcon/data-lectures/issues/122) dtype vocabulary; the shared manifest-driven `validate()` and this workflow are [#119](https://github.com/QuantEcon/data-lectures/issues/119) +- [x] PR validation — manifest schema check + per-dataset invariant tests (expected columns/dtypes, row-count floor, date-range recency, no all-NaN columns, overlap-window agreement with the previous vintage) on every PR touching data. The schema decisions these tests force — column patterns for wide files, `known_nulls` exact-vs-ceiling, a canonical dtype vocabulary — have their own issues under the [#14](https://github.com/QuantEcon/data-lectures/issues/14) tracker: [#120](https://github.com/QuantEcon/data-lectures/issues/120) column patterns, [#121](https://github.com/QuantEcon/data-lectures/issues/121) `known_nulls`, [#122](https://github.com/QuantEcon/data-lectures/issues/122) dtype vocabulary; the shared manifest-driven `validate()` and this workflow are [#119](https://github.com/QuantEcon/data-lectures/issues/119) **Landed 2026-09-07 (#119)**: `builders/_validate.py` reads the sidecar `schema` block as its spec and is shared by the dynamic builders and by `.github/workflows/validate-datasets.yml`, which runs `scripts/validate_datasets.py` on every PR — conformance for all 44 manifests, byte validation for every CSV; non-CSV formats are conformance-only until their range reads are built. - [x] Retrofit `builders/business_cycle.py` to the four-stage builder contract — **done 2026-09-01**, with the two provenance dumps moved out of the published tree to `provenance/` ([#13](https://github.com/QuantEcon/data-lectures/issues/13)). Its `validate()` is the first to face a *revised* upstream: it bounds the overlap window (5 pp) and prints the revision summary rather than asserting equality, which is the review surface the refresh-as-PR workflow below will use. It previously had fetch/transform/write but no validate stage. Builder architecture and a copy-able template: [#14](https://github.com/QuantEcon/data-lectures/issues/14) - [x] Scheduled refresh workflow for dynamic datasets — **landed 2026-09-01** as `.github/workflows/refresh-snapshots.yml`, manifest-driven rather than cron-per-class: a weekly run asks `scripts/snapshots.py due` which `dynamic-snapshot` datasets have their cadence elapsed (or are `diverged`, or were never refreshed), runs each builder in place, stamps the manifest (`retrieved`, `sha256`, `integrity.upstream: verified`, `date_range.end`), regenerates the catalog, and opens a PR on `refresh/` whose body is the builder's overlap summary. Nothing auto-merges; the first consumer is `business_cycle_data.csv`, not UNRATE — the pilot's order inverted once the World Bank file turned out to be the one already here - [x] Weekly sources-alive canary: fetch + validate, no commit, opens an issue on failure — **landed 2026-09-01** as the `canary` job of the same workflow: every dynamic snapshot's builder runs with `--out-dir`, and a failure opens or updates one `upstream-break` issue classified by exit code (2 = the data broke the contract, a human; anything else = the fetch, a retry). Covers the live APIs only as their snapshot twins land here — the 23 live-API lectures without a twin are still guarded by nothing but their own CI diff --git a/builders/README.md b/builders/README.md index c291b7f..27c4f43 100644 --- a/builders/README.md +++ b/builders/README.md @@ -30,7 +30,16 @@ template is [`_template.py`](_template.py) (not a builder — the underscore keeps it out of any manifest). Shared fetch code lives beside it under the same convention: [`_fred.py`](_fred.py) is the `Fred` class every FRED builder should use (`fred.series('UNRATE')`, `fred.frame([...])`), so a fetch -stage is a line and `validate()` is the only thing worth reading. One +stage is a line and `validate()` is the only thing worth reading — and most of +*that* is shared too: [`_validate.py`](_validate.py) reads the manifest's +`schema` block as the spec (columns and `pattern` runs, dtype families, exact +`known_nulls`, the `nulls:` placement rule, `row_count_floor`, `date_range`) +and measures the overlap window against the previous vintage +([#119](https://github.com/QuantEcon/data-lectures/issues/119)). A builder +calls `validate(frame.reset_index(), manifest, previous)` and adds only what +a schema cannot say — value bands, a grid, recency, the revision *bound*. The +same function runs over every committed CSV on every PR +(`scripts/validate_datasets.py`, `validate-datasets.yml`). One builder per **source** for a lecture's data, writing a composite file where the lecture reads the series together (decided 2026-09-01 on #26). diff --git a/builders/_template.py b/builders/_template.py index 6fb70e9..8543d6e 100644 --- a/builders/_template.py +++ b/builders/_template.py @@ -32,7 +32,7 @@ (an end year, an observed range, a row count) -- those live in the fields scripts/snapshots.py stamps, and only there -Requires pandas plus whatever the source needs (add it to requirements.txt). +Requires pandas and PyYAML plus whatever the source needs (add it to requirements.txt). """ import argparse import datetime as dt @@ -41,6 +41,9 @@ import sys import pandas as pd +import yaml + +from _validate import ValidationError, validate as validate_schema CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.dirname(CURRENT_FILE_DIR) @@ -52,10 +55,6 @@ MAX_STALENESS = None # newest observation must be at least this recent -class ValidationError(Exception): - """The fetched data broke the published contract -- exit code 2.""" - - def _check(condition, message): if not condition: raise ValidationError(message) @@ -73,22 +72,32 @@ def pre_process(raw): def validate(frame, previous=None): - """Assert the contract the manifest's schema block promises; return the - run summary. Every failure is a ValidationError with a message a human - can act on from the canary issue.""" - _check(len(frame) > 0, 'empty frame') - # columns / dtypes / known_nulls / units / recency ... + """Two layers (QuantEcon/data-lectures#119). The manifest's schema block + is the spec -- columns and pattern runs, dtype families, exact known_nulls, + the `nulls:` placement rule, row_count_floor, date_range, and the overlap + window MEASURED against `previous` -- enforced by builders/_validate.py. + Add here only what a schema cannot say: value bands, a grid, recency, and + the revision BOUND. Every failure is a ValidationError with a message a + human can act on from the canary issue.""" + with open(os.path.join(PUBLISHED_DIR, OUT_FILE + '.yml')) as f: + manifest = yaml.safe_load(f) + # pass the RAW shape: the period/label column as a column, not the index + raw = frame.reset_index() if frame.index.name else frame + prev_raw = previous.reset_index() if previous is not None and previous.index.name else previous + shared = validate_schema(raw, manifest, prev_raw) + # builder-specific: bands / grid / recency ... + raise NotImplementedError summary = { 'dataset': OUT_FILE, 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), - 'rows': int(frame.shape[0]), - 'columns': int(frame.shape[1]), - 'date_range': {'start': None, 'end': None}, - 'overlap': None, + 'rows': shared['rows'], + 'columns': shared['columns'], + 'date_range': shared['date_range'], + 'overlap': shared['overlap'], } if previous is not None: - # compare the shared window; report and BOUND revisions, never assert equality - raise NotImplementedError + _check(shared['overlap']['max_abs_change'] <= MAX_REVISION, + f'revision {shared["overlap"]["max_abs_change"]} exceeds {MAX_REVISION}') return summary diff --git a/builders/_validate.py b/builders/_validate.py new file mode 100644 index 0000000..130ac62 --- /dev/null +++ b/builders/_validate.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +""" +Shared, manifest-driven validation: the sidecar's `schema` block is the spec. + +Two callers use this unchanged (QuantEcon/data-lectures#119): + + * every dynamic-snapshot builder's validate stage -- the builder calls + `validate(frame, manifest, previous)` on the frame it is about to write + and layers its own checks (value bands, grid, per-series revision bounds) + on top of the summary this returns; + * the PR-validation workflow (`scripts/validate_datasets.py`), which reads + every committed CSV with `read_raw()` and runs the same `validate()` with + no `previous`, so a manifest that drifts from its bytes fails the PR. + +What is enforced, and the decision each rule records: + + columns #120 -- walked IN ORDER; a `name` entry claims one column, a + `pattern` entry claims the maximal run of CONSECUTIVE columns + that fullmatch it (Python `re`); at least one; every file + column must be claimed (exhaustive); a pattern's single + capture group is the period token `date_range` derives from + dtype #122 -- compared by FAMILY: str/string/object are one text + family, datetime64 matches any unit, a nullable bool read back + as object with bool values matches `bool`; the declared name + must be in the canonical vocabulary (checked by the CLI's + conformance pass, not here) + known_nulls #121 -- an integer is an EXACT count + nulls #121 -- placement for dynamic snapshots: `along` (rows | + columns) says which way a series runs; `leading` allows nulls + before a series' first observation; `ended` lists series that + stopped (nulls after their last observation); `recent` allows + nulls in the newest N periods of a live series; `inner` lists + accepted holes per series, as periods or {sparse_until: P}. + A null anywhere else fails + row_count_floor, date_range.start (and .end for non-dynamic files) + overlap against `previous` when given: a series may not disappear and + a populated cell may not go empty; revisions are MEASURED and + returned (cells_revised, max_abs_change, by series) for the + builder to bound -- never asserted equal + +The frame is the RAW table as written: no index column set, periods as a +column (`read_raw` does this; a builder passes `frame.reset_index()`). +Period labels are compared as text -- YAML hands back `datetime.date` objects +for date-shaped scalars, the same trap scripts/snapshots.py guards against. +""" +from __future__ import annotations + +import datetime as dt +import re + +import pandas as pd + + +class ValidationError(Exception): + """The data broke the published contract -- exit code 2 in a builder.""" + + +def _check(condition, message): + if not condition: + raise ValidationError(message) + + +# --------------------------------------------------------------------------- +# dtype families (#122) +# --------------------------------------------------------------------------- + +CANONICAL_DTYPES = {'str', 'float64', 'int64', 'bool', 'datetime64', + # storage types a non-CSV format genuinely carries + 'float32', 'int32', 'int16', 'int8'} + +_FAMILY = { + 'str': 'text', 'string': 'text', 'object': 'text', + 'float64': 'float', 'float32': 'float', 'float16': 'float', 'float': 'float', + 'int64': 'int', 'int32': 'int', 'int16': 'int', 'int8': 'int', 'int': 'int', + 'Int64': 'int', 'Int32': 'int', + 'bool': 'bool', 'boolean': 'bool', +} + + +def family(dtype) -> str: + s = str(dtype) + if s.startswith('datetime64'): + return 'datetime' + return _FAMILY.get(s, s) + + +def dtype_matches(series: pd.Series, declared: str) -> bool: + actual, want = family(series.dtype), family(declared) + if actual == want: + return True + # An all-null column gives pandas nothing to infer from (it reports + # float64); the declaration cannot be checked against the bytes, and the + # exact known_nulls count is what guards the column instead. + if series.isnull().all(): + return True + # pandas 3 reads text as `str`, pandas 2 as `object`: one family (above). + # A bool column with nulls comes back as object holding bools + NaN. + if want == 'bool' and actual == 'text': + vals = series.dropna() + return len(vals) == 0 or all(isinstance(v, (bool,)) or v in (True, False) for v in vals) + # An integer column with nulls is read as float; accept when every value + # is whole -- the declaration describes the bytes, not pandas' promotion. + if want == 'int' and actual == 'float': + vals = series.dropna() + return len(vals) == 0 or bool((vals == vals.round()).all()) + return False + + +# --------------------------------------------------------------------------- +# reading the raw file the way the manifest describes it +# --------------------------------------------------------------------------- + +def datetime_columns(schema: dict) -> list[str]: + return [c['name'] for c in schema.get('columns') or [] + if 'name' in c and str(c.get('dtype', '')).startswith('datetime64')] + + +def read_raw(path, manifest: dict) -> pd.DataFrame: + """Read a CSV as written, honouring `delimiter` and parsing the columns the + manifest declares `datetime64`. No index column: the manifest describes + the bytes on disk, not the frame a particular lecture builds.""" + schema = manifest.get('schema') or {} + fmt = schema.get('format') + if fmt != 'csv': + raise NotImplementedError(f'read_raw handles csv only, not {fmt!r}') + sep = schema.get('delimiter') or manifest.get('delimiter') or ',' + dates = datetime_columns(schema) + return pd.read_csv(path, sep=sep, parse_dates=dates or False) + + +# --------------------------------------------------------------------------- +# columns (#120) +# --------------------------------------------------------------------------- + +def match_columns(actual: list[str], entries: list[dict]) -> list[tuple[dict, list[str]]]: + """Walk the manifest's `columns` in order against the file's columns. + Returns [(entry, [claimed columns...]), ...]; raises on any mismatch.""" + # An empty header cell (a pandas index written without a name) comes back + # as `Unnamed: k`; manifests declare it as name: "" (or, older ones, as + # the pandas label itself). Compare a normalised form, keep the real label. + def _norm(c): + return '' if re.fullmatch(r'Unnamed: \d+', str(c)) else str(c) + matches, i, n = [], 0, len(actual) + for k, entry in enumerate(entries): + if 'name' in entry and 'pattern' in entry: + raise ValidationError(f'columns[{k}] carries both name and pattern') + if 'name' in entry: + _check(i < n, f'column {entry["name"]!r} missing: file ended after {actual[:i][-3:]}') + _check(_norm(actual[i]) == _norm(entry['name']), + f'column {i} is {actual[i]!r}, expected {entry["name"]!r}') + matches.append((entry, [actual[i]])); i += 1 + elif 'pattern' in entry: + rx = re.compile(entry['pattern']) + run = [] + while i < n and rx.fullmatch(str(actual[i])): + run.append(actual[i]); i += 1 + _check(run, f'pattern {entry["pattern"]!r} matched no column at position {i} ' + f'(next is {actual[i]!r})' if i < n else + f'pattern {entry["pattern"]!r} matched no column: wide part missing') + matches.append((entry, run)) + else: + raise ValidationError(f'columns[{k}] has neither name nor pattern') + _check(i == n, f'unexpected column(s) not claimed by any entry: {actual[i:i + 5]}') + return matches + + +def _period_from(pattern: str, label: str): + m = re.fullmatch(pattern, str(label)) + if m and m.groups(): + tok = m.group(1) + return int(tok) if tok.isdigit() else tok + return None + + +def _label(x) -> str: + """Period labels compared as text; dates normalised to ISO.""" + if isinstance(x, (pd.Timestamp, dt.datetime, dt.date)): + return pd.Timestamp(x).date().isoformat() + return str(x) + + +# --------------------------------------------------------------------------- +# nulls (#121) +# --------------------------------------------------------------------------- + +def _series_view(frame: pd.DataFrame, schema: dict, matches) -> tuple[pd.DataFrame, str]: + """Return (table, along): a frame whose ROWS are periods and COLUMNS are + series, whichever way the file is laid out, with period labels as text.""" + nulls = schema.get('nulls') or {} + along = nulls.get('along') + if along is None: + along = 'columns' if any('pattern' in e for e, _ in matches) else 'rows' + if along == 'columns': + pattern_entries = [(e, cols) for e, cols in matches if 'pattern' in e] + _check(len(pattern_entries) == 1, 'along: columns needs exactly one pattern entry') + period_cols = pattern_entries[0][1] + label_col = matches[0][1][0] # the first named column labels the series + table = frame.set_index(label_col)[period_cols].T + table.index = [_label(c) for c in table.index] + return table, along + dates = datetime_columns(schema) + if dates: + table = frame.set_index(dates[0]) + table.index = [_label(x) for x in table.index] + else: + table = frame.copy() + table.index = [str(i) for i in table.index] + return table, along + + +def check_known_nulls(frame: pd.DataFrame, schema: dict, dynamic: bool = False): + known = schema.get('known_nulls') or {} + if not dynamic: + # A frozen file declares every nulled column; one it does not declare + # must have none (the template's `known.get(col, 0)` rule, #121). + for col in frame.columns: + if col not in known: + have = int(frame[col].isnull().sum()) + _check(have == 0, f'{col}: {have} nulls but not declared under known_nulls') + for col, n in known.items(): + _check(col in frame.columns, f'known_nulls names {col!r}, not a column') + _check(isinstance(n, int) and not isinstance(n, bool), + f'known_nulls[{col!r}] must be an integer (exact count), got {n!r}') + have = int(frame[col].isnull().sum()) + _check(have == n, f'{col}: {have} nulls, manifest says exactly {n}') + return known + + +def check_placement(frame: pd.DataFrame, schema: dict, matches): + rule = schema.get('nulls') + if not rule: + return + table, along = _series_view(frame, schema, matches) + periods = list(table.index) + leading_ok = bool(rule.get('leading', False)) + recent = int(rule.get('recent', 0) or 0) + ended = set(map(str, rule.get('ended') or [])) + inner = rule.get('inner') or {} + recent_set = set(periods[-recent:]) if recent else set() + for series in table.columns: + s = table[series] + if not s.isnull().any(): + continue + first, last = s.first_valid_index(), s.last_valid_index() + _check(first is not None, f'{series}: no observation at all') + pos = {p: k for k, p in enumerate(periods)} + holes = inner.get(str(series), inner.get(series, [])) + sparse_until = holes.get('sparse_until') if isinstance(holes, dict) else None + hole_set = set() if isinstance(holes, dict) else {_label(h) for h in holes} + for p in periods: + if not pd.isnull(s[p]): + continue + if pos[p] < pos[first]: + _check(leading_ok, f'{series}: null at {p} before first observation, leading nulls not allowed') + continue + if pos[p] > pos[last]: + _check(str(series) in ended or p in recent_set, + f'{series}: null at {p} after its last observation {last}; not in `ended` and not within the newest {recent}') + continue + ok = p in hole_set or (sparse_until is not None and p < _label(sparse_until)) or p in recent_set + _check(ok, f'{series}: hole at {p} inside the series is not declared under nulls.inner') + + +# --------------------------------------------------------------------------- +# date_range, rows, overlap +# --------------------------------------------------------------------------- + +def derive_date_range(frame: pd.DataFrame, schema: dict, matches) -> dict: + for entry, cols in matches: + if 'pattern' in entry and re.compile(entry['pattern']).groups == 1: + toks = [_period_from(entry['pattern'], c) for c in cols] + return {'start': toks[0], 'end': toks[-1]} + dates = datetime_columns(schema) + if dates: + col = frame[dates[0]].dropna() + return {'start': _label(col.min()), 'end': _label(col.max())} + return {'start': None, 'end': None} + + +def _same_period(spec, derived) -> bool: + if spec is None or derived is None: + return True + a, b = _label(spec), _label(derived) + return a == b or b.startswith(a) or a.startswith(b) + + +def check_date_range(derived: dict, schema: dict, dynamic: bool): + spec = schema.get('date_range') or {} + _check(_same_period(spec.get('start'), derived['start']), + f'date_range.start is {derived["start"]}, manifest says {spec.get("start")}') + if not dynamic and spec.get('end') is not None: + _check(_same_period(spec.get('end'), derived['end']), + f'date_range.end is {derived["end"]}, manifest says {spec.get("end")}') + + +def check_rows(frame: pd.DataFrame, schema: dict): + floor = schema.get('row_count_floor') + if floor is not None: + _check(len(frame) >= floor, f'{len(frame)} rows, floor is {floor}') + + +def overlap(frame: pd.DataFrame, previous: pd.DataFrame, schema: dict, matches) -> dict: + """Measure the shared window. Asserts only that no series disappeared and + no populated cell went empty; revisions are returned, not judged.""" + new_t, _ = _series_view(frame, schema, matches) + prev_matches = match_columns(list(previous.columns), schema.get('columns') or []) + old_t, _ = _series_view(previous, schema, prev_matches) + _check(set(old_t.columns) <= set(new_t.columns), + f'series disappeared: {sorted(set(old_t.columns) - set(new_t.columns))}') + _check(set(old_t.index) <= set(new_t.index), + f'period(s) disappeared: {[p for p in old_t.index if p not in set(new_t.index)][:5]}') + common_p = [p for p in old_t.index if p in new_t.index] + _check(common_p, 'no period in common with the previous snapshot') + old = old_t.loc[common_p, old_t.columns] + new = new_t.loc[common_p, old_t.columns] + went_empty = old.notnull() & new.isnull() + _check(not went_empty.any().any(), + f'a populated cell went empty: {[(c, p) for c in old.columns for p in common_p if went_empty.loc[p, c]][:3]}') + diff = (old.apply(pd.to_numeric, errors='coerce') - new.apply(pd.to_numeric, errors='coerce')).abs() + by_series = {str(c): (round(float(diff[c].max()), 6) if diff[c].notnull().any() else 0.0) for c in old.columns} + return { + 'window': f'{common_p[0]}..{common_p[-1]}', + 'previous_end': old_t.index[-1], + 'cells_total': int(old.notnull().sum().sum()), + 'cells_revised': int((diff > 1e-9).sum().sum()), + 'max_abs_change': round(max(by_series.values()), 6) if by_series else 0.0, + 'max_abs_change_by_series': by_series, + 'new_columns': [p for p in new_t.index if p not in set(old_t.index)], + 'new_series': [str(c) for c in new_t.columns if c not in set(old_t.columns)], + } + + +# --------------------------------------------------------------------------- +# the entry point +# --------------------------------------------------------------------------- + +def validate(frame: pd.DataFrame, manifest: dict, previous: pd.DataFrame | None = None) -> dict: + """Enforce the manifest's schema block on a RAW-shaped frame; return the + run summary a builder writes as --summary-json.""" + schema = manifest.get('schema') or {} + dynamic = manifest.get('class') == 'dynamic-snapshot' + _check(len(frame) > 0, 'empty frame') + matches = match_columns([str(c) for c in frame.columns], schema.get('columns') or []) + for entry, cols in matches: + want = entry.get('dtype') + if want is None: + continue + for c in cols: + _check(dtype_matches(frame[c], want), + f'{c}: dtype {frame[c].dtype} is not in the {want!r} family') + check_rows(frame, schema) + check_known_nulls(frame, schema, dynamic) + if dynamic: + _check('nulls' in schema, 'a dynamic snapshot must declare a `nulls:` placement rule (#121)') + check_placement(frame, schema, matches) + derived = derive_date_range(frame, schema, matches) + check_date_range(derived, schema, dynamic) + summary = { + 'dataset': manifest.get('filename'), + 'rows': int(frame.shape[0]), + 'columns': int(frame.shape[1]), + 'date_range': derived, + 'overlap': None, + } + if previous is not None: + summary['overlap'] = overlap(frame, previous, schema, matches) + return summary diff --git a/builders/business_cycle.py b/builders/business_cycle.py index be8175b..f6df1e7 100644 --- a/builders/business_cycle.py +++ b/builders/business_cycle.py @@ -32,7 +32,8 @@ unemployment begin in 1971 and 1970), GDP growth is undefined in 1960 for everyone, and the newest year may not be published yet for every series. So the rule is structural, not a count: a null is allowed only BEFORE an -economy's first observation or in the newest MAX_TRAILING_YEARS, never inside +economy's first observation or in the newest years the manifest's `nulls.recent` +allows, never inside the series. A gap opening mid-series fails the refresh. Two provenance dumps (the GDP series' metadata, where the CC BY-4.0 licence @@ -50,6 +51,9 @@ import sys import pandas as pd +import yaml + +from _validate import ValidationError, validate as validate_schema import wbgapi as wb CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -62,7 +66,7 @@ FIRST_YEAR = 1960 YEAR_COL = re.compile(r'^YR(\d{4})$') MAX_STALENESS_YEARS = 2 -MAX_TRAILING_YEARS = 2 # the newest years may be unpublished for a series +# (the newest-years allowance is `nulls.recent` in each manifest, read by _validate) # One entry per published file. `economies` is the lecture's selection (the # union of every call that reads the series); `band` is the unit sanity check @@ -83,8 +87,9 @@ ] -class ValidationError(Exception): - """The fetched data broke the published contract -- exit code 2.""" +def _manifest(file): + with open(os.path.join(PUBLISHED_DIR, file + '.yml')) as f: + return yaml.safe_load(f) def _check(condition, message): @@ -115,68 +120,49 @@ def _years(frame): def validate(table, frame, previous=None): + """Two layers. The manifest's schema block is the spec for columns, dtypes, + exact and placed nulls, the row floor, date_range and the overlap window + (builders/_validate.py, #119); this function adds only what a schema + cannot say -- the unbroken year grid, the fixed economy set, the value + band, recency, GDP growth's first-year rule, and the revision BOUND.""" name = table['file'] - _check(list(frame.columns[:1]) == ['Country'], f'{name}: first columns {list(frame.columns[:3])}') + _check(frame.index.name == 'economy', f'{name}: index is {frame.index.name!r}') + raw = frame.reset_index() # the RAW shape: economy is a column + prev_raw = previous.reset_index() if previous is not None else None + try: + shared = validate_schema(raw, _manifest(name), prev_raw) + except ValidationError as exc: + raise ValidationError(f'{name}: {exc}') from None + years = _years(frame) - _check(len(years) == len(frame.columns) - 1, f'{name}: non-year column present') - _check(years[0] == FIRST_YEAR, f'{name}: first year {years[0]}') _check(years == list(range(FIRST_YEAR, years[-1] + 1)), f'{name}: gap in the year grid') year_cols = [f'YR{y}' for y in years] - _check(frame.index.name == 'economy', f'{name}: index is {frame.index.name!r}') _check(sorted(frame.index) == sorted(table['economies']), f'{name}: economies {sorted(frame.index)}') - _check(frame['Country'].notnull().all(), f'{name}: a Country label is missing') values = frame[year_cols] - _check(all(pd.api.types.is_float_dtype(values[c]) for c in year_cols), f'{name}: non-float year column') _check(values.abs().max().max() >= table['min_abs_max'], f'{name}: values look like ratios') lo, hi = table['band'] _check(values.stack().between(lo, hi).all(), f'{name}: value out of band [{lo}, {hi}]') _check(years[-1] >= dt.date.today().year - MAX_STALENESS_YEARS, f'{name}: newest year is {years[-1]}') - - # Nulls: only before an economy's first observation, or in the newest - # MAX_TRAILING_YEARS; never inside the series. GDP growth additionally has - # its first year empty for everyone. - trailing = set(year_cols[-MAX_TRAILING_YEARS:]) - for econ in frame.index: - row = values.loc[econ] - first = row.first_valid_index() - _check(first is not None, f'{name}: {econ} has no data at all') - inner = row.loc[first:] - bad = [c for c in inner.index if pd.isnull(inner[c]) and c not in trailing] - _check(not bad, f'{name}: {econ} has a gap inside its series at {bad[:3]}') - if table['first_year_null']: - _check(pd.isnull(row[year_cols[0]]), f'{name}: {econ} has a value in {year_cols[0]}') + if table['first_year_null']: + _check(values[year_cols[0]].isnull().all(), f'{name}: a value in {year_cols[0]} -- growth is undefined in the first year') summary = { 'dataset': name, 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), - 'rows': int(frame.shape[0]), - 'columns': int(frame.shape[1]), - 'date_range': {'start': years[0], 'end': years[-1]}, + 'rows': shared['rows'], + 'columns': shared['columns'], + 'date_range': shared['date_range'], # ints, from the pattern's capture group 'overlap': None, } if previous is not None: - prev_years = [f'YR{y}' for y in _years(previous)] - _check(set(prev_years) <= set(year_cols), f'{name}: a year column disappeared') - common = [e for e in previous.index if e in frame.index] - _check(common, f'{name}: no economy in common with the previous snapshot') - old = previous.loc[common, prev_years] - new = frame.loc[common, prev_years] - _check(not (old.notnull() & new.isnull()).any().any(), f'{name}: a populated cell went empty') - diff = (old - new).abs() - changed = int((diff > 1e-9).sum().sum()) - worst = float(diff.max().max()) if diff.notnull().any().any() else 0.0 - summary['overlap'] = { - 'window': f'{prev_years[0]}..{prev_years[-1]}', - 'previous_end': _years(previous)[-1], - 'cells_total': int(old.notnull().sum().sum()), - 'cells_revised': changed, - 'max_abs_change': round(worst, 4), - 'new_columns': sorted(set(year_cols) - set(prev_years)), - 'new_economies': sorted(set(frame.index) - set(previous.index)), - } - print(f'{name}: overlap {prev_years[0]}..{prev_years[-1]} over {common}: {changed} cells revised, ' - f'max |change| {worst:.3f}; new columns {summary["overlap"]["new_columns"] or "none"}; ' - f'new economies {summary["overlap"]["new_economies"] or "none"}') + ov = shared['overlap'] + ov['previous_end'] = _years(previous)[-1] # the int year the refresh PR title uses + ov['new_economies'] = sorted(set(frame.index) - set(previous.index)) + summary['overlap'] = ov + worst = ov['max_abs_change'] + print(f'{name}: overlap {ov["window"]}: {ov["cells_revised"]} of {ov["cells_total"]} cells revised, ' + f'max |change| {worst:.3f}; new columns {ov["new_columns"] or "none"}; ' + f'new economies {ov["new_economies"] or "none"}') _check(worst <= table['max_revision'], f'{name}: revision of {worst:.3f} exceeds {table["max_revision"]}') return summary diff --git a/builders/business_cycle_fred.py b/builders/business_cycle_fred.py index 218d78d..c346e6d 100644 --- a/builders/business_cycle_fred.py +++ b/builders/business_cycle_fred.py @@ -45,8 +45,10 @@ import sys import pandas as pd +import yaml from _fred import Fred +from _validate import ValidationError, validate as validate_schema CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.dirname(CURRENT_FILE_DIR) @@ -60,12 +62,10 @@ # is a different series. FIRST_OBS = {'UNRATE': '1948-01-01', 'USREC': START, 'UMCSENT': '1952-11-01', 'CPILFESL': '1957-01-01', 'INDPRO': START, 'M0892AUSM156SNBR': '1929-04-01'} -# Where each series becomes gap-free monthly. UMCSENT is quarterly/irregular -# before 1978; the historical unemployment series ends in 1942-06. -MONTHLY_FROM = {**{c: FIRST_OBS[c] for c in COLUMNS}, 'UMCSENT': '1978-01-01'} -LAST_OBS = {'M0892AUSM156SNBR': '1942-06-01'} -# Holes inside a series' monthly span that are known and accepted. -KNOWN_HOLES = {'UNRATE': ['2025-10-01'], 'CPILFESL': ['2025-10-01']} +# Where nulls may sit -- UMCSENT sparse before 1978-01, the historical series +# ended 1942-06, the 2025-10 shutdown hole in UNRATE and CPILFESL -- is the +# manifest's `nulls:` block (lectures/us_business_cycle_monthly.csv.yml), read +# by the shared validator; it is not repeated here (#121). # Value bands: percent, 0/1, index levels. BANDS = {'UNRATE': (0, 30), 'USREC': (0, 1), 'UMCSENT': (20, 150), 'CPILFESL': (20, 1000), 'INDPRO': (1, 300), 'M0892AUSM156SNBR': (0, 40)} @@ -75,10 +75,6 @@ MAX_STALENESS_MONTHS = 3 -class ValidationError(Exception): - """The fetched data broke the published contract -- exit code 2.""" - - def _check(condition, message): if not condition: raise ValidationError(message) @@ -102,28 +98,30 @@ def _months(index): def validate(frame, previous=None): - _check(list(frame.columns) == COLUMNS, f'columns {list(frame.columns)}') + """Two layers. The manifest's schema block (builders/_validate.py, #119) + checks the columns, dtype families, the exact null counts, the placement + of every null -- leading, the ended historical series, the declared holes, + UMCSENT's sparse early years -- the row floor and the date range; this + function adds what a schema cannot say: the first-of-month monthly grid, + each series' exact first observation, recency, value bands, USREC being + 0/1 and complete, and the per-series revision BOUND.""" _check(frame.index.name == 'DATE', f'index is {frame.index.name!r}') + raw = frame.reset_index() # the RAW shape: DATE is a column + prev_raw = previous.reset_index() if previous is not None else None + with open(os.path.join(PUBLISHED_DIR, OUT_FILE + '.yml')) as f: + manifest = yaml.safe_load(f) + shared = validate_schema(raw, manifest, prev_raw) + _check(frame.index[0] == pd.Timestamp(START), f'starts {frame.index[0].date()}') _check(frame.index.is_monotonic_increasing and (frame.index.day == 1).all(), 'not first-of-month') _check((_months(frame.index).diff().dropna() == 1).all(), 'gap in the monthly grid') today = dt.date.today() age = (today.year * 12 + today.month) - (frame.index[-1].year * 12 + frame.index[-1].month) _check(age <= MAX_STALENESS_MONTHS, f'newest month is {frame.index[-1].date()}') - for col in COLUMNS: s = frame[col] - first = pd.Timestamp(FIRST_OBS[col]) - _check(s.loc[:first - pd.offsets.MonthBegin(1)].isnull().all() if first > frame.index[0] else True, - f'{col}: observed before its first observation {first.date()}') - _check(s.first_valid_index() == first, f'{col}: first observation is {s.first_valid_index()}, not {first.date()}') - monthly_from = pd.Timestamp(MONTHLY_FROM[col]) - last = pd.Timestamp(LAST_OBS.get(col, frame.index[-1])) - span = s.loc[monthly_from:last] - holes = [str(d.date()) for d in span[span.isnull()].index] - _check(holes == KNOWN_HOLES.get(col, []), f'{col}: holes {holes} != known {KNOWN_HOLES.get(col, [])}') - if col in LAST_OBS: - _check(s.loc[last + pd.offsets.MonthBegin(1):].isnull().all(), f'{col}: observed after {last.date()}') + _check(s.first_valid_index() == pd.Timestamp(FIRST_OBS[col]), + f'{col}: first observation is {s.first_valid_index()}, not {FIRST_OBS[col]}') lo, hi = BANDS[col] _check(s.dropna().between(lo, hi).all(), f'{col}: out of band [{lo}, {hi}]') _check(frame['USREC'].notnull().all(), 'USREC has a missing month') @@ -132,30 +130,17 @@ def validate(frame, previous=None): summary = { 'dataset': OUT_FILE, 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), - 'rows': int(frame.shape[0]), - 'columns': int(frame.shape[1]), - 'date_range': {'start': str(frame.index[0].date()), 'end': str(frame.index[-1].date())}, + 'rows': shared['rows'], + 'columns': shared['columns'], + 'date_range': shared['date_range'], # ISO strings, from the DATE column 'overlap': None, } if previous is not None: - _check(list(previous.columns) == COLUMNS, 'previous snapshot has different columns') - common = previous.index.intersection(frame.index) - old, new = previous.loc[common], frame.loc[common] - _check(not (old.notnull() & new.isnull()).any().any(), 'a populated cell went empty') - diff = (old - new).abs() - worst = {c: float(diff[c].max()) if diff[c].notnull().any() else 0.0 for c in COLUMNS} - changed = int((diff > 1e-9).sum().sum()) - summary['overlap'] = { - 'window': f'{common[0].date()}..{common[-1].date()}', - 'previous_end': str(previous.index[-1].date()), - 'cells_total': int(old.notnull().sum().sum()), - 'cells_revised': changed, - 'max_abs_change': round(max(worst.values()), 4), - 'max_abs_change_by_series': {c: round(v, 4) for c, v in worst.items()}, - 'new_columns': [], - } - print(f'overlap window {common[0].date()}..{common[-1].date()}: {changed} cells revised; ' - f'max |change| by series {summary["overlap"]["max_abs_change_by_series"]}') + ov = shared['overlap'] + summary['overlap'] = ov + worst = ov['max_abs_change_by_series'] + print(f'overlap window {ov["window"]}: {ov["cells_revised"]} cells revised; ' + f'max |change| by series {worst}') for c in COLUMNS: _check(worst[c] <= MAX_REVISION[c], f'{c}: revision {worst[c]:.4f} exceeds {MAX_REVISION[c]}') return summary diff --git a/lectures/countries.csv.yml b/lectures/countries.csv.yml index d1e5204..d2e5c54 100644 --- a/lectures/countries.csv.yml +++ b/lectures/countries.csv.yml @@ -79,6 +79,12 @@ schema: # date_range omitted: country-attribute table, no time dimension. known_nulls: Capital: 248 # Capital column is empty throughout this export + Country code: 1 # NOT a null in the bytes: Namibia's ISO alpha-2 + # code is the string "NA", which a default + # pd.read_csv parses as missing. The schema + # describes a default read (#122), so it is + # declared; a lecture that needs Namibia's code + # must pass keep_default_na=False. Country (local): 3 Currency: 1 Currency code: 1 diff --git a/lectures/us_business_cycle_monthly.csv.yml b/lectures/us_business_cycle_monthly.csv.yml index 44729cb..3c7aa8b 100644 --- a/lectures/us_business_cycle_monthly.csv.yml +++ b/lectures/us_business_cycle_monthly.csv.yml @@ -112,22 +112,27 @@ schema: date_range: {start: 1919-01-01, end: 2026-07-01} # Every null is structural — before a series' first observation, in # UMCSENT's sparse pre-1978 years, after the historical series' last month, - # or the 2025-10 shutdown hole in UNRATE and CPILFESL. Three counts are - # stable (leading nulls plus the one hole) and are declared exact; - # M0892AUSM156SNBR's grows by one per month because the series ended in - # 1942-06, so it is covered by the placement rule alone (#121). - known_nulls: {UNRATE: 349, UMCSENT: 616, CPILFESL: 457} + # the 2025-10 shutdown hole in UNRATE and CPILFESL, or the newest month for + # a series that publishes with a lag. No exact counts: every one of them + # moves with the newest month (M0892AUSM156SNBR gains a trailing null every + # month; UMCSENT, CPILFESL and INDPRO gain one whenever the refresh runs + # before their mid-month release), so the contract is PLACEMENT (#121). + # First-vintage totals for the reader: UNRATE 349, UMCSENT 616, CPILFESL + # 457, M0892AUSM156SNBR 1132. + known_nulls: {} # Placement rule (#121, 2026-09-07): series run down the DATE index, one per # column. `leading`: empty before a series' first observation; `ended`: # empty after its last; `inner`: the accepted holes inside a series, as a # list of periods or `{sparse_until: P}` for an irregular early history; - # `recent: 0`: the newest month must be populated for every live series - # (the builder requires it). Mirrors FIRST_OBS / LAST_OBS / KNOWN_HOLES / - # MONTHLY_FROM in builders/business_cycle_fred.py. + # `recent: 1`: the newest month may be unpublished for a lagging series — + # UNRATE lands the first Friday, CPILFESL and INDPRO around the 15th, + # UMCSENT's final at month end — measured 2026-09-07, when the live frame + # ran to 2026-08 with only UNRATE and USREC populated there. Two empty + # months would mean a stalled series and fail. Read by builders/_validate.py. nulls: along: rows leading: true - recent: 0 + recent: 1 ended: [M0892AUSM156SNBR] inner: UNRATE: [2025-10-01] diff --git a/manifest-schema.yml b/manifest-schema.yml index d6bc527..99aa22f 100644 --- a/manifest-schema.yml +++ b/manifest-schema.yml @@ -20,7 +20,11 @@ # which doubles as the public dataset registry. # # COMPLETENESS: this file documents the fields that recur across the corpus. It -# is not a closed schema and nothing validates against it — a dataset whose +# is not a closed schema, but since 2026-09-07 the `schema` block IS validated: +# scripts/validate_datasets.py (run by .github/workflows/validate-datasets.yml +# on every PR) checks every manifest's conformance to the rules recorded below +# and, for CSVs, that the committed bytes satisfy their own schema block, via +# builders/_validate.py (#119). Provenance fields stay descriptive — a dataset whose # provenance needs a field not shown here should add one rather than distort # itself to fit, and a field that earns its place in several manifests should # then be documented back here. Two such extensions are already in wide use and diff --git a/scripts/validate_datasets.py b/scripts/validate_datasets.py new file mode 100644 index 0000000..daa56d9 --- /dev/null +++ b/scripts/validate_datasets.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +PR validation (PLAN Phase 5; QuantEcon/data-lectures#119): every manifest +conforms to manifest-schema.yml's executable rules, and every CSV's committed +bytes satisfy its own schema block, via the same builders/_validate.py the +dynamic builders run. + + python scripts/validate_datasets.py # everything under lectures/ + python scripts/validate_datasets.py lectures/gdp_growth_annual.csv.yml ... + +Two passes per manifest: + + conformance (all formats) class in the enum; every column entry has name + XOR pattern; a pattern compiles and has exactly + one capture group; dtype in the canonical set + (#122); known_nulls values are integers, never a + {max: N} ceiling (#121); a dynamic snapshot has + a `nulls:` block with known keys; known_nulls_total + appears only inside a `header: null` sheet entry + bytes (csv only) read_raw() + validate() with no previous vintage + +Formats other than csv get the conformance pass only and say so -- reading +xlsx/dta/npy/json ranges the way each lecture does is not built yet. + +Exit 1 if anything fails; every failure is a ::error annotation naming the +manifest, so it reads in the PR's checks tab. +""" +from __future__ import annotations + +import pathlib +import re +import sys + +import yaml + +REPO = pathlib.Path(__file__).resolve().parents[1] +LECTURES = REPO / 'lectures' +sys.path.insert(0, str(REPO / 'builders')) +from _validate import CANONICAL_DTYPES, ValidationError, read_raw, validate # noqa: E402 + +CLASSES = {'verbatim', 'constructed', 'dynamic-snapshot'} +NULLS_KEYS = {'along', 'leading', 'recent', 'ended', 'inner'} + + +def conformance(m: dict, path: pathlib.Path) -> list[str]: + errs = [] + if m.get('class') not in CLASSES: + errs.append(f'class {m.get("class")!r} not in {sorted(CLASSES)}') + if m.get('filename') != path.name[:-4]: + errs.append(f'filename {m.get("filename")!r} does not match sidecar name') + schema = m.get('schema') or {} + + def check_columns(cols, where): + for k, c in enumerate(cols or []): + has_name, has_pat = 'name' in c, 'pattern' in c + if has_name == has_pat and 'index' not in c: + errs.append(f'{where}[{k}]: needs exactly one of name / pattern') + if has_pat: + try: + rx = re.compile(c['pattern']) + if rx.groups != 1: + errs.append(f'{where}[{k}]: pattern {c["pattern"]!r} must have exactly one capture group (#120)') + except re.error as e: + errs.append(f'{where}[{k}]: pattern does not compile: {e}') + d = c.get('dtype') + if d is not None and str(d) not in CANONICAL_DTYPES: + errs.append(f'{where}[{k}] ({c.get("name") or c.get("pattern")}): dtype {d!r} not canonical (#122: {sorted(CANONICAL_DTYPES)})') + + check_columns(schema.get('columns'), 'schema.columns') + for t in schema.get('tables') or []: + check_columns(t.get('columns'), f'schema.tables[{t.get("name")}].columns') + for k, s in enumerate(schema.get('sheets') or []): + check_columns(s.get('columns'), f'schema.sheets[{k}].columns') + if 'known_nulls_total' in s and (s.get('read_as') or {}).get('header', 0) is not None: + errs.append(f'schema.sheets[{k}]: known_nulls_total is legal only with read_as.header: null (#121)') + if 'known_nulls_total' in schema: + errs.append('schema.known_nulls_total: retired for named columns -- use known_nulls (exact) and nulls (placement) (#121)') + for col, n in (schema.get('known_nulls') or {}).items(): + if not isinstance(n, int) or isinstance(n, bool): + errs.append(f'known_nulls[{col!r}] = {n!r}: must be an exact integer; there is no ceiling form (#121)') + if m.get('class') == 'dynamic-snapshot': + rule = schema.get('nulls') + if not isinstance(rule, dict): + errs.append('dynamic-snapshot without a `nulls:` placement block (#121)') + else: + bad = set(rule) - NULLS_KEYS + if bad: + errs.append(f'nulls: unknown key(s) {sorted(bad)}; allowed {sorted(NULLS_KEYS)}') + if rule.get('along') not in ('rows', 'columns'): + errs.append(f'nulls.along must be rows | columns, got {rule.get("along")!r}') + return errs + + +def main(argv: list[str]) -> int: + paths = [pathlib.Path(a) for a in argv] or sorted(LECTURES.glob('*.yml')) + failed = 0 + skipped_formats: dict[str, int] = {} + for path in paths: + m = yaml.safe_load(path.read_text(encoding='utf-8')) or {} + if 'filename' not in m: + continue + problems = conformance(m, path) + fmt = (m.get('schema') or {}).get('format') + status = 'conformance only' + if not problems and fmt == 'csv': + data = LECTURES / m['filename'] + if not data.exists(): + problems.append(f'{m["filename"]} not found beside its manifest') + else: + try: + s = validate(read_raw(data, m), m) + status = f'ok rows={s["rows"]} cols={s["columns"]} range={s["date_range"]["start"]}..{s["date_range"]["end"]}' + except ValidationError as e: + problems.append(str(e)) + except Exception as e: # a read failure is a failure of the contract too + problems.append(f'{type(e).__name__}: {e}') + elif not problems: + skipped_formats[fmt] = skipped_formats.get(fmt, 0) + 1 + status = f'conformance only ({fmt})' + if problems: + failed += 1 + for p in problems: + print(f'::error file={path.relative_to(REPO)}::{p}') + print(f'FAIL {path.name}: {len(problems)} problem(s)') + else: + print(f'{status:70s} {path.name}') + n = len(paths) + print(f'\n{n} manifest(s): {n - failed} pass, {failed} fail; bytes-validated formats: csv; ' + f'conformance-only: {skipped_formats}') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) From be76a6a308331372687ae7ac3ac17c5e412340c4 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Mon, 7 Sep 2026 13:45:37 +1000 Subject: [PATCH 2/2] Copilot review: label columns of a dynamic snapshot must have no undeclared nulls The placement rule governs a snapshot's series; economy, Country and DATE are labels, and the old business_cycle.py assertion on Country was dropped in the refactor. Now enforced in the shared validator for every dynamic file rather than restored per builder. Three new mutations caught. Co-Authored-By: Claude Fable 5.1 --- builders/_validate.py | 38 ++++++++++++++++++++++++++++---------- manifest-schema.yml | 3 +++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/builders/_validate.py b/builders/_validate.py index 130ac62..08af64a 100644 --- a/builders/_validate.py +++ b/builders/_validate.py @@ -24,7 +24,10 @@ as object with bool values matches `bool`; the declared name must be in the canonical vocabulary (checked by the CLI's conformance pass, not here) - known_nulls #121 -- an integer is an EXACT count + known_nulls #121 -- an integer is an EXACT count; a column with no entry + has none -- every column of a frozen file, and the LABEL + columns (economy, Country, DATE) of a dynamic snapshot, whose + placement rule governs only its series nulls #121 -- placement for dynamic snapshots: `along` (rows | columns) says which way a series runs; `leading` allows nulls before a series' first observation; `ended` lists series that @@ -208,15 +211,30 @@ def _series_view(frame: pd.DataFrame, schema: dict, matches) -> tuple[pd.DataFra return table, along -def check_known_nulls(frame: pd.DataFrame, schema: dict, dynamic: bool = False): +def series_columns(frame: pd.DataFrame, schema: dict, matches) -> set: + """The columns the `nulls:` placement rule governs: the pattern-claimed + run of a wide file, or every non-period column of a time-indexed one. + Everything else is a LABEL column (economy, Country, DATE) whose nulls + are exact -- zero unless declared -- in every class.""" + if not (schema.get('nulls') or {}) and not any('pattern' in e for e, _ in matches): + return set() + along = (schema.get('nulls') or {}).get('along') or 'columns' + if along == 'columns': + return {c for e, cols in matches if 'pattern' in e for c in cols} + return set(frame.columns) - set(datetime_columns(schema)) + + +def check_known_nulls(frame: pd.DataFrame, schema: dict, dynamic: bool = False, series: set = frozenset()): known = schema.get('known_nulls') or {} - if not dynamic: - # A frozen file declares every nulled column; one it does not declare - # must have none (the template's `known.get(col, 0)` rule, #121). - for col in frame.columns: - if col not in known: - have = int(frame[col].isnull().sum()) - _check(have == 0, f'{col}: {have} nulls but not declared under known_nulls') + # A column with no declared count must have none: every column of a + # frozen file (the template's `known.get(col, 0)` rule, #121), and the + # LABEL columns of a dynamic snapshot -- the placement rule governs only + # its series, so a blank economy, Country or DATE would otherwise pass. + for col in frame.columns: + if col in known or (dynamic and col in series): + continue + have = int(frame[col].isnull().sum()) + _check(have == 0, f'{col}: {have} nulls but not declared under known_nulls') for col, n in known.items(): _check(col in frame.columns, f'known_nulls names {col!r}, not a column') _check(isinstance(n, int) and not isinstance(n, bool), @@ -349,7 +367,7 @@ def validate(frame: pd.DataFrame, manifest: dict, previous: pd.DataFrame | None _check(dtype_matches(frame[c], want), f'{c}: dtype {frame[c].dtype} is not in the {want!r} family') check_rows(frame, schema) - check_known_nulls(frame, schema, dynamic) + check_known_nulls(frame, schema, dynamic, series_columns(frame, schema, matches)) if dynamic: _check('nulls' in schema, 'a dynamic snapshot must declare a `nulls:` placement rule (#121)') check_placement(frame, schema, matches) diff --git a/manifest-schema.yml b/manifest-schema.yml index 99aa22f..791d988 100644 --- a/manifest-schema.yml +++ b/manifest-schema.yml @@ -242,6 +242,9 @@ schema: # blanket no-nulls rule would reject). A dynamic snapshot may also declare # an exact count for a column whose nulls are stable by construction # (gdp_growth_annual's YR1960: growth is undefined in the first year). + # A column with no entry must have no nulls -- every column of a frozen + # file, and the LABEL columns of a dynamic snapshot (economy, Country, DATE: + # the placement rule below governs only its series). # There is deliberately NO ceiling form ({max: N}): a ceiling silently # accepts a column emptying out below the bound, which is the failure the # check exists to catch.