From a0ba3e10418dad025a1f9e3931b7a4e2c2510210 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 21 Jun 2026 09:29:07 -0700 Subject: [PATCH 01/85] feat(#218): canonical Panama food_acquired + sample; source-flag s-axis Phase 3 of GH #218. Rewrite Panama food_acquired to the canonical long form (index (t,i,j,u,s); columns [Quantity, Expenditure, Price]; materialize: make) and register it, add a minimal sample table so v joins at API time, and drop the dead derived script. Derived food_{expenditures,prices,quantities} now auto-surface via the framework's _FOOD_DERIVED. i/j swap fixed: legacy mapped form/hogar(household)->j and item->i; the new wave scripts map household->i, item->j. Country('Panama').food_acquired() now gives n_i=8634 households >> n_j=81 items. s-axis driven by the SOURCE-FLAG columns, not the obtained-quantity bucket (fixes the round-1 refutation). Per wave the non-purchased 15-day quantity (ga110a / gai10a / s11a10a) is split by the survey's yes/no source flags: - own production (ga111a / gai111 / s11a11a) -> s='produced' - gift / donation (ga111b / gai112 / s11a11b) -> s='inkind' - part-of-payment + business (ga111c/d, gai113/114, s11a11c/d) -> s='other' The MONTHLY recall bucket (ga109a / gai09a / s11a9a) is deliberately NOT used: it overlaps ~100% with the 15-day bucket and summing both double-counts. Purchased Expenditure reconciles to raw per wave (sentinel-masked): - 1997 356079.56 == raw ga106c<9999 356079.56 (exact) - 2003 396676.87 == raw gai06c minus the open-ended "otro" catch-all slots (which cannot map to a named item) 396676.88 (exact on the harmonizable universe; full raw 397466.44) - 2008 547159.13 == raw s11a6c<99999 547159.13 (exact) sample table (index (i,t), v=upm formatted via panama.py's v()): 1997 from INFORM.DTA (v only -- no weight var), 2003 from E03BASE.DTA (the food file lacks upm) with weight=factor, 2008 from 02hogar.dta with weight=factor. food_items.org: add the 2003 spelling for Condimentos (was '---', a gap; the named item now harmonizes, closing the 2003 reconciliation). Verified (LSMS_COUNTRIES_ROOT=worktree, LSMS_NO_CACHE=1): food_acquired index (t,v,i,j,u,s) with v joined; is_this_feature_sane().ok True (only the framework-joined-v warn); derived tables non-empty; Panama present (modal, not excluded) in the full Feature('food_acquired')() (18 countries) and in Feature('food_acquired')(['Panama','Uganda']). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../countries/Panama/1997/_/data_info.yml | 13 + .../countries/Panama/1997/_/food_acquired.py | 226 ++++++++++++++---- .../countries/Panama/2003/_/data_info.yml | 15 ++ .../countries/Panama/2003/_/food_acquired.py | 208 +++++++++++++--- .../countries/Panama/2008/_/data_info.yml | 14 ++ .../countries/Panama/2008/_/food_acquired.py | 205 ++++++++++++---- lsms_library/countries/Panama/_/Makefile | 15 +- .../countries/Panama/_/data_scheme.yml | 19 ++ .../countries/Panama/_/food_acquired.py | 60 +++-- .../countries/Panama/_/food_items.org | 2 +- ...food_prices_quantities_and_expenditures.py | 26 -- lsms_library/countries/Panama/_/panama.py | 14 +- 12 files changed, 637 insertions(+), 180 deletions(-) create mode 100644 lsms_library/countries/Panama/1997/_/data_info.yml create mode 100644 lsms_library/countries/Panama/2003/_/data_info.yml create mode 100644 lsms_library/countries/Panama/2008/_/data_info.yml create mode 100644 lsms_library/countries/Panama/_/data_scheme.yml delete mode 100644 lsms_library/countries/Panama/_/food_prices_quantities_and_expenditures.py diff --git a/lsms_library/countries/Panama/1997/_/data_info.yml b/lsms_library/countries/Panama/1997/_/data_info.yml new file mode 100644 index 000000000..ddcb793cd --- /dev/null +++ b/lsms_library/countries/Panama/1997/_/data_info.yml @@ -0,0 +1,13 @@ +Wave: '1997' +Country: Panama + +# Minimal sample table: maps each household to its primary sampling unit (upm), +# so the framework can join `v` onto food_acquired at API time. INFORM.DTA is +# the 1997 household cover file (one row per household, `form`), and carries the +# `upm` cluster id. No household expansion weight is present in this wave. +sample: + file: ../Data/INFORM.DTA + idxvars: + i: form + myvars: + v: upm # formatted by panama.py's v() (format_id wrapper) diff --git a/lsms_library/countries/Panama/1997/_/food_acquired.py b/lsms_library/countries/Panama/1997/_/food_acquired.py index 4fcaf05cb..1700618fb 100644 --- a/lsms_library/countries/Panama/1997/_/food_acquired.py +++ b/lsms_library/countries/Panama/1997/_/food_acquired.py @@ -1,63 +1,197 @@ #!/usr/bin/env python - +"""Panama 1997 food_acquired -> canonical long form. + +Index : (t, i, j, u, s) -- NO `v` (joined from sample() at API time) +Columns : [Quantity, Expenditure, Price] +i = household (form), j = harmonized food item, u = native unit, +s in {purchased, produced, inkind, other} (transformations.S_VALUES). + +Source: 1997/Data/GAST-A.DTA (consumption module GA, one row per +household x food item). The questionnaire records, per item: + + Purchases (last 15 days): + ga106a = quantity bought -> Quantity (s='purchased') + ga106b = unit code -> u + ga106c = total paid -> Expenditure (s='purchased') + + Acquisition WITHOUT purchase (last 15 days): + ga110a = quantity obtained -> Quantity (non-purchased) + ga110b = unit code -> u + The SOURCE of that non-purchased quantity is recorded in yes/no flags: + ga111a = OBT PROPI (own production) -> s='produced' + ga111b = OBT DONAC (gift / donation) -> s='inkind' + ga111c = OBT NEGO (own business) -> s='other' + ga111d = (unlabeled, parallels 2008 negocio)-> s='other' + A given obtained item may carry more than one flag; we emit one canonical + row per flagged source (the obtained quantity attributed to that source). + +NOTE on recall periods (the refuting defect from round 1): + ga109a ('NOR SINCO', unit ga109b='UNIDA MES') is the MONTHLY recall of the + same home-consumption quantity and OVERLAPS ~100% with ga110a (the 15-day + recall). Summing both double-counts. We use ONLY ga110a (15 days) so the + non-purchased quantity is on the SAME recall basis as the purchased ga106a. + ga110a is NOT 'produced' wholesale: it is total non-purchased acquisition; + the produced/inkind/other split is driven by the ga111* source flags. +""" import sys + sys.path.append('../../_/') -import pandas as pd -import numpy as np -import json sys.path.append('../../../_/') -from lsms_library.local_tools import to_parquet, df_from_orgfile, format_id, get_dataframe - -t = '1997' - -# Original used pyreadstat with default kwargs (no value-format application). -df = get_dataframe('../Data/GAST-A.DTA', convert_categoricals=False) -with open('../../_/units.json','r') as f: - unit_conversions = json.load(f) - -units = pd.DataFrame(unit_conversions["units"]) -df = df.loc[:, ['form','ga100', 'ga106a', 'ga106b', 'ga106c', 'ga110a', 'ga110b']] -df = df.rename({'form': 'j', 'ga100':'i', 'ga106a':'quantity bought', 'ga106b':'unitcode (bought)', 'ga106c':'total spent', 'ga110a':'quantity obtained', 'ga110b':'unitcode (obtained)'}, axis=1) -df = df.mask(df > 1e99).fillna(0) - -food_items = df_from_orgfile('../../_/food_items.org') -food_items = food_items.loc[:, ['Preferred Label', t]] +import json -if food_items[t].dtypes==object: - food_items[t] = food_items[t].str.strip() +import numpy as np +import pandas as pd -food_items = food_items.replace(['','---'],pd.NA).dropna() -food_items = food_items.set_index(t).dropna() -food_items = food_items.squeeze().str.strip().to_dict() +from lsms_library.local_tools import (df_from_orgfile, format_id, + get_dataframe, to_parquet) -unit_dict = units.loc[:, ["Unitcode", "Translation"]].set_index("Unitcode").to_dict()["Translation"] +t = '1997' -df['i'] = df['i'].replace(food_items) -df['j'] = df['j'].apply(format_id) -df = df.set_index(['j', 'i']) +# --- source ---------------------------------------------------------------- +df = get_dataframe('../Data/GAST-A.DTA', convert_categoricals=False) -df['unitcode (bought)'], df['unitcode (obtained)'] = df['unitcode (bought)'].astype(int).map(unit_dict), df['unitcode (obtained)'].astype(int).map(unit_dict) +cols = ['form', 'ga100', + 'ga106a', 'ga106b', 'ga106c', # purchased qty / unit / value + 'ga110a', 'ga110b', # non-purchased qty / unit (15d) + 'ga111a', 'ga111b', 'ga111c', 'ga111d'] # source flags +df = df.loc[:, cols].copy() -pound_dict = units.loc[:, ["Translation", "Conversion to Pounds"]].set_index("Translation").to_dict()["Conversion to Pounds"] -poundmappingb = df["unitcode (bought)"].map(pound_dict).fillna(1) * df['quantity bought'] -poundmappingo = df["unitcode (obtained)"].map(pound_dict).fillna(1) * df['quantity obtained'] +# Sentinel / missing handling. In 1997 the amount fields use 9999.x (e.g. +# 9999.5, 9999.9, 9999.99) as the explicit "NR"/missing sentinel (and >= 1e99 +# is Stata-missing). Mask everything >= 9999 -- matches the legacy script's +# 9999-band convention and reconciles purchased value to the raw < 9999 total. +for c in ['ga106a', 'ga106c', 'ga110a']: + df[c] = pd.to_numeric(df[c], errors='coerce') + df.loc[df[c] >= 9999, c] = np.nan -df['unitcode (bought)'] = np.where(poundmappingb != df["quantity bought"], 'pound', df["unitcode (bought)"]) -df['unitcode (obtained)'] = np.where(poundmappingo != df["quantity obtained"], 'pound', df["unitcode (obtained)"]) +# --- unit decode: numeric code -> English label via units.json ------------- +with open('../../_/units.json', 'r') as f: + units = pd.DataFrame(json.load(f)['units']) +unit_dict = (units.loc[:, ['Unitcode', 'Translation']] + .set_index('Unitcode')['Translation'].to_dict()) -df['quantity bought'] = poundmappingb -df['quantity obtained'] = poundmappingo -# replace numbers near 9999 which indicate missing -tolerance = 1 -numeric_cols = df.select_dtypes(include=[np.number]) -df[numeric_cols.columns] = numeric_cols.where((numeric_cols < (9999 - tolerance)) | (numeric_cols > (9999 + tolerance)), np.nan) +def decode_unit(code): + try: + return unit_dict.get(int(code), pd.NA) + except (ValueError, TypeError): + return pd.NA -df['unitcode (bought)'] = df['unitcode (bought)'].astype(str) -df['unitcode (obtained)'] = df['unitcode (obtained)'].astype(str) -df['price per unit'] = df['total spent']/df['quantity bought'] -df['price per unit'] = df['price per unit'].where(np.isfinite(df['price per unit'])) +df['u_bought'] = df['ga106b'].apply(decode_unit) +df['u_obtained'] = df['ga110b'].apply(decode_unit) -to_parquet(df, "food_acquired.parquet") +# --- harmonize food item j ------------------------------------------------- +food_items = df_from_orgfile('../../_/food_items.org') +food_items = food_items.loc[:, ['Preferred Label', t]] +food_items[t] = food_items[t].astype(str).str.strip() +food_items = food_items.replace(['', '---', 'nan'], pd.NA).dropna() +# 1997 column holds numeric item codes; key the lookup on the integer code. +item_map = {} +for code, label in zip(food_items[t], food_items['Preferred Label']): + try: + item_map[int(float(code))] = str(label).strip() + except (ValueError, TypeError): + continue + + +def harmonize_item(code): + try: + return item_map.get(int(code), pd.NA) + except (ValueError, TypeError): + return pd.NA + + +df['i'] = df['form'].apply(format_id) +df['j'] = df['ga100'].apply(harmonize_item) + +# --- build canonical rows -------------------------------------------------- +records = [] + +# Purchased rows +buy = df.loc[df['ga106a'].notna() | df['ga106c'].notna(), + ['i', 'j', 'u_bought', 'ga106a', 'ga106c']].copy() +buy = buy.rename(columns={'u_bought': 'u', 'ga106a': 'Quantity', + 'ga106c': 'Expenditure'}) +buy['s'] = 'purchased' +buy['Price'] = np.nan +records.append(buy) + +# Non-purchased rows, split by the source flags. +# Flag value 1 = "Sí" (yes); 2 = "No"; >=9 / NaN = not applicable. +flag_to_s = {'ga111a': 'produced', # OBT PROPI (own production) + 'ga111b': 'inkind', # OBT DONAC (gift / donation) + 'ga111c': 'other', # OBT NEGO (own business) + 'ga111d': 'other'} # unlabeled (parallels 2008 negocio) + +obtained = df.loc[df['ga110a'].notna() & (df['ga110a'] > 0)].copy() +for flag, s in flag_to_s.items(): + sub = obtained.loc[obtained[flag] == 1, + ['i', 'j', 'u_obtained', 'ga110a']].copy() + if sub.empty: + continue + sub = sub.rename(columns={'u_obtained': 'u', 'ga110a': 'Quantity'}) + sub['Expenditure'] = np.nan + sub['Price'] = np.nan + sub['s'] = s + records.append(sub) + +# Obtained quantity with NO source flag set -> default to 'produced' +# (own consumption with unrecorded source; small residual). +flagset = pd.Series(False, index=obtained.index) +for flag in flag_to_s: + flagset = flagset | (obtained[flag] == 1) +unflagged = obtained.loc[~flagset, ['i', 'j', 'u_obtained', 'ga110a']].copy() +if not unflagged.empty: + unflagged = unflagged.rename(columns={'u_obtained': 'u', + 'ga110a': 'Quantity'}) + unflagged['Expenditure'] = np.nan + unflagged['Price'] = np.nan + unflagged['s'] = 'produced' + records.append(unflagged) + +out = pd.concat(records, ignore_index=True) +out['t'] = t + +# Drop rows whose food item did not harmonize. +out = out[out['j'].notna()] +out['u'] = out['u'].astype('string') + +# Coerce value columns to plain float64 (defensive against arrow-backed dtypes). +for c in ['Quantity', 'Expenditure', 'Price']: + out[c] = pd.to_numeric(out[c], errors='coerce').astype('float64') + +# Zero amounts are "reported nothing" -> treat as missing. +out.loc[out['Quantity'] == 0, 'Quantity'] = np.nan +out.loc[out['Expenditure'] == 0, 'Expenditure'] = np.nan + +# Value-only convention (food-acquired skill / GhanaLSS): a purchase with a +# monetary value but no physical quantity/unit is carried as u='Value' with +# Quantity = Expenditure, so the framework can still sum it. This is what was +# previously leaking as a null `u` on ~zero-quantity purchase rows. +value_only = (out['Quantity'].isna() & out['u'].isna() + & out['Expenditure'].notna()) +out.loc[value_only, 'u'] = 'Value' +out.loc[value_only, 'Quantity'] = out.loc[value_only, 'Expenditure'] + +# Drop rows that carry no amount at all. +out = out[out['Quantity'].notna() | out['Expenditure'].notna()] + +# Purchased Price = total spent / quantity bought (per native unit). +buy_mask = out['s'] == 'purchased' +with np.errstate(divide='ignore', invalid='ignore'): + out.loc[buy_mask, 'Price'] = (out.loc[buy_mask, 'Expenditure'] + / out.loc[buy_mask, 'Quantity']) +out.loc[~np.isfinite(out['Price']), 'Price'] = np.nan + +idx = ['t', 'i', 'j', 'u', 's'] + +# Collapse duplicate (t,i,j,u,s) keys (same item/unit/source reported twice); +# sum the quantities/expenditures, mean the per-unit price. +agg = {'Quantity': 'sum', 'Expenditure': 'sum', 'Price': 'mean'} +out = out.groupby(idx, dropna=False).agg(agg) +out = out.dropna(how='all') + +if __name__ == '__main__': + to_parquet(out, 'food_acquired.parquet') diff --git a/lsms_library/countries/Panama/2003/_/data_info.yml b/lsms_library/countries/Panama/2003/_/data_info.yml new file mode 100644 index 000000000..baac722cf --- /dev/null +++ b/lsms_library/countries/Panama/2003/_/data_info.yml @@ -0,0 +1,15 @@ +Wave: '2003' +Country: Panama + +# Minimal sample table: maps each household to its primary sampling unit (upm). +# The food file (E03GA10B.DTA) does NOT carry `upm`; the household cover file +# E03BASE.DTA does (one row per household, keyed on `form`), along with the +# expansion factor (`factor`). +sample: + file: ../Data/E03BASE.DTA + idxvars: + i: form + myvars: + v: upm # formatted by panama.py's v() (format_id wrapper) + weight: factor + panel_weight: factor diff --git a/lsms_library/countries/Panama/2003/_/food_acquired.py b/lsms_library/countries/Panama/2003/_/food_acquired.py index 2e38569a8..c84c186fd 100644 --- a/lsms_library/countries/Panama/2003/_/food_acquired.py +++ b/lsms_library/countries/Panama/2003/_/food_acquired.py @@ -1,52 +1,196 @@ +#!/usr/bin/env python +"""Panama 2003 food_acquired -> canonical long form. + +Index : (t, i, j, u, s) -- NO `v` (joined from sample() at API time) +Columns : [Quantity, Expenditure, Price] +i = household (form), j = harmonized food item, u = native unit, +s in {purchased, produced, inkind, other} (transformations.S_VALUES). + +Source: 2003/Data/E03GA10B.DTA (consumption module GAI, one row per +household x food item). Per item: + + Purchases (last 15 days): + gai06a = number of presentations bought + gai06b1 = size per presentation (conversion to the unit in gai06b2) + gai06b2 = unit (LITRO/GRAMO/LIBRA/...) -> u + gai06c = total paid -> Expenditure (s='purchased') + Quantity = gai06a * gai06b1 in the unit gai06b2. + + Acquisition WITHOUT purchase (last 15 days): + gai10a = number of presentations obtained + gai10b1 = size per presentation + gai10b2 = unit -> u + Quantity = gai10a * gai10b1. + The SOURCE of the non-purchased quantity is the yes/no flag block + (parallels 1997 ga111* and 2008 s11a11*): + gai111 = own production -> s='produced' + gai112 = gift / donation -> s='inkind' + gai113 = part of payment -> s='other' + gai114 = own business -> s='other' + One canonical row per flagged source. + +NOTE on recall periods (round-1 refuting defect): gai09a is the MONTHLY +recall of the same home-consumption quantity and OVERLAPS ~100% with the +15-day gai10a; summing both double-counts. We use ONLY gai10a so the +non-purchased quantity shares the 15-day recall basis of purchased gai06a. +""" import sys + sys.path.append('../../_/') -import pandas as pd -import numpy as np -import json sys.path.append('../../../_/') -from lsms_library.local_tools import to_parquet, df_from_orgfile, format_id, get_dataframe + +import numpy as np +import pandas as pd + +from lsms_library.local_tools import (df_from_orgfile, format_id, + get_dataframe, to_parquet) t = '2003' +# Read categorically so gai00 (item) and gai06b2/gai10b2 (unit text) decode. df = get_dataframe('../Data/E03GA10B.DTA', convert_categoricals=True) -columns_dict = {"form": "j", "gai00": "i", "gai06a": "quantity (bought, in original units)", "gai06b1": "conversionb", "gai06b2": "unitcode (bought)", - "gai06c": "total spent", "gai10a": "quantity (obtained, in original units)", "gai10b1": "conversiono", "gai10b2": "unitcode (obtained)"} +cols = ['form', 'gai00', + 'gai06a', 'gai06b1', 'gai06b2', 'gai06c', # purchased + 'gai10a', 'gai10b1', 'gai10b2', # non-purchased (15d) + 'gai111', 'gai112', 'gai113', 'gai114'] # source flags +df = df.loc[:, cols].copy() + +# Numeric amounts. Sentinel 99999 == "NR". +for c in ['gai06a', 'gai06b1', 'gai06c', 'gai10a', 'gai10b1']: + df[c] = pd.to_numeric(df[c], errors='coerce') + df.loc[df[c] >= 99999, c] = np.nan + +# Source flags come back as categoricals (Sí/No/NR) under convert_categoricals; +# normalize to a yes-boolean. In the raw data 1=Sí, 2=No, 9=NR. +for c in ['gai111', 'gai112', 'gai113', 'gai114']: + df[c] = df[c].astype(str).str.strip().str.lower() -food_items = df_from_orgfile('../../_/food_items.org') -food_items = food_items.loc[:, ['Preferred Label', t]] -food_items[t] = food_items[t].str.strip() -food_items = food_items.replace(['','---'],pd.NA).dropna() -food_items = food_items.set_index(t).dropna() -food_items = food_items.squeeze().str.strip().to_dict() - -df = df.loc[:, columns_dict.keys()] -df = df.rename(columns_dict, axis=1) - -df['i'] = df['i'].replace(food_items) -df['j'] = df['j'].apply(format_id) -df = df.set_index(['j', 'i']) -df['quantity bought'] = df['quantity (bought, in original units)'].astype(float)*df['conversionb'].astype(float) -df['quantity obtained'] = df['quantity (obtained, in original units)'].astype(float)*df['conversiono'].astype(float) - -df = df.loc[:, ['quantity bought', 'unitcode (bought)', 'total spent', 'quantity obtained', 'unitcode (obtained)']] -df['total spent'] = df['total spent'].astype(float).mask(df['total spent'].astype(float) >= 99999) -df['quantity bought'] = df['quantity bought'].mask(df['quantity bought'] >= 99999) -df['quantity obtained'] = df['quantity obtained'].mask(df['quantity obtained'] >= 99999) -df.loc[df['unitcode (bought)'] == 'FRAMO', 'unitcode (bought)'] = 'GRAMO' +def is_yes(series): + return series.isin(['1', '1.0', 'sí', 'si']) + + +# --- unit decode: Spanish text -> English label ---------------------------- unit_dict = {'GALON': 'gallon', 'GRAMO': 'gram', + 'FRAMO': 'gram', # typo for GRAMO in the raw data 'KILOGRAMO': 'kilogram', 'LIBRA': 'pound', 'LITRO': 'liter', 'ONZA': 'ounce', 'MILILITRO': 'milliliter'} -df['unitcode (bought)'] = df['unitcode (bought)'].map(unit_dict).astype(str) -df['unitcode (obtained)'] = df['unitcode (obtained)'].map(unit_dict).astype(str) -df['price per unit'] = df['total spent']/df['quantity bought'] -df['price per unit'] = df['price per unit'].where(np.isfinite(df['price per unit'])) +def decode_unit(label): + if pd.isna(label): + return pd.NA + return unit_dict.get(str(label).strip().upper(), pd.NA) + + +df['u_bought'] = df['gai06b2'].apply(decode_unit) +df['u_obtained'] = df['gai10b2'].apply(decode_unit) + +# --- harmonize food item j ------------------------------------------------- +food_items = df_from_orgfile('../../_/food_items.org') +food_items = food_items.loc[:, ['Preferred Label', t]] +food_items[t] = food_items[t].astype(str).str.strip() +food_items = food_items.replace(['', '---', 'nan'], pd.NA).dropna() +item_map = dict(zip(food_items[t], food_items['Preferred Label'].str.strip())) + + +def harmonize_item(label): + if pd.isna(label): + return pd.NA + return item_map.get(str(label).strip(), pd.NA) + + +df['i'] = df['form'].apply(format_id) +df['j'] = df['gai00'].apply(harmonize_item) + +# Total quantity in native units = count * size-per-presentation. +df['qty_bought'] = df['gai06a'] * df['gai06b1'] +df['qty_obtained'] = df['gai10a'] * df['gai10b1'] + +# --- build canonical rows -------------------------------------------------- +records = [] + +# Purchased rows +buy = df.loc[df['qty_bought'].notna() | df['gai06c'].notna(), + ['i', 'j', 'u_bought', 'qty_bought', 'gai06c']].copy() +buy = buy.rename(columns={'u_bought': 'u', 'qty_bought': 'Quantity', + 'gai06c': 'Expenditure'}) +buy['s'] = 'purchased' +buy['Price'] = np.nan +records.append(buy) + +# Non-purchased rows split by source flags. +flag_to_s = {'gai111': 'produced', # own production + 'gai112': 'inkind', # gift / donation + 'gai113': 'other', # part of payment + 'gai114': 'other'} # own business + +obtained = df.loc[df['qty_obtained'].notna() & (df['qty_obtained'] > 0)].copy() +for flag, s in flag_to_s.items(): + sub = obtained.loc[is_yes(obtained[flag]), + ['i', 'j', 'u_obtained', 'qty_obtained']].copy() + if sub.empty: + continue + sub = sub.rename(columns={'u_obtained': 'u', 'qty_obtained': 'Quantity'}) + sub['Expenditure'] = np.nan + sub['Price'] = np.nan + sub['s'] = s + records.append(sub) + +# Obtained quantity with no flag -> default 'produced' (small residual). +flagset = pd.Series(False, index=obtained.index) +for flag in flag_to_s: + flagset = flagset | is_yes(obtained[flag]) +unflagged = obtained.loc[~flagset, ['i', 'j', 'u_obtained', 'qty_obtained']].copy() +if not unflagged.empty: + unflagged = unflagged.rename(columns={'u_obtained': 'u', + 'qty_obtained': 'Quantity'}) + unflagged['Expenditure'] = np.nan + unflagged['Price'] = np.nan + unflagged['s'] = 'produced' + records.append(unflagged) + +out = pd.concat(records, ignore_index=True) +out['t'] = t + +out = out[out['j'].notna()] +out['u'] = out['u'].astype('string') + +# Coerce value columns to plain float64 (the categorical read can hand back +# arrow-backed float32, which rejects scalar/array .loc assignment below). +for c in ['Quantity', 'Expenditure', 'Price']: + out[c] = pd.to_numeric(out[c], errors='coerce').astype('float64') + +# Zero amounts are "reported nothing" -> treat as missing. +out.loc[out['Quantity'] == 0, 'Quantity'] = np.nan +out.loc[out['Expenditure'] == 0, 'Expenditure'] = np.nan + +# Value-only convention (food-acquired skill / GhanaLSS): a purchase with a +# monetary value but no physical quantity/unit is carried as u='Value' with +# Quantity = Expenditure, so the framework can still sum it. +value_only = (out['Quantity'].isna() & out['u'].isna() + & out['Expenditure'].notna()) +out.loc[value_only, 'u'] = 'Value' +out.loc[value_only, 'Quantity'] = out.loc[value_only, 'Expenditure'] + +# Drop rows that carry no amount at all. +out = out[out['Quantity'].notna() | out['Expenditure'].notna()] + +buy_mask = out['s'] == 'purchased' +with np.errstate(divide='ignore', invalid='ignore'): + out.loc[buy_mask, 'Price'] = (out.loc[buy_mask, 'Expenditure'] + / out.loc[buy_mask, 'Quantity']) +out.loc[~np.isfinite(out['Price']), 'Price'] = np.nan + +idx = ['t', 'i', 'j', 'u', 's'] +agg = {'Quantity': 'sum', 'Expenditure': 'sum', 'Price': 'mean'} +out = out.groupby(idx, dropna=False).agg(agg) +out = out.dropna(how='all') -to_parquet(df, "food_acquired.parquet") +if __name__ == '__main__': + to_parquet(out, 'food_acquired.parquet') diff --git a/lsms_library/countries/Panama/2008/_/data_info.yml b/lsms_library/countries/Panama/2008/_/data_info.yml new file mode 100644 index 000000000..754abda28 --- /dev/null +++ b/lsms_library/countries/Panama/2008/_/data_info.yml @@ -0,0 +1,14 @@ +Wave: '2008' +Country: Panama + +# Minimal sample table: maps each household to its primary sampling unit (upm). +# 02hogar.dta is the 2008 household file (one row per household, keyed on +# `hogar`) and carries both the `upm` cluster id and the expansion factor. +sample: + file: ../Data/02hogar.dta + idxvars: + i: hogar + myvars: + v: upm # formatted by panama.py's v() (format_id wrapper) + weight: factor + panel_weight: factor diff --git a/lsms_library/countries/Panama/2008/_/food_acquired.py b/lsms_library/countries/Panama/2008/_/food_acquired.py index bee16deaf..454461df1 100644 --- a/lsms_library/countries/Panama/2008/_/food_acquired.py +++ b/lsms_library/countries/Panama/2008/_/food_acquired.py @@ -1,54 +1,179 @@ #!/usr/bin/env python +"""Panama 2008 food_acquired -> canonical long form. +Index : (t, i, j, u, s) -- NO `v` (joined from sample() at API time) +Columns : [Quantity, Expenditure, Price] +i = household (hogar), j = harmonized food item, u = native unit, +s in {purchased, produced, inkind, other} (transformations.S_VALUES). + +Source: 2008/Data/05alimentos.dta (Section 11A food, one row per +household x food item). Per item: + + Purchases (last 15 days): + s11a6a = quantity bought -> Quantity (s='purchased') + s11a6b = unit code (numeric) -> u + s11a6c = total comprado (total paid) -> Expenditure (s='purchased') + + Acquisition WITHOUT purchase (last 15 days): + s11a10a = quantity obtained (15 days) -> Quantity + s11a10b = unit code (numeric) -> u + The SOURCE of the non-purchased quantity is the yes/no flag block: + s11a11a = producción propia (own production) -> s='produced' + s11a11b = regalo o donación (gift) -> s='inkind' + s11a11c = parte del pago (part of payment) -> s='other' + s11a11d = del negocio (own business) -> s='other' + One canonical row per flagged source. + +NOTE on recall periods (round-1 refuting defect): s11a9a is the MONTHLY +recall of the same home-consumption quantity (parallels 1997 ga109a / +2003 gai09a) and overlaps the 15-day s11a10a; summing both double-counts. +We use ONLY s11a10a so the non-purchased quantity shares the 15-day recall +basis of purchased s11a6a. s11a10a is NOT 'produced' wholesale: it is total +non-purchased acquisition; the split is driven by the s11a11* source flags. +""" import sys + sys.path.append('../../_/') -import pandas as pd -import numpy as np -import json sys.path.append('../../../_/') -from lsms_library.local_tools import to_parquet, df_from_orgfile, format_id, get_dataframe + +import numpy as np +import pandas as pd + +from lsms_library.local_tools import (df_from_orgfile, format_id, + get_dataframe, to_parquet) t = '2008' -df = get_dataframe('../Data/05alimentos.dta', convert_categoricals=True) +# Read numerically: producto/units/flags are all numeric codes here. +df = get_dataframe('../Data/05alimentos.dta', convert_categoricals=False) + +cols = ['hogar', 'producto', + 's11a6a', 's11a6b', 's11a6c', # purchased + 's11a10a', 's11a10b', # non-purchased (15d) + 's11a11a', 's11a11b', 's11a11c', 's11a11d'] # source flags +df = df.loc[:, cols].copy() + +# Numeric amounts. Sentinel 99999 == missing; 0 amounts treated as missing. +for c in ['s11a6a', 's11a6c', 's11a10a']: + df[c] = pd.to_numeric(df[c], errors='coerce') + df.loc[df[c] >= 99999, c] = np.nan -df = df.loc[:, ['hogar','producto', 's11a6a', 's11a6b', 's11a6c', 's11a10a', 's11a10b']] -df = df.rename({'hogar': 'j', 'producto':'i', 's11a6a':'quantity bought', 's11a6b':'unitcode (bought)', - 's11a6c':'total spent', 's11a10a':'quantity obtained', 's11a10b':'unitcode (obtained)'}, axis=1) +# --- unit decode: numeric code -> English label ---------------------------- +# Codes from the Stata value labels on s11a6b/s11a10b. +unit_code = {12: 'gallon', 13: 'gram', 20: 'pound', 21: 'liter', + 27: 'ounce', 39: 'unit', 41: 'milliliter'} -cat_columns = df.iloc[:, 2:].select_dtypes(['category']).columns -df[cat_columns] = df[cat_columns].apply(lambda x: x.cat.codes) -df = df.replace({-1: np.nan}) -num_cols = df.select_dtypes(include=[np.number]).columns -df[num_cols] = df[num_cols].where(df[num_cols] < 99999) -df[num_cols] = df[num_cols].where(df[num_cols] != 0) +def decode_unit(code): + try: + return unit_code.get(int(code), pd.NA) + except (ValueError, TypeError): + return pd.NA + + +df['u_bought'] = df['s11a6b'].apply(decode_unit) +df['u_obtained'] = df['s11a10b'].apply(decode_unit) + +# --- harmonize food item j ------------------------------------------------- +# 2008 food_items.org column holds the lower-case product TEXT; decode the +# numeric producto code to text first via the source value labels. +dfc = get_dataframe('../Data/05alimentos.dta', convert_categoricals=True) +prod_text = dfc['producto'].astype(str).str.strip() food_items = df_from_orgfile('../../_/food_items.org') food_items = food_items.loc[:, ['Preferred Label', t]] -food_items[t] = food_items[t].str.strip() -food_items = food_items.replace(['','---'],pd.NA).dropna() -food_items = food_items.set_index(t).dropna() -food_items = food_items.squeeze().str.strip().to_dict() - -unit_dict = {'galón': 'gallon', - 'gramos': 'gram', - 'libra': 'pound', - 'litro': 'liter', - 'onzas': 'ounce', - 'unidad':'unit', - 'centímetro cúbico o mililitro': 'milliliter'} - -df['i'] = df['i'].replace(food_items) -df['j'] = df['j'].apply(format_id) -df = df.set_index(['j', 'i']) - -df['unitcode (bought)'] = df['unitcode (bought)'].map(unit_dict).astype(str) -df['unitcode (obtained)'] = df['unitcode (obtained)'].map(unit_dict).astype(str) -df['quantity bought'] = pd.to_numeric(df['quantity bought'], errors='coerce') -df['quantity obtained'] = pd.to_numeric(df['quantity obtained'], errors='coerce') - -df['price per unit'] = df['total spent']/df['quantity bought'] -df['price per unit'] = df['price per unit'].where(np.isfinite(df['price per unit'])) - -to_parquet(df, "food_acquired.parquet") +food_items[t] = food_items[t].astype(str).str.strip() +food_items = food_items.replace(['', '---', 'nan'], pd.NA).dropna() +item_map = dict(zip(food_items[t], food_items['Preferred Label'].str.strip())) + + +def harmonize_item(label): + if pd.isna(label) or str(label).strip().lower() in ('', 'nan'): + return pd.NA + return item_map.get(str(label).strip(), pd.NA) + + +df['i'] = df['hogar'].apply(format_id) +df['j'] = prod_text.apply(harmonize_item).values + +# --- build canonical rows -------------------------------------------------- +records = [] + +# Purchased rows +buy = df.loc[df['s11a6a'].notna() | df['s11a6c'].notna(), + ['i', 'j', 'u_bought', 's11a6a', 's11a6c']].copy() +buy = buy.rename(columns={'u_bought': 'u', 's11a6a': 'Quantity', + 's11a6c': 'Expenditure'}) +buy['s'] = 'purchased' +buy['Price'] = np.nan +records.append(buy) + +# Non-purchased rows split by source flags. 1 = yes (sí), 2 = no. +flag_to_s = {'s11a11a': 'produced', # producción propia + 's11a11b': 'inkind', # regalo o donación + 's11a11c': 'other', # parte del pago + 's11a11d': 'other'} # del negocio + +obtained = df.loc[df['s11a10a'].notna() & (df['s11a10a'] > 0)].copy() +for flag, s in flag_to_s.items(): + sub = obtained.loc[obtained[flag] == 1, + ['i', 'j', 'u_obtained', 's11a10a']].copy() + if sub.empty: + continue + sub = sub.rename(columns={'u_obtained': 'u', 's11a10a': 'Quantity'}) + sub['Expenditure'] = np.nan + sub['Price'] = np.nan + sub['s'] = s + records.append(sub) + +# Obtained quantity with no flag -> default 'produced' (small residual). +flagset = pd.Series(False, index=obtained.index) +for flag in flag_to_s: + flagset = flagset | (obtained[flag] == 1) +unflagged = obtained.loc[~flagset, ['i', 'j', 'u_obtained', 's11a10a']].copy() +if not unflagged.empty: + unflagged = unflagged.rename(columns={'u_obtained': 'u', + 's11a10a': 'Quantity'}) + unflagged['Expenditure'] = np.nan + unflagged['Price'] = np.nan + unflagged['s'] = 'produced' + records.append(unflagged) + +out = pd.concat(records, ignore_index=True) +out['t'] = t + +out = out[out['j'].notna()] +out['u'] = out['u'].astype('string') + +# Coerce value columns to plain float64 (defensive against arrow-backed dtypes). +for c in ['Quantity', 'Expenditure', 'Price']: + out[c] = pd.to_numeric(out[c], errors='coerce').astype('float64') + +# Zero amounts are "reported nothing" -> treat as missing. +out.loc[out['Quantity'] == 0, 'Quantity'] = np.nan +out.loc[out['Expenditure'] == 0, 'Expenditure'] = np.nan + +# Value-only convention (food-acquired skill / GhanaLSS): a purchase with a +# monetary value but no physical quantity/unit is carried as u='Value' with +# Quantity = Expenditure, so the framework can still sum it. +value_only = (out['Quantity'].isna() & out['u'].isna() + & out['Expenditure'].notna()) +out.loc[value_only, 'u'] = 'Value' +out.loc[value_only, 'Quantity'] = out.loc[value_only, 'Expenditure'] + +# Drop rows that carry no amount at all. +out = out[out['Quantity'].notna() | out['Expenditure'].notna()] + +buy_mask = out['s'] == 'purchased' +with np.errstate(divide='ignore', invalid='ignore'): + out.loc[buy_mask, 'Price'] = (out.loc[buy_mask, 'Expenditure'] + / out.loc[buy_mask, 'Quantity']) +out.loc[~np.isfinite(out['Price']), 'Price'] = np.nan + +idx = ['t', 'i', 'j', 'u', 's'] +agg = {'Quantity': 'sum', 'Expenditure': 'sum', 'Price': 'mean'} +out = out.groupby(idx, dropna=False).agg(agg) +out = out.dropna(how='all') + +if __name__ == '__main__': + to_parquet(out, 'food_acquired.parquet') diff --git a/lsms_library/countries/Panama/_/Makefile b/lsms_library/countries/Panama/_/Makefile index fd2b156a3..b17f3af46 100644 --- a/lsms_library/countries/Panama/_/Makefile +++ b/lsms_library/countries/Panama/_/Makefile @@ -7,13 +7,13 @@ COUNTRY := $(shell basename "$$(cd .. && pwd)") LSMS_DATA_ROOT ?= $(if $(LSMS_DATA_DIR),$(LSMS_DATA_DIR),$(if $(XDG_DATA_HOME),$(XDG_DATA_HOME),$(HOME)/.local/share)/lsms_library) VAR_DIR ?= $(LSMS_DATA_ROOT)/$(COUNTRY)/var -var = $(VAR_DIR)/food_expenditures.parquet $(VAR_DIR)/food_quantities.parquet $(VAR_DIR)/food_prices.parquet \ - $(VAR_DIR)/nutrition.parquet $(VAR_DIR)/fct.parquet - -all: $(parquet) $(var) +# food_expenditures / food_quantities / food_prices are NOT built here: +# they are auto-derived at runtime by the framework's _FOOD_DERIVED from the +# canonical food_acquired.parquet (cf. Uganda post-#245, GhanaLSS #109). +# Registering or building them is forbidden. +var = $(VAR_DIR)/food_acquired.parquet -$(VAR_DIR)/food_quantities.parquet $(VAR_DIR)/food_expenditures.parquet $(VAR_DIR)/food_prices.parquet &: $(VAR_DIR)/food_acquired.parquet food_acquired.py food_prices_quantities_and_expenditures.py - python food_prices_quantities_and_expenditures.py +all: $(parquet) $(var) food_acquired = $(shell find $(rounds) -name food_acquired.py) food_acquired_parquet := $(food_acquired:.py=.parquet) @@ -24,9 +24,6 @@ food_acquired_parquet := $(food_acquired:.py=.parquet) $(VAR_DIR)/food_acquired.parquet: food_acquired.py $(food_acquired_parquet) python food_acquired.py -$(VAR_DIR)/fct.parquet $(VAR_DIR)/nutrition.parquet : nutrition.py $(VAR_DIR)/food_quantities.parquet food_items.org - python nutrition.py - %.parquet: %.py food_items.org panama.py (cd $(@D) && python ./$(/_/food_acquired.py``) emits the canonical +long form with index ``(t, i, j, u, s)`` and columns +``[Quantity, Expenditure, Price]`` (Phase 3 of GH #218). ``i`` is the +household, ``j`` the harmonized food item, ``u`` the native unit, and ``s`` +the acquisition source (``{purchased, produced, inkind, other}``). ``v`` is +NOT baked in -- it is joined from ``sample()`` at API time. + +The three Panama waves (1997, 2003, 2008) are independent cross-sections (no +cross-wave household panel linkage), so this is a plain concatenation. + +``food_expenditures``/``food_prices``/``food_quantities`` are no longer built +here -- they are auto-derived at runtime by the framework's ``_FOOD_DERIVED`` +from this canonical ``food_acquired``. """ import sys + sys.path.append('../../_') -from lsms_library.local_tools import to_parquet + import pandas as pd -import numpy as np - -fa = [] -for t in ['1997', '2003', '2008']: - df = get_dataframe('../'+t+'/_/food_acquired.parquet').squeeze() - df['t'] = t - df = df.reset_index() - df['j'] = t + df['j'].map(str) - df['unitcode (bought)'] = df['unitcode (bought)'].astype(str) - - df = df.set_index(['j', 't', 'i', 'unitcode (bought)']) - df.index = df.index.rename({'unitcode (bought)': 'u'}) - df.columns.name = 'k' - #df = id_walk(df,t,Waves) - fa.append(df) - -fa = pd.concat(fa) - -fa = fa.replace(0,np.nan) -fa = fa.groupby(['j','t','i','u']).sum() -to_parquet(fa,'../var/food_acquired.parquet') + +from lsms_library.local_tools import get_dataframe, to_parquet + +WAVES = ['1997', '2003', '2008'] + +dfs = [] +for t in WAVES: + df = get_dataframe('../' + t + '/_/food_acquired.parquet') + if 't' not in df.index.names: + df = df.reset_index() + df['t'] = t + df = df.set_index(['t', 'i', 'j', 'u', 's']) + dfs.append(df) + +fa = pd.concat(dfs) + +to_parquet(fa, '../var/food_acquired.parquet') diff --git a/lsms_library/countries/Panama/_/food_items.org b/lsms_library/countries/Panama/_/food_items.org index 18803cc0b..97bae33d9 100644 --- a/lsms_library/countries/Panama/_/food_items.org +++ b/lsms_library/countries/Panama/_/food_items.org @@ -66,7 +66,7 @@ | Alimento Infantil | alimento infantil | Alimento Infantil | 1 | 19093 | | Sopa Enlatada | sopa enlatada y en sobres | Sopa enlatada y en sobres | 61 | 20007 | | Pastas De Tomate | pastas de tomate y salsas de tomate | Pasta y salsa de tomate | 62 | 20071 | -| Condimentos | condimentos (orégano, pimienta, ...) | --- | 64 | 22017 | +| Condimentos | condimentos (orégano, pimienta, ...) | Condimentos | 64 | 22017 | | Sal | sal | Sal | 65 | 22021 | | Sodas, Refrescos Y Jugos | sodas, refrescos y jugos | Sodas y refrescos | 66 | 17018 | | Helados | helados | Helados | 67 | 18015 | diff --git a/lsms_library/countries/Panama/_/food_prices_quantities_and_expenditures.py b/lsms_library/countries/Panama/_/food_prices_quantities_and_expenditures.py deleted file mode 100644 index caec9d3ce..000000000 --- a/lsms_library/countries/Panama/_/food_prices_quantities_and_expenditures.py +++ /dev/null @@ -1,26 +0,0 @@ -from lsms_library.local_tools import to_parquet -from lsms_library.local_tools import get_dataframe -"""Calculate food prices for different items across rounds; allow -different prices for different units. -""" -import pandas as pd -import numpy as np -import json - -df = get_dataframe('../var/food_acquired.parquet') - -x = df[['total spent']].rename({'total spent': 'total expenditure'}) -x = x.replace(0,np.nan).dropna() -z = x.droplevel('u').groupby(['j','t','m', 'i']).sum() -to_parquet(z, '../var/food_expenditures.parquet') - -p = df['price per unit'].replace(0,np.nan).dropna() - -p = p.groupby(['t','m','i','u']).median() -to_parquet(p.to_frame('Prices'), '../var/food_prices.parquet') - -q = x.join(p,on=['t','m','i','u']) -q = q['total spent']/q['price per unit'] -q = q.dropna() - -to_parquet(pd.DataFrame({'Quantity':q}), '../var/food_quantities.parquet') diff --git a/lsms_library/countries/Panama/_/panama.py b/lsms_library/countries/Panama/_/panama.py index 335690b7b..3bb0af314 100644 --- a/lsms_library/countries/Panama/_/panama.py +++ b/lsms_library/countries/Panama/_/panama.py @@ -1,7 +1,19 @@ import pandas as pd import numpy as np import json -from lsms_library.local_tools import get_dataframe +from lsms_library.local_tools import get_dataframe, format_id + + +def v(upm): + """Format the sampling-unit id (upm) for the sample table's `v` column. + + The framework resolves a formatting function whose name matches the + myvar (`v`) and applies it to the single source column declared in the + sample `data_info.yml` (``v: upm``). `format_id` is NOT applied to + myvars automatically (only to idxvars), so without this the numeric upm + would stringify to e.g. '9.0' instead of '9' -- breaking the v-join key. + """ + return format_id(upm) def _household_roster_from_df(df, sex, age, HHID, sex_converter=None, age_converter=None, From 23fb11faaa0c5e02677bfedde38711aa55f26cef Mon Sep 17 00:00:00 2001 From: Ethan Ligon & Sue Coder Date: Thu, 25 Jun 2026 15:36:24 -0700 Subject: [PATCH 02/85] fix(#218): register Panama in data_info.yml Currency section (PAB) PR #581 added the Panama config + registration but omitted Panama from the Currency: section, so test_currency.py::test_every_country_has_currency[Panama] failed (Panama missing from Currency: section). Panama's ENV survey is denominated in Balboas (PAB), pegged 1:1 to USD; USD also circulates (dual currency), mirroring the Liberia entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- lsms_library/data_info.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lsms_library/data_info.yml b/lsms_library/data_info.yml index 762393d4b..bfa4c23e2 100644 --- a/lsms_library/data_info.yml +++ b/lsms_library/data_info.yml @@ -343,6 +343,7 @@ Currency: Niger: XOF Nigeria: NGN Pakistan: PKR + Panama: PAB # Balboa pegged 1:1 to USD; USD also circulates (dual currency) Senegal: XOF Serbia: RSD # 2007 wave (RSD since 2006) 'Serbia and Montenegro': # 2002, 2003 waves From d97ad8d6bf1c838c22b52e68c0f5880c30967c44 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 06:27:06 -0700 Subject: [PATCH 03/85] feat(coverage): adopt the `blessed` tier (bless-on-first-use) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `blessed` tier was already fully implemented -- loader, tier promotion, colour, the `★` glyph, docs -- but its file was a 21-byte header, so the top of the ladder had never once been used. `sane` and `blessed` answer different questions, and the gap between them is where quiet errors live: sane the automated checks passed. No human has necessarily looked at a single number. blessed a human read the actual numbers for that cell and believes them. A feature can build cleanly, pass every sanity check, and still be wired to the wrong source column or carry an unverified unit conversion. For anything feeding published analysis, `sane` is not enough. Adopting the tier cost zero library code: - blessed.csv schema extended to `country,feature,wave,blessed_by,date,note`. `load_blessed()` reads via `r.get(...)`, so the provenance columns are free -- and they are what make a blessing auditable rather than a bare assertion. - The rule, documented in CLAUDE.md and docs/guide/coverage.md: if you used a cell in real analysis and looked at its numbers, bless it in the same PR. Seeded empty, deliberately. Blessings accrete; they are never bulk-seeded. An empty blessing file is honest -- bulk-blessing the 1208 `sane` cells would make `blessed` a synonym for `sane`, destroy the only distinction the tier draws, and assert a human review that never happened. Two things found while wiring it: - CLAUDE.md never mentioned the coverage matrix at all (it landed in June and the guide never learned about it). Added a section. - `load_blessed()` calls `pd.read_csv(..., keep_default_na=False)` with no `comment=` argument, so a `#` comment line in blessed.csv parses as a *row* -- a phantom blessed cell. Documented; file kept header-only rather than touching the loader. Also records the corollary that bit two live issues: a `sane` cell is not proof an issue is fixed. The matrix is a cold build, so it structurally cannot see #588's warm-cache-only divergence, and it does not check currency labels (#589). Both look green and are genuinely open. Co-Authored-By: Claude Opus 4.8 (1M context) --- .coder/coverage/blessed.csv | 2 +- CLAUDE.md | 17 +++++++++++++++++ docs/guide/coverage.md | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/.coder/coverage/blessed.csv b/.coder/coverage/blessed.csv index e288837ba..378087be8 100644 --- a/.coder/coverage/blessed.csv +++ b/.coder/coverage/blessed.csv @@ -1 +1 @@ -country,feature,wave +country,feature,wave,blessed_by,date,note diff --git a/CLAUDE.md b/CLAUDE.md index 4a4248140..358b3f3b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,6 +156,23 @@ Auto-unlock decrypts `s3_reader_creds.gpg` with an obfuscated passphrase at impo **EHCVM note**: EHCVM 2018-19 countries lack a continuous months variable. The binary `s01q12` ("lived continuously 6+ months?") is mapped to 0/12. Guinea-Bissau may need Portuguese keys (`Sim`/`Não`) alongside `Oui`/`Non`. See CONTENTS.org files for per-country documentation. +## Coverage Matrix (v0.9.0+) + +`make matrix` grades every `(country, feature, wave)` cell on a tier ladder — `absent` / `dropped` / `broken` / `builds` / `sane` / `blessed` — and commits a snapshot to `.coder/coverage/latest.csv`. `ll.coverage()` reads it back. See `docs/guide/coverage.md`. + +**`sane` is not `blessed`, and the difference matters.** + +- **`sane`** — the automated checks passed. *No human has necessarily looked at a single number.* +- **`blessed`** — a human read the actual numbers for that cell and believes them. Recorded in the git-tracked `.coder/coverage/blessed.csv`. + +A feature can build cleanly, pass every sanity check, and still be quietly wrong — wired to the wrong source column, or carrying an unverified unit conversion. So **for anything feeding published analysis, `sane` is not enough.** + +> **The rule: if you used a cell in real analysis and looked at its numbers, bless it in the same PR.** +> `country,feature,wave,blessed_by,date,note` (`wave` blank for country-level features). Blessings accrete; never bulk-seed them. An empty blessing file is honest — a file full of blessings nobody gave makes `blessed` a synonym for `sane` and destroys the tier. +> Do **not** put `#` comments in `blessed.csv`: `load_blessed()` reads it without a `comment=` arg, so a comment line becomes a phantom blessed cell. + +**Do not trust a `sane` cell as proof an issue is fixed.** Know what the grader does *not* look at — it is a cold build (so it cannot see warm-cache-only divergences) and it does not check currency labels. + ## Derived Tables `household_characteristics`, `food_expenditures`, `food_prices`, and `food_quantities` are **auto-derived at runtime** via `_ROSTER_DERIVED` and `_FOOD_DERIVED` in `country.py` (source transforms live in `transformations.py`). **Do NOT register them in `data_scheme.yml`** — `Country.data_scheme` auto-surfaces them when the source table (`food_acquired` or `household_roster`) is present. A `!derived` YAML tag was considered and rejected (2026-04-18): it would create a migration burden with no gain over the hardcoded dicts + auto-discovery. diff --git a/docs/guide/coverage.md b/docs/guide/coverage.md index c4f38ce38..71fb982cd 100644 --- a/docs/guide/coverage.md +++ b/docs/guide/coverage.md @@ -29,6 +29,42 @@ re-implements the sanity checks. | `sane` | The wave slice is non-empty and passes every sanity check (warnings allowed). | | `blessed` | `sane` **and** listed in the git-tracked blessing file `.coder/coverage/blessed.csv`. | +## Blessing a cell + +`sane` and `blessed` answer different questions: + +- **`sane`** — *the automated checks passed.* No human has necessarily looked at + a single number. +- **`blessed`** — *a human read the actual numbers for this cell and believes + them.* + +For anything feeding published analysis, `sane` is not enough. A feature can +build cleanly, pass every sanity check, and still be quietly wrong — wired to +the wrong source column, or carrying a unit conversion nobody verified. + +**The rule: if you used a cell in real analysis and looked at its numbers, bless +it in the same PR.** + +```csv +country,feature,wave,blessed_by,date,note +Uganda,food_expenditures,2019-20,ligon,2026-07-12,used in demand estimation; totals reconcile +``` + +`wave` is blank for country-level features. Only `country`, `feature`, and +`wave` are read by the matrix — `blessed_by` / `date` / `note` are provenance +for humans, and they are what make a blessing auditable rather than a bare +assertion. + +Blessings accrete; they are never bulk-seeded. An empty blessing file is honest. +A file full of blessings nobody actually gave is worse than no file at all, +because it would make `blessed` a synonym for `sane` and destroy the only +distinction the tier exists to draw. + +!!! warning "Do not put `#` comments in `blessed.csv`" + `load_blessed()` reads it with `pd.read_csv(..., keep_default_na=False)` and + no `comment=` argument, so a comment line is parsed as a **row** — a phantom + blessed cell. Header and data only. + The grid cell shows a worst-first glyph tally over a country's waves (e.g. `⚠2 ✓5 –1` = 2 `builds`, 5 `sane`, 1 `absent`) and is coloured by the most actionable tier present. From e55146c201ea12fb5d84888575ab38c8e156b189 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 06:28:13 -0700 Subject: [PATCH 04/85] =?UTF-8?q?ledger:=20household=5Fcharacteristics=20r?= =?UTF-8?q?esidence=20filter=20=E2=80=94=20roster=5Fto=5Fcharacteristics?= =?UTF-8?q?=20is=20the=20only=20site;=20BF-2014=20French=20labels=20are=20?= =?UTF-8?q?a=20config=20fix=20(reuse=20the=20EHCVM=20mapping:=20idiom),=20?= =?UTF-8?q?the=20per-t=20all-NaN=20guard=20is=20the=20one=20'new'=20piece?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NtARb9se2un1DLFJrrpjHe --- .../roster-characteristics-wave-deletion.md | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 .coder/ledger/roster-characteristics-wave-deletion.md diff --git a/.coder/ledger/roster-characteristics-wave-deletion.md b/.coder/ledger/roster-characteristics-wave-deletion.md new file mode 100644 index 000000000..b79a89225 --- /dev/null +++ b/.coder/ledger/roster-characteristics-wave-deletion.md @@ -0,0 +1,113 @@ +# Prior-Art Ledger — roster-characteristics-wave-deletion + +> Per-task ledger. Inherits the repo §0 baseline in `STANDING.md`. Living +> snapshot; git history is the journal. + +**Search tier used:** ripgrep + git floor (gitnexus not consulted this session), +plus an empirical census: `Country(c).household_characteristics()` per-wave row +counts for all 40 countries that declare `household_roster`, warm cache. + +## §1 Task, restated + +`household_characteristics` is a **derived** table (`Country._ROSTER_DERIVED`, +`STANDING.md §2`): `country.py` aggregates `household_roster` across *all* waves +into one frame and hands it to `transformations.roster_to_characteristics()`, +which drops non-resident members using whichever residence-duration column is +present (`MonthsSpent` / `MonthsAway` / `WeeksAway`, per `CLAUDE.md` +§"MonthsSpent / MonthsAway / WeeksAway"). Because the frame handed to the +transform is the **country-level concat**, a residence column contributed by +*one* wave is unioned onto *every* wave. Where the resolved months series is +unusable for a wave, the keep-mask is all-False **for that whole wave** and +`household_characteristics` silently returns nothing for it, even though +`household_roster` is fully populated. 8 wave-cells are currently destroyed. +The task fixes the resolution + the guard, and repairs one wave whose +`MonthsSpent` is an untranslated French label — in **config**, not in code. + +## §2 Existing machinery (this task's area) + +| symbol | path:line | what it does | tested? | reuse / extend / new | +|--------|-----------|--------------|---------|----------------------| +| `roster_to_characteristics` | `lsms_library/transformations.py:177` | roster → household sex×age counts + `log HSize`; owns the residence filter (L251-265) | `tests/test_roster_to_characteristics_movers.py`, `tests/test_age_intervals.py`, `tests/test_uganda_api_vs_replication.py` | **extend** (the residence-resolution block only) | +| `Country._ROSTER_DERIVED` + dispatch | `lsms_library/country.py:3325`, `3515-3530` | resolves the transform by *name* at call time and passes the **all-waves** roster with `final_index` = the roster's own index levels | integration surface | reuse unchanged | +| YAML `mapping:` on a `myvars` entry | e.g. `Mali/2017-18/_/data_info.yml:76`, `Mali/2018-19/_/data_info.yml:163` (`MonthsSpent: [s1q13, {mapping: {Oui: 12, Non: 0}}]`) | maps a categorical source label to the canonical numeric months value **at config level** | per-country builds | **reuse** — this is the sanctioned pattern for D3 | +| `mover_sentinel` / NaN-in-`final_index` handling | `transformations.py:287-337` | GH #197 / #268: keeps movers rather than silently dropping them | `tests/test_roster_to_characteristics_movers.py` | untouched (same *class* of silent-drop bug, one layer down) | + +Searched by concept — "months present", "residence", "drop departed", "keep +mask" — not just by identifier. No other implementation of a residence filter +exists in the library; `roster_to_characteristics` is the only site. + +## §3 Definitions & conventions in force + +- **MonthsSpent / MonthsAway / WeeksAway semantics** — `CLAUDE.md` + §"MonthsSpent / MonthsAway / WeeksAway" (cited, not restated): `MonthsSpent` + = months present 0–12; `MonthsAway` → `12 - value`; `WeeksAway` → + `12 - weeks/(52/12)`. Filter excludes NaN (question not asked) and 0 months, + **except** infants (age < 1). **"Countries without any residence column are + unaffected — the old count-everyone behavior continues."** That sentence is + the definition the D2 guard restores at *wave* granularity. +- **Ethiopia W4–W5 switched months→weeks** — same section: "`WeeksAway`: … + Used by Ethiopia W4–W5 and Cambodia, where the questionnaire switched from + months to weeks." So Ethiopia legitimately carries **both** columns at country + level, disjointly populated. The `elif` chain contradicts this documented fact. +- **EHCVM binary → 12/0** — same section: "The binary `s01q12` … is mapped to + 0/12", done in the wave `data_info.yml` `mapping:` block. D3's fix follows + this pattern verbatim (Burkina Faso 2014 EMC `B3A`, a 6-month binary). +- **Derived tables are not registered** — `STANDING.md §3`; the fix must stay in + `transformations.py` + config, with no `data_scheme.yml` change. + +## §4 Invariants & assumptions + +- **The 1315-HH Uganda drift is the reason the filter exists** (`CLAUDE.md`, + same section). The all-NaN fallback is *exactly* the old count-everyone + behaviour, so it must fire **only** where the wave has no usable residence + datum at all — never for a wave where the filter currently works. Enforced by + the per-`t` (not global) guard + the before/after census. +- The roster handed to the transform is a **multi-wave** frame with `t` in the + index (`country.py:3523` derives `final_index` from `['t','v','i','m']`). + Any per-wave logic must group on `t` — and must not assume `t` is present + (the transform is also called directly in tests with a `(i, pid)` index). +- Residence columns arrive with mixed dtypes across waves (BF: `'6 mois ou + plus'`, `'12'`, `12.0`) — the country concat unions object + float. Resolution + must stay `pd.to_numeric(..., errors='coerce')`-based. +- `STANDING.md §4` repo-wide invariants unchanged: no `inplace=`, `pd.NA` for + string/categorical missing, config resolved via `countries_root()`. + +## §5 Reuse decision + +| quantity | decision | reason | +|----------|----------|--------| +| months-present series | **extend** `roster_to_characteristics` | only implementation; the bug is in its resolution step (per-column `elif` → per-row coalesce) | +| per-wave "no usable residence data" guard | **new** (inside the same function) | nothing in the library expresses "this column is unioned-in from another wave"; it is a property of the country-level concat, invisible at wave level. Kept ~10 lines, local to the transform. | +| Burkina Faso 2014 French labels → 12/0 | **reuse** the YAML `mapping:` pattern (EHCVM `Oui`/`Non`) | `CLAUDE.md` prescribes it; keeps language-specific labels out of `transformations.py` | + +## §6 Open questions for the human + +- BF 2014 `B3A` is a **6-month binary**, not a months count (`6 mois ou plus` + 77,268 / `moins de 6 mois` 943 / NaN 332). Mapping it to 12/0 matches the + EHCVM convention (and drops the 943 short-stay members), but it is coarser + than Uganda's true months count. Flagged, not decided — if the project would + rather keep short-stay members in BF 2014, change the mapping to `12 / 6`. +- CotedIvoire 1985–89 and Mali 2021-22 now count **everyone** in the roster + (no residence question in those waves). That is the documented behaviour for + a country with no residence column, applied per-wave. It does mean those + waves are not filter-comparable with their EHCVM siblings. + +--- +### Phase 3 — verification (fill at task end) + +- `roster_to_characteristics` (residence-resolution block) — **OK (anchored on + §3)**: per-row coalesce `MonthsSpent → MonthsAway → WeeksAway` matches the + documented per-column semantics and is the only reading consistent with + "Ethiopia W4–W5 switched … to weeks" while W1–W3 use months. +- `roster_to_characteristics` (per-`t` all-NaN guard) — **OK (anchored on §3/§4)**: + restores "countries without any residence column are unaffected — the old + count-everyone behavior continues" at wave granularity; the census shows it + fires for exactly the 7 wave-cells with 0 non-null residence data and for no + other wave (Uganda's 8 waves byte-identical). +- Burkina Faso 2014 `MonthsSpent` mapping — **OK (anchored on §3, reuse §2)**: + same config-level `mapping:` idiom as the EHCVM `Oui`/`Non` → 12/0 waves; no + French in library code. +- No **REINVENTION**: no other residence/months-present filter exists in the + library (ripgrep over `months`, `spent`, `away`, `resident`). +- No **CONTRADICTION**: the filter still fires wherever it fires today; the only + waves whose counts move are the 8 that currently return nothing. From 8dcd2bcbb28c5028d6e00d94529c76d85aa7e474 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 06:50:25 -0700 Subject: [PATCH 05/85] =?UTF-8?q?feat(coverage):=20Phase=200=20=E2=80=94?= =?UTF-8?q?=20fix=20the=20ruler=20(absent=20verdicts=20+=20wave-slice=20gr?= =?UTF-8?q?ader)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage matrix measures, but it does not measure right. A census (2026-07-11) found that 595 of its 1849 cells -- 32% -- carry a tier that is wrong or unactionable, and that its errors bias in BOTH directions: it cries wolf about what we have, and stays silent about what we lack. A workplan built on those numbers is built on sand. This is the first half of fixing the instrument. Two defects, and the sequencing between them matters. ## 1. The wave-slice grader applied a COUNTRY-level check (128 cells) `coverage_matrix.grade_feature` grades a wave by slicing the country frame on `t` and calling `is_this_feature_sane` on the slice -- but `_check_no_all_null_columns` is a *country-level* check. A question simply not fielded in a given wave is legitimately all-null within that wave's slice, and was graded `builds` (a defect). It is a grading bug, not a data bug: of the 124 columns it flagged, ZERO are all-null country-wide. Every one is populated in some other wave. Fix: `is_this_feature_sane` gains an `extra_optional=` kwarg (default None -> historical behaviour byte-for-byte; it has many callers). The slice grader passes the set of columns populated somewhere in the parent frame. A column that is all-null across the WHOLE country still (correctly) fails -- we relaxed the slice, not the country. Verified on Malawi/Uganda/Niger: 434 cells, 37 changed, all `builds`-> `sane`, none unexplained by `no_all_null_columns`, ZERO regressions, and the 5 genuinely-broken cells (`no_null_index_levels` -- the `v`-join failure) correctly stayed `builds`. The fix is surgical. ### The sequencing rule this taught us A regrade is NOT a fix. Re-tiering 128 cells to `sane` would have SILENTLY BURIED two genuine gaps that the buggy grading was catching only by accident. Both were inventoried as issues BEFORE this landed: GH #591 Nigeria `food_prices()` silently drops ~99% of rows in 6 of 8 waves. A `Quantity == 0` sentinel (never decoded to NA) makes `Price = Expenditure/0 = inf`, which transformations.py:933 deletes. 675-1106 rows survive out of 120k-160k. All 6 cells were graded `sane`. GH #592 18 waves across 6 countries have ZERO GPS in cluster_features. Whenever a fix changes how cells are GRADED rather than what they CONTAIN, first inventory what the old grading was catching by accident. ## 2. `absent` was a black hole (456 cells) `absent` said only "not declared for this wave", conflating states that could not be more different -- so the number could never reach zero, and every probe's findings evaporated into the prober's head. Verdicts now live in `.coder/coverage/absent_verdicts.csv`: todo data IS there; config missing. Stays `absent` (still work) but is now sized and SOURCED. asked-not-distributed the instrument asked; the shipped extract lacks the variables. An ACQUISITION problem -- a different queue. CLOSES the cell. not-asked genuinely never asked. CLOSES the cell. unsure a required check could not run. Stays in the queue and RECORDS WHY. Silence is never evidence. `asked-not-distributed` is not a hypothetical: Albania 2004's questionnaire lists MODULE 2 DURABLE GOODS, and the variables are simply not in the shipped .dta. Data-side checks cannot see that, which is why C4 (the questionnaire) is mandatory before any permanent close. ### Evidence is load-bearing, and this already went wrong `load_verdicts()` REFUSES a closing verdict with an empty `evidence` field. A closing verdict is a permanent, unsupervised write; an unevidenced negative is unfalsifiable and therefore permanent whether or not it is true. `Albania/_/data_scheme.yml` asserted "earlier waves have no shocks module". It is FALSE. Verified against source: Albania 2005's `migrationE_cl.dta` carries `m6e_q00 = 'Type of Shock Code'` with ten shock types; 2008 is identical; 2003 and 2004 carry "faced any shocks in last 12 months". The claim was wrong, nobody could catch it (nothing recorded HOW it was reached), and it silently suppressed ~5 cells. Comment corrected; the four waves are seeded as `todo` with their file+variable citations, and 2002 as an honest `unsure` with its blocker named. A 38-cell pilot returned 25 todo / 13 unsure / **0 not-asked** -- so `absent` is not hiding "the survey never asked", it is hiding a backlog. This mechanism does not shrink the work. It makes the work knowable. Tests: 12 new (grader exemption, country-wide-null still fails, back-compat default, all four verdicts, and the evidence refusal). Docs in CLAUDE.md + docs/guide/coverage.md. Design: slurm_logs/DESIGN_coverage_discipline_2026-07-11.org Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 17 ++ bench/matrix.py | 9 + docs/guide/coverage.md | 64 +++++ .../countries/Albania/_/data_scheme.yml | 29 ++- lsms_library/coverage_matrix.py | 224 +++++++++++++++--- lsms_library/diagnostics.py | 21 +- tests/test_coverage_matrix.py | 170 ++++++++++++- 7 files changed, 494 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 358b3f3b7..21f08fb82 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -173,6 +173,23 @@ A feature can build cleanly, pass every sanity check, and still be quietly wrong **Do not trust a `sane` cell as proof an issue is fixed.** Know what the grader does *not* look at — it is a cold build (so it cannot see warm-cache-only divergences) and it does not check currency labels. +### Adjudicating `absent` cells + +`absent` says only "not declared for this wave" — which conflates *"the survey never asked"* with *"nobody wrote the config yet"*. Verdicts go in the git-tracked `.coder/coverage/absent_verdicts.csv` (`country,feature,wave,verdict,checks_run,evidence,adjudicated_by,date`): + +| verdict | meaning | closes the cell? | +|---|---|---| +| `todo` | data is there, config missing | no — stays `absent`, but now sized + sourced | +| `asked-not-distributed` | instrument asked; the shipped extract lacks it | **yes** → acquisition queue | +| `not-asked` | genuinely never asked | **yes** → closed forever | +| `unsure` | a required check could not be run | no — stays in the queue, records why | + +> **A closing verdict REQUIRES evidence — `load_verdicts()` refuses one without it.** A closing verdict is a permanent, unsupervised write. An unevidenced negative is unfalsifiable, and therefore permanent whether or not it is true. +> +> This already went wrong: `Albania/_/data_scheme.yml` asserted *"earlier waves have no shocks module"*, but Albania 2005's `migrationE_cl.dta` carries `m6e_q00 = 'Type of Shock Code'` with ten shock types. False, uncatchable (nothing recorded *how* it was reached), and it suppressed ~5 cells of work. **Never write an unevidenced "no module here" claim, in a verdict file or in a YAML comment.** + +Before closing a cell, run the four checks (see `docs/guide/coverage.md`). The two that are most often skipped and most often wrong: **C2 (sibling-wave differential) is necessary but NEVER sufficient** — module vocabularies change completely between waves; and **C4 (the questionnaire) is mandatory**, because *absence in the shipped `.dta` is not absence in the instrument*. Only the questionnaire separates `not-asked` from `asked-not-distributed`. + ## Derived Tables `household_characteristics`, `food_expenditures`, `food_prices`, and `food_quantities` are **auto-derived at runtime** via `_ROSTER_DERIVED` and `_FOOD_DERIVED` in `country.py` (source transforms live in `transformations.py`). **Do NOT register them in `data_scheme.yml`** — `Country.data_scheme` auto-surfaces them when the source table (`food_acquired` or `household_roster`) is present. A `!derived` YAML tag was considered and rejected (2026-04-18): it would create a migration burden with no gain over the hardcoded dicts + auto-discovery. diff --git a/bench/matrix.py b/bench/matrix.py index c6e43694b..c83f10c08 100644 --- a/bench/matrix.py +++ b/bench/matrix.py @@ -39,7 +39,15 @@ # Presentation only (the model is colour-agnostic). TIER_COLOR = { "n/a": "#e9ecef", + # `absent` is the LIVE QUEUE (an un-adjudicated gap), so it must not read as + # quietly settled. The adjudicated tiers below are the settled ones. "absent": "#f1f3f5", + # Adjudicated-and-closed: dimmer than `absent`, because there is no work here. + "not-asked": "#dee2e6", + # Adjudicated as an ACQUISITION problem (asked, but not in the shipped + # extract) -- a distinct colour, because it routes to a different queue + # entirely and should never be mistaken for either config work or a close. + "asked-not-distributed": "#c5b3e6", "declared": "#cfe2ff", "dropped": "#f5c2c7", "broken": "#dc3545", @@ -50,6 +58,7 @@ TIER_GLYPH = { "n/a": "·", "absent": "–", "declared": "?", "dropped": "✗!", "broken": "✗", "builds": "⚠", "sane": "✓", "blessed": "★", + "not-asked": "∅", "asked-not-distributed": "⤓", } diff --git a/docs/guide/coverage.md b/docs/guide/coverage.md index 71fb982cd..1098b52f5 100644 --- a/docs/guide/coverage.md +++ b/docs/guide/coverage.md @@ -28,6 +28,70 @@ re-implements the sanity checks. | `builds` | The wave slice is non-empty but fails a sanity check (`SanityReport.ok is False`). | | `sane` | The wave slice is non-empty and passes every sanity check (warnings allowed). | | `blessed` | `sane` **and** listed in the git-tracked blessing file `.coder/coverage/blessed.csv`. | +| `not-asked` | Adjudicated: the survey instrument genuinely never asked. Closed. | +| `asked-not-distributed` | Adjudicated: the instrument **did** ask, but the shipped extract does not carry the variables. An *acquisition* problem. | + +## Adjudicating an `absent` cell + +`absent` alone says only *"the feature is not declared for this wave."* That +conflates states that could not be more different, and until they are separated +the number can never reach zero: + +| verdict | meaning | closes the cell? | routes to | +|---|---|---|---| +| `todo` | the data **is** there; nobody wrote the config | no — stays `absent` | `add-feature` | +| `asked-not-distributed` | asked, but the extract lacks it | **yes** | acquisition / `add-wave` | +| `not-asked` | genuinely never asked | **yes** | closed forever | +| `unsure` | a required check could not be run | no — stays `absent` | human / OCR | + +Verdicts live in the git-tracked `.coder/coverage/absent_verdicts.csv`: + +```csv +country,feature,wave,verdict,checks_run,evidence,adjudicated_by,date +Albania,shocks,2005,todo,C1;C2,Data/migrationE_cl.dta m6e_q00 = 'Type of Shock Code' (10 types),sue,2026-07-12 +``` + +### Evidence is not optional + +A **closing** verdict (`not-asked` / `asked-not-distributed`) is a *permanent, +unsupervised write*: it removes a cell from the work queue with nobody +reviewing it. So `load_verdicts()` **refuses** a closing verdict whose +`evidence` field is empty, and warns. + +This is not defensive pedantry. It already went wrong: +`Albania/_/data_scheme.yml` asserted *"earlier waves have no shocks module"* — +and Albania 2005's `migrationE_cl.dta` carries `m6e_q00 = 'Type of Shock Code'` +with ten shock types. The claim was false, nobody could catch it (nothing +recorded *how* it was reached), and it silently suppressed work on ~5 cells. + +**An unevidenced negative is unfalsifiable, and therefore permanent whether or +not it is true.** + +### The four checks + +A cell may be closed only when **all applicable** checks were run and all came +back negative. Record which ran in `checks_run`. A check you *cannot* run makes +the verdict `unsure` — silence is never evidence. + +1. **C1 — variable/value-label sweep.** Metadata-only reads across every source + file for the wave. Be extension-agnostic (LSMS ships Stata as `.tab`, SPSS as + `.sav`) and **search in the instrument's own language** (French, Spanish, + Portuguese). Distinguish *"no labels found"* from *"the files carry no labels + at all"* — the latter means the check did not run. +2. **C2 — sibling-wave differential.** Necessary whenever the country has the + feature for another wave, but **never sufficient**: module vocabularies change + completely between waves (Iraq's 2007 `"Problem: theft"` shares no words with + 2012's phrasing, yet both waves have the module). +3. **C3 — Harmonized LSMS-ISA Ag cross-check.** `countries/Harmonized_LSMS-ISA_Ag/lsms_ag.parquet` + (7 ISA countries). **Positive-only**: a non-null harmonized column proves the + concept was asked; a *null* one proves nothing — it means the WB harmonizers + didn't code it. +4. **C4 — the questionnaire. Mandatory before any permanent close.** + *Absence in the shipped `.dta` is not absence in the instrument.* Every + data-side check can be defeated by a data-*distribution* gap, so no amount of + data-side evidence may close a cell as `not-asked` — only the questionnaire + can separate `not-asked` from `asked-not-distributed`, and those route to + completely different queues. ## Blessing a cell diff --git a/lsms_library/countries/Albania/_/data_scheme.yml b/lsms_library/countries/Albania/_/data_scheme.yml index 77135ee6b..570644709 100644 --- a/lsms_library/countries/Albania/_/data_scheme.yml +++ b/lsms_library/countries/Albania/_/data_scheme.yml @@ -83,10 +83,31 @@ Data Scheme: aggregation: visit: first - # Occurrence-only shocks module (2012 wave only; earlier waves have no - # shocks module). No impact (Affected*) or coping (HowCoped*) detail in - # the source; ``Years`` is the comma-joined list of years 2008-2012 in - # which each experienced shock was reported. + # Occurrence-only shocks module. WIRED for the 2012 wave only. No impact + # (Affected*) or coping (HowCoped*) detail in the source; ``Years`` is the + # comma-joined list of years 2008-2012 in which each experienced shock was + # reported. + # + # CORRECTION 2026-07-12: this comment previously asserted "2012 wave only; + # earlier waves have no shocks module". That claim was FALSE and it + # suppressed work on ~5 cells. Verified against source: + # + # 2005 Data/migrationE_cl.dta m6e_q00 "Type of Shock Code" + # (10 types: Dispossesion of Land, + # Unexpected death of income earner, + # Job loss, Flood damage, Pyramid + # scheme, ...) + m6e_q1989..q1999 + # year-occurrence grid + # 2008 Data/Modul_6E_migration_e.sav m6e_q00, identical + # 2003 Data/w2_hh_all.dta B10A_Q10 "faced any shocks in last + # 12 months" + 7 shock types + # 2004 Data/w3_hh_all.dta m10a_q09, same + # + # 2002 is UNRESOLVED (no household questionnaire in the repo to check). + # + # These waves are therefore TODO, not "no module". Tracked in the absent-cell + # adjudication (.coder/coverage/absent_verdicts.csv); do not re-close them + # without evidence. See slurm_logs/DESIGN_coverage_discipline_2026-07-11.org. shocks: index: (t, i, Shock) Years: str diff --git a/lsms_library/coverage_matrix.py b/lsms_library/coverage_matrix.py index 32153f416..eabc00b67 100644 --- a/lsms_library/coverage_matrix.py +++ b/lsms_library/coverage_matrix.py @@ -43,15 +43,61 @@ # Tier ladder # --------------------------------------------------------------------------- TIER_ORDER = [ - "n/a", "absent", "declared", "dropped", "broken", "builds", "sane", "blessed", + "n/a", "not-asked", "asked-not-distributed", "absent", "declared", + "dropped", "broken", "builds", "sane", "blessed", ] # Worst → best, for rolling several wave tiers up into one grid cell. The most # actionable (a real defect) sorts first so problems surface in the summary. +# +# ``absent`` (= an *un-adjudicated* gap) sorts as actionable, because it is the +# live queue. ``not-asked`` and ``asked-not-distributed`` are *adjudicated* and +# sort with ``n/a`` at the quiet end: they are settled, not pending. ROLLUP_PRIORITY = [ - "broken", "dropped", "builds", "declared", "sane", "blessed", "absent", "n/a", + "broken", "dropped", "builds", "absent", "declared", "sane", "blessed", + "asked-not-distributed", "not-asked", "n/a", ] COLUMNS = ["country", "feature", "wave", "tier", "coverage", "n_rows", "detail"] +# --------------------------------------------------------------------------- +# Absent-cell verdicts (GH #593) +# --------------------------------------------------------------------------- +# ``absent`` means only "this feature is not declared for this wave". It +# conflates states that could not be more different, and until they are +# separated the number can never reach zero: +# +# todo the data IS there; nobody has written the config. +# Real, actionable work. Stays ``absent`` in the grid +# (it is the live queue) but is now *sized and sourced*. +# asked-not-distributed the instrument DID ask, but the shipped extract does +# not carry the variables. An ACQUISITION problem, not +# a config one -- a different queue entirely. +# not-asked the instrument genuinely never asked. Closed forever. +# unsure a required check could not be run. STAYS in the queue, +# and records why. Silence is never evidence. +# +# A 38-cell pilot (2026-07-11) returned 25 todo / 13 unsure / **0 not-asked** -- +# so ``absent`` is not hiding "the survey never asked", it is hiding a backlog. +# ``not-asked`` may be near-empty; the mechanism earns its keep by making every +# probe TERMINATE in a recorded, re-checkable verdict rather than evaporating. +# +# The evidence bar is non-negotiable, because a verdict here is a PERMANENT, +# UNSUPERVISED write. An unevidenced negative is unfalsifiable, and therefore +# forever. This is not hypothetical: ``Albania/_/data_scheme.yml`` asserted +# "earlier waves have no shocks module" -- and Albania 2005's +# ``migrationE_cl.dta`` carries ``m6e_q00 = 'Type of Shock Code'`` with ten shock +# types. The claim was wrong, nobody could catch it, and it suppressed work. +# +# See ``docs/guide/coverage.md`` and +# ``slurm_logs/DESIGN_coverage_discipline_2026-07-11.org``. +ADJUDICATED_TIERS = {"not-asked", "asked-not-distributed"} +VERDICTS = {"todo", "asked-not-distributed", "not-asked", "unsure"} +# Verdicts that CLOSE a cell (change its tier away from plain ``absent``). +# ``todo`` and ``unsure`` deliberately do not: they are still open work. +_VERDICT_TO_TIER = { + "not-asked": "not-asked", + "asked-not-distributed": "asked-not-distributed", +} + # --------------------------------------------------------------------------- # Snapshot / blessing file locations @@ -76,6 +122,72 @@ def default_blessed_path() -> Path: return _repo_root() / ".coder" / "coverage" / "blessed.csv" +def default_verdicts_path() -> Path: + return _repo_root() / ".coder" / "coverage" / "absent_verdicts.csv" + + +def load_verdicts(path: Path | None = None) -> dict[tuple[str, str, str], dict]: + """Adjudications of ``absent`` cells, keyed ``(country, feature, wave)``. + + CSV with header + ``country,feature,wave,verdict,checks_run,evidence,adjudicated_by,date`` + (``wave`` blank for country-level features). Missing file → empty dict. + + ``verdict`` must be one of :data:`VERDICTS`. A row is IGNORED (with a + warning) when its verdict is unknown, or when it claims a closing verdict + (``not-asked`` / ``asked-not-distributed``) with an **empty ``evidence``** + field. + + That last rule is the point of the whole mechanism. A closing verdict is a + permanent, unsupervised write: it removes a cell from the work queue with + nobody reviewing it. An unevidenced negative cannot be challenged, and is + therefore permanent whether or not it is true. So we simply do not accept + one -- an evidence-free close is not a close, it is a red cell that lies. + """ + path = path or default_verdicts_path() + if not Path(path).exists(): + return {} + df = pd.read_csv(path, dtype=str, keep_default_na=False) + out: dict[tuple[str, str, str], dict] = {} + for _, r in df.iterrows(): + key = (r.get("country", ""), r.get("feature", ""), r.get("wave", "")) + verdict = (r.get("verdict", "") or "").strip() + if verdict not in VERDICTS: + warnings.warn( + f"absent_verdicts: ignoring {key} -- unknown verdict " + f"{verdict!r} (expected one of {sorted(VERDICTS)})", + stacklevel=2, + ) + continue + if verdict in _VERDICT_TO_TIER and not (r.get("evidence", "") or "").strip(): + warnings.warn( + f"absent_verdicts: ignoring {key} -- verdict {verdict!r} closes " + "the cell permanently and so REQUIRES a non-empty `evidence` " + "field. An unevidenced negative is unfalsifiable.", + stacklevel=2, + ) + continue + out[key] = dict(r) + return out + + +def _absent_tier(country, feature, wave, verdicts) -> tuple[str, str]: + """Grade a not-declared cell: ``(tier, detail)``. + + The single place ``absent`` is decided. Without an adjudication the cell is + plain ``absent`` -- an *open, un-triaged* gap. With one it may be closed + (``not-asked`` / ``asked-not-distributed``); ``todo`` and ``unsure`` stay + ``absent`` because they are still work, but carry their evidence forward so + the next person does not re-derive it. + """ + v = verdicts.get((country, feature, str(wave) if wave is not None else "")) + if not v: + return "absent", "source not declared for wave" + verdict = v.get("verdict", "") + detail = f"{verdict}: {v.get('evidence', '')}".strip().rstrip(":") + return _VERDICT_TO_TIER.get(verdict, "absent"), detail + + def load_blessed(path: Path | None = None) -> set[tuple[str, str, str]]: """Return the git-tracked set of blessed ``(country, feature, wave)`` cells. @@ -167,7 +279,8 @@ def _safe_build(co, feature, env): def grade_feature(country_name, feature, waves, co, env, *, - avail_by_wave=None, blessed=frozenset(), readiness=True) -> list[dict]: + avail_by_wave=None, blessed=frozenset(), verdicts=None, + readiness=True) -> list[dict]: """Per-wave cells for one ``(country, feature)``. Builds the country-level table at most once (``readiness=True``) and grades @@ -187,50 +300,83 @@ def grade_feature(country_name, feature, waves, co, env, *, except Exception: # noqa: BLE001 — a broken wave declares nothing avail_by_wave[w] = set() covered: dict[str, bool] = {w: feature in avail_by_wave[w] for w in waves} + verdicts = {} if verdicts is None else verdicts + + def _absent(w): + """The single place an un-declared cell is graded (see _absent_tier).""" + return _absent_tier(country_name, feature, w, verdicts) if not readiness: - return [ - _cell(country_name, feature, w, - tier="declared" if covered[w] else "absent", - coverage="declared" if covered[w] else "absent", - detail="coverage-only run") - for w in waves - ] + out = [] + for w in waves: + if covered[w]: + out.append(_cell(country_name, feature, w, "declared", "declared", + detail="coverage-only run")) + else: + tier, detail = _absent(w) + out.append(_cell(country_name, feature, w, tier, "absent", + detail=detail)) + return out df, err = _safe_build(co, feature, env) if err is not None or df is None or len(df) == 0: why = err or "feature built empty" - return [ - _cell(country_name, feature, w, - tier="broken" if covered[w] else "absent", - coverage="declared" if covered[w] else "absent", - detail=why if covered[w] else "source not declared for wave") - for w in waves - ] + out = [] + for w in waves: + if covered[w]: + out.append(_cell(country_name, feature, w, "broken", "declared", + detail=why)) + else: + tier, detail = _absent(w) + out.append(_cell(country_name, feature, w, tier, "absent", + detail=detail)) + return out names = list(df.index.names or []) if "t" not in names: rep = env["is_this_feature_sane"](df, country_name, feature) base = tier_from_report(rep) - cells = [ - _cell(country_name, feature, w, - tier="n/a" if covered[w] else "absent", - coverage="declared" if covered[w] else "absent", - detail=(f"no per-wave (t) axis; country-level grade={base}" - if covered[w] else "source not declared for wave")) - for w in waves - ] + cells = [] + for w in waves: + if covered[w]: + cells.append(_cell(country_name, feature, w, "n/a", "declared", + detail=f"no per-wave (t) axis; " + f"country-level grade={base}")) + else: + tier, detail = _absent(w) + cells.append(_cell(country_name, feature, w, tier, "absent", + detail=detail)) cells.append(_cell(country_name, feature, None, base, "declared", n_rows=len(df), detail=_report_summary(rep))) return cells t_values = {str(v) for v in df.index.get_level_values("t").unique()} t_level = df.index.get_level_values("t").astype(str) + + # Columns populated SOMEWHERE in the country frame. Exempt these from the + # per-wave ``no_all_null_columns`` check: it is a *country-level* check, and + # a question simply not fielded in a given wave is legitimately all-null in + # that wave's slice. Without this, such cells were graded ``builds`` (a + # defect) rather than ``sane`` -- 128 of the 138 ``builds`` cells in the + # 2026-06-26 snapshot were this artefact, and *zero* of the columns they + # flagged were all-null country-wide. + # + # A column that IS all-null across the whole country stays non-optional and + # still (correctly) fails -- we are relaxing the slice, not the country. + # + # NB: this deliberately hides *wave-level* column gaps from the sanity + # grade, because a sanity grade is the wrong instrument for them. Two such + # gaps that this exposure was catching only by accident were inventoried as + # GH #591 (Nigeria food_prices) and GH #592 (18 waves with no GPS) BEFORE + # this change landed, so the regrade cannot bury them. + populated = {c for c in df.columns if df[c].notna().any()} + cells = [] for w in waves: if not covered[w]: - cells.append(_cell(country_name, feature, w, "absent", "absent", - detail="source not declared for wave")) + tier, detail = _absent(w) + cells.append(_cell(country_name, feature, w, tier, "absent", + detail=detail)) continue if str(w) not in t_values: cells.append(_cell(country_name, feature, w, "dropped", "declared", @@ -241,7 +387,8 @@ def grade_feature(country_name, feature, waves, co, env, *, cells.append(_cell(country_name, feature, w, "dropped", "declared", detail="declared, present in t-index but 0 rows")) continue - rep = env["is_this_feature_sane"](sl, country_name, feature) + rep = env["is_this_feature_sane"](sl, country_name, feature, + extra_optional=populated) tier = tier_from_report(rep) if tier == "sane" and (country_name, feature, str(w)) in blessed: tier = "blessed" @@ -251,7 +398,8 @@ def grade_feature(country_name, feature, waves, co, env, *, def grade_country_level(country_name, feature, co, env, *, - blessed=frozenset(), readiness=True) -> dict: + blessed=frozenset(), verdicts=None, + readiness=True) -> dict: """Grade a country-level-only feature (panel_ids / updated_ids): wave=None.""" if not readiness: return _cell(country_name, feature, None, "n/a", "n/a", @@ -265,8 +413,11 @@ def grade_country_level(country_name, feature, co, env, *, detail=f"{type(e).__name__}: {e}") n = len(val) if hasattr(val, "__len__") else 0 if n == 0: # avoid `not val` — ambiguous if a future feature returns a frame - return _cell(country_name, feature, None, "absent", "absent", - detail="empty / no entries") + tier, detail = _absent_tier(country_name, feature, None, + verdicts or {}) + if detail == "source not declared for wave": # un-adjudicated + detail = "empty / no entries" + return _cell(country_name, feature, None, tier, "absent", detail=detail) tier = "blessed" if (country_name, feature, "") in blessed else "sane" return _cell(country_name, feature, None, tier, "declared", n_rows=n, detail=f"{n} entries") @@ -276,7 +427,8 @@ def grade_country_level(country_name, feature, co, env, *, # Whole-matrix build # --------------------------------------------------------------------------- def build_matrix(countries=None, features=None, *, readiness=True, - blessed=None, log=lambda _m: None) -> pd.DataFrame: + blessed=None, verdicts=None, + log=lambda _m: None) -> pd.DataFrame: """Build the full (country × feature × wave) status table. Parameters @@ -297,6 +449,8 @@ def build_matrix(countries=None, features=None, *, readiness=True, countries_root = env["countries_root"] if blessed is None: blessed = load_blessed() + if verdicts is None: + verdicts = load_verdicts() if countries is None: countries = catalog.countries() feat_filter = set(features) if features else None @@ -328,14 +482,16 @@ def build_matrix(countries=None, features=None, *, readiness=True, for f in sel: rows.extend(grade_feature(c, f, waves, co, env, avail_by_wave=avail_by_wave, - blessed=blessed, readiness=readiness)) + blessed=blessed, verdicts=verdicts, + readiness=readiness)) for clf in country_level_only: if feat_filter is not None and clf not in feat_filter: continue if _has_country_level_feature(c, clf, countries_root): rows.append(grade_country_level(c, clf, co, env, - blessed=blessed, readiness=readiness)) + blessed=blessed, verdicts=verdicts, + readiness=readiness)) df = pd.DataFrame(rows, columns=COLUMNS) df["tier"] = pd.Categorical(df["tier"], categories=TIER_ORDER, ordered=True) diff --git a/lsms_library/diagnostics.py b/lsms_library/diagnostics.py index 8b2778f68..8d327e9cb 100644 --- a/lsms_library/diagnostics.py +++ b/lsms_library/diagnostics.py @@ -592,6 +592,8 @@ def is_this_feature_sane( df: pd.DataFrame, country: str, feature: str, + *, + extra_optional: set[str] | None = None, ) -> SanityReport: """Run all sanity checks on a feature DataFrame. @@ -603,6 +605,21 @@ def is_this_feature_sane( Country name (e.g., ``'Uganda'``). feature : str Feature/table name (e.g., ``'food_acquired'``). + extra_optional : set[str], optional + Column names to exempt from ``no_all_null_columns``, *in addition* to + those declared ``optional`` in the country's scheme. + + This exists for **grading a slice of a larger frame** — notably the + coverage matrix, which grades one wave at a time by slicing the + country-level table on ``t``. ``no_all_null_columns`` is a + *country-level* check: a column that is legitimately not fielded in + one wave is all-null *within that wave's slice* and would be reported + as a failure, even though the country as a whole populates it. + + Callers grading a slice should pass the set of columns that are + populated **somewhere in the parent frame**; a column that is all-null + across the whole country is then still (correctly) reported as a + failure. Default ``None`` preserves the historical behaviour exactly. Returns ------- @@ -612,7 +629,9 @@ def is_this_feature_sane( """ scheme = _load_scheme(country) report = SanityReport(country=country, feature=feature) - optional = scheme.get(feature, {}).get("optional", set()) + optional = set(scheme.get(feature, {}).get("optional", set())) + if extra_optional: + optional |= set(extra_optional) report.checks.append(_check_not_empty(df)) report.checks.append(_check_has_index(df)) diff --git a/tests/test_coverage_matrix.py b/tests/test_coverage_matrix.py index bb63c5c93..cff82f51a 100644 --- a/tests/test_coverage_matrix.py +++ b/tests/test_coverage_matrix.py @@ -51,7 +51,9 @@ def load_feature(_co, _feature): raise load_result return load_result - def is_sane(_df, _c, _f): + def is_sane(_df, _c, _f, **_kw): + # **_kw absorbs ``extra_optional=`` (the populated-column set the + # wave-slice grader passes; see coverage_matrix.grade_feature). return report return { @@ -251,3 +253,169 @@ def test_build_matrix_covers_declared_features(monkeypatch): # household_roster is declared for Uganda; food_expenditures is derived-surfaced assert "household_roster" in feats assert "food_expenditures" in feats + + +# --------------------------------------------------------------------------- +# Wave-slice grading must not apply the COUNTRY-level all-null-column check +# to a single wave (the 2026-07-12 regrade; 128 of 138 `builds` cells). +# --------------------------------------------------------------------------- +def test_wave_slice_exempts_columns_populated_elsewhere_in_the_country(): + """A column not fielded in ONE wave must not fail that wave. + + ``_check_no_all_null_columns`` is a country-level check. Grading a wave by + slicing the country frame on ``t`` made any question that a given wave did + not ask look like a defect, because it is legitimately all-null *within + that slice*. ``grade_feature`` must exempt columns that are populated + somewhere in the parent frame. + """ + from lsms_library.diagnostics import is_this_feature_sane + + # Wave A fields `Latitude`; wave B does not. Country-wide it IS populated. + idx = pd.MultiIndex.from_tuples( + [("A", "hh1"), ("A", "hh2"), ("B", "hh3"), ("B", "hh4")], + names=["t", "i"], + ) + df = pd.DataFrame({"Latitude": [1.5, 2.5, None, None]}, index=idx) + + sl_b = df[df.index.get_level_values("t") == "B"] # the all-null slice + populated = {c for c in df.columns if df[c].notna().any()} + assert populated == {"Latitude"} + + # Without the exemption the slice fails; with it, it passes. + naive = is_this_feature_sane(sl_b, "Uganda", "household_roster") + fixed = is_this_feature_sane(sl_b, "Uganda", "household_roster", + extra_optional=populated) + naive_null = [c for c in naive.checks if c.name == "no_all_null_columns"] + fixed_null = [c for c in fixed.checks if c.name == "no_all_null_columns"] + assert naive_null and naive_null[0].status == "fail" # the old, wrong grade + assert fixed_null and fixed_null[0].status == "pass" # the corrected grade + + +def test_column_all_null_country_wide_still_fails(): + """We relaxed the SLICE, not the country. A truly dead column still fails.""" + from lsms_library.diagnostics import is_this_feature_sane + + idx = pd.MultiIndex.from_tuples( + [("A", "hh1"), ("A", "hh2"), ("B", "hh3")], names=["t", "i"] + ) + df = pd.DataFrame({"Latitude": [None, None, None]}, index=idx) # never populated + populated = {c for c in df.columns if df[c].notna().any()} + assert populated == set() # nothing to exempt + + rep = is_this_feature_sane(df, "Uganda", "household_roster", + extra_optional=populated) + null_check = [c for c in rep.checks if c.name == "no_all_null_columns"] + assert null_check and null_check[0].status == "fail" + + +def test_extra_optional_default_preserves_historical_behaviour(): + """Omitting ``extra_optional`` must grade exactly as before (back-compat).""" + from lsms_library.diagnostics import is_this_feature_sane + + idx = pd.MultiIndex.from_tuples([("A", "hh1"), ("A", "hh2")], names=["t", "i"]) + df = pd.DataFrame({"Latitude": [None, None]}, index=idx) + + a = is_this_feature_sane(df, "Uganda", "household_roster") + b = is_this_feature_sane(df, "Uganda", "household_roster", extra_optional=None) + pick = lambda r: [(c.name, c.status) for c in r.checks] + assert pick(a) == pick(b) + + +# --------------------------------------------------------------------------- +# Absent-cell verdicts (GH #593) -- the four-way split of the `absent` tier. +# --------------------------------------------------------------------------- +def _verdicts_file(tmp_path, rows): + p = tmp_path / "absent_verdicts.csv" + hdr = "country,feature,wave,verdict,checks_run,evidence,adjudicated_by,date\n" + p.write_text(hdr + "".join(rows)) + return p + + +def test_not_asked_closes_the_cell(tmp_path): + p = _verdicts_file(tmp_path, [ + "C,foo,w2,not-asked,C1;C2;C4,no module in questionnaire s3,sue,2026-07-12\n"]) + v = cov.load_verdicts(p) + co = _FakeCountry({"w1": ["foo"], "w2": []}) + cells = cov.grade_feature("C", "foo", ["w1", "w2"], co, + _env(_df_with_t({"w1": 3}), _ok_report()), + readiness=True, verdicts=v) + assert _tiers(cells)[("foo", "w2")] == "not-asked" + + +def test_asked_not_distributed_is_its_own_tier(tmp_path): + """The state the pilot found: instrument asked, extract does not carry it. + + Neither `todo` (nothing to configure) nor `not-asked` (it WAS asked) -- + it is an ACQUISITION problem and routes to a different queue. + """ + p = _verdicts_file(tmp_path, [ + "C,foo,w2,asked-not-distributed,C1;C2;C4," + "questionnaire MODULE 2 lists it; vars absent from shipped dta,sue,2026-07-12\n"]) + v = cov.load_verdicts(p) + co = _FakeCountry({"w1": ["foo"], "w2": []}) + cells = cov.grade_feature("C", "foo", ["w1", "w2"], co, + _env(_df_with_t({"w1": 3}), _ok_report()), + readiness=True, verdicts=v) + assert _tiers(cells)[("foo", "w2")] == "asked-not-distributed" + + +@pytest.mark.parametrize("verdict", ["todo", "unsure"]) +def test_todo_and_unsure_stay_in_the_queue(tmp_path, verdict): + """`todo` and `unsure` are OPEN work -- they must NOT close the cell.""" + p = _verdicts_file(tmp_path, [ + f"C,foo,w2,{verdict},C1,found s08q01 in EACIACT_p1.dta,sue,2026-07-12\n"]) + v = cov.load_verdicts(p) + co = _FakeCountry({"w1": ["foo"], "w2": []}) + cells = cov.grade_feature("C", "foo", ["w1", "w2"], co, + _env(_df_with_t({"w1": 3}), _ok_report()), + readiness=True, verdicts=v) + cell = [c for c in cells if c["wave"] == "w2"][0] + assert cell["tier"] == "absent" # still in the queue + assert verdict in cell["detail"] # but the evidence is carried + + +def test_closing_verdict_without_evidence_is_REFUSED(tmp_path): + """The load-bearing rule. + + A closing verdict is a permanent, unsupervised write. An unevidenced + negative is unfalsifiable and therefore permanent whether or not it is + true -- exactly the failure already sitting in Albania's data_scheme.yml + ("earlier waves have no shocks module"; Albania 2005 in fact carries + m6e_q00 = 'Type of Shock Code'). So we refuse it. + """ + p = _verdicts_file(tmp_path, ["C,foo,w2,not-asked,C1,,sue,2026-07-12\n"]) + with pytest.warns(UserWarning, match="unevidenced negative|REQUIRES"): + v = cov.load_verdicts(p) + assert v == {} # the row is dropped, not honoured + + co = _FakeCountry({"w1": ["foo"], "w2": []}) + cells = cov.grade_feature("C", "foo", ["w1", "w2"], co, + _env(_df_with_t({"w1": 3}), _ok_report()), + readiness=True, verdicts=v) + assert _tiers(cells)[("foo", "w2")] == "absent" # stays red. good. + + +def test_todo_without_evidence_is_allowed(tmp_path): + """Only CLOSING verdicts need evidence; `todo` keeps the cell open anyway.""" + p = _verdicts_file(tmp_path, ["C,foo,w2,todo,C1,,sue,2026-07-12\n"]) + v = cov.load_verdicts(p) + assert ("C", "foo", "w2") in v + + +def test_unknown_verdict_is_refused(tmp_path): + p = _verdicts_file(tmp_path, ["C,foo,w2,definitely-not,C1,x,sue,2026-07-12\n"]) + with pytest.warns(UserWarning, match="unknown verdict"): + assert cov.load_verdicts(p) == {} + + +def test_verdicts_missing_file_is_empty(tmp_path): + assert cov.load_verdicts(tmp_path / "nope.csv") == {} + + +def test_absent_stays_absent_with_no_verdicts(): + """Back-compat: no verdicts file -> the historical grading, unchanged.""" + co = _FakeCountry({"w1": ["foo"], "w2": []}) + cells = cov.grade_feature("C", "foo", ["w1", "w2"], co, + _env(_df_with_t({"w1": 3}), _ok_report()), + readiness=True) + assert _tiers(cells)[("foo", "w2")] == "absent" From 85bdf4a57516a29d837a4fee5f718ca430930916 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 06:50:59 -0700 Subject: [PATCH 06/85] fix(household_characteristics): stop the residence filter deleting whole waves `roster_to_characteristics()` is handed the *country-level* concat of every wave's `household_roster`, so a residence-duration column contributed by one wave is unioned (all-NaN) onto every other wave. Three defects followed, each silently returning NOTHING for a wave whose roster is fully populated: D1 The resolution was an if/elif chain over the columns *present*, not *populated*. Ethiopia carries both MonthsAway (W1-W3) and WeeksAway (W4-W5, when the questionnaire switched to weeks -- exactly as CLAUDE.md documents). MonthsAway won the elif, resolved to all-NaN on W4-W5, and both waves vanished. -> per-ROW coalesce across the three sources, priority MonthsSpent > MonthsAway > WeeksAway. D2 No all-NaN guard. CotedIvoire 1985-89 and Mali 2021-22 never asked a residence question; their MonthsSpent column exists only because a later wave supplies it. The keep-mask was all-False -> 5 waves vanished. -> per-`t` fallback to the documented "count everyone" behaviour ("countries without any residence column are unaffected", CLAUDE.md), applied at wave granularity, and it now warns rather than deleting silently. D3 Burkina Faso 2014's MonthsSpent (EMC B3A) is the raw French 6-month binary '6 mois ou plus' / 'moins de 6 mois'; to_numeric(errors='coerce') NaN'd the wave away. -> fixed in CONFIG (the wave's data_info.yml `mapping:`), following the EHCVM Oui/Non -> 12/0 pattern. No French in library code. The filter itself is untouched wherever it has data: the fallback fires only for a wave with no usable residence datum at all. Census over all 34 countries that expose household_characteristics: the 8 dead wave-cells come back (+34,974 households) and all 88 other waves are byte-identical, Uganda included -- so the 1315-HH Uganda drift the filter was introduced to fix stays fixed. Ledger: .coder/ledger/roster-characteristics-wave-deletion.md Tests: tests/test_roster_residence_filter.py (6 of them fail pre-fix) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NtARb9se2un1DLFJrrpjHe --- .../Burkina_Faso/2014/_/data_info.yml | 14 +- lsms_library/transformations.py | 101 ++++++- tests/test_roster_residence_filter.py | 251 ++++++++++++++++++ 3 files changed, 354 insertions(+), 12 deletions(-) create mode 100644 tests/test_roster_residence_filter.py diff --git a/lsms_library/countries/Burkina_Faso/2014/_/data_info.yml b/lsms_library/countries/Burkina_Faso/2014/_/data_info.yml index a23d197c1..780d4cbc1 100644 --- a/lsms_library/countries/Burkina_Faso/2014/_/data_info.yml +++ b/lsms_library/countries/Burkina_Faso/2014/_/data_info.yml @@ -33,7 +33,19 @@ household_roster: Sex: B2 Age: B4 Relationship: B5 - MonthsSpent: B3A + # B3A is a 6-month binary ("Depuis combien de temps [NOM] vit dans + # ce menage?"), not a months count: '6 mois ou plus' (77,268) / + # 'moins de 6 mois' (943) / NaN (332). Same semantics as the EHCVM + # waves' s01q12/s01q13 (>=6 months y/n), so it is mapped to the same + # 12/0 canonical MonthsSpent -- see CLAUDE.md "EHCVM note" and e.g. + # Burkina_Faso/2018-19, Mali/2017-18. Left untranslated, the French + # labels coerce to NaN in roster_to_characteristics() and the whole + # wave drops out of household_characteristics. + MonthsSpent: + - B3A + - mapping: + '6 mois ou plus': 12 + 'moins de 6 mois': 0 individual_education: # 2014 EMC: highest formal-education level attained is B14 ("Quel est le diff --git a/lsms_library/transformations.py b/lsms_library/transformations.py index 4af5bd638..37a367746 100644 --- a/lsms_library/transformations.py +++ b/lsms_library/transformations.py @@ -226,6 +226,22 @@ def roster_to_characteristics(df, age_cuts=(4, 9, 14, 19, 31, 51), drop='pid', pandas.DataFrame Household-level counts with one column per sex × age bucket and a ``log HSize`` column. + + Notes + ----- + **Residence filter.** Non-resident members are dropped using whichever + residence-duration column the roster carries — ``MonthsSpent`` (months + present), ``MonthsAway`` (→ ``12 - value``) or ``WeeksAway`` + (→ ``12 - weeks/(52/12)``); see ``CLAUDE.md`` §"MonthsSpent / + MonthsAway / WeeksAway". ``df`` here is the *country-level* concat of + every wave, so those columns are unioned across waves: a column + supplied by one wave appears (all-NaN) on the others. The resolution + is therefore a **per-row coalesce** across the three sources, and a + wave with no usable residence datum at all falls back to counting every + roster member — the documented behaviour for a country with no + residence column, applied per-wave. Without those two rules an entire + wave's ``household_characteristics`` silently vanishes while its + ``household_roster`` is fully populated. """ roster_df = df.copy() roster_df.columns = roster_df.columns.str.lower() @@ -248,20 +264,83 @@ def roster_to_characteristics(df, age_cuts=(4, 9, 14, 19, 31, 51), drop='pid', # Uganda uses MonthsSpent (months present, 0-12). East African # surveys (Ethiopia, Tanzania, Malawi) use MonthsAway (months # absent, 0-12); convert to months-present for a uniform filter. + # Ethiopia W4-W5 and Cambodia use WeeksAway instead. + # + # The resolution is a per-ROW coalesce, not a per-column choice: this + # frame is the *country-level* concat of every wave, so a column + # contributed by one wave is unioned onto all the others. Ethiopia + # carries BOTH MonthsAway (W1-W3) and WeeksAway (W4-W5), disjointly + # populated -- picking the first column *present* would resolve W4-W5 + # to an all-NaN MonthsAway and delete both waves. Priority order on + # the (rare) rows where more than one source is populated: + # MonthsSpent (asked directly) > MonthsAway > WeeksAway (coarser). + # Work positionally (numpy) rather than through pandas alignment: the + # roster index is a MultiIndex with duplicate entries in the wild. + def _numeric(col): + return pd.to_numeric(roster_df[col], errors='coerce').astype('float64').to_numpy() + ms = None + _sources = [] if 'monthsspent' in roster_df.columns: - ms = pd.to_numeric(roster_df['monthsspent'], errors='coerce') - elif 'monthsaway' in roster_df.columns: - ma = pd.to_numeric(roster_df['monthsaway'], errors='coerce') - ms = 12 - ma - ms = ms.clip(lower=0) # guard against >12 outliers - elif 'weeksaway' in roster_df.columns: - wa = pd.to_numeric(roster_df['weeksaway'], errors='coerce') - ms = 12 - (wa / (52 / 12)) - ms = ms.clip(lower=0) + _sources.append(_numeric('monthsspent')) + if 'monthsaway' in roster_df.columns: + # months present = 12 - months away; clip guards >12 outliers + _sources.append(np.clip(12 - _numeric('monthsaway'), 0, None)) + if 'weeksaway' in roster_df.columns: + _sources.append(np.clip(12 - (_numeric('weeksaway') / (52 / 12)), 0, None)) + if _sources: + ms = _sources[0] + for _alt in _sources[1:]: + ms = np.where(np.isnan(ms), _alt, ms) if ms is not None: - age = pd.to_numeric(roster_df['age'], errors='coerce') - keep = ms.notna() & ((ms > 0) | (age < 1)) + age = _numeric('age') + with np.errstate(invalid='ignore'): + keep = ~np.isnan(ms) & ((ms > 0) | (age < 1)) + # A wave can end up with a residence column that is entirely NaN + # *within that wave* -- either because the survey never asked the + # question (CotedIvoire 1985-89, Mali 2021-22: the column exists in + # this frame only because the country-level concat unioned it in + # from a later wave) or because the labels did not parse. ``keep`` + # is then all-False for the wave and household_characteristics + # silently returns NOTHING for it while household_roster is fully + # populated. Guard per ``t``: a wave with no usable residence datum + # at all reverts to the documented pre-MonthsSpent behaviour -- + # "countries without any residence column are unaffected -- the old + # count-everyone behavior continues" (CLAUDE.md) -- applied at wave + # granularity. Waves that DO have residence data are filtered + # exactly as before, so the Uganda drift the filter was introduced + # to fix stays fixed. + has_ms = ~np.isnan(ms) + idx_names = roster_df.index.names or [] + if 't' in idx_names: + t_vals = pd.Index(roster_df.index.get_level_values('t')) + wave_has_ms = ( + pd.Series(has_ms) + .groupby(t_vals, dropna=False) + .transform('any') + .to_numpy() + ) + else: + wave_has_ms = np.full(len(roster_df), bool(has_ms.any())) + if not wave_has_ms.all(): + import warnings as _warnings + if 't' in idx_names: + _dead = sorted( + {str(t) for t, ok in zip(t_vals, wave_has_ms) if not ok} + ) + else: + _dead = [''] + _warnings.warn( + f"household_characteristics: no usable residence duration " + f"(MonthsSpent / MonthsAway / WeeksAway) for wave(s) {_dead}; " + f"counting every roster member there instead of filtering to " + f"resident members. The column is present in the country-level " + f"frame only because another wave supplies it. Waves with " + f"residence data are filtered as usual.", + UserWarning, + stacklevel=3, + ) + keep = keep | ~wave_has_ms roster_df = roster_df[keep] roster_df = roster_df.dropna(subset=['sex', 'age']) roster_df['age_interval'] = age_intervals(roster_df['age'], age_cuts) diff --git a/tests/test_roster_residence_filter.py b/tests/test_roster_residence_filter.py new file mode 100644 index 000000000..1da6ff6c0 --- /dev/null +++ b/tests/test_roster_residence_filter.py @@ -0,0 +1,251 @@ +""" +Regression tests for the ``roster_to_characteristics`` residence filter -- +the silent per-wave deletion of ``household_characteristics``. + +``Country.household_characteristics()`` hands the transform the *country-level* +concat of every wave's ``household_roster``, so a residence-duration column +contributed by one wave is unioned (all-NaN) onto every other wave. Three +defects followed from that, each destroying an entire wave-cell while the +underlying roster stayed fully populated: + +* **D1** the resolution was an ``if/elif`` chain over the columns *present*, + not *populated*. Ethiopia carries both ``MonthsAway`` (W1-W3) and + ``WeeksAway`` (W4-W5, when the questionnaire switched to weeks -- see + ``CLAUDE.md`` §"MonthsSpent / MonthsAway / WeeksAway"). ``MonthsAway`` won + the ``elif``, resolved to all-NaN on W4-W5, and both waves vanished. + Fixed by a per-**row** coalesce across the three sources. +* **D2** no all-NaN guard. CotedIvoire 1985-89 and Mali 2021-22 never asked a + residence question; their ``MonthsSpent`` column exists only because a later + wave supplies it. The keep-mask was all-False -> 5 waves vanished. Fixed by + a per-``t`` fallback to the documented "count everyone" behaviour. +* **D3** untranslated labels. Burkina Faso 2014's ``MonthsSpent`` was the raw + French ``'6 mois ou plus'`` / ``'moins de 6 mois'``; ``to_numeric(..., + errors='coerce')`` turned both into NaN. Fixed in *config* (the wave's + ``data_info.yml`` ``mapping:``), following the EHCVM ``Oui``/``Non`` -> 12/0 + pattern -- asserted here so a config regression is caught. + +The adversarial constraint: the filter exists because counting everyone +produced a 1315-household drift on Uganda vs. the replication pipeline. The D2 +fallback IS that old behaviour, so it must fire **only** for a wave with no +usable residence datum at all -- never for a wave where the filter works. +``TestFilterStillFiresWhereItShould`` pins that. + +Synthetic rosters only -- no World Bank Microdata access required (except the +config assertion, which reads YAML). +""" +import warnings + +import pandas as pd +import pytest +import yaml + +from lsms_library.paths import countries_root +from lsms_library.transformations import roster_to_characteristics + +_IDX = ['t', 'v', 'i', 'pid'] + + +def _roster(rows, columns): + """rows: list of ((t, v, i, pid), (col values...)).""" + keys, vals = zip(*rows) + idx = pd.MultiIndex.from_tuples(keys, names=_IDX) + return pd.DataFrame(list(vals), index=idx, columns=columns) + + +def _hh(result): + """Households present per wave in a household_characteristics frame.""" + return result.groupby(level='t').size().to_dict() + + +def _members(result): + """Members counted per wave (sum of the sex x age buckets).""" + buckets = [c for c in result.columns if c != 'log HSize'] + return result[buckets].sum(axis=1).groupby(level='t').sum().astype(int).to_dict() + + +def _quiet(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + return roster_to_characteristics(*args, **kwargs) + + +class TestD1PerRowCoalesce: + """MonthsAway (W1) + WeeksAway (W2), disjointly populated -- the Ethiopia + shape. Both waves must survive, and both must still be filtered.""" + + @pytest.fixture + def roster(self): + # w1 asks MonthsAway; w2 asks WeeksAway. Each wave is NaN in the + # other's column, exactly as the country-level concat produces. + rows = [ + # ---- wave 1: MonthsAway populated, WeeksAway NaN + (('w1', 'V1', 'HH1', 1), ('MALE', 35, 0.0, pd.NA)), # resident + (('w1', 'V1', 'HH1', 2), ('FEMALE', 33, 2.0, pd.NA)), # resident + (('w1', 'V1', 'HH1', 3), ('MALE', 20, 12.0, pd.NA)), # away all year -> drop + (('w1', 'V2', 'HH2', 1), ('FEMALE', 40, 1.0, pd.NA)), # resident + # ---- wave 2: WeeksAway populated, MonthsAway NaN + (('w2', 'V1', 'HH1', 1), ('MALE', 36, pd.NA, 0.0)), # resident + (('w2', 'V1', 'HH1', 2), ('FEMALE', 34, pd.NA, 4.0)), # resident + (('w2', 'V1', 'HH1', 3), ('MALE', 21, pd.NA, 52.0)), # away all year -> drop + (('w2', 'V2', 'HH2', 1), ('FEMALE', 41, pd.NA, 2.0)), # resident + ] + return _roster(rows, ['sex', 'age', 'monthsaway', 'weeksaway']) + + def test_both_waves_survive(self, roster): + result = _quiet(roster) + assert _hh(result) == {'w1': 2, 'w2': 2}, ( + "D1: the WeeksAway wave must not be deleted by an all-NaN " + "MonthsAway column unioned in from the other wave" + ) + + def test_filter_still_fires_in_both_waves(self, roster): + result = _quiet(roster) + # The away-all-year member is dropped in each wave; nobody else is. + assert _members(result) == {'w1': 3, 'w2': 3} + + def test_no_fallback_warning(self, roster): + """Every wave has a usable datum -> the D2 fallback must NOT fire.""" + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always", UserWarning) + roster_to_characteristics(roster) + assert not [w for w in captured + if 'no usable residence duration' in str(w.message)] + + def test_monthsspent_wins_over_monthsaway_on_the_same_row(self): + """Priority order when >1 source is populated: MonthsSpent first.""" + rows = [ + # MonthsSpent says resident (12); MonthsAway would say away (12 -> 0) + (('w1', 'V1', 'HH1', 1), ('MALE', 35, 12.0, 12.0)), + (('w1', 'V2', 'HH2', 1), ('FEMALE', 33, 12.0, 12.0)), + ] + roster = _roster(rows, ['sex', 'age', 'monthsspent', 'monthsaway']) + result = _quiet(roster) + assert _members(result) == {'w1': 2} + + +class TestD2AllNaNWaveGuard: + """A wave whose residence column is entirely NaN reverts to counting + everyone -- the documented behaviour for a country with no residence + column, applied per-wave (CotedIvoire 1985-89 / Mali 2021-22 shape).""" + + @pytest.fixture + def roster(self): + rows = [ + # ---- old wave: residence question never asked (all NaN) + (('w1', 'V1', 'HH1', 1), ('MALE', 35, pd.NA)), + (('w1', 'V1', 'HH1', 2), ('FEMALE', 33, pd.NA)), + (('w1', 'V2', 'HH2', 1), ('FEMALE', 40, pd.NA)), + # ---- new wave: MonthsSpent asked + (('w2', 'V1', 'HH1', 1), ('MALE', 36, 12.0)), + (('w2', 'V1', 'HH1', 2), ('FEMALE', 34, 0.0)), # non-resident -> drop + (('w2', 'V2', 'HH2', 1), ('FEMALE', 41, 6.0)), + ] + return _roster(rows, ['sex', 'age', 'monthsspent']) + + def test_all_nan_wave_is_not_deleted(self, roster): + result = _quiet(roster) + assert 'w1' in _hh(result), ( + "D2: a wave with no usable residence datum must fall back to " + "counting everyone, not vanish" + ) + assert _hh(result) == {'w1': 2, 'w2': 2} + + def test_all_nan_wave_counts_everyone(self, roster): + result = _quiet(roster) + assert _members(result)['w1'] == 3 + + def test_populated_wave_is_still_filtered(self, roster): + """The fallback is per-``t``: the wave that DOES have data keeps its + filter (this is the Uganda-drift guarantee).""" + result = _quiet(roster) + assert _members(result)['w2'] == 2 + + def test_warns_naming_the_wave(self, roster): + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always", UserWarning) + roster_to_characteristics(roster) + msgs = [str(w.message) for w in captured + if 'no usable residence duration' in str(w.message)] + assert msgs, "the silent-deletion fallback must announce itself" + assert "'w1'" in msgs[-1] + assert "'w2'" not in msgs[-1] + + def test_single_wave_all_nan_without_t_level(self): + """Same guard when the caller passes a roster with no ``t`` level.""" + idx = pd.MultiIndex.from_tuples( + [('HH1', 1), ('HH1', 2)], names=['i', 'pid']) + roster = pd.DataFrame( + [('MALE', 35, pd.NA), ('FEMALE', 33, pd.NA)], + index=idx, columns=['sex', 'age', 'monthsspent']) + result = _quiet(roster, final_index=['i']) + assert len(result) == 1 + assert result[[c for c in result.columns + if c != 'log HSize']].sum(axis=1).iloc[0] == 2 + + +class TestFilterStillFiresWhereItShould: + """The adversarial check: the residence filter exists to fix a 1315-HH + Uganda drift. Nothing above may weaken it on a wave that has data.""" + + @pytest.fixture + def roster(self): + # Single Uganda-shaped wave: everyone has MonthsSpent. + rows = [ + (('w1', 'V1', 'HH1', 1), ('MALE', 35, 12.0)), + (('w1', 'V1', 'HH1', 2), ('FEMALE', 33, 12.0)), + (('w1', 'V1', 'HH1', 3), ('MALE', 8, 0.0)), # departed -> drop + (('w1', 'V1', 'HH1', 4), ('FEMALE', 0.5, 0.0)), # infant -> KEEP + (('w1', 'V1', 'HH1', 5), ('MALE', 20, pd.NA)), # not asked -> drop + ] + return _roster(rows, ['sex', 'age', 'monthsspent']) + + def test_zero_months_dropped_nan_dropped_infant_kept(self, roster): + result = _quiet(roster) + assert _members(result) == {'w1': 3} + row = result.iloc[0] + assert row['FEMALE 00-03'] == 1 # the infant survived + # The departed 8-y-o and the not-asked 20-y-o are gone: their buckets + # never even appear (``dummies`` emits a column only for an observed + # sex x age combination). + assert 'MALE 04-08' not in result.columns + assert 'MALE 19-30' not in result.columns + + def test_no_fallback_warning_when_wave_has_data(self, roster): + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always", UserWarning) + roster_to_characteristics(roster) + assert not [w for w in captured + if 'no usable residence duration' in str(w.message)] + + def test_string_months_still_parse(self): + """Country concat can leave MonthsSpent as strings ('12') or floats + ('12.0') -- both must still resolve.""" + rows = [ + (('w1', 'V1', 'HH1', 1), ('MALE', 35, '12')), + (('w1', 'V1', 'HH1', 2), ('FEMALE', 33, '0')), # -> drop + (('w1', 'V2', 'HH2', 1), ('FEMALE', 41, '12.0')), + ] + roster = _roster(rows, ['sex', 'age', 'monthsspent']) + assert _members(_quiet(roster)) == {'w1': 2} + + +class TestD3BurkinaFaso2014Config: + """The French 6-month binary must be mapped to 12/0 in *config*, exactly + as the EHCVM waves map ``Oui``/``Non`` (CLAUDE.md §"EHCVM note"). Left + raw, ``to_numeric(errors='coerce')`` NaNs the whole wave away.""" + + def test_monthsspent_mapping_declared(self): + path = (countries_root() / 'Burkina_Faso' / '2014' / '_' + / 'data_info.yml') + spec = yaml.safe_load(path.read_text()) + ms = spec['household_roster']['myvars']['MonthsSpent'] + assert isinstance(ms, list), ( + "Burkina Faso 2014 MonthsSpent (B3A) is the French 6-month binary " + "'6 mois ou plus' / 'moins de 6 mois'; it needs a mapping: block, " + "or roster_to_characteristics() coerces the wave to all-NaN and " + "household_characteristics returns nothing for 2014." + ) + assert ms[0] == 'B3A' + mapping = ms[-1]['mapping'] + assert mapping['6 mois ou plus'] == 12 + assert mapping['moins de 6 mois'] == 0 From e22b4429d64732582bfe28b574c8fe0087175ff7 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 06:55:10 -0700 Subject: [PATCH 07/85] =?UTF-8?q?docs(coverage):=20design=20doc=20?= =?UTF-8?q?=E2=80=94=20the=20discipline=20and=20workplan=20for=20filling?= =?UTF-8?q?=20the=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six defects in the coverage instrument, the four-way adjudication of `absent`, the NOT-ASKED evidence standard (and the pilot that refuted my original premise: 25 todo / 13 unsure / 0 not-asked), and a workplan ordered by leverage. Cited by PR #593. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../DESIGN_coverage_discipline_2026-07-11.org | 827 ++++++++++++++++++ 1 file changed, 827 insertions(+) create mode 100644 slurm_logs/DESIGN_coverage_discipline_2026-07-11.org diff --git a/slurm_logs/DESIGN_coverage_discipline_2026-07-11.org b/slurm_logs/DESIGN_coverage_discipline_2026-07-11.org new file mode 100644 index 000000000..4b0f1df1e --- /dev/null +++ b/slurm_logs/DESIGN_coverage_discipline_2026-07-11.org @@ -0,0 +1,827 @@ +#+TITLE: Filling the coverage matrix: a discipline and a workplan +#+AUTHOR: Sue (Claude Opus 4.8) +#+DATE: [2026-07-11 Sat] + +* Motivation + +We already /measure/ coverage. =make matrix= grades every +=(country, feature, wave)= cell on an eight-tier ladder and commits the +result to =.coder/coverage/latest.csv=; =ll.coverage()= reads it back. +That machinery landed in June (#584, #586, #590) and it works. + +What we do /not/ have is a discipline for closing the cells it reddens. +The snapshot has 1849 cells: + +| tier | cells | meaning | +|---------+-------+---------------------------------------------| +| sane | 1208 | builds, passes sanity | +| absent | 456 | "source not declared for this wave" | +| builds | 138 | builds, /fails/ a sanity check | +| dropped | 39 | declared for the wave, missing from output | +| broken | 8 | build raises, or the feature is empty | +| blessed | 0 | human-signed-off --- the tier is never used | + +This document argues that the matrix does not fill because of defects in +the /instrument/, not any shortage of effort. A census run on +[2026-07-11 Sat] found *six*, and they bias in both directions at once: +the matrix reports work that does not exist, and hides work that does. + +** The corrected ledger + +Once the six defects below are accounted for, the cube's real state is +materially different from what it reports: + +| tier as reported | cells | what it actually is | +|------------------+-------+---------------------------------------------------------------------------| +| sane | 1208 | genuine | +| builds | 138 | *131 are a grading bug* (Defect 5); ~7 real, incl. 2 buried data gaps | +| broken | 8 | *all 8 mis-tiered* --- Armenia/Nepal have no microdata at all (Defect 6) | +| dropped | 39 | *genuine, and the most dangerous tier* --- silent data loss | +| absent | 456 | *unknowable* --- conflates "never asked" with "not yet done" (Defect 1) | +| blessed | 0 | the top tier has never once been used | + +: really-clean ~1339 (1208 + the 131 misgraded) +: really-broken ~46 (39 dropped + ~7 builds) + 24 data gaps hiding inside the regrade +: genuinely unknown 456 <- and a 38-cell pilot says ~2/3 of these are REAL WORK + +*The 456 is not a pile of "the survey never asked."* A pilot running the +full evidence standard over 38 of them returned *25 TODO, 13 UNSURE, and +zero NOT-ASKED* (§Defect 1). The uncomfortable implication: the backlog is +*larger* than the matrix's red suggests, not smaller. Nothing here shrinks +the work. It makes it knowable, sized, and ordered --- which is the most an +instrument can honestly do. + +*Cells whose reported tier is wrong or unactionable: 595 of 1849 --- 32% of +the cube.* And the cube's /denominator/ is itself wrong: it omits every +wave the World Bank publishes that we have not downloaded (Defect 2), and +eleven countries whose data we already hold (Defect 4). + +The matrix cannot drive a workplan until it can be trusted. *Fix the +ruler, then measure.* That is Phase 0, and it is about a day's work. + +* Six defects in the measurement, before any workplan + +** Defect 1 --- =absent= is a black hole (the keystone) + +=absent= means "the feature is not declared for this wave." That single +tier conflates two states that could not be more different: + +- *NOT-ASKED* --- the survey instrument never asked the question. There + is no work here. There will never be work here. +- *TODO* --- the data is sitting in =Data/=; nobody has written the + config. This is real, actionable work. + +Nothing in the repo distinguishes them. Two consequences, and the second +is worse than the first. + +*The queue never converges.* A genuinely not-asked cell stays red +forever, so the number cannot reach zero, so nobody can tell progress +from noise. The queue is not a queue --- it is a mood. + +*And the probe's findings evaporate.* Whoever investigates a cell learns +something real --- "this wave's module is in =migrationE_cl.dta=, not +=Data/=" or "this instrument genuinely dropped the question in 2005" --- +and has *nowhere to put it*. The knowledge dies in their head, and the +next person pays the same cost to rediscover it. Over 456 cells that is +not inefficiency; it is a repository that cannot learn. + +The evidence that this has already happened, and produced a *false* +conclusion now sitting in the codebase, is in "Exhibit A" below. It is +the reason the fix is an /evidence/ file, not merely a /verdict/ file. + +*Fix.* Add a git-tracked adjudication file =.coder/coverage/absent_verdicts.csv=, +keyed =(country, feature, wave)= with a *mandatory* =evidence= column, a +=checks_run= column, and =adjudicated_by= / =date=. It mirrors the +existing =blessed.csv= mechanism --- same loader, same shape --- so it costs +about fifteen lines in =coverage_matrix.py= (=absent= is emitted at five +sites in =grade_feature=; factor a single =_absent_tier()= helper). + +The property this buys is the one that matters: + +#+begin_quote +Every probe of an =absent= cell now *terminates* --- in config, or in a +recorded verdict with evidence. A probe that ends without one of those +did not happen. +#+end_quote + +*** What the pilot proved (and what it refuted) + +A 38-cell pilot (=shocks=, =assets=; 2026-07-11) ran the full evidence +standard of §D2. Its headline *refutes the premise this section was +originally written on*: + +: 25 TODO 13 UNSURE 0 NOT-ASKED + +*The =absent= tier is not hiding a pile of "the survey never asked." It +is hiding a backlog.* On this evidence NOT-ASKED may be close to an +/empty category/, and =actionable_todo= will land near 456, not far below +it. The honest consequence is uncomfortable and should be stated plainly: +*the gap is bigger than it looked, not smaller.* The mechanism proposed +here does not shrink the work. It makes the work /knowable/, which is a +different and lesser claim than the one I started with. + +But the pilot found a state I had missed, and it is not rare: + +*** The third state: ASKED-NOT-DISTRIBUTED + +- *Albania 2004 =assets=* --- the label sweep and the sibling differential + both come back negative, and *both are correct about the data*: the + variables genuinely are not in the shipped =.dta=. But the + questionnaire (=alb04hhq2.pdf=) has =MODULE 2: DWELLING, UTILITIES AND + DURABLE GOODS=, itemising =105 Refrigerator 106 Freezer 107 Washing + machine=. *The module was asked. The extract does not carry it.* +- *EthiopiaRHS R5--R7* --- the archive ships only aggregates; no raw + section files exist to search. + +This is neither TODO (there is nothing to configure) nor NOT-ASKED (the +instrument asked). It is a *data-acquisition* problem --- chase the +extract --- and it belongs to a different queue and a different skill +(=add-wave=), not to the config backlog. Conflating it with either would +be a third silent error. + +So the verdict is *four-way*, not binary: + +| verdict | meaning | terminates? | routes to | +|-------------------------+----------------------------------------+-------------+----------------| +| =todo= | data present, config missing | no | =add-feature= | +| =asked-not-distributed= | instrument asked; extract lacks it | *yes* | acquisition | +| =not-asked= | instrument genuinely never asked | *yes* | closed forever | +| =unsure= | a required check could not be run | no | human / OCR | + +*** Exhibit A --- the failure is not hypothetical. It already happened. + +=lsms_library/countries/Albania/_/data_scheme.yml:86= states: + +#+begin_quote +"Occurrence-only shocks module (2012 wave only; *earlier waves have no +shocks module*)." +#+end_quote + +That claim is *false*. Verified directly against the source: +=Albania/2005/Data/migrationE_cl.dta= carries +=m6e_q00 = 'Type of Shock Code'= with ten shock types (Dispossession of +Land, Unexpected death of income earner, Job loss, Flood damage, Pyramid +scheme, ...), plus a =m6e_q1989..q1999= year-occurrence grid. Albania +2008 carries the identical variable; 2003 and 2004 carry +="faced any shocks in last 12 months"=. + +*An evidence-free NOT-ASKED claim already exists in this repo, it is +wrong, and it has been quietly suppressing work on roughly five cells.* +Nobody could catch it, because nothing recorded /how/ it was reached. + +This is the entire argument for the evidence column, made for me, by the +codebase, before I proposed it. The lesson is not "someone was careless" +--- it is that *an unevidenced negative is unfalsifiable*, and therefore +permanent. Which is exactly what we are about to license 456 times. + +** Defect 2 --- the denominator is self-congratulatory + +=build_matrix= enumerates waves with =waves = list(co.waves)= +(=coverage_matrix.py:308=), and =Country.waves= is derived from +directories that exist /on disk/. So a wave the World Bank publishes and +we have never downloaded is not a red cell. It is *no cell at all*. + +Two perverse incentives follow. Downloading a new wave makes coverage +look /worse/ (fresh =absent= cells appear). Never downloading anything +looks perfect. A matrix that rewards not acquiring data is measuring +the wrong thing. + +*Fix.* We already have =data_access.discover_waves(country)=, which +returns the WB catalog annotated with a =local= bool. Feed it in and +emit a =missing-wave= row for every upstream wave we lack. The cube +gains its honest denominator. (Two configured countries --- =EthiopiaRHS= +and =Serbia and Montenegro= --- are missing from =_COUNTRY_CODES= and so +are blind to discovery; that is a two-line fix.) + +** Defect 3 --- no wave records its provenance (blocks the fix for Defect 2) + +=discover_waves()= decides whether we already hold a catalog entry by +/string-matching/ a fabricated wave label (=_catalog_to_wave_label= vs +=_local_waves=) against directory names. It is wrong in both directions: +of 41 flagged entries, *12 were false positives* and *at least 2 were +false negatives*. + +The false negatives are the dangerous ones, because they are silent: + +- *Nigeria 2018-19* --- catalog id 3827 is the /Nigeria Living Standards + Survey/. =Nigeria/2018-19/= actually holds GHS-Panel W4 (id 3557), a + *different survey*. The label collides, so we believe we have the LSS. + We do not. +- *Ethiopia 2013-14* --- id 2671 (Land & Soil Experimental Research) is + masked by ESS W2 (id 2247). +- *GhanaLSS / GhanaSPS* both map to ISO =GHA=, so each sees the other's + waves as its own missing waves. + +Root cause: *none of the 136 wave directories records the WB catalog id it +came from.* =add_wave()= writes a =SOURCE.org=; not one pre-existing wave +has one (=find ... -name SOURCE.org= → 0). + +*Fix (prerequisite for Defect 2).* Stamp =catalog_id= per wave dir and +match on the id, not on a reconstructed label. Backfilling 136 waves is a +bounded, mostly-mechanical job. Until it is done, any "missing wave" +column in the matrix is guesswork. + +** Defect 4 --- eleven countries are invisible + +=catalog._country_dirs()= admits a directory only if it carries +=_/data_scheme.yml=. Eleven country directories hold *downloaded survey +data and no config at all*: + +: Afghanistan(2 waves) Bosnia-Herzegovina(5) Brazil(1) Bulgaria(5) +: KenyaLPS(4) Kyrgyz Republic(4) Nicaragua(3) Panama(3) Peru(4) +: Rwanda(2) Tanzania_Kegera(2) + +Thirty-five waves of paid-for, downloaded microdata that the coverage +matrix reports nothing about, because the matrix's own gate hides them. +The matrix says 35 countries; the repo has 46. + +*Fix.* Enumerate country /directories/, not configured countries, and +emit an =unconfigured= country-level row for the config-less ones. They +should be visibly red, not absent from the report. + +** Defect 5 --- the wave-slice grader applies country-level checks + +The single biggest numerical error in the matrix. +=coverage_matrix._wave_cells= (L244) grades a wave by slicing the country +frame on =t= and calling =diagnostics.is_this_feature_sane= on the slice. +But =_check_no_all_null_columns= (=diagnostics.py:280=) is a *country-level* +check. A question that simply was not fielded in a given wave is +legitimately all-null /in that slice/ --- and gets graded =builds=. + +*Proof it is a grading bug and not a data bug:* of the 124 offending +columns, *zero* are all-null country-wide. Every one is populated in at +least one other wave. + +*131 of the 138 =builds= cells are false.* The matrix is crying wolf on +7% of the entire cube. + +*Fix.* When grading a slice, pass the country frame's populated-column +set as =optional=. Small. + +*** But sequence this correctly --- a regrade is not a fix + +Fixing the grader re-tiers 131 cells =builds= → =sane= in one commit. +Two *genuine* data gaps are currently hiding inside that bucket and would +be *silently buried* by it: + +- *Nigeria =food_acquired.Quantity_kg= is all-null in 6 of 8 waves.* The + kg conversion is missing, so the /default/ =food_prices(units='kgvalue')= + --- =Expenditure / Quantity_kg= --- returns *NaN* for those waves. A + live, user-facing data defect with no tracking issue. +- *18 cells of missing GPS =Latitude=/=Longitude=* in =cluster_features= + (CotedIvoire, Nigeria, Ethiopia, Malawi, Niger, Uganda). + +*Extract both as their own issues BEFORE regrading.* Otherwise the +cleanup destroys the only evidence that they exist. This is the general +rule: /whenever a fix changes how cells are graded rather than what they +contain, first inventory what the old grading was catching by accident./ + +** Defect 6 --- =broken= is mis-tiered for the known-no-microdata countries + +All 8 =broken= cells are =PathMissingError= from *Armenia* (2) and *Nepal* +(6) --- both of which CLAUDE.md already documents under /Countries Without +Microdata/. They are not broken; they have no data. They need a +known-no-data skip list and an =absent=/=n/a= tier, not a bug fix. + +(Secondary harness bug found alongside: all three Nepal waves report the +/1995-96/ path --- the country-level build raised once and the single +exception was fanned out to every wave, so per-wave detail in =broken= +cells is not trustworthy.) + +** Aside --- a live data-integrity bug found while counting + +The ten wave-shaped directories at the =countries/= root (=2009-10/=, +=2010-11/=, ...) are *not* misplaced data. All 33 =.dta= in them are +md5-identical to DVC-tracked sources under =Uganda/=, =Nigeria/=, +=Tanzania/=. They are output of a cwd bug: the country-level =_/= script +idiom =../{wave}/Data/...= resolves against the /process cwd/ when a +script runs from the country dir rather than =_/=, and +=_resolve_data_path()= (=local_tools.py:719=) has no pattern for that +form, so it passes the path through unrewritten. + +The litter is harmless; the /latent hazard/ is not. +=_is_polluted_workspace_copy()= (=local_tools.py:679=) refuses an in-tree +copy only when a *sister =.dvc= sidecar* exists. These strays have none +--- so =get_dataframe()= would accept them as legitimate local data and +silently shadow the DVC blob with a stale copy. + +*This is a correctness bug, not a coverage bug.* File it separately: +teach =_resolve_data_path= to handle (or loudly reject) the +=../{wave}/Data/...= form, and/or resolve country-script paths against +=__file__= rather than cwd. Cleaning the 47 MB is the easy half. + +** Why these come first + +Every one is a defect in the /instrument/, not the data. Fixing them is +cheap --- about a day in total --- and until they are fixed, every workplan +built on this matrix is built on numbers that are 32% wrong and a +denominator that cannot reach zero. Fix the ruler, then measure. + +Note the *asymmetry of the errors*, which is what makes them insidious: + +- Defects 1, 5, 6 make the matrix *cry wolf* --- 595 cells whose tier is + wrong or unactionable. The cost is wasted effort and, worse, + desensitisation: a dashboard that is a third noise stops being read. +- Defects 2, 3, 4 make the matrix *flatter itself* --- upstream waves we + never downloaded and countries whose data we already hold simply do not + appear. The cost is invisible work, and an incentive gradient that + /punishes acquiring data/ (a new wave only ever adds red cells). + +A measure that is simultaneously alarmist about what we have and silent +about what we lack is worse than no measure, because it is trusted. + +* The discipline + +** D1. Every red cell terminates in a recorded state, or the probe did not happen + +| state | meaning | artifact | +|-------------------------+-----------------------------------+----------------------------------------------| +| =sane= | config written, builds, passes | the config + a green =make matrix= | +| =not-asked= | instrument genuinely never asked | =absent_verdicts.csv= row + *questionnaire* | +| =asked-not-distributed= | asked; the extract lacks it | =absent_verdicts.csv= row → acquisition queue | +| =unsure= | a required check could not be run | =absent_verdicts.csv= row + the named blocker | +| =issue= | data is there, the build is wrong | a GH issue → the =backlog-workflow= loop | + +There is no "I looked at it once." A probe that ends without one of these +artifacts did not happen, and its cost will be paid again by the next +person. =unsure= is a first-class outcome, not a failure --- it keeps the +cell in the queue and *records why*, which is the whole point. + +** D2. The evidence standard (no human sign-off, but a serious search) + +*Constraint (Ethan, 2026-07-11):* his eyes are /not/ required on every +=not_asked= row --- bandwidth is finite --- but "a serious search should +happen before this conclusion is reached." + +So the bar cannot be "an agent looked." =not_asked= is a *permanent, +unsupervised write*: it removes a cell from the queue forever, with nobody +checking. That makes it the single most dangerous write in this design, +and it earns a hard, enumerated protocol. The failure mode is specific +and it is not exotic: + +#+begin_quote +*A renamed variable is indistinguishable from an absent one.* +=hh_s11q01= becomes =s11q1a= between waves; a naive label search finds +nothing and concludes the survey stopped asking. It did not. We then +close the cell permanently and lose the data forever, silently. +#+end_quote + +Every check below exists to catch that. A cell may be written to +=not_asked.csv= only when *all applicable* checks are run and *all* come +back negative; the row records *which ran*, in a =checks= column. Any +check that cannot be run downgrades the verdict to =UNSURE= --- which +stays in the queue. Silence is never evidence. + +*Each rule below was found by the pilot being wrong first. They are not +theory.* + +*** Check 1 --- variable-label sweep (always; necessary, never sufficient) +No matching variable or value label in any source file for that wave. +Metadata-only reads (=pyreadstat.read_dta(..., metadataonly=True)=), blob +fetched by the lock-free direct-S3 bypass. Three corrections the pilot +paid for: + +1. *Be extension-agnostic.* LSMS ships Stata binaries named =.tab= + (EthiopiaRHS) and SPSS =.sav= (Albania 2008/2012). Try =read_dta= → + =read_sav= → text, regardless of suffix. A suffix-driven reader + silently reads garbage. +2. *Report label /availability/ separately from label /hits/.* + CotedIvoire's =.DCT= fixed-width files carry *no variable labels at + all* (names are opaque: =C01Q01=). The first pass scored "0 hits + across 80 files" and read as a clean negative. It was the probe having + nothing to read. *Silence must be structurally distinguishable from + absence* --- if the corpus is unlabelled, the check did not run. +3. *Multilingual, or it is not a search.* Albania 2012's durables are + =Frigorifer=, =Lavatrice=; CotedIvoire/Mali shocks are =sinistre=, + =sécheresse=. English-only missed Albania 2012 entirely. + +Also: *the wave's =Data/= dir is not the source universe.* EthiopiaRHS +wave dirs hold 4--6 files; its =_dataverse_archive/= holds 251. Scoping +the probe to =Data/= alone would have produced seven confident false +negatives. + +*** Check 2 --- sibling differential (necessary for PARTIAL pairs; *not sufficient*) +Take the working sibling wave's source variable and establish its +/analogue/ is absent --- names do not survive renaming, so match on *item +content* (the value-label lexicon of =j= / =Shock=), not variable names. + +*The pilot proved this check alone manufactures false NOT-ASKEDs.* It +returned "content absent" (0/23 lexicon hits) for *Iraq 2006-07* --- a wave +carrying a full 11-shock + 24-coping module. The 2007 labels +(="Problem: theft"=) share no vocabulary with 2012's (="Loss of asset or +livestock due to theft/accident/death"=). Only the *filename* +(=2007ihses18_risks.dta=) caught it. A lexicon differential is necessary, +never sufficient. + +Module renumbering is the norm, not the exception: Nigeria =sect5= (w4) → +=sect10= (w5) for assets, =sect15a= → =sect12= for shocks; Iraq =risks= +(2007) → =shocks= (2012), wide → long. + +*** Check 3 --- Harmonized LSMS-ISA Ag cross-check (mandatory where it applies) +An *independent* oracle, and the one Ethan specifically asked for. The +WB Harmonised Panel is already in the repo as a 48 MB DVC blob at +=countries/Harmonized_LSMS-ISA_Ag/lsms_ag.parquet= (281,803 x 223; +materializes lock-free via =get_dataframe=). It covers *Ethiopia, +Malawi, Mali, Niger, Nigeria, Tanzania, Uganda* and carries harmonized ag +variables --- =total_labor_days=, =seed_kg=, =inorganic_fertilizer_value=, +=harvest_kg=, =plot_area=, ... --- keyed by =country= x =wave=. + +This is *decisive* for the ag features, which are *137 of the 456 absent +cells (30%)*: =plot_features= (30), =livestock= (30), =crop_production= +(27), =plot_inputs= (25), =plot_labor= (25). If the WB extracted +=total_labor_days= for Tanzania wave 3, then that wave *has* plot-labor +data and any NOT-ASKED claim for it is simply false. + +And it is better than a veto. The WB's per-wave *do-files* +(=lsms-worldbank/LSMS-ISA-harmonised-dataset-on-agricultural-productivity-and-welfare=) +name the *raw file and variable* behind each harmonized column. So a +positive hit does not merely reject NOT-ASKED --- it hands us the source +file and variable name, i.e. most of the config. Check 3 is both the +strictest gate and the biggest accelerator on this list. + +*The oracle is /positive-only/.* A non-null harmonized column proves the +concept was asked. *A null column proves nothing* --- it means the WB's +harmonizers did not code it, not that the survey did not ask. The pilot +nailed this: Mali 2014-15 has =hh_asset_index= *0% non-null*, yet the +durables roster is plainly there (=EACIACT_p1.dta:s08q01=, 25 items). Had +Check 3 been allowed to vote "no", it would have closed a live cell. + +*Check 3 may corroborate TODO. It may never establish NOT-ASKED.* + +/Wave mapping (solved):/ no external map is needed --- the oracle's own +=survey= column carries the label. For Tanzania: =wave 1.0 → 'NPS 2008 - +2009'=, =2.0 → 'NPS 2010 - 2011'=, ... Group by =(country, survey)=. + +/Coverage limit:/ 7 countries, and it *stops at GHS 2018-19* --- so it gives +no help on Nigeria's 2023-24 rounds, which is exactly where Nigeria's +absent ag cells live. Check 1 found them anyway (=sect10_plantingw5.dta=, +=sect12_harvestw5.dta=). + +/Housekeeping:/ this parquet is currently an *orphan* --- nothing in the +codebase references it. Wiring it into the probe finally gives it a job. + +*** Check 4 --- the questionnaire. *Mandatory before any permanent close.* +Originally listed last and optional. *The pilot promoted it to the +decisive check*, and it is the one that must never be skipped. + +*Albania 2004 =assets=*: Check 1 and Check 2 both returned negative, and +both were *correct about the data* --- the variables really are absent from +the shipped =.dta=. Three independent data-side checks unanimously said +NOT-ASKED. The questionnaire said =MODULE 2: ... DURABLE GOODS=, +=105 Refrigerator 106 Freezer 107 Washing machine=. They were all wrong. + +#+begin_quote +*Absence in the shipped =.dta= is not absence in the instrument.* +Every data-side check can be defeated by a data-/distribution/ gap. Only +the questionnaire can separate NOT-ASKED from ASKED-NOT-DISTRIBUTED --- and +those two route to completely different queues. +#+end_quote + +So: *no data-side evidence, however unanimous, may close a cell as +NOT-ASKED.* Only the questionnaire can. Where no questionnaire exists, +or it is a pre-1995 =CCITTFaxDecode= scan with no text layer (CotedIvoire +1985--89; needs OCR), the verdict is =unsure= and the cell *stays in the +queue*. The pilot's own control makes the point: Albania 2002's +study-level DDI scores zero hits for "durable" even though 2002 +demonstrably /has/ =durables_cl.dta=. Its silence carries no information. + +/Implementation note:/ =pdftotext=/=pypdf= are absent on Savio compute +nodes. The pilot's pure-python extractor (zlib-inflate the =FlateDecode= +streams, pull =Tj=/=TJ= text ops) works on modern PDFs and is enough. + +*** What gets recorded +: country, feature, wave, verdict, checks_run, evidence, adjudicated_by, date +=evidence= must cite something a human can re-check in under a minute --- a +questionnaire section number, or "no variable or value label matching +{shock|choc|conmoción}/... across the 14 =.dta= in =Data/=; sibling wave +2013-14 uses =hh_s16q01=, which has no analogue here." Not "searched, not +found." + +*** Why this is enough without Ethan's eyes +The row is a PR. It is diffable, it names the checks it ran, and its +evidence is re-verifiable in a minute by anyone who doubts it. A wrong +=not_asked= is therefore *recoverable* --- which is the property that makes +unsupervised writes acceptable. What would /not/ be acceptable is an +evidence-free row, because nothing about it can ever be challenged. + +** D3. Probe by =(country, feature)=, not by cell + +The 456 absent cells collapse to *151 =(country, feature)= pairs*, and +they split unevenly: + +- *338 cells / 130 pairs are PARTIAL* --- the country /already has/ that + feature for at least one other wave. The schema is settled, the source + structure is known, a working =data_info.yml= sits one directory over. + The only open question is whether /this/ wave's instrument asked. +- *118 cells / 21 pairs are WHOLE* --- the feature is missing for every + wave. A from-scratch build. + +*74% of the gap is the cheap kind.* A single probe of one pair typically +resolves 2--5 cells at once, because the answer ("the 2005 and 2008 waves +dropped the assets module") is a property of the pair, not of the cell. +Probing cell-by-cell is a factor-of-three waste. + +** D4. Reuse the existing execution loop; do not build a new one + +We already have the machinery to /fix/ a cell and prove it: + +- =.claude/skills/add-feature/= --- the authoring path, with sub-skills + for =shocks=, =assets=, =sample=, =food-acquired=, =housing=, =pp-ph=, + =panel-ids=. +- =.claude/skills/add-wave/= --- the acquisition path (=discover_waves=, + =add_wave=, DVC push). +- =bench/feature_audit/= --- the regression net. +- =.claude/skills/backlog-workflow/= --- worktree-isolated fix → verify → + red-team → PR. + +The matrix's job is to be the *worklist generator* for that loop, not to +grow a parallel one. The only new machinery this document proposes is +=not_asked.csv= and two enumeration fixes. + +** D5. =blessed= means a human looked [ADOPTED 2026-07-12] + +=sane= and =blessed= answer different questions, and the gap between them +is where quiet errors live: + +- *=sane=* --- the automated checks passed. /No human has necessarily looked + at a single number./ +- *=blessed=* --- a human read the actual numbers for that cell and believes + them. + +A feature can build cleanly, pass every check, and still be wired to the +wrong source column or carry an unverified unit conversion. *For anything +feeding published analysis, =sane= is not enough.* + +*The tier was already fully implemented* --- loader, promotion, colour, =★= +glyph, docs. Only the file was empty (21 bytes). Adopting it cost *zero +code*: + +1. Schema extended to =country,feature,wave,blessed_by,date,note=. + =load_blessed= reads via =r.get(...)=, so the provenance columns are + free --- and they are what make a blessing /auditable/ rather than a bare + assertion. Verified: extra columns ignored, blank-=wave= country-level + key still works. +2. *The rule* (=CLAUDE.md= + =docs/guide/coverage.md=): /if you used a cell + in real analysis and looked at its numbers, bless it in the same PR./ + +*Seeded empty, deliberately* (Ethan, 2026-07-12). Blessings /accrete/; +they are never bulk-seeded. An empty blessing file is honest. Bulk- +blessing the 1208 =sane= cells would make =blessed= a synonym for =sane=, +destroy the only distinction the tier draws, and assert a human review +that never happened. + +*Trap found while wiring it:* =load_blessed= calls =pd.read_csv(..., +keep_default_na=False)= with *no* =comment== argument, so a ~#~-comment +line in =blessed.csv= is parsed as a *row* --- a phantom blessed cell. +Header and data only. Documented in both places. + +*** Corollary: a =sane= cell is not proof an issue is fixed +Know what the grader does /not/ look at. It is a *cold* build, so it +cannot see warm-cache-only divergences (issue #588); and it does not check +currency labels (#589). Both look clean in the matrix and are genuinely +open. This is the same discipline as D2: /an unevidenced negative is +worthless, whoever --- or whatever --- produced it./ + +* Workplan + +Ordered by leverage --- cells closed per unit of effort --- not by country. + +** Phase 0 --- Fix the ruler (~1 day, do this first, in this order) + +0. *Inventory before regrading* (Defect 5): file the Nigeria =Quantity_kg= + gap and the GPS Lat/Lon gap as issues /first/, or step 4 buries them. +1. *Absent-verdict tiers* (Defect 1) --- =.coder/coverage/absent_verdicts.csv= + + loader, mirroring =blessed=; =absent= is emitted at five sites in + =grade_feature=, so factor a single =_absent_tier()= helper. Four + verdicts, not two: =todo= / =asked-not-distributed= / =not-asked= / + =unsure=. Teach the HTML readout and =ll.coverage()= the tiers. Headline + number = =todo + unsure= (the live queue). +2. *Wave provenance* (Defect 3) --- stamp the WB =catalog_id= into each wave + dir (=add_wave= already writes =SOURCE.org=; *0 of 136 existing waves has + one*); match =discover_waves.local= on the id, not a reconstructed label. + Add =_COUNTRY_CODES["Serbia and Montenegro"] = "SCG"=; disambiguate the + shared =GHA= code that makes GhanaLSS and GhanaSPS each see the other's + waves as missing. +3. *Upstream wave axis* (Defect 2) --- fold =discover_waves()= into + =build_matrix= as =missing-wave= rows. Only meaningful after step 2. +4. *Fix the wave-slice grader* (Defect 5) --- pass the country frame's + populated-column set as =optional= when grading a slice. Regrades 131. +5. *Known-no-data skip list* (Defect 6) --- Armenia, Nepal → =n/a=, not =broken=. +6. *Surface the 10 in-remit config-less countries* (Defect 4) as =unconfigured=. + +*Exit test:* no tier lies, and every number is one we can burn down. +Concretely: =builds= drops from 138 to ~7; =broken= from 8 to 0; a +=missing-wave= count appears; the 456 splits four ways; and the ten +in-remit orphan countries show up red instead of not at all. + +7. *Correct the false claim in* =Albania/_/data_scheme.yml:86= (see Defect 1, + Exhibit A) --- and grep the other 34 countries' =_/= comments for + unevidenced "no X module" assertions. If one is wrong, assume others are. + +** Phase 1 --- Adjudicate the 456 (the probe sweep) + +Run the four-check probe over every absent cell. Read-only, embarrassingly +parallel, and *the single highest-value item on this list*: it converts 456 +undifferentiated red cells into a costed, ordered, evidence-backed queue. + +*Cost (measured, not guessed).* The 456 cells collapse to *78 distinct +=(country, wave)= folders* --- metadata is read once per folder and shared +across the ~5.8 absent features in it, so the folder, not the cell, is the +unit. Check 1 over all 78: *~3 min warm, ~12 min cold* (S3 required; the +lock-free bypass worked, no lock contention). Checks 2 and 4 are the real +cost --- ~78 questionnaires, and a lexicon spec per =(country, feature)=. + +: total 12--20 agent-hours, parallelizable ~8-way by country +: -> ~2--3 hours wall-clock + +*Expected yield (from the pilot, and it is not what I predicted).* +Roughly *60--75% TODO, 20--30% UNSURE, and very few NOT-ASKED.* Plan for +the sweep to /confirm work/, not to retire it. Its value is that the work +becomes *sized and sourced* --- 25 of the pilot's 38 cells came back with +the exact file and variable name attached, which is most of the config +already written. + +*Exit:* every absent cell carries a verdict + evidence, or is =unsure= with +a named blocker. Nothing is left in the undifferentiated bucket. + +*Known blockers the sweep will surface (from the pilot's 13 UNSURE):* +- *CotedIvoire 1985--89 (8 cells)* --- =.DAT=/=.DCT= fixed-width with zero + variable labels, and questionnaires that are =CCITTFaxDecode= *scans* + with no text layer. No check can run. Needs OCR + a section-number → + module map. This is the hardest corner of the cube. +- *EthiopiaRHS R5--R7 (3 cells)* --- archive ships aggregates only. Is the + raw section data obtainable from IFPRI Dataverse, or aggregate-only + forever? A question for a human, not an agent. +- *Albania 2004 assets / 2002 shocks* --- chase the missing extract; locate + the 2002 household questionnaire. + +** Phase 2 --- Harvest the PARTIAL TODOs + +The =data_info.yml=-next-door cases. Highest yield per hour in the repo. +Dispatch through the existing =add-feature= + =backlog-workflow= loop, +worktree-isolated, verified against the audit harness. + +*** Start with Nigeria's ag block --- 50 cells, one country + +The most concentrated gap in the cube. Nigeria has ten PP/PH round +directories (=2010Q3=, =2011Q1=, ... =2024Q1=) and: + +| feature | absent | builds | dropped | +|-----------------+--------+--------+---------| +| crop_production | 10 | 0 | 0 | +| livestock | 10 | 0 | 0 | +| plot_inputs | 10 | 0 | 0 | +| plot_labor | 10 | 0 | 0 | +| plot_features | 0 | 5 | 5 | + +Why this is the right first target: + +- Nigeria is a core LSMS-ISA country; its GHS-Panel unquestionably /has/ + ag modules. The 40 =absent= cells are TODO, not NOT-ASKED, and the + Harmonized-Ag oracle (Check 3) proves it --- it carries Nigeria waves + 1--4 with =harvest_kg=, =seed_kg=, =total_labor_days=. +- The WB do-files *name the raw file and variable per wave*, so most of + the config is pre-derivable rather than reverse-engineered. +- =plot_features= is already wired for all ten rounds, so the PP/PH + script pattern (=.claude/skills/add-feature/pp-ph/=) exists in-repo --- + the four missing features follow it. +- Its =plot_features= being 5 =builds= + 5 =dropped= is very likely the + PP/PH distinct-=t= / duplicate-index bug that skill already documents. + Fix the root and the degraded half of the block closes with it. + +One country, one script pattern, ~50 cells. Nothing else in the cube has +that density. + +** Phase 3 --- The degraded cells, in severity order + +The census (2026-07-11) already clustered these. Do them in *this* order +--- it is not the order the tier ladder suggests. + +*** 3a. =roster_to_characteristics= deletes entire waves (8 cells, TOP SEVERITY) + +The one unambiguous silent-data-loss bug in the cube. +=transformations.py:251-265= --- the residence filter added with +=MonthsSpent= --- drops every row of a wave when the resolved months +column is unusable, so =household_characteristics= returns *nothing* for +that wave while the underlying roster is fully populated. Three distinct +defects in one 14-line block: + +1. *The =elif= chain picks the first column /present/, not /populated/.* + Ethiopia's country frame carries both =MonthsAway= (0 non-null in + 2018-19 and 2021-22) and =WeeksAway= (28,634 / 20,504 non-null --- W4/W5 + switched to weeks, exactly as CLAUDE.md documents). =monthsaway= wins + the =elif=, =ms= is all-NaN, *both waves are deleted.* Fix: coalesce + per row rather than per column. +2. *No all-NaN guard.* CotedIvoire 1985-89 and Mali 2021-22 have + =MonthsSpent= 0% non-null --- the column exists in the frame only because + the country-level concat unions it in from other waves. =keep= is + all-False; *five waves deleted.* +3. *Untranslated French labels.* Burkina Faso 2014 =MonthsSpent= is + ='6 mois ou plus'= / ='moins de 6 mois'= (77,268 / 943 rows). + =errors='coerce'= → NaN → *wave deleted.* A one-line YAML mapping, the + same =Oui=/=Non= → 12/0 pattern CLAUDE.md already prescribes for EHCVM. + +Defects 1 and 2 collapse into one guard: /if the resolved =ms= is entirely +NaN within a =t= group, skip the filter for that group/ (reverting to the +documented pre-=MonthsSpent= count-everyone behaviour). *One function, 8 +cells, top severity.* It is a post-read derived transform (=_ROSTER_DERIVED=), +so per =backlog-workflow= it reproduces *warm* --- no cold rebuild needed. + +*** 3b. Extract the two genuine gaps hiding inside the =builds= bucket +Nigeria =Quantity_kg= (6 cells; breaks the default =food_prices=) and the +GPS Lat/Lon gap (18 cells). File before regrading (see Defect 5). + +*** 3c. Fix the grader (Defect 5) --- regrades 131 cells + +*** 3d. Nigeria's PP/PH =dropped= bloc (22 cells) --- and correct issue #587 +#587 attributes these to a "multi-round wave-label ↔ =t= mismatch." *That +diagnosis is refuted.* The built =t= values /are/ valid Nigeria wave +labels --- they are simply the labels of the *opposite round*. Each feature +is being built for only one round of each PP/PH pair: +=community_prices= is all Q1 and no Q3; =plot_features= is all Q3 and no +Q1; =food_coping= all Q3. These are not false positives; they are real +half-missing features. Re-diagnose before dispatching. + +*** 3e. The mis-tiered =broken= (8) and the CotedIvoire derived-feature over-declaration (4) +Armenia/Nepal need a known-no-data skip list (Defect 6). CotedIvoire's +=food_expenditures= 1985-89 is =dropped= only because its /source/ +=food_acquired= is =absent= for those waves --- derived features should +inherit their source's per-wave declaration in the coverage layer. + +*** Stale issues to close (the matrix says they are already done) +=#571= (Guatemala =food_acquired=), =#572= (Serbia), =#574= (GhanaSPS) --- +all =sane=, with their derived tables =sane= too. =#573= (Panama) is +genuinely still open. Note =#588= and =#589= look clean in the matrix but +are *not* stale: the matrix is a cold build and structurally cannot see +#588's warm-cache-only divergence, nor #589's currency labels. *A cell +being =sane= is not proof an issue is fixed --- know what the grader does +not look at.* + +*** Tracking +*146 of the 185 degraded cells have no tracking issue at all.* Only the 39 +=dropped= are tracked, by =#587= alone --- and #587's hypothesis for its +largest bloc is wrong (3d). + +** Phase 4 --- The 21 WHOLE-feature pairs + +Real feature work through the =add-feature= sub-skills; schedule +deliberately. + +** Phase 5 --- Bootstrap the ten in-remit orphan countries + +*Scope decision (Ethan, 2026-07-11): these are in remit.* They are +"kissing cousins" --- WB data carrying many of the LSMS modules. Mostly +/classic/ LSMS (the Deaton-era ENNIV / EMNV / ENV / PPV surveys) rather +than LSMS-ISA, so their instrument profile differs; that difference is +exactly what =not_asked.csv= is for, and needs no new mechanism. + +Ordered by cost-to-first-green (from the census): + +| # | country | waves | survey | why here | +|---+------------------+-------+-------------------+---------------------------------------------------------| +| 1 | Panama | 3 | ENV | *cheapest win.* Already has =panama.py=, =food_acquired.py=, =food_items.org=, =units.py=; only =data_scheme.yml= + wave =data_info.yml= missing | +| 2 | Kyrgyz Republic | 4 | PMS / LSMS | textbook LSMS modules (expenditure, diet, anthropometry, community) | +| 3 | Peru | 4 | ENNIV | =food_acquired= already scoped in a country =data_info.yml=; friction: 1990 is =.ssp=, and its =v=-in-index predates =_join_v_from_sample()= | +| 4 | Bosnia-Herzegovina | 5 | BiH LSMS | genuine 4-year panel; =2001-04/= is a panel-file dir → use the multi-round-waves pattern, not a naive wave | +| 5 | Nicaragua | 3 | EMNV | 1993 / 1998-99 clean =.DTA=; 2001 is =.dat= + =.dct= (needs =infile=); 2005 has no microdata | +| 6 | Tanzania_Kagera | 2 | KHDS | real long panel; dir name is a *typo* (=Kegera=). Note the WB census found KHDS W1--W6 upstream (ids 359, 79, 2251) --- acquire and bootstrap together | +| 7 | Brazil | 1 | PPV | clean one-off cross-section; cheap, low leverage | +| 8 | Afghanistan | 2 | ALCS | partial bootstrap exists (=afghanistan.py=, =Makefile=) | +| 9 | Rwanda | 2 | EICV | 2000-01 workable (=.sav=); 2005-06 is bare =.dat=, no dictionary | +| 10 | Bulgaria | 5 | Integrated HHS | repeated cross-sections; sidecars mix raw with WB-*constructed* aggregates --- sorting raw from derived is the real cost | + +*Recommend dropping =KenyaLPS=* (5 dirs, 158 MB). It is the Kenya Life +Panel Survey --- a Miguel/Kremer /individual/ deworming-tracking study, not +an LSMS household survey. The canonical schema (=household_roster=, +=food_acquired=, =sample=/=v=) does not map onto it. Keeping it in the +denominator just manufactures permanent red. + +Note there is *no =add-country= skill* --- =add-feature= covers the +"create a =data_scheme.yml= if absent" case in a paragraph. Ten +bootstraps justify writing one. + +* Decisions taken + +1. *Scope of the config-less countries* --- [2026-07-11] *in remit.* They + are WB "kissing cousins" carrying many LSMS modules. See Phase 5. + /Open sub-question:/ is =KenyaLPS= the one exception? It is an + individual tracking panel, not a household survey, and the canonical + schema does not map onto it. +2. *Adjudication authority* --- [2026-07-11] *no human sign-off required*, + but only against the enumerated evidence standard of §D2, which the + pilot then hardened: Check 4 (the questionnaire) is *mandatory* before + any permanent close, because every data-side check can be defeated by a + data-/distribution/ gap (Albania 2004). An agent may write a verdict; + it may not write an /unevidenced/ one. +3. *=blessed=* --- [2026-07-12] *adopted, seeded empty, bless-on-first-use.* + Implemented; see §D5. + +* Open questions for Ethan + +1. *=KenyaLPS=* --- drop from the denominator? (Recommend: yes.) +2. *EthiopiaRHS R5--R7* --- is the raw section data obtainable from IFPRI + Dataverse, or is the archive aggregate-only forever? Three cells hang + on this and no agent can answer it. +3. *The =Harmonized_LSMS-ISA_Ag= parquet* is an orphan --- 48 MB, DVC-tracked, + referenced by nothing. The probe (§D2 Check 3) gives it a job. Keep it + where it is, or move it out of =countries/=? It is not a country. From 6af3d3289b54fbf670d0b48214bd87e2f0a8b7ce Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 07:02:31 -0700 Subject: [PATCH 08/85] =?UTF-8?q?feat(coverage):=20Phase=200=20cont.=20?= =?UTF-8?q?=E2=80=94=20unconfigured=20countries,=20no-microdata=20tier,=20?= =?UTF-8?q?and=20stop=20a=20scoped=20run=20from=20destroying=20the=20snaps?= =?UTF-8?q?hot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three more defects in the coverage instrument, plus one found while fixing them. ## Defect 4 — eleven countries were INVISIBLE `catalog.countries()` admits a directory only if it carries `_/data_scheme.yml`, so country dirs holding downloaded microdata and no config never appeared in the matrix at all. That hid **11 countries and 35 waves of already-downloaded data** (Afghanistan, Bosnia-Herzegovina, Brazil, Bulgaria, KenyaLPS, Kyrgyz Republic, Nicaragua, Panama, Peru, Rwanda, Tanzania_Kegera) -- ten of which Ethan has confirmed are in remit. New `unconfigured` tier + `unconfigured_countries()`. They are now visibly red rather than absent from the report. A denominator that omits the work you have not started is not a denominator. ## Defect 6 — `broken` was mis-tiered for the known-no-microdata countries All 8 `broken` cells in the cube were Armenia (2) and Nepal (6) -- both already documented in CLAUDE.md as having no source data in the repo. They cannot build; that is a data-availability fact, not a defect. Declared in the canonical `data_info.yml` (config, not code -- GH #436) and graded `n/a`. This matters beyond tidiness: the most alarming tier in the ladder contained *nothing actionable*, so a genuine build failure would have been lost among them. ## Found while fixing the above: a scoped run DESTROYED the snapshot `save_snapshot()` wrote wholesale, so `make matrix C="Uganda"` -- which grades only Uganda -- silently erased every cell it did not grade. `docs/guide/coverage.md` documents exactly that as the "spot refresh" procedure, immediately followed by `git add .coder/coverage/latest.csv && git commit`. Following the guide verbatim therefore committed a 67-cell snapshot over the authoritative 1849-cell one. I did it to myself while testing the renderer. `save_snapshot(..., merge=True)` (now the default) upserts on `(country, feature, wave)` and leaves un-graded cells untouched. A full sweep still rewrites everything it grades, so the authoritative run is unaffected -- it simply no longer depends on *not* being scoped. **A partial measurement must never be able to erase a complete one.** ## Notes - Two full-suite failures investigated, neither caused by this work: `test_feature_ghana_per_wave` is pre-existing and already tracked as **GH #589** (its title is literally that test name); the Albania `sample` failure was a parquet-cache race from running three heavy builds concurrently right after a `data_scheme.yml` edit invalidated Albania's content hash. Passes in isolation. - 11 new tests (34 in test_coverage_matrix.py total). Co-Authored-By: Claude Opus 4.8 (1M context) --- bench/matrix.py | 5 +- docs/guide/coverage.md | 8 ++ lsms_library/coverage_matrix.py | 192 ++++++++++++++++++++++++++++---- lsms_library/data_info.yml | 16 ++- tests/test_coverage_matrix.py | 106 ++++++++++++++++++ 5 files changed, 302 insertions(+), 25 deletions(-) diff --git a/bench/matrix.py b/bench/matrix.py index c83f10c08..f1ef4dd76 100644 --- a/bench/matrix.py +++ b/bench/matrix.py @@ -48,6 +48,9 @@ # extract) -- a distinct colour, because it routes to a different queue # entirely and should never be mistaken for either config work or a close. "asked-not-distributed": "#c5b3e6", + # Data downloaded, zero config -- work not yet STARTED. Must read as work, + # not as a quiet gap. + "unconfigured": "#fd7e14", "declared": "#cfe2ff", "dropped": "#f5c2c7", "broken": "#dc3545", @@ -58,7 +61,7 @@ TIER_GLYPH = { "n/a": "·", "absent": "–", "declared": "?", "dropped": "✗!", "broken": "✗", "builds": "⚠", "sane": "✓", "blessed": "★", - "not-asked": "∅", "asked-not-distributed": "⤓", + "not-asked": "∅", "asked-not-distributed": "⤓", "unconfigured": "⌀", } diff --git a/docs/guide/coverage.md b/docs/guide/coverage.md index 1098b52f5..3be176b81 100644 --- a/docs/guide/coverage.md +++ b/docs/guide/coverage.md @@ -165,6 +165,14 @@ to refresh what readers see is: lsms-library cache clear --country Uganda # repeat per country, or clear all make matrix C="Uganda" # omit C= for the full cube ``` + + !!! note "A scoped run upserts; it does not replace" + `save_snapshot()` **merges** on `(country, feature, wave)`, so + `make matrix C="Uganda"` updates Uganda's cells and leaves every other + country's alone. Before 2026-07-12 it replaced the file wholesale -- + which meant that following this very procedure with `C=` committed a + 67-cell snapshot over the authoritative 1849-cell one. A partial + measurement must never be able to erase a complete one. 2. **Commit the journal** (it is tracked despite the global `*.csv` ignore): ```bash git add .coder/coverage/latest.csv diff --git a/lsms_library/coverage_matrix.py b/lsms_library/coverage_matrix.py index eabc00b67..5b169ef91 100644 --- a/lsms_library/coverage_matrix.py +++ b/lsms_library/coverage_matrix.py @@ -14,27 +14,49 @@ Tier ladder (worst→best; each derived from existing machinery only): -========== =========================================================== -tier meaning -========== =========================================================== -n/a no per-wave readiness applies (country-level-only feature, or - the built table has no ``t`` axis) -absent feature applies to the country but its source is not declared - for this wave (``feature`` ∉ ``Wave.data_scheme`` ∪ derived) -declared source present for the wave; readiness not assessed - (coverage-only run) -dropped source declared for the wave but the wave is missing / empty in - the built table (the silent-drop bug class, e.g. Iraq #532) -broken the country-level build raised, or the whole feature is empty -builds wave slice is non-empty but ``SanityReport.ok is False`` -sane wave slice is non-empty and ``SanityReport.ok is True`` -blessed ``sane`` and listed in the git-tracked blessing file -========== =========================================================== +===================== ==================================================== +tier meaning +===================== ==================================================== +n/a no per-wave readiness applies: a country-level-only + feature, a table with no ``t`` axis, or a country with + no microdata in the repo at all (Armenia, Nepal — see + :func:`countries_without_microdata`) +not-asked adjudicated: the instrument genuinely never asked. + Closed. *Requires evidence* (:func:`load_verdicts`). +asked-not-distributed adjudicated: the instrument DID ask, but the shipped + extract does not carry the variables. An acquisition + problem, not a config one. *Requires evidence.* +absent feature applies to the country but its source is not + declared for this wave — and the gap is UN-ADJUDICATED + (or adjudicated ``todo``/``unsure``, which stay open). + This is the live queue. +declared source present for the wave; readiness not assessed + (coverage-only run) +dropped source declared for the wave but the wave is missing / + empty in the built table (the silent-drop bug class) +broken the country-level build raised, or the feature is empty +builds wave slice is non-empty but ``SanityReport.ok is False`` +sane wave slice is non-empty and ``SanityReport.ok is True`` + — i.e. the automated checks passed. NOT a claim that + any human has looked at a single number. +blessed ``sane`` **and** a human read the numbers and believed + them (git-tracked ``blessed.csv``). Required for any + cell feeding published analysis. +===================== ==================================================== + +Two rules the tiers exist to enforce, both learned the hard way: + +- *An unevidenced negative is unfalsifiable, and therefore permanent whether or + not it is true.* So a closing verdict without evidence is refused outright. +- *A regrade is not a fix.* When a change alters how cells are GRADED rather + than what they CONTAIN, first inventory what the old grading was catching by + accident (see GH #591, #592). """ from __future__ import annotations import os import warnings +from functools import lru_cache from pathlib import Path import pandas as pd @@ -43,8 +65,8 @@ # Tier ladder # --------------------------------------------------------------------------- TIER_ORDER = [ - "n/a", "not-asked", "asked-not-distributed", "absent", "declared", - "dropped", "broken", "builds", "sane", "blessed", + "n/a", "not-asked", "asked-not-distributed", "unconfigured", "absent", + "declared", "dropped", "broken", "builds", "sane", "blessed", ] # Worst → best, for rolling several wave tiers up into one grid cell. The most # actionable (a real defect) sorts first so problems surface in the summary. @@ -53,8 +75,8 @@ # live queue. ``not-asked`` and ``asked-not-distributed`` are *adjudicated* and # sort with ``n/a`` at the quiet end: they are settled, not pending. ROLLUP_PRIORITY = [ - "broken", "dropped", "builds", "absent", "declared", "sane", "blessed", - "asked-not-distributed", "not-asked", "n/a", + "broken", "dropped", "builds", "unconfigured", "absent", "declared", + "sane", "blessed", "asked-not-distributed", "not-asked", "n/a", ] COLUMNS = ["country", "feature", "wave", "tier", "coverage", "n_rows", "detail"] @@ -204,6 +226,68 @@ def load_blessed(path: Path | None = None) -> set[tuple[str, str, str]]: } +def unconfigured_countries(countries_root) -> dict[str, int]: + """Country dirs with downloaded microdata but **no** ``_/data_scheme.yml``. + + Returns ``{country: n_waves_with_data}``. + + :func:`catalog.countries` admits a directory only if it carries a + ``_/data_scheme.yml``, so these are invisible to ``Country()``, to + ``Feature()``, and — until now — to this matrix. As of 2026-07-11 that hid + **ten in-remit countries and ~35 waves of already-downloaded microdata** + (Afghanistan, Bosnia-Herzegovina, Brazil, Bulgaria, Kyrgyz Republic, + Nicaragua, Panama, Peru, Rwanda, Tanzania_Kegera). + + A denominator that omits the work you have not started is not a denominator, + so the matrix now reports them as ``unconfigured`` rather than not at all. + + A directory counts only if at least one of its wave subdirs has a ``Data/`` + (with or without files — DVC sidecars alone are enough, since the blobs are + fetched lazily). That keeps stray non-country dirs out. + """ + out: dict[str, int] = {} + try: + root = Path(countries_root()) + except Exception: # noqa: BLE001 + return out + for p in sorted(root.iterdir()): + if not p.is_dir() or p.name.startswith((".", "_")): + continue + if (p / "_" / "data_scheme.yml").exists(): + continue # configured; not our problem + n = sum(1 for w in p.iterdir() if w.is_dir() and (w / "Data").is_dir()) + if n: + out[p.name] = n + return out + + +@lru_cache(maxsize=1) +def countries_without_microdata() -> dict[str, str]: + """Countries whose config exists but whose source microdata is not in-repo. + + Declared in the canonical ``lsms_library/data_info.yml`` (config, not code — + GH #436), mirroring the "Countries Without Microdata" table in CLAUDE.md. + + Their features *cannot* build. That is a fact about data availability, not + a defect, so the matrix grades their cells ``n/a`` rather than ``broken``. + Before this, **all 8 ``broken`` cells in the cube were these two countries** + (Armenia ×2, Nepal ×6) — which meant the most alarming tier in the ladder + contained nothing actionable, and a genuine build failure would have been + lost among them. + + Returns ``{country: reason}``. + """ + try: + from importlib.resources import files + from .yaml_utils import load_yaml + with open(files("lsms_library") / "data_info.yml", encoding="utf-8") as fh: + info = load_yaml(fh) + section = (info or {}).get("Countries Without Microdata", {}) or {} + return dict(section.get("countries", {}) or {}) + except Exception: # noqa: BLE001 — a missing/old data_info.yml must not break grading + return {} + + # --------------------------------------------------------------------------- # Lazy bundle of lsms_library symbols (avoids import cycles at package init) # --------------------------------------------------------------------------- @@ -321,10 +405,17 @@ def _absent(w): df, err = _safe_build(co, feature, env) if err is not None or df is None or len(df) == 0: why = err or "feature built empty" + # A country with no microdata in the repo cannot build. That is a data- + # availability fact, not a defect -- grade `n/a`, not `broken`, so the + # `broken` tier stays meaningful. (See countries_without_microdata.) + no_data = countries_without_microdata().get(country_name) + fail_tier = "n/a" if no_data else "broken" + if no_data: + why = f"no microdata in repo: {no_data}" out = [] for w in waves: if covered[w]: - out.append(_cell(country_name, feature, w, "broken", "declared", + out.append(_cell(country_name, feature, w, fail_tier, "declared", detail=why)) else: tier, detail = _absent(w) @@ -409,6 +500,10 @@ def grade_country_level(country_name, feature, co, env, *, warnings.simplefilter("ignore") val = env["load_feature"](co, feature) except Exception as e: # noqa: BLE001 + no_data = countries_without_microdata().get(country_name) + if no_data: + return _cell(country_name, feature, None, "n/a", "declared", + detail=f"no microdata in repo: {no_data}") return _cell(country_name, feature, None, "broken", "declared", detail=f"{type(e).__name__}: {e}") n = len(val) if hasattr(val, "__len__") else 0 @@ -456,6 +551,21 @@ def build_matrix(countries=None, features=None, *, readiness=True, feat_filter = set(features) if features else None rows: list[dict] = [] + + # Country dirs holding downloaded microdata but NO config are invisible to + # `catalog.countries()` (it requires `_/data_scheme.yml`), so the matrix has + # never reported them at all -- 10 in-remit countries and ~35 waves of + # paid-for data that simply did not appear. Surface them as `unconfigured`: + # visibly red, not absent from the report. A denominator that omits the work + # you have not started is not a denominator. + for c, nwaves in unconfigured_countries(countries_root).items(): + if countries is not None and c not in countries: + continue + rows.append(_cell(c, "", None, "unconfigured", "absent", + n_rows=nwaves, + detail=f"microdata present ({nwaves} waves) but no " + f"_/data_scheme.yml — country not configured")) + for c in countries: try: co = Country(c, preload_panel_ids=False) @@ -498,12 +608,48 @@ def build_matrix(countries=None, features=None, *, readiness=True, return df -def save_snapshot(df: pd.DataFrame, path: Path | None = None) -> Path: - """Write the status table to the git-tracked snapshot CSV.""" +def save_snapshot(df: pd.DataFrame, path: Path | None = None, *, + merge: bool = True) -> Path: + """Write the status table to the git-tracked snapshot CSV. + + ``merge=True`` (default) **upserts** ``df``'s cells into the existing + snapshot on the ``(country, feature, wave)`` key, leaving every other cell + untouched. ``merge=False`` replaces the file wholesale. + + Why merge is the default + ------------------------ + A *scoped* run (``make matrix C="Uganda"``) grades only some cells. Writing + that result wholesale silently **destroys** every cell it did not grade -- + and ``docs/guide/coverage.md`` documents exactly that as the "spot refresh" + procedure, immediately followed by ``git add .coder/coverage/latest.csv && + git commit``. Following the guide verbatim therefore committed a 67-cell + snapshot over the authoritative 1849-cell one. (Observed 2026-07-12.) + + A partial measurement must never be able to erase a complete one. A full + sweep still rewrites everything it grades, so the authoritative run is + unaffected; it simply no longer depends on *not* being scoped. + """ path = Path(path) if path is not None else default_snapshot_path() path.parent.mkdir(parents=True, exist_ok=True) out = df.copy() out["tier"] = out["tier"].astype(str) + + if merge and path.exists(): + try: + prev = pd.read_csv(path, dtype=str, keep_default_na=False) + except Exception: # noqa: BLE001 — an unreadable snapshot is simply replaced + prev = None + if prev is not None and not prev.empty: + key = ["country", "feature", "wave"] + fresh = out.copy() + for k in key: # align dtypes for the anti-join + fresh[k] = fresh[k].astype(str) + keep = prev.merge(fresh[key].drop_duplicates(), on=key, + how="left", indicator=True) + keep = keep[keep["_merge"] == "left_only"].drop(columns="_merge") + out = pd.concat([keep, fresh], ignore_index=True) + + out = out.reindex(columns=COLUMNS) out.sort_values(["country", "feature", "wave"]).to_csv(path, index=False) return path diff --git a/lsms_library/data_info.yml b/lsms_library/data_info.yml index 762393d4b..c80c81805 100644 --- a/lsms_library/data_info.yml +++ b/lsms_library/data_info.yml @@ -355,4 +355,18 @@ Currency: Tanzania: TZS Timor-Leste: USD # USD official since 2000 Togo: XOF - Uganda: UGX \ No newline at end of file + Uganda: UGX +# Countries whose config exists but whose source microdata is NOT in this +# repository. Their features cannot build; that is a fact about data +# availability, not a defect. The coverage matrix grades their cells `n/a` +# rather than `broken`, so a real build failure elsewhere is not lost in the +# noise (before this, all 8 `broken` cells in the cube were these two). +# +# Keep this in sync with the "Countries Without Microdata" table in CLAUDE.md. +Countries Without Microdata: + note: >- + Config present, source .dta absent. See CLAUDE.md. Adding the data is an + acquisition task; it is not a bug in the country's config. + countries: + Armenia: No data files downloaded; WB catalog / external hosting. + Nepal: NSO hosts the data, not the WB (https://microdata.nsonepal.gov.np/). diff --git a/tests/test_coverage_matrix.py b/tests/test_coverage_matrix.py index cff82f51a..e58649e76 100644 --- a/tests/test_coverage_matrix.py +++ b/tests/test_coverage_matrix.py @@ -419,3 +419,109 @@ def test_absent_stays_absent_with_no_verdicts(): _env(_df_with_t({"w1": 3}), _ok_report()), readiness=True) assert _tiers(cells)[("foo", "w2")] == "absent" + + +# --------------------------------------------------------------------------- +# `unconfigured` + known-no-microdata (Defects 4 and 6) +# --------------------------------------------------------------------------- +def test_unconfigured_countries_finds_data_without_config(tmp_path): + """A country dir with microdata but no _/data_scheme.yml must be REPORTED. + + `catalog.countries()` requires a data_scheme.yml, so these were invisible to + the whole library -- 10 in-remit countries and ~35 waves of already- + downloaded data that the matrix simply never mentioned. A denominator that + omits the work you have not started is not a denominator. + """ + root = tmp_path / "countries" + # configured country -> NOT unconfigured + (root / "Uganda" / "_").mkdir(parents=True) + (root / "Uganda" / "_" / "data_scheme.yml").write_text("Country: Uganda\n") + (root / "Uganda" / "2019-20" / "Data").mkdir(parents=True) + # data, no config -> unconfigured + (root / "Peru" / "1994" / "Data").mkdir(parents=True) + (root / "Peru" / "1991" / "Data").mkdir(parents=True) + # no Data/ at all -> not a country dir; must be ignored + (root / "stray").mkdir() + + got = cov.unconfigured_countries(lambda: root) + assert got == {"Peru": 2} + + +def test_known_no_microdata_countries_are_na_not_broken(): + """Armenia/Nepal have NO source data. That is a fact, not a defect. + + Before this, ALL 8 `broken` cells in the cube were these two countries -- + so the most alarming tier in the ladder contained nothing actionable, and a + genuine build failure would have been lost among them. + """ + known = cov.countries_without_microdata() + assert set(known) >= {"Armenia", "Nepal"} + + co = _FakeCountry({"1996": ["household_roster"]}) + env = _env(FileNotFoundError("PathMissingError: INDSECA.dta"), _ok_report()) + + cells = cov.grade_feature("Armenia", "household_roster", ["1996"], co, env, + readiness=True) + cell = cells[0] + assert cell["tier"] == "n/a" # not `broken` + assert "no microdata in repo" in cell["detail"] + + # ...but a country that SHOULD have data still reports `broken`. + cells = cov.grade_feature("Uganda", "household_roster", ["1996"], co, env, + readiness=True) + assert cells[0]["tier"] == "broken" + + +def test_new_tiers_are_registered_in_the_ladder(): + for t in ("not-asked", "asked-not-distributed", "unconfigured"): + assert t in cov.TIER_ORDER, t + assert t in cov.ROLLUP_PRIORITY, t + # every tier the model can emit must be rankable by the rollup + assert set(cov.TIER_ORDER) == set(cov.ROLLUP_PRIORITY) + + +def test_scoped_snapshot_MERGES_and_does_not_destroy_other_cells(tmp_path): + """A partial measurement must never erase a complete one. + + `make matrix C="Uganda"` grades only Uganda. Writing that wholesale wiped + every other country -- and docs/guide/coverage.md documented exactly that as + the "spot refresh" procedure, followed by `git add latest.csv && commit`. + Following the guide verbatim replaced the authoritative 1849-cell snapshot + with a 67-cell one. (Observed 2026-07-12.) + """ + snap = tmp_path / "latest.csv" + full = pd.DataFrame( + [{"country": "Uganda", "feature": "housing", "wave": "2019-20", + "tier": "builds", "coverage": "declared", "n_rows": "10", "detail": "old"}, + {"country": "Malawi", "feature": "housing", "wave": "2016-17", + "tier": "sane", "coverage": "declared", "n_rows": "99", "detail": "keep me"}], + columns=cov.COLUMNS) + cov.save_snapshot(full, snap, merge=False) + + # a SCOPED re-run: only Uganda was graded + scoped = pd.DataFrame( + [{"country": "Uganda", "feature": "housing", "wave": "2019-20", + "tier": "sane", "coverage": "declared", "n_rows": "10", "detail": "new"}], + columns=cov.COLUMNS) + cov.save_snapshot(scoped, snap) + + got = pd.read_csv(snap, dtype=str, keep_default_na=False) + assert len(got) == 2, "the un-graded country was destroyed" + uga = got[got.country == "Uganda"].iloc[0] + mwi = got[got.country == "Malawi"].iloc[0] + assert uga["tier"] == "sane" and uga["detail"] == "new" # upserted + assert mwi["tier"] == "sane" and mwi["detail"] == "keep me" # untouched + + +def test_save_snapshot_merge_false_still_replaces(tmp_path): + snap = tmp_path / "latest.csv" + cov.save_snapshot(pd.DataFrame( + [{"country": "Malawi", "feature": "housing", "wave": "w", "tier": "sane", + "coverage": "declared", "n_rows": "1", "detail": ""}], + columns=cov.COLUMNS), snap, merge=False) + cov.save_snapshot(pd.DataFrame( + [{"country": "Uganda", "feature": "housing", "wave": "w", "tier": "sane", + "coverage": "declared", "n_rows": "1", "detail": ""}], + columns=cov.COLUMNS), snap, merge=False) + got = pd.read_csv(snap, dtype=str) + assert list(got.country) == ["Uganda"] From 1f48cb376edc7e03cc19df9f5a4e15ff536d1ee5 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Sun, 12 Jul 2026 07:43:27 -0700 Subject: [PATCH 09/85] =?UTF-8?q?docs(CLAUDE):=20correct=20the=20.pth=20/?= =?UTF-8?q?=20PYTHONPATH=20worktree=20guidance=20=E2=80=94=20it=20was=20wr?= =?UTF-8?q?ong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addendum #3 claimed "`PYTHONPATH` alone does NOT redirect imports of `lsms_library` to a worktree". That is FALSE -- PYTHONPATH does beat the `.pth`. The wrong claim sent agents to a needless mitigation (build a fresh venv inside the worktree, or verify by static diff only). The ACTUAL trap is `python