From c7dee21242e041ac24f4f047007162c98255df70 Mon Sep 17 00:00:00 2001 From: Ethan Ligon Date: Tue, 21 Jul 2026 13:04:14 -0700 Subject: [PATCH] docs(#637): record the (t, i, pid) key review at the six EHCVM people_last7days sites Key-soundness review of the six copy-pasted EHCVM `people_last7days` `.groupby(['t','i','pid'], dropna=False).first()` sites (Benin, Burkina Faso, CotedIvoire, Guinea-Bissau, Senegal, Togo). Verdict: no key is broken. The collapse is a strict NO-OP in all six countries -- there are zero duplicate (t, i, pid) groups, so `.first()`'s per-column skipna=True can form no composite row at all. This commit adds only the evidence: a comment at each site recording that the key was reviewed, what was checked, and the counts. No behaviour change. Measured on the SOURCE directly rather than by instrumenting the build. These tables are `materialize: make`, so they run out-of-process via subprocess; the earlier in-process `DataFrameGroupBy.first` monkeypatch probe never observed them, which is why it reported a meaningless "0" (see the #637 correction thread). country s04 rows s01 rows dup key groups merge fan-out Benin 35,042 42,343 0 / 0 0 Burkina Faso 45,612 45,612 0 / 0 0 CotedIvoire 61,116 61,116 0 / 0 0 Guinea-Bissau 42,839 42,839 0 / 0 0 Senegal 66,120 66,119 0 / 0 0 Togo 25,282 27,480 0 / 0 0 Every country's household-id formatter is injective over the full roster (8,012 / 7,010 / 12,992 / 5,351 / 7,156 / 6,171 distinct (grappe, menage) pairs -> the same number of distinct ids), so no two households -- and hence no two persons -- can be merged onto one key. Cold rebuilds confirm end to end: the wave parquets carry exactly the pre-groupby row counts (Togo 25,282; Benin 35,042; Senegal 66,120) with zero duplicate index entries. Deliberately NOT changed: * `.first()` is left as-is. `.first(skipna=False)` was withdrawn on #637 as a regression -- NaN is absence, not contradiction, and it would return for values the survey recorded. * No `aggregation:` YAML key (GH #323 D1 forbids it). Also recorded at the sites, so they are not rediscovered as defects: * `dropna=False` governs NA *group keys* only. Rows with NA i/pid are already filtered out immediately above and `t` is a literal, so the kwarg is inert at all six sites. * Senegal: exactly one s04 person-key (grappe 481, menage 9, line 17) has no s01 roster row, so that person's Age/working_age is NA. A one-row roster gap, not a key defect. * Senegal's API-level row count is 65,723, not the parquet's 66,120. The 397-row difference is NOT the v-join and NOT id_walk (both were checked and drop nothing): it is the universal hollow-row safety net `df.dropna(how='all')` at `country.py:2243` removing 397 persons whose every declared column is NA. Unrelated to the key. Refs #637. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Benin/2018-19/_/people_last7days.py | 22 ++++++++++++++++ .../2018-19/_/people_last7days.py | 23 +++++++++++++++++ .../CotedIvoire/2018-19/_/people_last7days.py | 24 ++++++++++++++++++ .../2018-19/_/people_last7days.py | 23 +++++++++++++++++ .../Senegal/2018-19/_/people_last7days.py | 25 +++++++++++++++++++ .../countries/Togo/2018/_/people_last7days.py | 21 ++++++++++++++++ 6 files changed, 138 insertions(+) diff --git a/lsms_library/countries/Benin/2018-19/_/people_last7days.py b/lsms_library/countries/Benin/2018-19/_/people_last7days.py index bfbe5b57c..190eb742d 100644 --- a/lsms_library/countries/Benin/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Benin/2018-19/_/people_last7days.py @@ -97,6 +97,28 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Benin data: there is nothing to + # collapse, so `.first()`'s per-column skipna=True can form no composite + # row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_ben2018 rows 35,042; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_ben2018 rows 42,343; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 35,042) + # benin.i() is INJECTIVE: 8,012 distinct (grappe, menage) pairs in the + # full roster -> 8,012 distinct ids (menage runs to 562, so the + # zero-padded-3 form is exercised; no collision) + # groups on (t, i, pid) 35,042 == rows 35,042 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py b/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py index a2d8ad451..f3c96c1b9 100644 --- a/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py @@ -90,6 +90,29 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Burkina Faso data: there is + # nothing to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_bfa2018 rows 45,612; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_bfa2018 rows 45,612; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 45,612) + # ehcvm_i() is INJECTIVE: 7,010 distinct (grappe, menage) pairs -> + # 7,010 distinct ids (menage runs to 527, i.e. the three-digit case + # that broke the older `i()`, and the '0'-separated form collides on + # none of it) + # groups on (t, i, pid) 45,612 == rows 45,612 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py b/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py index 2d36e2ebb..8e1068851 100644 --- a/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py +++ b/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py @@ -96,6 +96,30 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped CotedIvoire data: there is + # nothing to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_CIV2018 rows 61,116; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_CIV2018 rows 61,116; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 61,116) + # _i() is INJECTIVE: 12,992 distinct (grappe, menage) pairs -> 12,992 + # distinct ids. Worth stating why, since _i() concatenates with NO + # separator: menage tops out at 30, so zeropadding=3 always yields + # exactly three characters and the split point is unambiguous. A + # future CotedIvoire wave with menage > 999 would break that. + # groups on (t, i, pid) 61,116 == rows 61,116 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py b/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py index 6237e1615..06a016235 100644 --- a/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py @@ -106,6 +106,29 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Guinea-Bissau data: there is + # nothing to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_gnb2018 rows 42,839; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_gnb2018 rows 42,839; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 42,839) + # i() is INJECTIVE: 5,351 distinct (grappe, menage) pairs -> 5,351 + # distinct ids. Note Guinea-Bissau's grappe is a wide code (max + # 27,711,109, not a 3-digit cluster number as in the other EHCVM + # countries); the '0'-separated form still collides on none of it. + # groups on (t, i, pid) 42,839 == rows 42,839 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Senegal/2018-19/_/people_last7days.py b/lsms_library/countries/Senegal/2018-19/_/people_last7days.py index ef98e2155..a89f614a2 100644 --- a/lsms_library/countries/Senegal/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Senegal/2018-19/_/people_last7days.py @@ -103,6 +103,31 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Senegal data: there is nothing + # to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_sen2018 rows 66,120; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_sen2018 rows 66,119; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 66,120) + # i() is INJECTIVE: 7,156 distinct (grappe, menage) pairs -> 7,156 + # distinct ids + # groups on (t, i, pid) 66,120 == rows 66,120 -> 0 duplicate groups + # Separate, non-key observation from the same probe: exactly ONE s04 + # person-key -- (grappe 481, menage 9, line 17) -- has no s01 roster row, + # so that person's Age (and hence working_age) is NA. A one-row roster + # gap, not a key defect; recorded so it is not rediscovered as one. + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Togo/2018/_/people_last7days.py b/lsms_library/countries/Togo/2018/_/people_last7days.py index 393add06b..2a4a6d339 100644 --- a/lsms_library/countries/Togo/2018/_/people_last7days.py +++ b/lsms_library/countries/Togo/2018/_/people_last7days.py @@ -107,6 +107,27 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Togo data: there is nothing to + # collapse, so `.first()`'s per-column skipna=True can form no composite + # row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_tgo2018 rows 25,282; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_tgo2018 rows 27,480; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 25,282) + # i() is INJECTIVE: 6,171 distinct (grappe, menage) pairs in the full + # roster -> 6,171 distinct ids + # groups on (t, i, pid) 25,282 == rows 25,282 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df