diff --git a/lsms_library/countries/Benin/2018-19/_/people_last7days.py b/lsms_library/countries/Benin/2018-19/_/people_last7days.py index bfbe5b57c..190eb742d 100644 --- a/lsms_library/countries/Benin/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Benin/2018-19/_/people_last7days.py @@ -97,6 +97,28 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Benin data: there is nothing to + # collapse, so `.first()`'s per-column skipna=True can form no composite + # row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_ben2018 rows 35,042; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_ben2018 rows 42,343; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 35,042) + # benin.i() is INJECTIVE: 8,012 distinct (grappe, menage) pairs in the + # full roster -> 8,012 distinct ids (menage runs to 562, so the + # zero-padded-3 form is exercised; no collision) + # groups on (t, i, pid) 35,042 == rows 35,042 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py b/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py index a2d8ad451..f3c96c1b9 100644 --- a/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py @@ -90,6 +90,29 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Burkina Faso data: there is + # nothing to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_bfa2018 rows 45,612; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_bfa2018 rows 45,612; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 45,612) + # ehcvm_i() is INJECTIVE: 7,010 distinct (grappe, menage) pairs -> + # 7,010 distinct ids (menage runs to 527, i.e. the three-digit case + # that broke the older `i()`, and the '0'-separated form collides on + # none of it) + # groups on (t, i, pid) 45,612 == rows 45,612 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py b/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py index 2d36e2ebb..8e1068851 100644 --- a/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py +++ b/lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py @@ -96,6 +96,30 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped CotedIvoire data: there is + # nothing to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_CIV2018 rows 61,116; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_CIV2018 rows 61,116; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 61,116) + # _i() is INJECTIVE: 12,992 distinct (grappe, menage) pairs -> 12,992 + # distinct ids. Worth stating why, since _i() concatenates with NO + # separator: menage tops out at 30, so zeropadding=3 always yields + # exactly three characters and the split point is unambiguous. A + # future CotedIvoire wave with menage > 999 would break that. + # groups on (t, i, pid) 61,116 == rows 61,116 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py b/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py index 6237e1615..06a016235 100644 --- a/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py @@ -106,6 +106,29 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Guinea-Bissau data: there is + # nothing to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_gnb2018 rows 42,839; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_gnb2018 rows 42,839; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 42,839) + # i() is INJECTIVE: 5,351 distinct (grappe, menage) pairs -> 5,351 + # distinct ids. Note Guinea-Bissau's grappe is a wide code (max + # 27,711,109, not a 3-digit cluster number as in the other EHCVM + # countries); the '0'-separated form still collides on none of it. + # groups on (t, i, pid) 42,839 == rows 42,839 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Senegal/2018-19/_/people_last7days.py b/lsms_library/countries/Senegal/2018-19/_/people_last7days.py index ef98e2155..a89f614a2 100644 --- a/lsms_library/countries/Senegal/2018-19/_/people_last7days.py +++ b/lsms_library/countries/Senegal/2018-19/_/people_last7days.py @@ -103,6 +103,31 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Senegal data: there is nothing + # to collapse, so `.first()`'s per-column skipna=True can form no + # composite row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_sen2018 rows 66,120; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_sen2018 rows 66,119; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 66,120) + # i() is INJECTIVE: 7,156 distinct (grappe, menage) pairs -> 7,156 + # distinct ids + # groups on (t, i, pid) 66,120 == rows 66,120 -> 0 duplicate groups + # Separate, non-key observation from the same probe: exactly ONE s04 + # person-key -- (grappe 481, menage 9, line 17) -- has no s01 roster row, + # so that person's Age (and hence working_age) is NA. A one-row roster + # gap, not a key defect; recorded so it is not rediscovered as one. + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df diff --git a/lsms_library/countries/Togo/2018/_/people_last7days.py b/lsms_library/countries/Togo/2018/_/people_last7days.py index 393add06b..2a4a6d339 100644 --- a/lsms_library/countries/Togo/2018/_/people_last7days.py +++ b/lsms_library/countries/Togo/2018/_/people_last7days.py @@ -107,6 +107,27 @@ def _finish_people_last7days(df, t): keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work', 'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age'] df = df[keep] + # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ---------- + # This collapse is a NO-OP on the shipped Togo data: there is nothing to + # collapse, so `.first()`'s per-column skipna=True can form no composite + # row here. Measured on the SOURCE directly (this table is + # `materialize: make`, so it builds out-of-process and an in-process + # groupby probe cannot see it -- see the #637 thread): + # s04_me_tgo2018 rows 25,282; duplicate (grappe, menage, s01q00a) + # groups 0 + # s01_me_tgo2018 rows 27,480; duplicate person-key groups 0 + # -> the how='left' merge fans out 0 rows (merged == 25,282) + # i() is INJECTIVE: 6,171 distinct (grappe, menage) pairs in the full + # roster -> 6,171 distinct ids + # groups on (t, i, pid) 25,282 == rows 25,282 -> 0 duplicate groups + # So the key merges no distinct persons -- the ONLY situation in which the + # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix + # for a merged entity is the identifier, never a reducer). + # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is + # absence, not contradiction, and skipna=False would return for + # values the survey actually recorded. + # `dropna=False` governs NA *group keys* only; rows with NA i/pid are + # already dropped above and `t` is a literal, so it too is inert here. df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first()) df = df.set_index(['t', 'i', 'pid']) return df