docs(#637): EHCVM people_last7days (t, i, pid) key review -- no key is broken

lsms_library/countries/Benin/2018-19/_/people_last7days.py

@@ -97,6 +97,28 @@ def _finish_people_last7days(df, t):
     keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work',
             'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age']
     df = df[keep]
+    # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ----------
+    # This collapse is a NO-OP on the shipped Benin data: there is nothing to
+    # collapse, so `.first()`'s per-column skipna=True can form no composite
+    # row here.  Measured on the SOURCE directly (this table is
+    # `materialize: make`, so it builds out-of-process and an in-process
+    # groupby probe cannot see it -- see the #637 thread):
+    #   s04_me_ben2018 rows 35,042; duplicate (grappe, menage, s01q00a)
+    #     groups 0
+    #   s01_me_ben2018 rows 42,343; duplicate person-key groups 0
+    #     -> the how='left' merge fans out 0 rows (merged == 35,042)
+    #   benin.i() is INJECTIVE: 8,012 distinct (grappe, menage) pairs in the
+    #     full roster -> 8,012 distinct ids (menage runs to 562, so the
+    #     zero-padded-3 form is exercised; no collision)
+    #   groups on (t, i, pid) 35,042 == rows 35,042  ->  0 duplicate groups
+    # So the key merges no distinct persons -- the ONLY situation in which the
+    # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix
+    # for a merged entity is the identifier, never a reducer).
+    # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is
+    # absence, not contradiction, and skipna=False would return <NA> for
+    # values the survey actually recorded.
+    # `dropna=False` governs NA *group keys* only; rows with NA i/pid are
+    # already dropped above and `t` is a literal, so it too is inert here.
     df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first())
     df = df.set_index(['t', 'i', 'pid'])
     return df

lsms_library/countries/Burkina_Faso/2018-19/_/people_last7days.py

@@ -90,6 +90,29 @@ def _finish_people_last7days(df, t):
     keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work',
             'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age']
     df = df[keep]
+    # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ----------
+    # This collapse is a NO-OP on the shipped Burkina Faso data: there is
+    # nothing to collapse, so `.first()`'s per-column skipna=True can form no
+    # composite row here.  Measured on the SOURCE directly (this table is
+    # `materialize: make`, so it builds out-of-process and an in-process
+    # groupby probe cannot see it -- see the #637 thread):
+    #   s04_me_bfa2018 rows 45,612; duplicate (grappe, menage, s01q00a)
+    #     groups 0
+    #   s01_me_bfa2018 rows 45,612; duplicate person-key groups 0
+    #     -> the how='left' merge fans out 0 rows (merged == 45,612)
+    #   ehcvm_i() is INJECTIVE: 7,010 distinct (grappe, menage) pairs ->
+    #     7,010 distinct ids (menage runs to 527, i.e. the three-digit case
+    #     that broke the older `i()`, and the '0'-separated form collides on
+    #     none of it)
+    #   groups on (t, i, pid) 45,612 == rows 45,612  ->  0 duplicate groups
+    # So the key merges no distinct persons -- the ONLY situation in which the
+    # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix
+    # for a merged entity is the identifier, never a reducer).
+    # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is
+    # absence, not contradiction, and skipna=False would return <NA> for
+    # values the survey actually recorded.
+    # `dropna=False` governs NA *group keys* only; rows with NA i/pid are
+    # already dropped above and `t` is a literal, so it too is inert here.
     df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first())
     df = df.set_index(['t', 'i', 'pid'])
     return df

lsms_library/countries/CotedIvoire/2018-19/_/people_last7days.py

@@ -96,6 +96,30 @@ def _finish_people_last7days(df, t):
     keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work',
             'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age']
     df = df[keep]
+    # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ----------
+    # This collapse is a NO-OP on the shipped CotedIvoire data: there is
+    # nothing to collapse, so `.first()`'s per-column skipna=True can form no
+    # composite row here.  Measured on the SOURCE directly (this table is
+    # `materialize: make`, so it builds out-of-process and an in-process
+    # groupby probe cannot see it -- see the #637 thread):
+    #   s04_me_CIV2018 rows 61,116; duplicate (grappe, menage, s01q00a)
+    #     groups 0
+    #   s01_me_CIV2018 rows 61,116; duplicate person-key groups 0
+    #     -> the how='left' merge fans out 0 rows (merged == 61,116)
+    #   _i() is INJECTIVE: 12,992 distinct (grappe, menage) pairs -> 12,992
+    #     distinct ids.  Worth stating why, since _i() concatenates with NO
+    #     separator: menage tops out at 30, so zeropadding=3 always yields
+    #     exactly three characters and the split point is unambiguous.  A
+    #     future CotedIvoire wave with menage > 999 would break that.
+    #   groups on (t, i, pid) 61,116 == rows 61,116  ->  0 duplicate groups
+    # So the key merges no distinct persons -- the ONLY situation in which the
+    # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix
+    # for a merged entity is the identifier, never a reducer).
+    # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is
+    # absence, not contradiction, and skipna=False would return <NA> for
+    # values the survey actually recorded.
+    # `dropna=False` governs NA *group keys* only; rows with NA i/pid are
+    # already dropped above and `t` is a literal, so it too is inert here.
     df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first())
     df = df.set_index(['t', 'i', 'pid'])
     return df

lsms_library/countries/Guinea-Bissau/2018-19/_/people_last7days.py

@@ -106,6 +106,29 @@ def _finish_people_last7days(df, t):
     keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work',
             'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age']
     df = df[keep]
+    # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ----------
+    # This collapse is a NO-OP on the shipped Guinea-Bissau data: there is
+    # nothing to collapse, so `.first()`'s per-column skipna=True can form no
+    # composite row here.  Measured on the SOURCE directly (this table is
+    # `materialize: make`, so it builds out-of-process and an in-process
+    # groupby probe cannot see it -- see the #637 thread):
+    #   s04_me_gnb2018 rows 42,839; duplicate (grappe, menage, s01q00a)
+    #     groups 0
+    #   s01_me_gnb2018 rows 42,839; duplicate person-key groups 0
+    #     -> the how='left' merge fans out 0 rows (merged == 42,839)
+    #   i() is INJECTIVE: 5,351 distinct (grappe, menage) pairs -> 5,351
+    #     distinct ids.  Note Guinea-Bissau's grappe is a wide code (max
+    #     27,711,109, not a 3-digit cluster number as in the other EHCVM
+    #     countries); the '0'-separated form still collides on none of it.
+    #   groups on (t, i, pid) 42,839 == rows 42,839  ->  0 duplicate groups
+    # So the key merges no distinct persons -- the ONLY situation in which the
+    # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix
+    # for a merged entity is the identifier, never a reducer).
+    # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is
+    # absence, not contradiction, and skipna=False would return <NA> for
+    # values the survey actually recorded.
+    # `dropna=False` governs NA *group keys* only; rows with NA i/pid are
+    # already dropped above and `t` is a literal, so it too is inert here.
     df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first())
     df = df.set_index(['t', 'i', 'pid'])
     return df

lsms_library/countries/Senegal/2018-19/_/people_last7days.py

@@ -103,6 +103,31 @@ def _finish_people_last7days(df, t):
     keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work',
             'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age']
     df = df[keep]
+    # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ----------
+    # This collapse is a NO-OP on the shipped Senegal data: there is nothing
+    # to collapse, so `.first()`'s per-column skipna=True can form no
+    # composite row here.  Measured on the SOURCE directly (this table is
+    # `materialize: make`, so it builds out-of-process and an in-process
+    # groupby probe cannot see it -- see the #637 thread):
+    #   s04_me_sen2018 rows 66,120; duplicate (grappe, menage, s01q00a)
+    #     groups 0
+    #   s01_me_sen2018 rows 66,119; duplicate person-key groups 0
+    #     -> the how='left' merge fans out 0 rows (merged == 66,120)
+    #   i() is INJECTIVE: 7,156 distinct (grappe, menage) pairs -> 7,156
+    #     distinct ids
+    #   groups on (t, i, pid) 66,120 == rows 66,120  ->  0 duplicate groups
+    # Separate, non-key observation from the same probe: exactly ONE s04
+    # person-key -- (grappe 481, menage 9, line 17) -- has no s01 roster row,
+    # so that person's Age (and hence working_age) is NA.  A one-row roster
+    # gap, not a key defect; recorded so it is not rediscovered as one.
+    # So the key merges no distinct persons -- the ONLY situation in which the
+    # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix
+    # for a merged entity is the identifier, never a reducer).
+    # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is
+    # absence, not contradiction, and skipna=False would return <NA> for
+    # values the survey actually recorded.
+    # `dropna=False` governs NA *group keys* only; rows with NA i/pid are
+    # already dropped above and `t` is a literal, so it too is inert here.
     df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first())
     df = df.set_index(['t', 'i', 'pid'])
     return df

lsms_library/countries/Togo/2018/_/people_last7days.py

@@ -107,6 +107,27 @@ def _finish_people_last7days(df, t):
     keep = ['t', 'i', 'pid', 'farm_work', 'SOB_work', 'wage_work',
             'farm_hrs', 'SB_hrs', 'wage_hrs', 'Industry', 'working_age']
     df = df[keep]
+    # --- (t, i, pid) key soundness reviewed 2026-07-21 -- GH #637 ----------
+    # This collapse is a NO-OP on the shipped Togo data: there is nothing to
+    # collapse, so `.first()`'s per-column skipna=True can form no composite
+    # row here.  Measured on the SOURCE directly (this table is
+    # `materialize: make`, so it builds out-of-process and an in-process
+    # groupby probe cannot see it -- see the #637 thread):
+    #   s04_me_tgo2018 rows 25,282; duplicate (grappe, menage, s01q00a)
+    #     groups 0
+    #   s01_me_tgo2018 rows 27,480; duplicate person-key groups 0
+    #     -> the how='left' merge fans out 0 rows (merged == 25,282)
+    #   i() is INJECTIVE: 6,171 distinct (grappe, menage) pairs in the full
+    #     roster -> 6,171 distinct ids
+    #   groups on (t, i, pid) 25,282 == rows 25,282  ->  0 duplicate groups
+    # So the key merges no distinct persons -- the ONLY situation in which the
+    # composite would be wrong (GH #637 correction thread; GH #323 D1: the fix
+    # for a merged entity is the identifier, never a reducer).
+    # Do NOT "fix" this to `.first(skipna=False)`: in this codebase NaN is
+    # absence, not contradiction, and skipna=False would return <NA> for
+    # values the survey actually recorded.
+    # `dropna=False` governs NA *group keys* only; rows with NA i/pid are
+    # already dropped above and `t` is a literal, so it too is inert here.
     df = (df.groupby(['t', 'i', 'pid'], dropna=False, as_index=False).first())
     df = df.set_index(['t', 'i', 'pid'])
     return df