fix(#323): Ethiopia + Niger geo — 3 of Ethiopia's 5 waves had ZERO coordinates (unblocks #627)

lsms_library/countries/Ethiopia/2011-12/_/data_info.yml

@@ -1,6 +1,21 @@
 Country: Ethiopia
 Wave: 2011-12
 
+# cluster_features (GH #323 site 4).  Both sources are HOUSEHOLD-grain, so
+# they are merged on the household id (1:1 -- household_id is unique and blank-
+# free in BOTH frames: 3,969 cover rows / 3,917 geo rows, 0 keys duplicated in
+# both) and reduced to the (t, v) EA grain by Wave.cluster_features' existing
+# GH #161 collapse -- mean() for Latitude/Longitude (the cluster centroid),
+# first() for Region/District/Rural.  That is why `i` is in `final_index`:
+# the collapse only fires when `i` is an index level.  Merging on the CLUSTER
+# key v instead made the join many-to-many (333 EAs shared by ~3.9k households
+# on each side) -- a cartesian product.
+#
+# The geo file spells its coordinates LAT_DD_MOD / LON_DD_MOD (UPPERCASE) in
+# THIS wave only; asking for the lowercase names raised a KeyError that the
+# GH #515 optional-sub-df fallback swallowed, silently serving cluster_features
+# with Latitude/Longitude 100% absent.  Verified against the source: the file
+# carries LAT_DD_MOD/LON_DD_MOD non-null for all 3,917 rows.
 cluster_features:
     dfs:
         - df_main
@@ -9,6 +24,7 @@ cluster_features:
         file: sect_cover_hh_w1.dta
         idxvars:
             v: ea_id
+            i: household_id
         myvars:
             Region: saq01
             District: saq02
@@ -30,15 +46,23 @@ cluster_features:
     df_geo:
         file: Pub_ETH_HouseholdGeovariables_Y1.dta
         idxvars:
-            v: ea_id
+            i: household_id
         myvars:
-            Latitude: lat_dd_mod
-            Longitude: lon_dd_mod
+            # UPPERCASE in W1 -- see the header comment.
+            Latitude: LAT_DD_MOD
+            Longitude: LON_DD_MOD
     merge_on:
-        - v
+        - i
+    # The cover page is authoritative for which households exist; the geo file
+    # is a strict enrichment.  `left` keeps orphan geo households (households
+    # the cover does not carry) from arriving with a null v and collapsing
+    # into one phantom cluster.  W1 has 0 such orphans, but the declaration is
+    # uniform across the five waves, which do (W2 25, W4 124, W5 35).
+    merge_how: left
     final_index:
         - t
         - v
+        - i
 
 household_roster:
     file: sect1_hh_w1.dta

lsms_library/countries/Ethiopia/2013-14/_/data_info.yml

@@ -1,6 +1,25 @@
 Country: Ethiopia
 Wave: 2013-14
 
+# cluster_features (GH #323 site 4).  Both sources are HOUSEHOLD-grain, so
+# they are merged on the household id (1:1) and reduced to the (t, v) EA grain
+# by Wave.cluster_features' existing GH #161 collapse -- mean() for
+# Latitude/Longitude (the cluster centroid), first() for Region/District/Rural.
+# That is why `i` is in `final_index`: the collapse only fires when `i` is an
+# index level.
+#
+# Merging on the CLUSTER key v instead made the join many-to-many and produced
+# a CARTESIAN PRODUCT: 5,262 cover households x 5,287 geo households over 433
+# EAs -> 65,508 rows, of which 60,221 were phantom.  The downstream
+# groupby().first() then tidied the evidence away, so the table reported clean.
+#
+# The key MUST be household_id2 (the W2 re-identified id), not household_id:
+# household_id is the W1 baseline id and is BLANK for households new to the
+# panel, so it is non-unique on the empty value -- re-keying to it would trade
+# an EA cartesian for a NULL-KEY cartesian (pd.merge matches nulls).  Verified:
+# household_id2 is unique and blank-free in BOTH frames (5,262 / 5,287 rows,
+# 0 keys duplicated in both).  This matches the wave's v (ea_id2) and the
+# i that sample/household_roster emit -- see _/CONTENTS.org.
 cluster_features:
     dfs:
         - df_main
@@ -9,6 +28,7 @@ cluster_features:
         file: sect_cover_hh_w2.dta
         idxvars:
             v: ea_id2
+            i: household_id2
         myvars:
             Region: saq01
             District: saq02
@@ -31,15 +51,21 @@ cluster_features:
     df_geo:
         file: Pub_ETH_HouseholdGeovars_Y2.dta
         idxvars:
-            v: ea_id2
+            i: household_id2
         myvars:
             Latitude: lat_dd_mod
             Longitude: lon_dd_mod
     merge_on:
-        - v
+        - i
+    # The cover page is authoritative for which households exist; the geo file
+    # is a strict enrichment and carries 25 households the cover does not (and
+    # a 434th EA the cover has never heard of).  `left` keeps those orphans
+    # from arriving with a null v and collapsing into one phantom cluster.
+    merge_how: left
     final_index:
         - t
         - v
+        - i
 
 panel_ids:
     file:

lsms_library/countries/Ethiopia/2015-16/_/data_info.yml

@@ -1,6 +1,22 @@
 Country: Ethiopia
 Wave: 2015-16
 
+# cluster_features (GH #323 site 4).  Both sources are HOUSEHOLD-grain, so
+# they are merged on the household id (1:1) and reduced to the (t, v) EA grain
+# by Wave.cluster_features' existing GH #161 collapse -- mean() for
+# Latitude/Longitude (the cluster centroid), first() for Region/District/Rural.
+# That is why `i` is in `final_index`: the collapse only fires when `i` is an
+# index level.
+#
+# Merging on the CLUSTER key v instead made the join many-to-many and produced
+# a CARTESIAN PRODUCT: 4,954 x 4,954 households over 432 EAs -> 57,786 rows, of
+# which 52,832 were phantom.  The downstream groupby().first() then tidied the
+# evidence away, so the table reported clean.
+#
+# The key is household_id2 (the W2 re-identified id, which W3 inherits) -- the
+# only id this geo file carries, and the i that sample/household_roster emit
+# for this wave (see _/CONTENTS.org).  Verified unique and blank-free in BOTH
+# frames (4,954 / 4,954 rows, 0 keys duplicated in both).
 cluster_features:
     dfs:
         - df_main
@@ -9,6 +25,7 @@ cluster_features:
         file: sect_cover_hh_w3.dta
         idxvars:
             v: ea_id2
+            i: household_id2
         myvars:
             Region: saq01
             District: saq02
@@ -31,15 +48,21 @@ cluster_features:
     df_geo:
         file: ETH_HouseholdGeovars_y3.dta
         idxvars:
-            v: ea_id2
+            i: household_id2
         myvars:
             Latitude: lat_dd_mod
             Longitude: lon_dd_mod
     merge_on:
-        - v
+        - i
+    # The cover page is authoritative for which households exist; the geo file
+    # is a strict enrichment.  `left` keeps orphan geo households from arriving
+    # with a null v and collapsing into one phantom cluster.  W3 has 0 such
+    # orphans, but the declaration is uniform across the five waves, which do.
+    merge_how: left
     final_index:
         - t
         - v
+        - i
 
 panel_ids:
     # Built by bespoke script (materialize: make in data_scheme.yml).

lsms_library/countries/Ethiopia/2018-19/_/data_info.yml

@@ -1,6 +1,21 @@
 Country: Ethiopia
 Wave: 2018-19
 
+# cluster_features (GH #323 site 4).  Both sources are HOUSEHOLD-grain, so
+# they are merged on household_id (1:1 -- unique and blank-free in BOTH frames:
+# 6,770 cover rows / 6,804 geo rows, 0 keys duplicated in both) and reduced to
+# the (t, v) EA grain by Wave.cluster_features' existing GH #161 collapse --
+# mean() for Latitude/Longitude (the cluster centroid), first() for
+# Region/District/Rural.  That is why `i` is in `final_index`: the collapse
+# only fires when `i` is an index level.  Merging on the CLUSTER key v instead
+# was a latent cartesian (535 EAs shared by ~6.8k households on each side); it
+# never fired only because the geo sub-df never loaded at all -- see below.
+#
+# W4 spells its coordinates lat_mod / lon_mod -- NO '_dd_' infix, unlike every
+# other ESS wave.  Asking for lat_dd_mod raised a KeyError that the GH #515
+# optional-sub-df fallback swallowed, silently serving cluster_features with
+# Latitude/Longitude 100% absent.  Verified against the source: the file
+# carries lat_mod/lon_mod non-null for 6,637 of its 6,804 rows.
 cluster_features:
     dfs:
         - df_main
@@ -9,6 +24,7 @@ cluster_features:
         file: sect_cover_hh_w4.dta
         idxvars:
             v: ea_id
+            i: household_id
         myvars:
             Region: saq01
             District: saq02
@@ -24,15 +40,22 @@ cluster_features:
     df_geo:
         file: ETH_HouseholdGeovariables_Y4.dta
         idxvars:
-            v: ea_id
+            i: household_id
         myvars:
-            Latitude: lat_dd_mod
-            Longitude: lon_dd_mod
+            # W4 drops the '_dd_' infix -- see the header comment.
+            Latitude: lat_mod
+            Longitude: lon_mod
     merge_on:
-        - v
+        - i
+    # The cover page is authoritative for which households exist; the geo file
+    # is a strict enrichment and carries 124 households the cover does not.
+    # `left` keeps those orphans from arriving with a null v and collapsing
+    # into one phantom cluster.
+    merge_how: left
     final_index:
         - t
         - v
+        - i
 
 household_roster:
     file: sect1_hh_w4.dta

lsms_library/countries/Ethiopia/2021-22/_/data_info.yml

@@ -1,6 +1,22 @@
 Country: Ethiopia
 Wave: 2021-22
 
+# cluster_features (GH #323 site 4).  Both sources are HOUSEHOLD-grain, so
+# they are merged on household_id (1:1 -- unique and blank-free in BOTH frames:
+# 4,959 cover rows / 4,890 geo rows, 0 keys duplicated in both) and reduced to
+# the (t, v) EA grain by Wave.cluster_features' existing GH #161 collapse --
+# mean() for Latitude/Longitude (the cluster centroid), first() for
+# Region/District/Rural.  That is why `i` is in `final_index`: the collapse
+# only fires when `i` is an index level.
+#
+# The W5 geo file has NO ea_id column AT ALL -- it is keyed on household_id
+# only (52 columns, checked).  So the old `v: ea_id` idxvar on df_geo asked it
+# for a column it does not have; the KeyError was swallowed by the GH #515
+# optional-sub-df fallback and cluster_features was served with
+# Latitude/Longitude 100% absent.  The COORDINATES themselves were never
+# missing: lat_dd_mod / lon_dd_mod are present and non-null for 4,847 of the
+# 4,890 rows.  It was the KEY that was wrong, not the data -- so this is wired
+# up, NOT marked optional.  Joining on household_id lets the cover supply v.
 cluster_features:
     dfs:
         - df_main
@@ -9,6 +25,7 @@ cluster_features:
         file: sect_cover_hh_w5.dta
         idxvars:
             v: ea_id
+            i: household_id
         myvars:
             Region: saq01
             District: saq02
@@ -24,17 +41,24 @@ cluster_features:
                     urban: Urban
                     2. URBAN: Urban
     df_geo:
+        # Keyed on household_id ONLY -- this file carries no ea_id.
         file: eth_householdgeovariables_y5.dta
         idxvars:
-            v: ea_id
+            i: household_id
         myvars:
             Latitude: lat_dd_mod
             Longitude: lon_dd_mod
     merge_on:
-        - v
+        - i
+    # The cover page is authoritative for which households exist; the geo file
+    # is a strict enrichment and carries 35 households the cover does not.
+    # `left` keeps those orphans from arriving with a null v and collapsing
+    # into one phantom cluster.
+    merge_how: left
     final_index:
         - t
         - v
+        - i
 
 panel_ids:
     file:

lsms_library/countries/Ethiopia/_/CONTENTS.org

@@ -454,6 +454,54 @@ We therefore DEFER Latitude/Longitude -- they are NOT emitted.  Note
 the GPS-*measured field area* (square metres; =pp_s3q05_a= W1-W3,
 =s3q08= W4-W5) is NOT redacted and is the primary Area source.
 
+This applies to =plot_features= ONLY.  The HOUSEHOLD geovariables
+files are a different release and DO carry usable (offset) coordinates
+-- see the next section.  Do not conflate the two.
+
+** Household geovariables: four spellings across five waves (GH #323)
+
+=cluster_features= gets its Latitude/Longitude from the per-wave
+household geovariables file.  The coordinates are the WB's standard
+*modified* (randomly offset) decimal degrees -- real and usable at the
+EA/cluster grain, and NOT redacted like the plot GPS above.
+
+The catch: the WB renamed the coordinate columns in almost every wave,
+and W5 dropped the EA key entirely.  All four variants are live:
+
+| Wave    | Geovariables file                     | Lat / Lon columns          | Keys present            |
+|---------+---------------------------------------+----------------------------+-------------------------|
+| 2011-12 | =Pub_ETH_HouseholdGeovariables_Y1=    | =LAT_DD_MOD= / =LON_DD_MOD= (UPPERCASE!) | household_id, ea_id     |
+| 2013-14 | =Pub_ETH_HouseholdGeovars_Y2=         | =lat_dd_mod= / =lon_dd_mod= | household_id(2), ea_id(2) |
+| 2015-16 | =ETH_HouseholdGeovars_y3=             | =lat_dd_mod= / =lon_dd_mod= | household_id2, ea_id2   |
+| 2018-19 | =ETH_HouseholdGeovariables_Y4=        | =lat_mod= / =lon_mod= (no =_dd_=!) | household_id, ea_id     |
+| 2021-22 | =eth_householdgeovariables_y5=        | =lat_dd_mod= / =lon_dd_mod= | household_id ONLY -- *no ea_id* |
+
+Every one of these files is HOUSEHOLD-grain, not EA-grain.  So
+=cluster_features= merges the cover page to the geo file on the
+HOUSEHOLD id (1:1; unique and blank-free in both frames in all five
+waves) and lets =Wave.cluster_features='s GH #161 collapse reduce to
+the =(t, v)= EA grain -- =mean()= for Latitude/Longitude (the cluster
+centroid), =first()= for Region/District/Rural.  This is why =i= is in
+each wave's =final_index=: that collapse only fires when =i= is an
+index level.
+
+Merging on the CLUSTER key =v= instead -- which is what the config used
+to do -- is a many-to-many join and produces a CARTESIAN PRODUCT
+(2013-14: 5,262 x 5,287 households over 433 EAs -> 65,508 rows, 60,221
+of them phantom; 2015-16: 57,786 rows, 52,832 phantom).  The downstream
+=groupby().first()= then tidied the evidence away, so the table
+reported clean.  Use =household_id2= for W2/W3, NOT =household_id=:
+the latter is the W1 baseline id and is BLANK for households new to the
+panel, so re-keying to it would trade an EA cartesian for a NULL-KEY
+cartesian.
+
+=merge_how: left= is declared in all five waves: the cover page is
+authoritative for which households exist, and the geo file carries
+households the cover does not (W2 25, W4 124, W5 35).  Under the
+default =outer= those orphans arrive with a null =v= and collapse into
+one phantom cluster -- W2 gained a spurious 434th EA
+(=030412088800105=) exactly this way.
+
 ** Area and the local-unit gap
 
 Area is GPS square-metres / 10000 -> hectares where present

lsms_library/countries/Niger/2011-12/_/data_info.yml

@@ -66,14 +66,46 @@ cluster_features:
                     urbain: Urban
                     URBAIN: Urban
     df_geo:
-        file: NER_2011_ECVMA_v01_M_Stata8/NER_HouseholdGeovars_Y1.dta
+        # GH #323 site 4 -- THE COORDINATES WERE IN THE WRONG FILE.
+        #
+        # This used to point at NER_HouseholdGeovars_Y1.dta and ask it for
+        # lat_dd_mod / lon_dd_mod.  That file has 43 columns and NOT ONE of
+        # them is a coordinate: it ships only DERIVED raster covariates
+        # (dist_road, dist_market, af_bio_*, srtm, twi, sq1..sq7, EVI/NDVI
+        # series).  The KeyError was swallowed by the GH #515 optional-sub-df
+        # fallback, and cluster_features has been served with
+        # Latitude/Longitude 100% ABSENT ever since.
+        #
+        # The coordinates ship in a SIBLING file, NER_EA_Offsets.dta -- the
+        # WB's standard offset-EA-centroid release: 271 rows x (grappe,
+        # LAT_DD_MOD, LON_DD_MOD), 270 non-null grappes, coordinates inside
+        # Niger's bounding box (lat 11.88-18.75 N, lon 0.40-13.70 E).  It is
+        # already DVC-tracked alongside the rest of the wave.
+        #
+        # So this is NOT a "the survey never measured GPS" case and the
+        # columns are NOT marked `optional:` -- the data is real, available,
+        # and merely mis-addressed.  Wired up.
+        #
+        # Grain: EA_Offsets is one row per grappe (270 unique / 270), while
+        # df_main (ecvmamen_p1) is household-grain (3,968 rows over the same
+        # 270 grappes).  So the merge on the cluster key v is many-to-ONE --
+        # NOT a cartesian (0 keys duplicated in BOTH frames).  Every grappe in
+        # df_main is present in the offsets file and vice versa (0 orphans
+        # either way).  The coordinate is an EA-level constant, so the
+        # downstream collapse to the (t, v) cluster grain is exactly lossless.
+        file: NER_2011_ECVMA_v01_M_Stata8/NER_EA_Offsets.dta
         idxvars:
             v: grappe
         myvars:
-            Latitude: lat_dd_mod
-            Longitude: lon_dd_mod
+            Latitude: LAT_DD_MOD
+            Longitude: LON_DD_MOD
     merge_on:
         - v
+    # `left` is load-bearing: the offsets file has ONE trailing row with a
+    # null grappe (and null coordinates).  Under the default `outer` that row
+    # arrives as a phantom cluster with a null v; `left` drops it, since
+    # df_main is authoritative for which clusters exist.
+    merge_how: left
     final_index:
         - t
         - v