From d4f39e8625448edfb14b916d890354520dd18d8a Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:45:39 -0400 Subject: [PATCH 01/10] add check for multiple shape ids per guids --- R/process_dist_pop_data.R | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index a487638..089133f 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -318,7 +318,7 @@ load_all_patches <- function(pakistan_file_path = "GID/PEB/SIR/Data/pop/pop raw/ #' Loads and formats the population growth rate file for use in R. #' #' -#' @param file_loc `str` Path to WPP Excel file. +#' @param .file_loc `str` Path to WPP Excel file. #' @details #' The Excel file uses the "Estimates" tab. The first few rows (16 rows or so) are deleted for ease of loading into #' R. The source of this dataset is in the [World Population Prospects](https://population.un.org/wpp/downloads?folder=Standard%20Projections&group=Most%20used) @@ -327,11 +327,10 @@ load_all_patches <- function(pakistan_file_path = "GID/PEB/SIR/Data/pop/pop raw/ #' @return `tibble` Growth rates for each country by year. #' @export load_growth_rates <- function( - file_loc = "GID/PEB/SIR/Data/pop/pop raw/WPP2024_GEN_F01_DEMOGRAPHIC_INDICATORS_COMPACT.xlsx", + .file_loc = "GID/PEB/SIR/Data/pop/pop raw/WPP2024_GEN_F01_DEMOGRAPHIC_INDICATORS_COMPACT.xlsx", edav = TRUE ) { - wpp_raw <- sirfunctions::sirfunctions_io("read", NULL, file_loc, edav = edav, sheet = 1) - wpp_raw <- wpp_raw$Estimates + wpp_raw <- sirfunctions::sirfunctions_io("read", NULL, file_loc = .file_loc, edav = edav, sheet = 1) # Select and standardize output growth_rates <- wpp_raw |> @@ -611,7 +610,7 @@ process_dist_pop_data <- function(pop_data, dplyr::tibble() |> dplyr::select(#WHO_REGION, apparently missing in the new SF dplyr::ends_with("_NAME"), dplyr::ends_with("_GUID"), - yr.st, yr.end, active.year.01, GUID) |> + yr.st, yr.end, active.year.01, GUID, ADM2_SHAPE_ID) |> dplyr::select(-dplyr::ends_with("VIZ_NAME")) rm(district_long) @@ -620,7 +619,8 @@ process_dist_pop_data <- function(pop_data, # Transform POLIS wide and 0-5Y/U0-15Y/UALL to Under5Pop/Under15Pop/Total polis_pop <- pop_data |> dplyr::arrange(year) |> - dplyr::select(-CREATEDDATE, -UPDATEDDATE, -STARTDATE, -ENDDATE, -is_forward_fill) |> + dplyr::select(-CREATEDDATE, -UPDATEDDATE, -STARTDATE, -ENDDATE, -is_forward_fill, + -FK_DataSetId) |> tidyr::pivot_wider(names_from = AgeGroupCode, values_from = Value) |> dplyr::rename( ADM0_GUID = adm0guid, @@ -647,6 +647,19 @@ process_dist_pop_data <- function(pop_data, ADM1_GUID = dplyr::coalesce(sf_adm1guid, ADM1_GUID)) |> dplyr::select(-dplyr::starts_with("sf_")) + non_stable_adm2_shape_ids <- polis_pop |> + dplyr::group_by(GUID) |> + dplyr::summarize(multiple_shape_ids = length(unique(ADM2_SHAPE_ID))) |> + dplyr::filter(multiple_shape_ids > 1) + + if (nrow(non_stable_adm2_shape_ids) > 0) { + cli::cli_alert_warning("An adm2guid has multiple shape IDs! Please refer to the error folder.") + sirfunctions::sirfunctions_io("read", NULL, file.path(pop_dir, "errors", + paste0(Sys.time(), "_adm2guids_multiple_shape_ids.csv"))) + } else { + cli::cli_alert_success("adm2guids all have unique shape IDs.") + } + # Input Non-POLIS data and removal of forward-filled repeats Values non_polis_pop <- load_all_patches(pakistan_file_path, somalia_2022_file_path, From ae23aa4e0c12ce5a46b2d8d6324764004dd677ea Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:45:49 -0400 Subject: [PATCH 02/10] Update renv.lock --- renv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/renv.lock b/renv.lock index 45e0386..915b57a 100644 --- a/renv.lock +++ b/renv.lock @@ -3154,7 +3154,7 @@ }, "sirfunctions": { "Package": "sirfunctions", - "Version": "2.1.3", + "Version": "2.1.4", "Source": "GitHub", "Title": "Key Functions to Analyze Global Polio Surveillance Data", "Authors@R": "c( person(\"Nishant\", \"Kishore\", , \"ynm2@cdc.gov\", role = c(\"aut\"), comment = c(ORCID = \"0000-0003-0408-2747\")), person(\"Mervin Keith\", \"Cuadera\", , \"xrg9@cdc.gov\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4898-2659\")), person(\"Nicholas\", \"Heaghney\", , \"uic3@cdc.gov\", role = \"aut\"), person(\"Elizabeth\", \"Krow-Lucal\", , \"yxn9@cdc.gov\", role = \"aut\"), person(\"Smita\", \"Chavan\", , \"wsy2@cdc.gov\", role = \"aut\") )", @@ -3225,8 +3225,8 @@ "RemoteHost": "api.github.com", "RemoteRepo": "sirfunctions", "RemoteUsername": "CDCGov", - "RemoteRef": "hotfix", - "RemoteSha": "05e8abceef088714ecb2386287e3bcd4f4260e06", + "RemoteRef": "HEAD", + "RemoteSha": "55e13d5685ddb6714a9eda3f8ec5dc50b1965f3b", "NeedsCompilation": "no", "Author": "Nishant Kishore [aut] (ORCID: ), Mervin Keith Cuadera [aut, cre] (ORCID: ), Nicholas Heaghney [aut], Elizabeth Krow-Lucal [aut], Smita Chavan [aut]", "Maintainer": "Mervin Keith Cuadera ", From 838f2d42d81c1d8f35146a2504520e776b55632d Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:46:03 -0400 Subject: [PATCH 03/10] add documentation --- man/load_growth_rates.Rd | 4 ++-- man/survpopdata-package.Rd | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/man/load_growth_rates.Rd b/man/load_growth_rates.Rd index df4256b..dedf928 100644 --- a/man/load_growth_rates.Rd +++ b/man/load_growth_rates.Rd @@ -5,13 +5,13 @@ \title{Load population growth rates} \usage{ load_growth_rates( - file_loc = + .file_loc = "GID/PEB/SIR/Data/pop/pop raw/WPP2024_GEN_F01_DEMOGRAPHIC_INDICATORS_COMPACT.xlsx", edav = TRUE ) } \arguments{ -\item{file_loc}{\code{str} Path to WPP Excel file.} +\item{.file_loc}{\code{str} Path to WPP Excel file.} } \value{ \code{tibble} Growth rates for each country by year. diff --git a/man/survpopdata-package.Rd b/man/survpopdata-package.Rd index 26ea5d4..33ed44c 100644 --- a/man/survpopdata-package.Rd +++ b/man/survpopdata-package.Rd @@ -6,6 +6,8 @@ \alias{survpopdata-package} \title{survpopdata: Create Cleaned Population Datasets Used In GPEI Analyses} \description{ +\if{html}{\figure{logo.png}{options: style='float: right' alt='logo' width='120'}} + The package aims to clean the population datasets used in multiple GPEI analyses, from the ADM0 through the ADM2 levels. } \seealso{ From 207a81d06520fa601031d3f54442d48bfb6c22ca Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 10:38:55 -0400 Subject: [PATCH 04/10] add check to ensure that geo names and adm0-1 match between pop and sf --- R/process_dist_pop_data.R | 91 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index 089133f..2d1dfae 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -639,14 +639,90 @@ process_dist_pop_data <- function(pop_data, sf_adm0guid = ADM0_GUID, sf_adm1guid = ADM1_GUID), polis_pop) |> - dplyr::distinct() |> - dplyr::mutate(ADM0_NAME = dplyr::coalesce(sf_adm0_name, ADM0_NAME), - ADM1_NAME = dplyr::coalesce(sf_adm1_name, ADM1_NAME), - ADM2_NAME = dplyr::coalesce(sf_adm2_name, ADM2_NAME), - ADM0_GUID = dplyr::coalesce(sf_adm0guid, ADM0_GUID), - ADM1_GUID = dplyr::coalesce(sf_adm1guid, ADM1_GUID)) |> + dplyr::distinct() + + # Check when the pop names are not matching with the shapefile + ctry_name_mismatch <- polis_pop |> + dplyr::filter(sf_adm0_name != ADM0_NAME) + prov_name_mismatch <- polis_pop |> + dplyr::filter(sf_adm1_name != ADM1_NAME) + dist_name_mismatch <- polis_pop |> + dplyr::filter(sf_adm2_name != ADM2_NAME) + + # Check when adm0guid are not matching the shapefile + adm0guid_mismatch <- polis_pop |> + dplyr::filter(sf_adm0guid != ADM0_GUID) + adm1guid_mismatch <- polis_pop |> + dplyr::filter(sf_adm1guid != ADM1_GUID) + + if (nrow(ctry_name_mismatch) > 0) { + cli::cli_alert_warning("Country name mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_ctry_name_mismatches.csv")), + obj = ctry_name_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in country names between pop and shapefile.") + } + + if (nrow(prov_name_mismatch) > 0) { + cli::cli_alert_warning("Province name mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_prov_name_mismatches.csv")), + obj = prov_name_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in province names between pop and shapefile.") + } + + if (nrow(dist_name_mismatch) > 0) { + cli::cli_alert_warning("District name mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_dist_name_mismatches.csv")), + obj = dist_name_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in district names between pop and shapefile.") + } + + if (nrow(adm0guid_mismatch) > 0) { + cli::cli_alert_warning("adm0guid mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_adm0guid_mismatches.csv")), + obj = adm0guid_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in adm0guid between pop and shapefile.") + } + + if (nrow(adm1guid_mismatch) > 0) { + cli::cli_alert_warning("adm1guid mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_adm1guid_mismatches.csv")), + obj = adm1guid_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in adm1guid between pop and shapefile.") + } + + polis_pop <- polis_pop |> + dplyr::mutate(ADM0_NAME = sf_adm0_name, + ADM1_NAME = sf_adm1_name, + ADM2_NAME = sf_adm2_name, + ADM0_GUID = sf_adm0guid, + ADM1_GUID = sf_adm1guid) |> dplyr::select(-dplyr::starts_with("sf_")) + # Each adm2 shape ID can have multiple GUIDs, typically when a province + # changes shape. However, an adm2guid should not have multiple Shape IDs. What + # that would mean is that a district changed shape without redistricting, which + # is atypical. + non_stable_adm2_shape_ids <- polis_pop |> dplyr::group_by(GUID) |> dplyr::summarize(multiple_shape_ids = length(unique(ADM2_SHAPE_ID))) |> @@ -661,6 +737,7 @@ process_dist_pop_data <- function(pop_data, } # Input Non-POLIS data and removal of forward-filled repeats Values + # All these data came from WHO non_polis_pop <- load_all_patches(pakistan_file_path, somalia_2022_file_path, somalia_2023_file_path, @@ -823,7 +900,7 @@ process_dist_pop_data <- function(pop_data, dplyr::ungroup()) formatted_result <- result |> - dplyr::select(-FK_DataSetId, -ISO_3_CODE) |> + dplyr::select(-ISO_3_CODE) |> tidyr::replace_na(list(used_growth_ALL = FALSE, `used_growth_0-15Y` = FALSE, `used_growth_0-5Y` = FALSE, From 58e9eab182f57f5f739610e8f4a24d465150429d Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:31:30 -0400 Subject: [PATCH 05/10] default to ADM2_SHAPE_ID for growth rates --- R/process_dist_pop_data.R | 2 +- man/apply_growth_rate.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index 2d1dfae..ccde0c5 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -448,7 +448,7 @@ remove_forward_fill_non_polis <- function(non_polis_pop) { #' @returns `tibble` Population data with population filled based on growth rates #' @keywords internal #' -apply_growth_rate <- function(base_data, pop_column, grouping_col = "ADM2_GUID") { +apply_growth_rate <- function(base_data, pop_column, grouping_col = "ADM2_SHAPE_ID") { # Create anchor year vars base_data_formatted <- base_data |> diff --git a/man/apply_growth_rate.Rd b/man/apply_growth_rate.Rd index 0646daa..992772c 100644 --- a/man/apply_growth_rate.Rd +++ b/man/apply_growth_rate.Rd @@ -4,7 +4,7 @@ \alias{apply_growth_rate} \title{Applies the growth rate to each population columns} \usage{ -apply_growth_rate(base_data, pop_column, grouping_col = "ADM2_GUID") +apply_growth_rate(base_data, pop_column, grouping_col = "ADM2_SHAPE_ID") } \arguments{ \item{base_data}{\code{tibble} Combined population data with the growth rate columns.} From 098fc7336e0e342d9732699dfcd30929bf91cec9 Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:38:21 -0400 Subject: [PATCH 06/10] Update process_dist_pop_data.R --- R/process_dist_pop_data.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index ccde0c5..acc39a1 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -911,12 +911,16 @@ process_dist_pop_data <- function(pop_data, adm0guid = "ADM0_GUID", adm1guid = "ADM1_GUID", adm2guid = "ADM2_GUID", + adm2_shape_id = "ADM2_SHAPE_ID", ctry = "ADM0_NAME", prov = "ADM1_NAME", dist = "ADM2_NAME", u15pop = "0-15Y", u5pop = "0-5Y", totpop = "ALL", + u15.anchor.year = "0-15Y_anchor_year", + u5.anchor.year = "0-5Y_anchor_year", + tot.anchor.year = "ALL_anchor_year", used_growth_rate_tot = "used_growth_ALL", used_growth_rate_u5 = "used_growth_0-5Y", used_growth_rate_u15 = "used_growth_0-15Y" @@ -944,6 +948,8 @@ process_dist_pop_data <- function(pop_data, dplyr::between(u15pop, 100000, 499999) ~ "100,000-499,999", u15pop >= 500000 ~ ">=500,000") ) |> + dplyr::relocate(u15pop, u5pop, totpop, .after = datasource) |> + dplyr::relocate(growth.rate, .before = used.growth.rate) |> dplyr::mutate(pop.cat = factor(pop.cat, levels = c("Missing", "<25,000", "25,000-49,999", "50,000-99,999", From c1486b7c6a3b62f24059e9379c64ca67f3f899c1 Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:52:30 -0400 Subject: [PATCH 07/10] format output for prov pop --- R/process_dist_pop_data.R | 2 +- R/process_prov_pop_data.R | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index acc39a1..70c0dc2 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -911,7 +911,7 @@ process_dist_pop_data <- function(pop_data, adm0guid = "ADM0_GUID", adm1guid = "ADM1_GUID", adm2guid = "ADM2_GUID", - adm2_shape_id = "ADM2_SHAPE_ID", + adm2.shape.id = "ADM2_SHAPE_ID", ctry = "ADM0_NAME", prov = "ADM1_NAME", dist = "ADM2_NAME", diff --git a/R/process_prov_pop_data.R b/R/process_prov_pop_data.R index 7c8ce26..a88df11 100644 --- a/R/process_prov_pop_data.R +++ b/R/process_prov_pop_data.R @@ -249,9 +249,12 @@ process_prov_pop_data <- function(pop_data, u15pop = "0-15Y", u5pop = "0-5Y", totpop = "ALL", - used_growth_rate_tot = "used_growth_ALL", - used_growth_rate_u5 = "used_growth_0-5Y", - used_growth_rate_u15 = "used_growth_0-15Y" + u15.anchor.year = "0-15Y_anchor_year", + u5.anchor.year = "0-5Y_anchor_year", + tot.anchor.year = "ALL_anchor_year", + used.growth.rate.tot = "used_growth_ALL", + used.growth.rate.u5 = "used_growth_0-5Y", + used.growth.rate.u15 = "used_growth_0-15Y" ) |> dplyr::relocate(who.region, ctry, prov, adm0guid, adm1guid, .after = year) |> dplyr::mutate( @@ -276,6 +279,8 @@ process_prov_pop_data <- function(pop_data, dplyr::between(u15pop, 500000, 999999) ~ "500,000-999,999", u15pop >= 1000000 ~ ">=1,000,000") ) |> + dplyr::relocate(u15pop, u5pop, totpop, .after = datasource) |> + dplyr::relocate(growth.rate, .before = used.growth.rate) |> dplyr::mutate(pop.cat = factor(pop.cat, levels = c("Missing", "<100,000", From 2781c63516eaec5d71a4a701f78cbbb6fed07714 Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:57:43 -0400 Subject: [PATCH 08/10] add check to prov pop --- R/process_dist_pop_data.R | 10 +++---- R/process_prov_pop_data.R | 55 ++++++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index 70c0dc2..7f8d9af 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -659,7 +659,7 @@ process_dist_pop_data <- function(pop_data, cli::cli_alert_warning("Country name mismatches between pop and shapefile. See 'errors' folder.") sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", paste0(Sys.Date(), - "_ctry_name_mismatches.csv")), + "_ctry_name_mismatches_in_dist_pop.csv")), obj = ctry_name_mismatch, edav = edav) } else { @@ -670,7 +670,7 @@ process_dist_pop_data <- function(pop_data, cli::cli_alert_warning("Province name mismatches between pop and shapefile. See 'errors' folder.") sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", paste0(Sys.Date(), - "_prov_name_mismatches.csv")), + "_prov_name_mismatches_in_dist_pop.csv")), obj = prov_name_mismatch, edav = edav) } else { @@ -681,7 +681,7 @@ process_dist_pop_data <- function(pop_data, cli::cli_alert_warning("District name mismatches between pop and shapefile. See 'errors' folder.") sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", paste0(Sys.Date(), - "_dist_name_mismatches.csv")), + "_dist_name_mismatches_in_dist_pop.csv")), obj = dist_name_mismatch, edav = edav) } else { @@ -692,7 +692,7 @@ process_dist_pop_data <- function(pop_data, cli::cli_alert_warning("adm0guid mismatches between pop and shapefile. See 'errors' folder.") sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", paste0(Sys.Date(), - "_adm0guid_mismatches.csv")), + "_adm0guid_mismatches_in_dist_pop.csv")), obj = adm0guid_mismatch, edav = edav) } else { @@ -703,7 +703,7 @@ process_dist_pop_data <- function(pop_data, cli::cli_alert_warning("adm1guid mismatches between pop and shapefile. See 'errors' folder.") sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", paste0(Sys.Date(), - "_adm1guid_mismatches.csv")), + "_adm1guid_mismatches_in_dist_pop.csv")), obj = adm1guid_mismatch, edav = edav) } else { diff --git a/R/process_prov_pop_data.R b/R/process_prov_pop_data.R index a88df11..21a2214 100644 --- a/R/process_prov_pop_data.R +++ b/R/process_prov_pop_data.R @@ -129,8 +129,6 @@ process_prov_pop_data <- function(pop_data, active.year.01 = year ) - # Only patch for province is the NPAFP rate indicator and potentially indonesia patch - # will check if we can get data from POLIS only first # Prefer adm0 guid and names from the shapefile polis_pop <- dplyr::left_join(province_long_subset |> @@ -138,10 +136,55 @@ process_prov_pop_data <- function(pop_data, sf_adm1_name = ADM1_NAME, sf_adm0guid = ADM0_GUID), polis_pop) |> - dplyr::distinct() |> - dplyr::mutate(ADM0_NAME = dplyr::coalesce(sf_adm0_name, ADM0_NAME), - ADM1_NAME = dplyr::coalesce(sf_adm1_name, ADM1_NAME), - ADM0_GUID = dplyr::coalesce(sf_adm0guid, ADM0_GUID)) |> + dplyr::distinct() + + # Check when the pop names are not matching with the shapefile + ctry_name_mismatch <- polis_pop |> + dplyr::filter(sf_adm0_name != ADM0_NAME) + prov_name_mismatch <- polis_pop |> + dplyr::filter(sf_adm1_name != ADM1_NAME) + + # Check when adm0guid are not matching the shapefile + adm0guid_mismatch <- polis_pop |> + dplyr::filter(sf_adm0guid != ADM0_GUID) + + if (nrow(ctry_name_mismatch) > 0) { + cli::cli_alert_warning("Country name mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_ctry_name_mismatches_in_prov_pop.csv")), + obj = ctry_name_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in country names between pop and shapefile.") + } + + if (nrow(prov_name_mismatch) > 0) { + cli::cli_alert_warning("Province name mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_prov_name_mismatches_in_prov_pop.csv")), + obj = prov_name_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in province names between pop and shapefile.") + } + + if (nrow(adm0guid_mismatch) > 0) { + cli::cli_alert_warning("adm0guid mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_adm0guid_mismatches_in_prov_pop.csv")), + obj = adm0guid_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in adm0guid between pop and shapefile.") + } + + polis_pop <- polis_pop |> + dplyr::mutate(ADM0_NAME = sf_adm0_name, + ADM1_NAME = sf_adm1_name, + ADM0_GUID = sf_adm0guid) |> dplyr::select(-dplyr::starts_with("sf_"), -FK_DataSetId) base_data <- polis_pop |> From 5efc7af4807ccf2759ab8ec5ae2cfb90de6c5a4d Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 12:03:50 -0400 Subject: [PATCH 09/10] add check to ctry pop --- R/process_ctry_pop_data.R | 23 +++++++++++++++++++++-- R/process_prov_pop_data.R | 6 +++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/R/process_ctry_pop_data.R b/R/process_ctry_pop_data.R index fe33568..4207352 100644 --- a/R/process_ctry_pop_data.R +++ b/R/process_ctry_pop_data.R @@ -92,8 +92,25 @@ process_ctry_pop_data <- function(pop_data, polis_pop <- dplyr::left_join(country_long_subset |> dplyr::rename(sf_adm0_name = ADM0_NAME), polis_pop) |> - dplyr::distinct() |> - dplyr::mutate(ADM0_NAME = dplyr::coalesce(sf_adm0_name, ADM0_NAME)) |> + dplyr::distinct() + + # Check when the pop names are not matching with the shapefile + ctry_name_mismatch <- polis_pop |> + dplyr::filter(sf_adm0_name != ADM0_NAME) + + if (nrow(ctry_name_mismatch) > 0) { + cli::cli_alert_warning("Country name mismatches between pop and shapefile. See 'errors' folder.") + sirfunctions::sirfunctions_io("write", NULL, file.path(pop_dir, "errors", + paste0(Sys.Date(), + "_ctry_name_mismatches_in_ctry_pop.csv")), + obj = ctry_name_mismatch, + edav = edav) + } else { + cli::cli_alert_success("No mismatches in country names between pop and shapefile.") + } + + polis_pop <- polis_pop |> + dplyr::mutate(ADM0_NAME = sf_adm0_name) |> dplyr::select(-dplyr::starts_with("sf_"), -FK_DataSetId) base_data <- polis_pop |> @@ -193,6 +210,8 @@ process_ctry_pop_data <- function(pop_data, miss.u15 = dplyr::if_else(is.na(u15pop), TRUE, FALSE), miss.totpop = dplyr::if_else(is.na(totpop), TRUE, FALSE) ) |> + dplyr::relocate(u15pop, u5pop, totpop, .after = datasource) |> + dplyr::relocate(growth.rate, .before = used.growth.rate) |> dplyr::select(-dplyr::contains("used_growth_rate_")) sirfunctions::sirfunctions_io("write", NULL, diff --git a/R/process_prov_pop_data.R b/R/process_prov_pop_data.R index 21a2214..c801701 100644 --- a/R/process_prov_pop_data.R +++ b/R/process_prov_pop_data.R @@ -295,9 +295,9 @@ process_prov_pop_data <- function(pop_data, u15.anchor.year = "0-15Y_anchor_year", u5.anchor.year = "0-5Y_anchor_year", tot.anchor.year = "ALL_anchor_year", - used.growth.rate.tot = "used_growth_ALL", - used.growth.rate.u5 = "used_growth_0-5Y", - used.growth.rate.u15 = "used_growth_0-15Y" + used_growth_rate_tot = "used_growth_ALL", + used_growth_rate_u5 = "used_growth_0-5Y", + used_growth_rate_u15 = "used_growth_0-15Y" ) |> dplyr::relocate(who.region, ctry, prov, adm0guid, adm1guid, .after = year) |> dplyr::mutate( From 8d99eb23990a6b4f99cbf66c01473c0845952236 Mon Sep 17 00:00:00 2001 From: Mervin Keith Cuadera <40894971+mcuadera@users.noreply.github.com> Date: Fri, 13 Mar 2026 12:57:20 -0400 Subject: [PATCH 10/10] update main function documentation --- R/map_pop_outlier.R | 2 ++ R/process_ctry_pop_data.R | 13 +++++++++---- R/process_dist_pop_data.R | 10 ++++++---- R/process_prov_pop_data.R | 12 +++++++----- man/process_ctry_pop_data.Rd | 14 ++++++++++---- man/process_dist_pop_data.Rd | 9 +++++---- man/process_prov_pop_data.Rd | 11 ++++++----- 7 files changed, 45 insertions(+), 26 deletions(-) diff --git a/R/map_pop_outlier.R b/R/map_pop_outlier.R index 69fd6d1..d5c26aa 100644 --- a/R/map_pop_outlier.R +++ b/R/map_pop_outlier.R @@ -56,11 +56,13 @@ map_pop_outlier <- function(pop_w_outlier_cat, dplyr::filter(dplyr::between(year, year_start, year_end)) if (!is.null(ctry_name)) { + ctry_name <- stringr::str_trim(stringr::str_to_upper(ctry_name)) sf_w_pop <- sf_w_pop |> dplyr::filter(ctry == ctry_name) } if (!is.null(who_region)) { + who_region <- stringr::str_trim(stringr::str_to_upper(who_region)) sf_w_pop <- sf_w_pop |> dplyr::filter(who.region == who_region) } diff --git a/R/process_ctry_pop_data.R b/R/process_ctry_pop_data.R index 4207352..5973a6f 100644 --- a/R/process_ctry_pop_data.R +++ b/R/process_ctry_pop_data.R @@ -1,16 +1,21 @@ # Public function ---- -#' Build country population (Admin0) in wide format +#' Build country population #' -#' Combines POLIS + patches + Jamal; joins to country-year shapes; deduplicates; -#' fills datasource across gaps within GUID; fills missing values using growth rates. -#' Aggregates remaining country-level gaps by summing all province totals per country. +#' Cleans the country level population data from the POLIS API and fill in +#' gaps in population counts using the application of growth rates. #' #' @param pop_data `tibble` Country population dataset pulled from the POLIS API. #' @inheritParams process_dist_pop_data #' @returns `tibble` Cleaned country population. #' #' @export +#' @examples +#' \dontrun{ +#' pop_data <- load_polis_pop("ctry") +#' ctry_pop <- process_ctry_pop_data(pop_data) +#' } +#' process_ctry_pop_data <- function(pop_data, pop_dir = "GID/PEB/SIR/Data/pop", ctry_file_path = "GID/PEB/SIR/Data/spatial/global.ctry.rds", diff --git a/R/process_dist_pop_data.R b/R/process_dist_pop_data.R index 7f8d9af..9460689 100644 --- a/R/process_dist_pop_data.R +++ b/R/process_dist_pop_data.R @@ -549,10 +549,11 @@ patch_polis_with_non_polis_pop <- function(polis_pop, non_polis_pop, patch_file) # Public function ---- -#' Build district population (Admin2) in wide format +#' Build district population #' -#' Combines POLIS + patches + Jamal; joins to district-year shapes; deduplicates; -#' fills datasource across gaps within GUID; fills missing values using growth rates. +#' @description +#' Cleans the district level population data from the POLIS API and fill in +#' gaps in population counts using several patch files and application of growth rates. #' #' @param pop_data `tibble` District population dataset pulled from the POLIS API. #' @param pop_dir `str` Default directory to the population folder. @@ -568,7 +569,8 @@ patch_polis_with_non_polis_pop <- function(polis_pop, non_polis_pop, patch_file) #' #' @examples #' \dontrun{ -#' dist_pop_data(pop_data, output_file = "Data/pop/dist_pop_admin2.rds") +#' pop_data <- load_polis_pop("dist") +#' dist_pop <- process_dist_pop_data(pop_data) #' } process_dist_pop_data <- function(pop_data, pop_dir = "GID/PEB/SIR/Data/pop", diff --git a/R/process_prov_pop_data.R b/R/process_prov_pop_data.R index c801701..8571057 100644 --- a/R/process_prov_pop_data.R +++ b/R/process_prov_pop_data.R @@ -53,11 +53,12 @@ load_indonesia_patch <- function(indonesia_file_path = "GID/PEB/SIR/Data/pop/pop # Public function ---- -#' Build province population (Admin1) in wide format +#' Build province population #' -#' Combines POLIS + patches + Jamal; joins to province-year shapes; deduplicates; -#' fills datasource across gaps within GUID; fills missing values using growth rates. -#' Aggregates remaining province-level gaps by summing all district totals per province. +#' @description +#' Cleans the province level population data from the POLIS API and fill in +#' gaps in population counts using district roll-ups where applicable and +#' application of growth rates. #' #' @param pop_data `tibble` Province population dataset pulled from the POLIS API. #' @param pop_dir `str` Default directory to the population folder. @@ -72,7 +73,8 @@ load_indonesia_patch <- function(indonesia_file_path = "GID/PEB/SIR/Data/pop/pop #' @export #' @examples #' \dontrun{ -#' prov_pop_data(pop_data, output_file = "Data/pop/prov_pop_admin1.rds") +#' pop_data <- load_polis_pop("prov") +#' prov_pop <- process_prov_pop_data(pop_data) #' } process_prov_pop_data <- function(pop_data, pop_dir = "GID/PEB/SIR/Data/pop", diff --git a/man/process_ctry_pop_data.Rd b/man/process_ctry_pop_data.Rd index f02fee7..a24dca0 100644 --- a/man/process_ctry_pop_data.Rd +++ b/man/process_ctry_pop_data.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/process_ctry_pop_data.R \name{process_ctry_pop_data} \alias{process_ctry_pop_data} -\title{Build country population (Admin0) in wide format} +\title{Build country population} \usage{ process_ctry_pop_data( pop_data, @@ -33,7 +33,13 @@ is \code{rds}.} \code{tibble} Cleaned country population. } \description{ -Combines POLIS + patches + Jamal; joins to country-year shapes; deduplicates; -fills datasource across gaps within GUID; fills missing values using growth rates. -Aggregates remaining country-level gaps by summing all province totals per country. +Cleans the country level population data from the POLIS API and fill in +gaps in population counts using the application of growth rates. +} +\examples{ +\dontrun{ +pop_data <- load_polis_pop("ctry") +ctry_pop <- process_ctry_pop_data(pop_data) +} + } diff --git a/man/process_dist_pop_data.Rd b/man/process_dist_pop_data.Rd index 3887d13..83073fe 100644 --- a/man/process_dist_pop_data.Rd +++ b/man/process_dist_pop_data.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/process_dist_pop_data.R \name{process_dist_pop_data} \alias{process_dist_pop_data} -\title{Build district population (Admin2) in wide format} +\title{Build district population} \usage{ process_dist_pop_data( pop_data, @@ -57,11 +57,12 @@ is \code{rds}.} \code{tibble} Cleaned district population dataset. } \description{ -Combines POLIS + patches + Jamal; joins to district-year shapes; deduplicates; -fills datasource across gaps within GUID; fills missing values using growth rates. +Cleans the district level population data from the POLIS API and fill in +gaps in population counts using several patch files and application of growth rates. } \examples{ \dontrun{ -dist_pop_data(pop_data, output_file = "Data/pop/dist_pop_admin2.rds") +pop_data <- load_polis_pop("dist") +dist_pop <- process_dist_pop_data(pop_data) } } diff --git a/man/process_prov_pop_data.Rd b/man/process_prov_pop_data.Rd index e01b790..61405c2 100644 --- a/man/process_prov_pop_data.Rd +++ b/man/process_prov_pop_data.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/process_prov_pop_data.R \name{process_prov_pop_data} \alias{process_prov_pop_data} -\title{Build province population (Admin1) in wide format} +\title{Build province population} \usage{ process_prov_pop_data( pop_data, @@ -37,12 +37,13 @@ process_prov_pop_data( \code{tibble} Tibble with GUID + province-year rows and final output naming. } \description{ -Combines POLIS + patches + Jamal; joins to province-year shapes; deduplicates; -fills datasource across gaps within GUID; fills missing values using growth rates. -Aggregates remaining province-level gaps by summing all district totals per province. +Cleans the province level population data from the POLIS API and fill in +gaps in population counts using district roll-ups where applicable and +application of growth rates. } \examples{ \dontrun{ -prov_pop_data(pop_data, output_file = "Data/pop/prov_pop_admin1.rds") +pop_data <- load_polis_pop("prov") +prov_pop <- process_prov_pop_data(pop_data) } }