diff --git a/data/bundle_antimicrobial_resistance/build.R b/data/bundle_antimicrobial_resistance/build.R index 568dd3444..6da198a2e 100644 --- a/data/bundle_antimicrobial_resistance/build.R +++ b/data/bundle_antimicrobial_resistance/build.R @@ -44,7 +44,8 @@ human_long <- human_agent %>% select( geography, time, source, genus, species_serotype, antimicrobial_class, antimicrobial, - test_method, pct_resistant, n_resistant, n_tested + test_method, pct_resistant, n_resistant, n_tested, + flag_not_tested, flag_no_isolates_tested ) # ----------------------------------------------------------------------------- @@ -106,6 +107,10 @@ resistance_agent <- bind_rows( animal_long, food_long ) %>% + mutate( + flag_not_tested = replace(flag_not_tested, is.na(flag_not_tested), 0L), + flag_no_isolates_tested = replace(flag_no_isolates_tested, is.na(flag_no_isolates_tested), 0L) + ) %>% # Convert geography FIPS to state names for display left_join(state_fips_lookup, by = c("geography" = "fips")) %>% mutate( @@ -142,7 +147,8 @@ resistance_pattern <- human_pattern %>% select(-geography_name) %>% select( geography, time, source, genus, species_serotype, - pattern, test_method, pct_resistant, n_resistant, n_tested + pattern, test_method, pct_resistant, n_resistant, n_tested, + flag_not_tested, flag_no_isolates_tested ) %>% arrange(geography, time, genus, pattern) diff --git a/data/bundle_antimicrobial_resistance/dist/resistance_by_agent.parquet b/data/bundle_antimicrobial_resistance/dist/resistance_by_agent.parquet index a3c998ccd..176d2ebe4 100644 Binary files a/data/bundle_antimicrobial_resistance/dist/resistance_by_agent.parquet and b/data/bundle_antimicrobial_resistance/dist/resistance_by_agent.parquet differ diff --git a/data/bundle_antimicrobial_resistance/dist/resistance_by_pattern.parquet b/data/bundle_antimicrobial_resistance/dist/resistance_by_pattern.parquet index 4fde50228..f549f8e38 100644 Binary files a/data/bundle_antimicrobial_resistance/dist/resistance_by_pattern.parquet and b/data/bundle_antimicrobial_resistance/dist/resistance_by_pattern.parquet differ diff --git a/data/narms/ingest.R b/data/narms/ingest.R index 084ff9a82..581692a65 100644 --- a/data/narms/ingest.R +++ b/data/narms/ingest.R @@ -8,6 +8,7 @@ library(dplyr) library(vroom) library(httr2) library(jsonlite) +library(tidyr) # Initialize process record (process.json is created by dcf::dcf_add_source()) process <- dcf::dcf_process_record() @@ -84,6 +85,88 @@ sites <- c( # Helper Functions # ============================================================================= +#' Convert a label into a column-name-safe slug +#' e.g. "Salmonella I 4,[5],12:i:-" -> "salmonella_i_4_5_12_i" +clean_name <- function(x) { + gsub("_$", "", tolower(gsub("[^A-Za-z0-9]+", "_", x))) +} + +#' Join organism name parts, dropping any that are missing +#' paste() would render a missing serotype as the literal string "NA" +#' ("Enterococcus faecalis NA"), so drop absent parts instead of pasting them. +combine_organism <- function(...) { + parts <- lapply(list(...), function(p) { + p <- as.character(p) + p[is.na(p) | p == "NA" | !nzchar(trimws(p))] <- "" + p + }) + out <- do.call(paste, c(parts, sep = " ")) + trimws(gsub(" +", " ", out)) +} + +#' Make every organism x measure combination explicit, and flag why it is empty +#' +#' Pivoting wide forces a cell to exist for every row and measure, so pairs that +#' had no long-format row surface as NA. Two different things hide in those NAs: +#' not_on_panel - the organism/measure pair appears nowhere in the file, so it +#' is not a gap but a combination that does not exist. Permanent. +#' not_tested - the pair appears elsewhere but not for this row. May fill in. +#' Where MIC columns are supplied, a row that has a real denominator but no MIC +#' is flagged tested_no_mic. FDA publishes a susceptibility interpretation for +#' some drugs (notably streptomycin) without publishing the concentration, so +#' the counts are usable even though the MIC was never measured. +#' All value columns are zero-filled; the flag carries the reason. +#' +#' @param data long data, one row per row_key x measure +#' @param row_keys columns identifying a row (must include organism_col) +#' @param measure_col column holding measure names (antimicrobial or pattern) +#' @param organism_col column defining panel membership +#' @param value_cols columns to zero-fill +#' @param mic_cols subset of value_cols that are MIC concentrations +#' @param flag_col existing flag column for rows present in the source +add_panel_flags <- function(data, row_keys, measure_col, organism_col, + value_cols, mic_cols = character(), + flag_col = NULL) { + panel <- data %>% + distinct(across(all_of(c(organism_col, measure_col)))) %>% + mutate(.on_panel = TRUE) + + grid <- tidyr::crossing( + distinct(data[row_keys]), + tibble::tibble(!!measure_col := unique(data[[measure_col]])) + ) + + # A row is "present" in the source if it carried a flag (human files) or a + # value (FDA files, which have no flag until now). + present_col <- if (is.null(flag_col)) value_cols[1] else flag_col + + out <- grid %>% + left_join(data, by = c(row_keys, measure_col)) %>% + left_join(panel, by = c(organism_col, measure_col)) + + base <- if (is.null(flag_col)) { + ifelse(!is.na(out[[present_col]]), "tested", NA_character_) + } else { + as.character(out[[flag_col]]) + } + + out$narms_flag <- dplyr::case_when( + !is.na(base) ~ base, + is.na(out$.on_panel) ~ "not_on_panel", + TRUE ~ "not_tested" + ) + + # A real denominator with no concentration recorded is still a real result + if (length(mic_cols) > 0) { + no_mic <- out$narms_flag == "tested" & is.na(out[[mic_cols[1]]]) + out$narms_flag[no_mic] <- "tested_no_mic" + } + + out %>% + mutate(across(all_of(value_cols), ~ replace(.x, is.na(.x), 0))) %>% + select(-.on_panel, -any_of(setdiff(flag_col, "narms_flag"))) +} + #' Build a site filter Where clause for Power BI queries #' Returns NULL if site_name is NULL (no filter = national "All") build_site_filter <- function(site_name) { @@ -702,6 +785,185 @@ parse_pattern_response <- function(response, genus, species, test_method) { do.call(rbind, rows) } +#' Build a corrected DSC query that groups by state using the NARMS fact table. +#' The standard agent query uses NARMSSiteName for state filtering, which causes +#' the ResistByAgentCell numerator to leak national counts for Shigella DSC. +#' This query uses the NARMS entity's SiteName for grouping, which works correctly. +build_dsc_state_query <- function(species, test_method, + year_from = YEAR_FROM_AST, year_to = YEAR_TO) { + list( + version = "1.0.0", + queries = list(list( + Query = list(Commands = list(list( + SemanticQueryDataShapeCommand = list( + Query = list( + Version = 2L, + From = list( + list(Name = "n", Entity = "NARMS", Type = 0L), + list(Name = "n1", Entity = "NARMSTest", Type = 0L), + list(Name = "n11", Entity = "NARMSLookupGenus", Type = 0L), + list(Name = "n111", Entity = "NARMSLookupSpecies", Type = 0L), + list(Name = "n2", Entity = "NARMSYear", Type = 0L), + list(Name = "n3", Entity = "NARMSAgent", Type = 0L), + list(Name = "r", Entity = "NARMSResultAST", Type = 0L) + ), + Select = list( + list( + Column = list(Expression = list(SourceRef = list(Source = "n")), + Property = "SiteName"), + Name = "NARMS.SiteName", NativeReferenceName = "SiteName" + ), + list( + Column = list(Expression = list(SourceRef = list(Source = "n2")), + Property = "DataYear"), + Name = "NARMSYear.DataYear", NativeReferenceName = "Year" + ), + list( + Measure = list(Expression = list(SourceRef = list(Source = "r")), + Property = "ResistByAgentCell"), + Name = "NARMSResultAST.ResistByAgentCell", + NativeReferenceName = "ResistByAgentCell" + ) + ), + Where = list( + list(Condition = list(In = list( + Expressions = list(list(Column = list( + Expression = list(SourceRef = list(Source = "n1")), + Property = "TestMethod"))), + Values = list(list(list(Literal = list( + Value = paste0("'", test_method, "'"))))) + ))), + list(Condition = list(In = list( + Expressions = list( + list(Column = list(Expression = list(SourceRef = list(Source = "n11")), + Property = "Genus")), + list(Column = list(Expression = list(SourceRef = list(Source = "n111")), + Property = "SpeciesSerotype")) + ), + Values = list(list( + list(Literal = list(Value = "'Shigella'")), + list(Literal = list(Value = paste0("'", species, "'"))) + )) + ))), + list(Condition = list(And = list( + Left = list(Comparison = list(ComparisonKind = 2L, + Left = list(Column = list(Expression = list(SourceRef = list(Source = "n2")), + Property = "DataYear")), + Right = list(Literal = list(Value = paste0(year_from, "D"))))), + Right = list(Comparison = list(ComparisonKind = 4L, + Left = list(Column = list(Expression = list(SourceRef = list(Source = "n2")), + Property = "DataYear")), + Right = list(Literal = list(Value = paste0(year_to, "D"))))) + ))), + list(Condition = list(In = list( + Expressions = list( + list(Column = list(Expression = list(SourceRef = list(Source = "n3")), + Property = "SearchType")), + list(Column = list(Expression = list(SourceRef = list(Source = "n3")), + Property = "Display")), + list(Column = list(Expression = list(SourceRef = list(Source = "n3")), + Property = "Antimicrobial Agent")) + ), + Values = list(list( + list(Literal = list(Value = "'By Agent'")), + list(Literal = list(Value = "'Select Agent'")), + list(Literal = list(Value = "'Ciprofloxacin (DSC)'")) + )) + ))) + ) + ), + Binding = list( + Primary = list(Groupings = list(list(Projections = list(0L)))), + Secondary = list(Groupings = list(list(Projections = list(1L, 2L)))), + DataReduction = list(DataVolume = 4L, + Primary = list(Window = list(Count = 200L)), + Secondary = list(Top = list(Count = 100L))), + Version = 1L + ), + ExecutionMetricsKind = 1L + ) + ))), + QueryId = "", + ApplicationContext = list( + DatasetId = POWERBI_DATASET_ID, + Sources = list(list(ReportId = POWERBI_REPORT_ID, + VisualId = AGENT_VISUAL_ID)) + ) + )), + cancelQueries = list(), + modelId = POWERBI_MODEL_ID + ) +} + +#' Parse DSC state query response (state × year matrix) into a data frame +parse_dsc_state_response <- function(response, species, test_method) { + dsr <- response$results[[1]]$result$data$dsr + + if (is.null(dsr$DS)) { + warning(sprintf("No data returned for Shigella / %s / %s (DSC state)", species, test_method)) + return(NULL) + } + + ds <- dsr$DS[[1]] + cell_values <- ds$ValueDicts$D0 + + # Years from secondary header + sh_key <- names(ds$SH[[1]])[grep("^DM", names(ds$SH[[1]]))] + years <- sapply(ds$SH[[1]][[sh_key]], function(x) x[[grep("^G", names(x))[1]]]) + + # Parse primary groups (states) + ph_key <- names(ds$PH[[1]])[grep("^DM", names(ds$PH[[1]]))] + groups <- ds$PH[[1]][[ph_key]] + + rows <- list() + for (g in groups) { + if (!is.null(g[["Ø"]])) next + + state_name <- g$G0 + if (is.null(state_name) || !is.character(state_name)) next + if (is.null(g$X)) next + + prev_value <- NULL + for (i in seq_along(g$X)) { + cell <- g$X[[i]] + + if (!is.null(cell$R)) { + cell_text <- prev_value + } else if (!is.null(cell$M0)) { + if (is.character(cell$M0)) { + cell_text <- cell$M0 + } else { + cell_text <- cell_values[[cell$M0 + 1]] + } + prev_value <- cell_text + } else { + cell_text <- NULL + prev_value <- NULL + } + + if (is.null(cell_text)) next + parsed <- parse_cell_value(cell_text) + + rows[[length(rows) + 1]] <- data.frame( + year = years[i], + genus = "Shigella", + species_serotype = species, + antimicrobial_class = "Quinolones", + antimicrobial_agent = "Ciprofloxacin (DSC)", + test_method = test_method, + narms_now_pct_resistant = parsed$pct_resistant, + narms_now_n_resistant = parsed$n_resistant, + narms_now_n_tested = parsed$n_tested, + site_name = state_name, + stringsAsFactors = FALSE + ) + } + } + + if (length(rows) == 0) return(NULL) + do.call(rbind, rows) +} + # ============================================================================= # Main Scraping Loop — writes raw data to raw/narms_now_agent.csv.gz # and raw/narms_now_pattern.csv.gz @@ -737,10 +999,8 @@ if (needs_scrape) { n_sites <- length(sites) total_queries <- length(organisms) * length(test_methods) * 2 * (n_sites + 1) - message(sprintf("Organisms: %d | Test methods: %d | Sites: %d (+ national) | Total queries: ~%d", - length(organisms), length(test_methods), n_sites, total_queries)) - message(sprintf("Estimated time: ~%.0f minutes (%.1fs delay between queries)", - total_queries * QUERY_DELAY / 60, QUERY_DELAY)) + message(sprintf("Organisms: %d | Test methods: %d | Sites: %d (+ national) | Total queries: ~%d (%.1fs delay between queries)", + length(organisms), length(test_methods), n_sites, total_queries, QUERY_DELAY)) all_agent_data <- list() all_pattern_data <- list() @@ -835,10 +1095,67 @@ if (needs_scrape) { new_agent_df <- bind_rows(existing, new_agent_df) } + # National rows carry site_name = NA, so repeated scrapes have historically + # accumulated exact copies of them. The copies never disagree, so dropping + # them here keeps the raw file from growing and makes it self-healing. + new_agent_df <- distinct(new_agent_df) + vroom::vroom_write(new_agent_df, "raw/narms_now_agent.csv.gz", delim = ",") message(sprintf("Wrote %d rows to raw/narms_now_agent.csv.gz", nrow(new_agent_df))) } + # --- Fix Shigella DSC state-level data --- + # The main scrape returns wrong state-level values for Ciprofloxacin (DSC) + # because the ResistByAgentCell measure leaks national counts when filtered + # by NARMSSiteName. Re-scrape using the NARMS fact table for state grouping, + # which returns correct values (6 queries total). + if (file.exists("raw/narms_now_agent.csv.gz")) { + message("=== Correcting Shigella DSC state-level data (6 queries) ===") + shigella_species <- c("flexneri", "other", "sonnei") + dsc_corrected <- list() + + for (sp in shigella_species) { + for (tm in test_methods) { + year_from <- if (tm == "WGS") YEAR_FROM_WGS else YEAR_FROM_AST + message(sprintf(" DSC fix: Shigella %s / %s", sp, tm)) + + tryCatch({ + query <- build_dsc_state_query(sp, tm, year_from = year_from, year_to = YEAR_TO) + response <- execute_powerbi_query(query) + parsed <- parse_dsc_state_response(response, sp, tm) + + if (!is.null(parsed) && nrow(parsed) > 0) { + dsc_corrected[[length(dsc_corrected) + 1]] <- parsed + message(sprintf(" -> %d rows", nrow(parsed))) + } + }, error = function(e) { + warning(sprintf(" -> DSC fix ERROR: %s", conditionMessage(e))) + }) + Sys.sleep(QUERY_DELAY) + } + } + + if (length(dsc_corrected) > 0) { + dsc_df <- do.call(rbind, dsc_corrected) %>% + rename(pct_resistant = narms_now_pct_resistant, + n_resistant = narms_now_n_resistant, + n_tested = narms_now_n_tested) + + agent_raw <- vroom::vroom("raw/narms_now_agent.csv.gz", show_col_types = FALSE) + + # Remove bad Shigella DSC state rows and replace with corrected data + agent_fixed <- agent_raw %>% + filter(!(genus == "Shigella" & + antimicrobial_agent == "Ciprofloxacin (DSC)" & + !is.na(site_name))) %>% + bind_rows(dsc_df) + + vroom::vroom_write(agent_fixed, "raw/narms_now_agent.csv.gz", delim = ",") + message(sprintf("DSC fix: replaced Shigella DSC state rows. %d -> %d total rows", + nrow(agent_raw), nrow(agent_fixed))) + } + } + if (length(all_pattern_data) > 0) { new_pattern_df <- do.call(rbind, all_pattern_data) %>% rename(pct_resistant = narms_now_pct_resistant, @@ -852,6 +1169,9 @@ if (needs_scrape) { new_pattern_df <- bind_rows(existing, new_pattern_df) } + # See note in the agent block above. + new_pattern_df <- distinct(new_pattern_df) + vroom::vroom_write(new_pattern_df, "raw/narms_now_pattern.csv.gz", delim = ",") message(sprintf("Wrote %d rows to raw/narms_now_pattern.csv.gz", nrow(new_pattern_df))) } @@ -898,15 +1218,53 @@ if (file.exists("raw/narms_now_agent.csv.gz")) { select(geography, site_name) agent_raw <- vroom::vroom("raw/narms_now_agent.csv.gz", show_col_types = FALSE) - agent_standard <- agent_raw %>% + agent_long <- agent_raw %>% left_join(site_to_fips, by = "site_name") %>% mutate( geography = if_else(is.na(site_name), "00", geography), - time = paste0(year, "-12-31") + time = paste0(year, "-12-31"), + narms_flag = case_when( + is.na(pct_resistant) & is.na(n_resistant) & is.na(n_tested) ~ "not_tested", + !is.na(n_tested) & n_tested == 0 ~ "no_isolates_tested", + TRUE ~ "tested" + ), + pct_resistant = replace(pct_resistant, is.na(pct_resistant), 0), + n_resistant = replace(n_resistant, is.na(n_resistant), 0), + n_tested = replace(n_tested, is.na(n_tested), 0), + genus_species_serotype = combine_organism(genus, species_serotype), + antimicrobial = clean_name(antimicrobial_agent) + ) %>% + select(geography, time, genus_species_serotype, + test_method, antimicrobial, + pct_resistant, n_resistant, n_tested, narms_flag) %>% + distinct() + + # Validate: warn if any pct_resistant > 100 + bad_rows <- agent_long %>% filter(pct_resistant > 100) + if (nrow(bad_rows) > 0) { + warning(sprintf( + "%d agent rows have pct_resistant > 100%%. Top offenders: %s", + nrow(bad_rows), + paste(unique(bad_rows$antimicrobial)[1:min(5, length(unique(bad_rows$antimicrobial)))], + collapse = ", ") + )) + } + + agent_standard <- agent_long %>% + add_panel_flags( + row_keys = c("geography", "time", "genus_species_serotype", "test_method"), + measure_col = "antimicrobial", + organism_col = "genus_species_serotype", + value_cols = c("pct_resistant", "n_resistant", "n_tested"), + flag_col = "narms_flag" ) %>% - select(geography, time, genus, species_serotype, - antimicrobial_class, antimicrobial_agent, test_method, - pct_resistant, n_resistant, n_tested) + pivot_wider( + id_cols = c(geography, time, genus_species_serotype, test_method), + names_from = antimicrobial, + values_from = c(pct_resistant, n_resistant, n_tested, narms_flag), + names_glue = "narms_{.value}_{antimicrobial}" + ) %>% + rename_with(~ gsub("narms_narms_flag", "narms_flag", .x)) vroom::vroom_write(agent_standard, "standard/data_resistance_agent.csv.gz", delim = ",") message(sprintf("Wrote %d rows to standard/data_resistance_agent.csv.gz", nrow(agent_standard))) @@ -927,28 +1285,82 @@ if (file.exists("raw/narms_now_pattern.csv.gz")) { } pattern_raw <- vroom::vroom("raw/narms_now_pattern.csv.gz", show_col_types = FALSE) - pattern_standard <- pattern_raw %>% + pattern_long <- pattern_raw %>% left_join(site_to_fips, by = "site_name") %>% mutate( geography = if_else(is.na(site_name), "00", geography), - time = paste0(year, "-12-31") + time = paste0(year, "-12-31"), + narms_flag = case_when( + is.na(pct_resistant) & is.na(n_resistant) & is.na(n_tested) ~ "not_tested", + !is.na(n_tested) & n_tested == 0 ~ "no_isolates_tested", + TRUE ~ "tested" + ), + pct_resistant = replace(pct_resistant, is.na(pct_resistant), 0), + n_resistant = replace(n_resistant, is.na(n_resistant), 0), + n_tested = replace(n_tested, is.na(n_tested), 0), + genus_species_serotype = combine_organism(genus, species_serotype), + pattern_name = clean_name(pattern) ) %>% - select(geography, time, genus, species_serotype, - pattern, test_method, - pct_resistant, n_resistant, n_tested) + select(geography, time, genus_species_serotype, + test_method, pattern_name, + pct_resistant, n_resistant, n_tested, narms_flag) %>% + distinct() + + bad_rows <- pattern_long %>% filter(pct_resistant > 100) + if (nrow(bad_rows) > 0) { + warning(sprintf( + "%d pattern rows have pct_resistant > 100%%. Top offenders: %s", + nrow(bad_rows), + paste(unique(bad_rows$pattern_name)[1:min(5, length(unique(bad_rows$pattern_name)))], + collapse = ", ") + )) + } + + pattern_standard <- pattern_long %>% + add_panel_flags( + row_keys = c("geography", "time", "genus_species_serotype", "test_method"), + measure_col = "pattern_name", + organism_col = "genus_species_serotype", + value_cols = c("pct_resistant", "n_resistant", "n_tested"), + flag_col = "narms_flag" + ) %>% + pivot_wider( + id_cols = c(geography, time, genus_species_serotype, test_method), + names_from = pattern_name, + values_from = c(pct_resistant, n_resistant, n_tested, narms_flag), + names_glue = "narms_{.value}_{pattern_name}" + ) %>% + rename_with(~ gsub("narms_narms_flag", "narms_flag", .x)) vroom::vroom_write(pattern_standard, "standard/data_resistance_pattern.csv.gz", delim = ",") message(sprintf("Wrote %d rows to standard/data_resistance_pattern.csv.gz", nrow(pattern_standard))) } +#' Resolve a worksheet name by regex +#' FDA renames sheets as they extend the year range (e.g. "2017-2021_data" +#' became "2017-2024_data"), so match on a stable pattern instead of the +#' literal name. Falls back to the first sheet, with a warning, if nothing matches. +resolve_sheet <- function(path, pattern) { + sheets <- readxl::excel_sheets(path) + matched <- grep(pattern, sheets, value = TRUE) + if (length(matched) == 0) { + warning(sprintf( + "No sheet matching '%s' in %s (found: %s); falling back to '%s'", + pattern, basename(path), paste(sheets, collapse = ", "), sheets[1] + )) + return(sheets[1]) + } + matched[1] +} + # ============================================================================= # Source 3: NARMS Retail Meats Data (FDA/CVM) # Source: FDA NARMS Integrated Reports/Summaries # URL: https://www.fda.gov/animal-veterinary/national-antimicrobial-resistance-monitoring-system/integrated-reportssummaries -# File: raw/cvm-narms-retail-meats.xlsx +# File: raw/narms-retail-meats.xlsx # ============================================================================= -retail_raw_path <- "raw/cvm-narms-retail-meats.xlsx" +retail_raw_path <- "raw/narms-retail-meats.xlsx" retail_url <- "https://www.fda.gov/files/animal%20%26%20veterinary/published/cvm-narms-retail-meats_0.xlsx" tryCatch( @@ -963,167 +1375,185 @@ tryCatch( ) current_retail_state <- list(hash = as.character(tools::md5sum(retail_raw_path))) -if (!identical(process$retail_meats_state, current_retail_state)) { - message("Processing NARMS retail meats data...") +# Always standardise from the local raw file. The hash is recorded for change +# reporting only -- gating on it would freeze the output at whatever format the +# code had when the raw data last changed. +message("Processing NARMS retail meats data...") + +library(readxl) +library(tidyr) + +# SIR (Susceptible / Intermediate / Resistant) column codes +sir_codes <- c( + "AMC", "AMI", "AMP", "ATM", "AVL", "AXO", "AZI", "BAC", + "CAZ", "CCV", "CEP", "CEQ", "CHL", "CIP", "CIP2", "CLI", + "COL", "COT", "CTC", "CTX", "DAP", "DOX", "ERY", "FEP", + "FFN", "FIS", "FLA", "FOX", "GEN", "IMI", "KAN", "LIN", + "LZD", "MER", "NAL", "NIT", "PEN", "PTZ", "QDA", "SAL", + "SMX", "STR", "SUF", "TEL", "TET", "TGC", "TIO", "TYL", "VAN" +) +sir_col_names <- paste0(sir_codes, " SIR") + +# Full antimicrobial names from the FDA NARMS data dictionary +# (https://www.fda.gov/media/110404/download) +# FLA, SAL, SUF, CIP2 are veterinary-specific and not in the standard +# data dictionary; identified from genus-specificity in the data: +# FLA/SAL = Enterococcus only; SUF/CIP2 = Salmonella/E. coli only +antimicrobial_names <- c( + AMC = "Amoxicillin-clavulanic acid", + AMI = "Amikacin", + AMP = "Ampicillin", + ATM = "Aztreonam", + AVL = "Avilamycin", + AXO = "Ceftriaxone", + AZI = "Azithromycin", + BAC = "Bacitracin", + CAZ = "Ceftazidime", + CCV = "Ceftiofur", + CEP = "Cephalothin", + CEQ = "Cefquinome", + CHL = "Chloramphenicol", + CIP = "Ciprofloxacin", + CIP2 = "Ciprofloxacin (2nd breakpoint)", + CLI = "Clindamycin", + COL = "Colistin", + COT = "Trimethoprim-sulfamethoxazole", + CTC = "Chlortetracycline", + CTX = "Cefotaxime", + DAP = "Daptomycin", + DOX = "Doxycycline", + ERY = "Erythromycin", + FEP = "Cefepime", + FFN = "Florfenicol", + FIS = "Sulfisoxazole", + FLA = "Flaveomycin", + FOX = "Cefoxitin", + GEN = "Gentamicin", + IMI = "Imipenem", + KAN = "Kanamycin", + LIN = "Lincomycin", + LZD = "Linezolid", + MER = "Meropenem", + NAL = "Nalidixic acid", + NIT = "Nitrofurantoin", + PEN = "Penicillin", + PTZ = "Piperacillin-tazobactam", + QDA = "Quinupristin-dalfopristin", + SAL = "Salinomycin", + SMX = "Sulfamethoxazole", + STR = "Streptomycin", + SUF = "Sulfonamides", + TEL = "Telithromycin", + TET = "Tetracycline", + TGC = "Tigecycline", + TIO = "Ceftiofur", + TYL = "Tylosin", + VAN = "Vancomycin" +) - library(readxl) - library(tidyr) +# FIPS lookup: state abbreviation -> 2-digit FIPS +all_fips <- vroom::vroom("../../resources/all_fips.csv.gz", show_col_types = FALSE) +state_fips_lookup <- all_fips %>% + filter(nchar(geography) == 2) %>% + select(geography, state) - # SIR (Susceptible / Intermediate / Resistant) column codes - sir_codes <- c( - "AMC", "AMI", "AMP", "ATM", "AVL", "AXO", "AZI", "BAC", - "CAZ", "CCV", "CEP", "CEQ", "CHL", "CIP", "CIP2", "CLI", - "COL", "COT", "CTC", "CTX", "DAP", "DOX", "ERY", "FEP", - "FFN", "FIS", "FLA", "FOX", "GEN", "IMI", "KAN", "LIN", - "LZD", "MER", "NAL", "NIT", "PEN", "PTZ", "QDA", "SAL", - "SMX", "STR", "SUF", "TEL", "TET", "TGC", "TIO", "TYL", "VAN" - ) - sir_col_names <- paste0(sir_codes, " SIR") - - # Full antimicrobial names from the FDA NARMS data dictionary - # (https://www.fda.gov/media/110404/download) - # FLA, SAL, SUF, CIP2 are veterinary-specific and not in the standard - # data dictionary; identified from genus-specificity in the data: - # FLA/SAL = Enterococcus only; SUF/CIP2 = Salmonella/E. coli only - antimicrobial_names <- c( - AMC = "Amoxicillin-clavulanic acid", - AMI = "Amikacin", - AMP = "Ampicillin", - ATM = "Aztreonam", - AVL = "Avilamycin", - AXO = "Ceftriaxone", - AZI = "Azithromycin", - BAC = "Bacitracin", - CAZ = "Ceftazidime", - CCV = "Ceftiofur", - CEP = "Cephalothin", - CEQ = "Cefquinome", - CHL = "Chloramphenicol", - CIP = "Ciprofloxacin", - CIP2 = "Ciprofloxacin (2nd breakpoint)", - CLI = "Clindamycin", - COL = "Colistin", - COT = "Trimethoprim-sulfamethoxazole", - CTC = "Chlortetracycline", - CTX = "Cefotaxime", - DAP = "Daptomycin", - DOX = "Doxycycline", - ERY = "Erythromycin", - FEP = "Cefepime", - FFN = "Florfenicol", - FIS = "Sulfisoxazole", - FLA = "Flaveomycin", - FOX = "Cefoxitin", - GEN = "Gentamicin", - IMI = "Imipenem", - KAN = "Kanamycin", - LIN = "Lincomycin", - LZD = "Linezolid", - MER = "Meropenem", - NAL = "Nalidixic acid", - NIT = "Nitrofurantoin", - PEN = "Penicillin", - PTZ = "Piperacillin-tazobactam", - QDA = "Quinupristin-dalfopristin", - SAL = "Salinomycin", - SMX = "Sulfamethoxazole", - STR = "Streptomycin", - SUF = "Sulfonamides", - TEL = "Telithromycin", - TET = "Tetracycline", - TGC = "Tigecycline", - TIO = "Ceftiofur", - TYL = "Tylosin", - VAN = "Vancomycin" - ) +retail_raw <- readxl::read_excel( + retail_raw_path, + sheet = resolve_sheet(retail_raw_path, "^Retail") +) - # FIPS lookup: state abbreviation -> 2-digit FIPS - all_fips <- vroom::vroom("../../resources/all_fips.csv.gz", show_col_types = FALSE) - state_fips_lookup <- all_fips %>% - filter(nchar(geography) == 2) %>% - select(geography, state) - - retail_raw <- readxl::read_excel(retail_raw_path, sheet = "Retail_Meats") - - # MIC concentration columns (bare antibiotic codes, no suffix) - mic_col_names <- sir_codes - - # Filter to positive cultures and add a row ID for joining - retail_filtered <- retail_raw %>% - filter(GROWTH == "YES") %>% - mutate(.row_id = row_number()) - - # Pivot SIR values to long format - sir_long <- retail_filtered %>% - select(.row_id, any_of(sir_col_names)) %>% - pivot_longer( - cols = any_of(sir_col_names), - names_to = "antimicrobial", - values_to = "sir" - ) %>% - mutate(antimicrobial = sub(" SIR$", "", antimicrobial)) - - # Pivot MIC values to long format - mic_long <- retail_filtered %>% - select(.row_id, any_of(mic_col_names)) %>% - pivot_longer( - cols = any_of(mic_col_names), - names_to = "antimicrobial", - values_to = "mic" - ) %>% - mutate(mic = as.numeric(mic)) - - # Join SIR + MIC by row and antibiotic, map codes to full names - retail_long <- sir_long %>% - left_join(mic_long, by = c(".row_id", "antimicrobial")) %>% - left_join( - retail_filtered %>% select(.row_id, YEAR, GENUS_NAME, SPECIES, SEROTYPE, SOURCE, STATE), - by = ".row_id" - ) %>% - mutate(antimicrobial = antimicrobial_names[antimicrobial]) %>% - filter(!is.na(sir)) %>% - select(-.row_id) - - # Aggregate by state, converting abbreviation to FIPS - retail_standard <- retail_long %>% - left_join(state_fips_lookup, by = c("STATE" = "state")) %>% - filter(!is.na(geography)) %>% - group_by(YEAR, GENUS_NAME, SPECIES, SEROTYPE, SOURCE, antimicrobial, geography) %>% - summarize( - n_tested = n(), - n_resistant = sum(sir == "R"), - mic50 = median(mic, na.rm = TRUE), - mic90 = quantile(mic, 0.90, na.rm = TRUE), - .groups = "drop" - ) %>% - mutate( - pct_resistant = n_resistant / n_tested * 100, - time = paste0(YEAR, "-12-31") - ) %>% - rename( - genus = GENUS_NAME, - species = SPECIES, - serotype = SEROTYPE, - meat_source = SOURCE - ) %>% - select( - geography, time, genus, species, serotype, meat_source, - antimicrobial, pct_resistant, n_resistant, n_tested, mic50, mic90 - ) +# MIC concentration columns (bare antibiotic codes, no suffix) +mic_col_names <- sir_codes + +# Filter to positive cultures and add a row ID for joining +retail_filtered <- retail_raw %>% + filter(GROWTH == "YES") %>% + mutate(.row_id = row_number()) + +# Pivot SIR values to long format +sir_long <- retail_filtered %>% + select(.row_id, any_of(sir_col_names)) %>% + pivot_longer( + cols = any_of(sir_col_names), + names_to = "antimicrobial", + values_to = "sir" + ) %>% + mutate(antimicrobial = sub(" SIR$", "", antimicrobial)) + +# Pivot MIC values to long format +mic_long <- retail_filtered %>% + select(.row_id, any_of(mic_col_names)) %>% + pivot_longer( + cols = any_of(mic_col_names), + names_to = "antimicrobial", + values_to = "mic" + ) %>% + mutate(mic = as.numeric(mic)) + +# Join SIR + MIC by row and antibiotic, map codes to full names +retail_long <- sir_long %>% + left_join(mic_long, by = c(".row_id", "antimicrobial")) %>% + left_join( + retail_filtered %>% select(.row_id, YEAR, GENUS_NAME, SPECIES, SEROTYPE, SOURCE, STATE), + by = ".row_id" + ) %>% + mutate(antimicrobial = antimicrobial_names[antimicrobial]) %>% + filter(!is.na(sir)) %>% + select(-.row_id) + +# Aggregate by state, converting abbreviation to FIPS +retail_agg <- retail_long %>% + left_join(state_fips_lookup, by = c("STATE" = "state")) %>% + filter(!is.na(geography)) %>% + group_by(YEAR, GENUS_NAME, SPECIES, SEROTYPE, SOURCE, antimicrobial, geography) %>% + summarize( + n_tested = n(), + n_resistant = sum(sir == "R"), + mic50 = median(mic, na.rm = TRUE), + mic90 = quantile(mic, 0.90, na.rm = TRUE), + .groups = "drop" + ) %>% + mutate( + pct_resistant = n_resistant / n_tested * 100, + time = paste0(YEAR, "-12-31"), + genus_species_serotype = combine_organism(GENUS_NAME, SPECIES, SEROTYPE), + antimicrobial = tolower(gsub("[^A-Za-z0-9]+", "_", antimicrobial)), + antimicrobial = gsub("_$", "", antimicrobial) + ) %>% + rename(meat_source = SOURCE) %>% + select( + geography, time, genus_species_serotype, meat_source, + antimicrobial, pct_resistant, n_resistant, n_tested, mic50, mic90 + ) - vroom::vroom_write( - retail_standard, - "standard/data_retail_meats.csv.gz", - delim = "," - ) - message(sprintf( - "Wrote %d rows to standard/data_retail_meats.csv.gz", - nrow(retail_standard) - )) +retail_standard <- retail_agg %>% + add_panel_flags( + row_keys = c("geography", "time", "genus_species_serotype", "meat_source"), + measure_col = "antimicrobial", + organism_col = "genus_species_serotype", + value_cols = c("pct_resistant", "n_resistant", "n_tested", "mic50", "mic90"), + mic_cols = c("mic50", "mic90") + ) %>% + pivot_wider( + id_cols = c(geography, time, genus_species_serotype, meat_source), + names_from = antimicrobial, + values_from = c(pct_resistant, n_resistant, n_tested, mic50, mic90, narms_flag), + names_glue = "narms_{.value}_{antimicrobial}" + ) %>% + rename_with(~ gsub("narms_narms_flag", "narms_flag", .x)) + +vroom::vroom_write( + retail_standard, + "standard/data_retail_meats.csv.gz", + delim = "," +) +message(sprintf( + "Wrote %d rows to standard/data_retail_meats.csv.gz", + nrow(retail_standard) +)) - process$retail_meats_state <- current_retail_state - dcf::dcf_process_record(updated = process) -} +process$retail_meats_state <- current_retail_state +dcf::dcf_process_record(updated = process) # ============================================================================= # Source 4: NARMS Animal Pathogen Data (FDA/CVM - Vet-LIRN/NAHLN) @@ -1146,64 +1576,83 @@ tryCatch( ) current_animal_path_state <- list(hash = as.character(tools::md5sum(animal_path_raw_path))) -if (!identical(process$animal_pathogen_state, current_animal_path_state)) { - message("Processing NARMS animal pathogen data...") +# Always standardise (see note in the retail meats section above). +message("Processing NARMS animal pathogen data...") - if (!requireNamespace("readxl", quietly = TRUE)) library(readxl) +if (!requireNamespace("readxl", quietly = TRUE)) library(readxl) - # FIPS lookup: full state names -> 2-digit FIPS - all_fips <- vroom::vroom("../../resources/all_fips.csv.gz", show_col_types = FALSE) - state_fips_lookup <- all_fips %>% - filter(nchar(geography) == 2) %>% - select(geography, geography_name) - - animal_raw <- readxl::read_excel(animal_path_raw_path, sheet = "2017-2021_data") - - # Data is already in long format with one row per isolate/drug - # Filter to interpretable results, exclude non-US (Canada) - animal_standard <- animal_raw %>% - filter(Interpretation != "Non-Interpretable") %>% - left_join(state_fips_lookup, by = c("State" = "geography_name")) %>% - filter(!is.na(geography)) %>% - mutate(MIC = as.numeric(MIC)) %>% - group_by( - geography, Year, Genus, `Host Species`, `Collection Source`, `Drug Name` - ) %>% - summarize( - n_tested = n(), - n_resistant = sum(Interpretation == "Resistant"), - mic50 = median(MIC, na.rm = TRUE), - mic90 = quantile(MIC, 0.90, na.rm = TRUE), - .groups = "drop" - ) %>% - mutate( - pct_resistant = n_resistant / n_tested * 100, - time = paste0(Year, "-12-31") - ) %>% - rename( - genus = Genus, - host_species = `Host Species`, - collection_source = `Collection Source`, - antimicrobial = `Drug Name` - ) %>% - select( - geography, time, genus, host_species, collection_source, - antimicrobial, pct_resistant, n_resistant, n_tested, mic50, mic90 - ) +# FIPS lookup: full state names -> 2-digit FIPS +all_fips <- vroom::vroom("../../resources/all_fips.csv.gz", show_col_types = FALSE) +state_fips_lookup <- all_fips %>% + filter(nchar(geography) == 2) %>% + select(geography, geography_name) - vroom::vroom_write( - animal_standard, - "standard/data_animal_pathogen.csv.gz", - delim = "," - ) - message(sprintf( - "Wrote %d rows to standard/data_animal_pathogen.csv.gz", - nrow(animal_standard) - )) +animal_raw <- readxl::read_excel( + animal_path_raw_path, + sheet = resolve_sheet(animal_path_raw_path, "_data$") +) - process$animal_pathogen_state <- current_animal_path_state - dcf::dcf_process_record(updated = process) -} +# Data is already in long format with one row per isolate/drug +# Filter to interpretable results, exclude non-US (Canada) +animal_agg <- animal_raw %>% + filter(Interpretation != "Non-Interpretable") %>% + left_join(state_fips_lookup, by = c("State" = "geography_name")) %>% + filter(!is.na(geography)) %>% + mutate(MIC = as.numeric(MIC)) %>% + group_by( + geography, Year, Genus, `Host Species`, `Collection Source`, `Drug Name` + ) %>% + summarize( + n_tested = n(), + n_resistant = sum(Interpretation == "Resistant"), + mic50 = median(MIC, na.rm = TRUE), + mic90 = quantile(MIC, 0.90, na.rm = TRUE), + .groups = "drop" + ) %>% + mutate( + pct_resistant = n_resistant / n_tested * 100, + time = paste0(Year, "-12-31"), + antimicrobial = tolower(gsub("[^A-Za-z0-9]+", "_", `Drug Name`)), + antimicrobial = gsub("_$", "", antimicrobial) + ) %>% + rename( + genus = Genus, + host_species = `Host Species`, + collection_source = `Collection Source` + ) %>% + select( + geography, time, genus, host_species, collection_source, + antimicrobial, pct_resistant, n_resistant, n_tested, mic50, mic90 + ) + +animal_standard <- animal_agg %>% + add_panel_flags( + row_keys = c("geography", "time", "genus", "host_species", "collection_source"), + measure_col = "antimicrobial", + organism_col = "genus", + value_cols = c("pct_resistant", "n_resistant", "n_tested", "mic50", "mic90"), + mic_cols = c("mic50", "mic90") + ) %>% + pivot_wider( + id_cols = c(geography, time, genus, host_species, collection_source), + names_from = antimicrobial, + values_from = c(pct_resistant, n_resistant, n_tested, mic50, mic90, narms_flag), + names_glue = "narms_{.value}_{antimicrobial}" + ) %>% + rename_with(~ gsub("narms_narms_flag", "narms_flag", .x)) + +vroom::vroom_write( + animal_standard, + "standard/data_animal_pathogen.csv.gz", + delim = "," +) +message(sprintf( + "Wrote %d rows to standard/data_animal_pathogen.csv.gz", + nrow(animal_standard) +)) + +process$animal_pathogen_state <- current_animal_path_state +dcf::dcf_process_record(updated = process) # ============================================================================= # Source 5: NARMS Food-Producing Animals (HACCP, Cecal, Minor Species) @@ -1215,28 +1664,28 @@ food_animal_files <- list( list( url = "https://www.fda.gov/media/93333/download?attachment", raw = "raw/narms-haccp-1997-2005.xlsx", - sheet = "HACCP_1997_2005", + sheet_pattern = "^HACCP_1997", source_label = "HACCP", year_col = "Year" ), list( url = "https://www.fda.gov/media/93344/download?attachment", raw = "raw/narms-haccp-2006-present.xlsx", - sheet = "HACCP_2006_present", + sheet_pattern = "^HACCP_2006", source_label = "HACCP", year_col = "Year" ), list( url = "https://www.fda.gov/media/93351/download?attachment", raw = "raw/narms-cecal-2013-present.xlsx", - sheet = "Cecal", + sheet_pattern = "^Cecal", source_label = "Cecal", year_col = "Year" ), list( url = "https://www.fda.gov/media/183419/download?attachment", raw = "raw/narms-minor-species.xlsx", - sheet = "Minor Species_2020-2022", + sheet_pattern = "^Minor Species", source_label = "Minor Species", year_col = "YEAR" ) @@ -1259,164 +1708,186 @@ for (f in food_animal_files) { } current_food_animal_state <- list(hash = paste(food_animal_hashes, collapse = "_")) -if (!identical(process$food_animal_state, current_food_animal_state)) { - message("Processing NARMS food-producing animal data...") - - library(readxl) - library(tidyr) - - # SIR column codes and antimicrobial name lookup (same as retail meats section) - sir_codes <- c( - "AMC", "AMI", "AMP", "ATM", "AVL", "AXO", "AZI", "BAC", - "CAZ", "CCV", "CEP", "CEQ", "CHL", "CIP", "CIP2", "CLI", - "COL", "COT", "CTC", "CTX", "DAP", "DOX", "ERY", "FEP", - "FFN", "FIS", "FLA", "FOX", "GEN", "IMI", "KAN", "LIN", - "LZD", "MER", "NAL", "NIT", "PEN", "PTZ", "QDA", "SAL", - "SMX", "STR", "SUF", "TEL", "TET", "TGC", "TIO", "TYL", "VAN" - ) - sir_col_names <- paste0(sir_codes, " SIR") - antimicrobial_names <- c( - AMC = "Amoxicillin-clavulanic acid", - AMI = "Amikacin", - AMP = "Ampicillin", - ATM = "Aztreonam", - AVL = "Avilamycin", - AXO = "Ceftriaxone", - AZI = "Azithromycin", - BAC = "Bacitracin", - CAZ = "Ceftazidime", - CCV = "Ceftiofur", - CEP = "Cephalothin", - CEQ = "Cefquinome", - CHL = "Chloramphenicol", - CIP = "Ciprofloxacin", - CIP2 = "Ciprofloxacin (2nd breakpoint)", - CLI = "Clindamycin", - COL = "Colistin", - COT = "Trimethoprim-sulfamethoxazole", - CTC = "Chlortetracycline", - CTX = "Cefotaxime", - DAP = "Daptomycin", - DOX = "Doxycycline", - ERY = "Erythromycin", - FEP = "Cefepime", - FFN = "Florfenicol", - FIS = "Sulfisoxazole", - FLA = "Flaveomycin", - FOX = "Cefoxitin", - GEN = "Gentamicin", - IMI = "Imipenem", - KAN = "Kanamycin", - LIN = "Lincomycin", - LZD = "Linezolid", - MER = "Meropenem", - NAL = "Nalidixic acid", - NIT = "Nitrofurantoin", - PEN = "Penicillin", - PTZ = "Piperacillin-tazobactam", - QDA = "Quinupristin-dalfopristin", - SAL = "Salinomycin", - SMX = "Sulfamethoxazole", - STR = "Streptomycin", - SUF = "Sulfonamides", - TEL = "Telithromycin", - TET = "Tetracycline", - TGC = "Tigecycline", - TIO = "Ceftiofur", - TYL = "Tylosin", - VAN = "Vancomycin" - ) +# Always standardise from the local raw files. The hash below records what was +# processed, but must NOT gate this block: gating it means a code change to the +# standardisation never reaches the output files on the automated monthly run. +message("Processing NARMS food-producing animal data...") + +library(readxl) +library(tidyr) + +# SIR column codes and antimicrobial name lookup (same as retail meats section) +sir_codes <- c( + "AMC", "AMI", "AMP", "ATM", "AVL", "AXO", "AZI", "BAC", + "CAZ", "CCV", "CEP", "CEQ", "CHL", "CIP", "CIP2", "CLI", + "COL", "COT", "CTC", "CTX", "DAP", "DOX", "ERY", "FEP", + "FFN", "FIS", "FLA", "FOX", "GEN", "IMI", "KAN", "LIN", + "LZD", "MER", "NAL", "NIT", "PEN", "PTZ", "QDA", "SAL", + "SMX", "STR", "SUF", "TEL", "TET", "TGC", "TIO", "TYL", "VAN" +) +sir_col_names <- paste0(sir_codes, " SIR") +antimicrobial_names <- c( + AMC = "Amoxicillin-clavulanic acid", + AMI = "Amikacin", + AMP = "Ampicillin", + ATM = "Aztreonam", + AVL = "Avilamycin", + AXO = "Ceftriaxone", + AZI = "Azithromycin", + BAC = "Bacitracin", + CAZ = "Ceftazidime", + CCV = "Ceftiofur", + CEP = "Cephalothin", + CEQ = "Cefquinome", + CHL = "Chloramphenicol", + CIP = "Ciprofloxacin", + CIP2 = "Ciprofloxacin (2nd breakpoint)", + CLI = "Clindamycin", + COL = "Colistin", + COT = "Trimethoprim-sulfamethoxazole", + CTC = "Chlortetracycline", + CTX = "Cefotaxime", + DAP = "Daptomycin", + DOX = "Doxycycline", + ERY = "Erythromycin", + FEP = "Cefepime", + FFN = "Florfenicol", + FIS = "Sulfisoxazole", + FLA = "Flaveomycin", + FOX = "Cefoxitin", + GEN = "Gentamicin", + IMI = "Imipenem", + KAN = "Kanamycin", + LIN = "Lincomycin", + LZD = "Linezolid", + MER = "Meropenem", + NAL = "Nalidixic acid", + NIT = "Nitrofurantoin", + PEN = "Penicillin", + PTZ = "Piperacillin-tazobactam", + QDA = "Quinupristin-dalfopristin", + SAL = "Salinomycin", + SMX = "Sulfamethoxazole", + STR = "Streptomycin", + SUF = "Sulfonamides", + TEL = "Telithromycin", + TET = "Tetracycline", + TGC = "Tigecycline", + TIO = "Ceftiofur", + TYL = "Tylosin", + VAN = "Vancomycin" +) - #' Process a single food-animal Excel file into long format - #' @param file_info list with raw, sheet, source_label, year_col - process_food_animal_file <- function(file_info) { - raw <- readxl::read_excel(file_info$raw, sheet = file_info$sheet) +#' Process a single food-animal Excel file into long format +#' @param file_info list with raw, sheet, source_label, year_col +process_food_animal_file <- function(file_info) { + raw <- readxl::read_excel( + file_info$raw, + sheet = resolve_sheet(file_info$raw, file_info$sheet_pattern) + ) - # Standardise year column name - if (file_info$year_col != "YEAR") { - raw <- raw %>% rename(YEAR = !!file_info$year_col) - } + # Standardise year column name + if (file_info$year_col != "YEAR") { + raw <- raw %>% rename(YEAR = !!file_info$year_col) + } - # Filter to positive cultures - filtered <- raw %>% - filter(GROWTH == "YES") %>% - mutate(.row_id = row_number()) - - # Pivot SIR columns - sir_long <- filtered %>% - select(.row_id, any_of(sir_col_names)) %>% - pivot_longer(cols = any_of(sir_col_names), - names_to = "antimicrobial", values_to = "sir") %>% - mutate(antimicrobial = sub(" SIR$", "", antimicrobial)) - - # Pivot MIC columns (coerce all to character first to avoid type conflicts) - mic_cols_present <- intersect(sir_codes, names(filtered)) - mic_data <- filtered %>% - select(.row_id, any_of(sir_codes)) %>% - mutate(across(any_of(mic_cols_present), as.character)) - mic_long <- mic_data %>% - pivot_longer(cols = any_of(sir_codes), - names_to = "antimicrobial", values_to = "mic") %>% - mutate(mic = as.numeric(mic)) - - # Join and attach metadata - sir_long %>% - left_join(mic_long, by = c(".row_id", "antimicrobial")) %>% - left_join( - filtered %>% select(.row_id, YEAR, GENUS_NAME, SPECIES, SEROTYPE, - HOST_SPECIES, SOURCE), - by = ".row_id" - ) %>% - mutate( - antimicrobial = antimicrobial_names[antimicrobial], - source_program = file_info$source_label - ) %>% - filter(!is.na(sir)) %>% - select(-.row_id) - } + # Filter to positive cultures + filtered <- raw %>% + filter(GROWTH == "YES") %>% + mutate(.row_id = row_number()) + + # Pivot SIR columns + sir_long <- filtered %>% + select(.row_id, any_of(sir_col_names)) %>% + pivot_longer(cols = any_of(sir_col_names), + names_to = "antimicrobial", values_to = "sir") %>% + mutate(antimicrobial = sub(" SIR$", "", antimicrobial)) + + # Pivot MIC columns (coerce all to character first to avoid type conflicts) + mic_cols_present <- intersect(sir_codes, names(filtered)) + mic_data <- filtered %>% + select(.row_id, any_of(sir_codes)) %>% + mutate(across(any_of(mic_cols_present), as.character)) + mic_long <- mic_data %>% + pivot_longer(cols = any_of(sir_codes), + names_to = "antimicrobial", values_to = "mic") %>% + mutate(mic = as.numeric(mic)) + + # Join and attach metadata + sir_long %>% + left_join(mic_long, by = c(".row_id", "antimicrobial")) %>% + left_join( + filtered %>% select(.row_id, YEAR, GENUS_NAME, SPECIES, SEROTYPE, + HOST_SPECIES, SOURCE), + by = ".row_id" + ) %>% + mutate( + antimicrobial = antimicrobial_names[antimicrobial], + source_program = file_info$source_label + ) %>% + filter(!is.na(sir)) %>% + select(-.row_id) +} - # Process all four files and combine - all_food_long <- bind_rows(lapply(food_animal_files, process_food_animal_file)) - - # Aggregate nationally (no state data in these files) - food_animal_standard <- all_food_long %>% - group_by(YEAR, GENUS_NAME, SPECIES, SEROTYPE, HOST_SPECIES, SOURCE, - source_program, antimicrobial) %>% - summarize( - n_tested = n(), - n_resistant = sum(sir == "R"), - mic50 = median(mic, na.rm = TRUE), - mic90 = quantile(mic, 0.90, na.rm = TRUE), - .groups = "drop" - ) %>% - mutate( - pct_resistant = n_resistant / n_tested * 100, - geography = "00", - time = paste0(YEAR, "-12-31") - ) %>% - rename( - genus = GENUS_NAME, - species = SPECIES, - serotype = SEROTYPE, - host_species = HOST_SPECIES, - source_type = SOURCE - ) %>% - select( - geography, time, source_program, source_type, genus, species, serotype, - host_species, antimicrobial, - pct_resistant, n_resistant, n_tested, mic50, mic90 - ) +# Process all four files and combine +all_food_long <- bind_rows(lapply(food_animal_files, process_food_animal_file)) + +# Aggregate nationally (no state data in these files) +food_animal_agg <- all_food_long %>% + group_by(YEAR, GENUS_NAME, SPECIES, SEROTYPE, HOST_SPECIES, SOURCE, + antimicrobial) %>% + summarize( + n_tested = n(), + n_resistant = sum(sir == "R"), + mic50 = median(mic, na.rm = TRUE), + mic90 = quantile(mic, 0.90, na.rm = TRUE), + .groups = "drop" + ) %>% + mutate( + pct_resistant = n_resistant / n_tested * 100, + geography = "00", + time = paste0(YEAR, "-12-31"), + genus_species_serotype = combine_organism(GENUS_NAME, SPECIES, SEROTYPE), + antimicrobial = tolower(gsub("[^A-Za-z0-9]+", "_", antimicrobial)), + antimicrobial = gsub("_$", "", antimicrobial) + ) %>% + rename( + host_species = HOST_SPECIES, + source_type = SOURCE + ) %>% + select( + geography, time, genus_species_serotype, + source_type, host_species, antimicrobial, + pct_resistant, n_resistant, n_tested, mic50, mic90 + ) - vroom::vroom_write( - food_animal_standard, - "standard/data_food_animals.csv.gz", - delim = "," - ) - message(sprintf( - "Wrote %d rows to standard/data_food_animals.csv.gz", - nrow(food_animal_standard) - )) +food_animal_standard <- food_animal_agg %>% + add_panel_flags( + row_keys = c("geography", "time", "genus_species_serotype", + "source_type", "host_species"), + measure_col = "antimicrobial", + organism_col = "genus_species_serotype", + value_cols = c("pct_resistant", "n_resistant", "n_tested", "mic50", "mic90"), + mic_cols = c("mic50", "mic90") + ) %>% + pivot_wider( + id_cols = c(geography, time, genus_species_serotype, + source_type, host_species), + names_from = antimicrobial, + values_from = c(pct_resistant, n_resistant, n_tested, mic50, mic90, narms_flag), + names_glue = "narms_{.value}_{antimicrobial}" + ) %>% + rename_with(~ gsub("narms_narms_flag", "narms_flag", .x)) + +vroom::vroom_write( + food_animal_standard, + "standard/data_food_animals.csv.gz", + delim = "," +) +message(sprintf( + "Wrote %d rows to standard/data_food_animals.csv.gz", + nrow(food_animal_standard) +)) - process$food_animal_state <- current_food_animal_state - dcf::dcf_process_record(updated = process) -} +process$food_animal_state <- current_food_animal_state +dcf::dcf_process_record(updated = process) diff --git a/data/narms/measure_info.json b/data/narms/measure_info.json index a607e45b7..779be3499 100644 --- a/data/narms/measure_info.json +++ b/data/narms/measure_info.json @@ -1,11 +1,50 @@ { + "_column_naming": { + "id": "_column_naming", + "short_name": "Column naming convention", + "long_name": "How measure columns are named in the standardized NARMS files", + "category": "antimicrobial_resistance", + "short_description": "Value columns are named narms_{measure}_{antimicrobial}", + "long_description": "All five standardized NARMS files are stored in wide format. Each row is one organism (plus geography, time, and any sampling context such as test method, meat source, host species, or collection source), and each antimicrobial agent or resistance pattern contributes its own set of columns. Column names follow the pattern narms_{measure}_{antimicrobial}, for example narms_pct_resistant_ciprofloxacin, narms_n_tested_ciprofloxacin, and narms_flag_ciprofloxacin. Antimicrobial and pattern names are lowercased with non-alphanumeric characters replaced by underscores. The entries below describe each {measure} prefix; they apply identically to every antimicrobial suffix.", + "measure_type": "Metadata", + "unit": "", + "time_resolution": "Year", + "sources": [ + { "id": "narms_now" }, + { "id": "narms_retail_meats" }, + { "id": "narms_animal_pathogen" }, + { "id": "narms_food_animals" } + ], + "citations": [] + }, + + "narms_flag": { + "id": "narms_flag", + "short_name": "Measurement status flag", + "long_name": "Why a measurement is present or absent for this organism and antimicrobial", + "category": "antimicrobial_resistance", + "short_description": "Categorical flag explaining whether the accompanying values are real measurements", + "long_description": "Every antimicrobial has a matching narms_flag_{antimicrobial} column recording whether the values beside it are real. The value columns are never NA; missing measurements are stored as 0, and this flag carries the reason. A 0 is only a genuine finding when the flag reads 'tested'. VALUES: (1) 'tested' - isolates were collected, the antimicrobial was on the panel, and susceptibility results came back. The denominator, numerator, and percentage are all real, and a 0 percent means the lab tested isolates and found none resistant. (2) 'no_isolates_tested' - the antimicrobial was on the panel and tracked for that organism, geography, and year, but no isolates were available to test. CDC reports this as an explicit zero, meaning 'we looked and there was nothing' as distinct from 'we did not look'. All value columns are placeholder zeros. Occurs only in the human clinical files; the FDA files aggregate isolate records, so a group with no isolates produces no row rather than a zero. (3) 'not_tested' - the antimicrobial was not part of the panel run for that specific combination of organism, geography, year, and test method, although it is used against that organism elsewhere in the file. Causes include antimicrobials added to or dropped from panels partway through the surveillance period, states running narrower panels, and recent years where collection and testing are still in progress. In the human files this is overwhelmingly a state-level phenomenon and is concentrated in the most recent years, which are preliminary and will partly resolve to 'tested' in future releases. All value columns are placeholder zeros. (4) 'not_on_panel' - the antimicrobial is never used against that organism, in any geography, year, or test method. This is not a gap but a combination that does not exist as a clinical question, and it will never fill in. It arises because each file holds the union of every organism's panel in one set of columns, so each row carries columns for antimicrobials that do not apply to it. This is a property of the organism and antimicrobial pairing rather than of any particular row. All value columns are placeholder zeros. (5) 'tested_no_mic' - isolates were tested and the resistance counts and percentage are real, but no minimum inhibitory concentration was published. Occurs only in the food-producing animals file, almost entirely for streptomycin, because three of the four FDA source files publish a susceptibility interpretation for that agent without publishing the underlying concentration. IMPORTANT: rows flagged 'tested_no_mic' contain valid resistance measurements. Filtering on narms_flag == 'tested' will silently exclude them. To select all real resistance measurements, match both 'tested' and 'tested_no_mic'. The five values are mutually exclusive; each cell carries exactly one. A single row will normally contain several different flag values across its antimicrobial columns.", + "statement": "Measurement status for {antimicrobial} was {value}", + "measure_type": "Category", + "unit": "", + "time_resolution": "Year", + "sources": [ + { "id": "narms_now" }, + { "id": "narms_retail_meats" }, + { "id": "narms_animal_pathogen" }, + { "id": "narms_food_animals" } + ], + "citations": [] + }, + "pct_resistant": { "id": "pct_resistant", "short_name": "Percent resistant to antimicrobial", "long_name": "Percentage of isolates resistant to a specific antimicrobial agent", "category": "antimicrobial_resistance", "short_description": "Percentage of isolates showing resistance to a specific antimicrobial agent", - "long_description": "Percentage of isolates tested by NARMS showing resistance to a specific antimicrobial agent, based on CLSI breakpoints. Calculated as n_resistant / n_tested * 100. Applies across all NARMS surveillance programs: human clinical isolates (CDC NARMS Now, AST and WGS methods), retail meat isolates (FDA, state-level), animal diagnostic pathogen isolates (FDA, state-level), and food-producing animal isolates (FDA HACCP/Cecal/Minor Species, national only). NA indicates the agent was on the testing panel but marked 'Not Tested' for that organism/year combination.", + "long_description": "Percentage of isolates tested by NARMS showing resistance to a specific antimicrobial agent, based on CLSI breakpoints. Calculated as n_resistant / n_tested * 100. Applies across all NARMS surveillance programs: human clinical isolates (CDC NARMS Now, AST and WGS methods), retail meat isolates (FDA, state-level), animal diagnostic pathogen isolates (FDA, state-level), and food-producing animal isolates (FDA HACCP/Cecal/Minor Species, national only). This column is never NA. Where no measurement exists the value is 0, and the matching narms_flag_{antimicrobial} column records why. A 0 is only a real finding when that flag reads 'tested' or 'tested_no_mic'; otherwise it is a placeholder and should be excluded from any calculation. Treating placeholder zeros as real will understate resistance.", "statement": "In {location}, {value}% of isolates were resistant to {antimicrobial}", "measure_type": "Percent", "unit": "Percent", @@ -27,13 +66,14 @@ } ] }, + "n_resistant": { "id": "n_resistant", "short_name": "Number of resistant isolates", "long_name": "Number of isolates resistant to a specific antimicrobial agent", "category": "antimicrobial_resistance", "short_description": "Count of isolates showing resistance to a specific antimicrobial agent", - "long_description": "Number of isolates classified as Resistant (R) by CLSI breakpoints for a specific antimicrobial agent. Numerator of the resistance percentage (pct_resistant). Applies across all NARMS surveillance programs. NA when the dashboard or source reports 'Not Tested' for that organism/year/agent combination. Zero when the agent was on the panel but no isolates were available.", + "long_description": "Number of isolates classified as Resistant (R) by CLSI breakpoints for a specific antimicrobial agent. Numerator of the resistance percentage (pct_resistant). Applies across all NARMS surveillance programs. This column is never NA. Where no measurement exists the value is 0, and the matching narms_flag_{antimicrobial} column records why. A 0 is only a real count when that flag reads 'tested' or 'tested_no_mic'; otherwise it is a placeholder.", "statement": "In {location}, {value} isolates were resistant to {antimicrobial}", "measure_type": "Count", "unit": "Isolates", @@ -46,13 +86,14 @@ ], "citations": [] }, + "n_tested": { "id": "n_tested", "short_name": "Number of isolates tested", "long_name": "Number of isolates tested for antimicrobial susceptibility", "category": "antimicrobial_resistance", "short_description": "Total number of isolates tested for resistance to a specific antimicrobial agent", - "long_description": "Total number of isolates tested for susceptibility to a specific antimicrobial agent. Denominator of the resistance percentage (pct_resistant). Applies across all NARMS surveillance programs. NA when the source reports 'Not Tested' for that organism/year/agent combination. Zero when the agent was on the panel but no isolates were available.", + "long_description": "Total number of isolates tested for susceptibility to a specific antimicrobial agent. Denominator of the resistance percentage (pct_resistant). Applies across all NARMS surveillance programs. This column is never NA. Where no measurement exists the value is 0, and the matching narms_flag_{antimicrobial} column records why. Note that a 0 here is ambiguous on its own: it covers both 'no_isolates_tested' (the antimicrobial was on the panel but no isolates were available) and 'not_tested' (the antimicrobial was not on the panel for that row), which are roughly equal in size in the human files. Only the flag distinguishes them. Selecting rows where n_tested > 0 is equivalent to selecting flag values of 'tested' or 'tested_no_mic'.", "statement": "In {location}, {value} isolates were tested for {antimicrobial}", "measure_type": "Count", "unit": "Isolates", @@ -65,13 +106,14 @@ ], "citations": [] }, + "mic50": { "id": "mic50", "short_name": "MIC50", "long_name": "Median minimum inhibitory concentration (MIC50)", "category": "antimicrobial_resistance", "short_description": "Median MIC value across isolates tested for a specific antimicrobial", - "long_description": "The MIC50 is the minimum inhibitory concentration (in micrograms per milliliter) at which 50% of isolates are inhibited — i.e., the median MIC value. Calculated from individual isolate-level MIC data. Available for retail meats, animal pathogen, and food-producing animal datasets. Not available for human clinical (NARMS Now) data, which reports only aggregate resistance percentages.", + "long_description": "The MIC50 is the minimum inhibitory concentration (in micrograms per milliliter) at which 50% of isolates are inhibited, i.e. the median MIC value. Calculated from individual isolate-level MIC data. Available for retail meats, animal pathogen, and food-producing animal datasets. Not available for human clinical (NARMS Now) data, which reports only aggregate resistance percentages. This column is never NA; where no concentration was measured the value is 0. A concentration of 0 micrograms per milliliter does not occur on the MIC scale, so a 0 here always means the measurement is absent rather than very low, and the matching narms_flag_{antimicrobial} column records why. In the food-producing animals file, streptomycin carries a susceptibility interpretation but no concentration in roughly 99 percent of its rows, because three of the four FDA source spreadsheets publish an 'STR SIR' column without a corresponding 'STR' concentration column. Those rows are flagged 'tested_no_mic'. Anyone computing MIC distributions or MIC trends for streptomycin from that file should be aware that only a small number of records carry a real value.", "statement": "The MIC50 for {antimicrobial} was {value} µg/mL", "measure_type": "Concentration", "unit": "µg/mL", @@ -83,13 +125,14 @@ ], "citations": [] }, + "mic90": { "id": "mic90", "short_name": "MIC90", "long_name": "90th percentile minimum inhibitory concentration (MIC90)", "category": "antimicrobial_resistance", "short_description": "90th percentile MIC value across isolates tested for a specific antimicrobial", - "long_description": "The MIC90 is the minimum inhibitory concentration (in micrograms per milliliter) at which 90% of isolates are inhibited — i.e., the 90th percentile MIC value. Calculated from individual isolate-level MIC data. Available for retail meats, animal pathogen, and food-producing animal datasets. Not available for human clinical (NARMS Now) data, which reports only aggregate resistance percentages.", + "long_description": "The MIC90 is the minimum inhibitory concentration (in micrograms per milliliter) at which 90% of isolates are inhibited, i.e. the 90th percentile MIC value. Calculated from individual isolate-level MIC data. Available for retail meats, animal pathogen, and food-producing animal datasets. Not available for human clinical (NARMS Now) data, which reports only aggregate resistance percentages. This column is never NA; where no concentration was measured the value is 0. As with mic50, a 0 always means absent rather than very low, and the matching narms_flag_{antimicrobial} column records why. The same streptomycin limitation described under mic50 applies here.", "statement": "The MIC90 for {antimicrobial} was {value} µg/mL", "measure_type": "Concentration", "unit": "µg/mL", @@ -110,7 +153,7 @@ "location_url": "https://app.powerbigov.us/view?r=eyJrIjoiZmU5ZjA2ZDItNTU0MS00M2EzLWEyZmQtZmY3Y2RlZjdjYTdjIiwidCI6IjljZTcwODY5LTYwZGItNDRmZC1hYmU4LWQyNzY3MDc3ZmM4ZiJ9", "organization": "Centers for Disease Control and Prevention", "organization_url": "https://www.cdc.gov", - "description": "Detailed antimicrobial resistance data from NARMS human clinical isolates, including resistance by individual antimicrobial agent (with CLSI class) and by resistance pattern (multidrug resistance profiles). Data available for 30 species/serotypes across 5 genera (Campylobacter, E. coli O157, Non-cholera Vibrio, Salmonella, Shigella). Both AST (phenotypic) and WGS (genotypic) test method results are available. Data extracted from the NARMS Now Power BI dashboard. Suggested citation: CDC. National Antimicrobial Resistance Monitoring System (NARMS) Now: Human Data. Atlanta, Georgia: U.S. Department of Health and Human Services.", + "description": "Detailed antimicrobial resistance data from NARMS human clinical isolates, including resistance by individual antimicrobial agent (with CLSI class) and by resistance pattern (multidrug resistance profiles). Data available for 30 species/serotypes across 5 genera (Campylobacter, E. coli O157, Non-cholera Vibrio, Salmonella, Shigella), tested against 22 antimicrobial agents and summarised into 29 resistance patterns. Both AST (phenotypic) and WGS (genotypic) test method results are available; WGS coverage begins in 2016. Data span 1999-2025. Records from years after 2021 are preliminary, since isolate collection and testing are still in progress, and a substantially higher share of those cells is flagged 'not_tested' as a result. Data extracted from the NARMS Now Power BI dashboard. Suggested citation: CDC. National Antimicrobial Resistance Monitoring System (NARMS) Now: Human Data. Atlanta, Georgia: U.S. Department of Health and Human Services.", "restrictions": "Public domain. CDC data is generally not subject to copyright restrictions.", "date_accessed": 2026 }, @@ -119,7 +162,7 @@ "url": "https://www.fda.gov/animal-veterinary/national-antimicrobial-resistance-monitoring-system/integrated-reportssummaries", "organization": "U.S. Food and Drug Administration, Center for Veterinary Medicine", "organization_url": "https://www.fda.gov/animal-veterinary", - "description": "Isolate-level antimicrobial susceptibility data from the FDA NARMS retail meats program. Samples collected from retail stores in participating states. Organisms tested include Campylobacter (coli, jejuni), Enterococcus (faecalis, faecium), Escherichia coli, and Salmonella (multiple serotypes). Meat sources include retail chicken, ground beef, ground turkey, pork chops, and veal. SIR (Susceptible/Intermediate/Resistant) calls based on CLSI breakpoints. Data span 2002-2021.", + "description": "Isolate-level antimicrobial susceptibility data from the FDA NARMS retail meats program. Samples collected from retail stores in participating states. Organisms tested include Campylobacter (coli, jejuni), Enterococcus (faecalis, faecium), Escherichia coli, and Salmonella (multiple serotypes), giving 221 distinct genus/species/serotype combinations tested against 31 antimicrobials. Meat sources include retail chicken, ground beef, ground turkey, pork chops, and veal. SIR (Susceptible/Intermediate/Resistant) calls based on CLSI breakpoints. Data span 2002-2021. Because this file is built by aggregating isolate-level records, a group with no isolates produces no row rather than a zero, so the 'no_isolates_tested' flag value does not occur here.", "restrictions": "Public domain. FDA data is generally not subject to copyright restrictions.", "date_accessed": 2025 }, @@ -128,7 +171,7 @@ "url": "https://www.fda.gov/animal-veterinary/national-antimicrobial-resistance-monitoring-system/integrated-reportssummaries", "organization": "U.S. Food and Drug Administration, Center for Veterinary Medicine", "organization_url": "https://www.fda.gov/animal-veterinary", - "description": "Antimicrobial susceptibility data from the FDA NARMS animal diagnostic pathogen surveillance program. Isolates submitted by veterinary diagnostic laboratories across the United States. Organisms tested include Campylobacter, E. coli, Enterococcus, and Salmonella from various host species (cattle, swine, poultry, etc.). SIR calls based on CLSI breakpoints. Data available at state level.", + "description": "Antimicrobial susceptibility data from the FDA NARMS animal diagnostic pathogen surveillance program. Isolates submitted by veterinary diagnostic laboratories across the United States, covering 4 genera (E. coli, Klebsiella spp, S. pseudintermedius, Salmonella) tested against 74 antimicrobials. SIR calls based on CLSI breakpoints. Data available at state level and span 2017-2024. Because testing is driven by what a veterinarian ordered for a specific clinical case rather than by a fixed sampling panel, roughly half of the 'not_tested' cells in this file fall in years where the same organism and antimicrobial pairing was tested in a different host species or collection source. The 'no_isolates_tested' flag value does not occur here, since the file is built by aggregating isolate-level records.", "restrictions": "Public domain. FDA data is generally not subject to copyright restrictions.", "date_accessed": 2026 }, @@ -137,7 +180,7 @@ "url": "https://www.fda.gov/animal-veterinary/national-antimicrobial-resistance-monitoring-system/integrated-reportssummaries", "organization": "U.S. Food and Drug Administration, Center for Veterinary Medicine", "organization_url": "https://www.fda.gov/animal-veterinary", - "description": "Isolate-level antimicrobial susceptibility data from four FDA NARMS food-producing animal surveillance programs: (1) HACCP slaughter surveillance (1997-present), covering federally inspected slaughter and processing plants; (2) Cecal sampling at slaughter (2013-present), testing cecal contents from food animals; (3) Minor Species (2020-2022), covering less common food animal species. Organisms include Campylobacter, E. coli, Enterococcus, and Salmonella. SIR calls based on CLSI breakpoints. National-level data only (no state breakdown). Data span 1997-2023.", + "description": "Isolate-level antimicrobial susceptibility data from four FDA NARMS food-producing animal surveillance programs: (1) HACCP slaughter surveillance (1997-present), covering federally inspected slaughter and processing plants; (2) Cecal sampling at slaughter (2013-present), testing cecal contents from food animals; (3) Minor Species (2020-2022), covering less common food animal species. Organisms include Campylobacter, E. coli, Enterococcus, and Salmonella, giving 638 distinct genus/species/serotype combinations, of which 628 are Salmonella serotypes, tested against 35 antimicrobials. SIR calls based on CLSI breakpoints. National-level data only (no state breakdown). Data span 1997-2023. This is the only file in which the 'tested_no_mic' flag value occurs, almost entirely for streptomycin: three of the four source spreadsheets publish an 'STR SIR' interpretation column without a corresponding 'STR' concentration column, so resistance counts are available while MIC values are not. Because the file is built by aggregating isolate-level records, the 'no_isolates_tested' flag value does not occur here.", "restrictions": "Public domain. FDA data is generally not subject to copyright restrictions.", "date_accessed": 2026 } diff --git a/data/narms/process.json b/data/narms/process.json index 637b4f062..22ebf2773 100644 --- a/data/narms/process.json +++ b/data/narms/process.json @@ -5,38 +5,81 @@ { "path": "ingest.R", "manual": false, - "last_run": "2026-07-31 19:32:15", "frequency": 0, - "run_time": 10.124, + "last_run": "2026-06-30 19:37:56", + "run_time": 10.87, "last_status": { - "log": "Sheet '2017-2021_data' not found", - "success": false + "log": [ + "Wrote 937244 rows to standard/data_resistance_agent.csv.gz", + "Wrote 1144329 rows to standard/data_resistance_pattern.csv.gz" + ], + "success": true } } ], - "checked": "2026-07-31 20:47:35", + "checked": "2026-06-30 19:53:26", "check_results": { "data/narms/standard/data_animal_pathogen.csv.gz": { - "measures": ["missing_info: genus", "missing_info: host_species", "missing_info: collection_source", "missing_info: antimicrobial"] + "measures": [ + "missing_info: genus", + "missing_info: host_species", + "missing_info: collection_source", + "missing_info: antimicrobial" + ] }, "data/narms/standard/data_food_animals.csv.gz": { - "measures": ["missing_info: source_program", "missing_info: source_type", "missing_info: genus", "missing_info: species", "missing_info: serotype", "missing_info: host_species", "missing_info: antimicrobial"] + "measures": [ + "missing_info: source_program", + "missing_info: source_type", + "missing_info: genus", + "missing_info: species", + "missing_info: serotype", + "missing_info: host_species", + "missing_info: antimicrobial" + ] }, "data/narms/standard/data_resistance_agent.csv.gz": { - "measures": ["missing_info: genus", "missing_info: species_serotype", "missing_info: antimicrobial_class", "missing_info: antimicrobial_agent", "missing_info: test_method"] + "measures": [ + "missing_info: genus", + "missing_info: species_serotype", + "missing_info: antimicrobial_class", + "missing_info: antimicrobial_agent", + "missing_info: test_method" + ] }, "data/narms/standard/data_resistance_pattern.csv.gz": { - "measures": ["missing_info: genus", "missing_info: species_serotype", "missing_info: pattern", "missing_info: test_method"] + "measures": [ + "missing_info: genus", + "missing_info: species_serotype", + "missing_info: pattern", + "missing_info: test_method" + ] + }, + "data/narms/standard/data_retail_meats.csv": { + "data": "not_compressed", + "measures": [ + "missing_info: genus", + "missing_info: species", + "missing_info: serotype", + "missing_info: meat_source", + "missing_info: antimicrobial" + ] }, "data/narms/standard/data_retail_meats.csv.gz": { - "measures": ["missing_info: genus", "missing_info: species", "missing_info: serotype", "missing_info: meat_source", "missing_info: antimicrobial"] + "measures": [ + "missing_info: genus", + "missing_info: species", + "missing_info: serotype", + "missing_info: meat_source", + "missing_info: antimicrobial" + ] } }, "narms_now_state": { - "last_scrape_date": "2026-07-03", - "n_agent_rows": 937262, + "last_scrape_date": "2026-07-07", + "n_agent_rows": 937244, "n_pattern_rows": 1144329, - "n_errors": 0, + "n_errors": 1, "n_sites": 52, "year_from_ast": 1999, "year_from_wgs": 2016, @@ -46,7 +89,7 @@ "hash": "1f782917ad56db9803836b6811a9018a" }, "animal_pathogen_state": { - "hash": "5a663c78c6f94bc39eeb9db0fd027142" + "hash": "7317cb767db2979f52d005d4e2f89342" }, "food_animal_state": { "hash": "5e0fdb8272e046e94f63a366f056c9bc_e6ccdcfac415b34210fd437cd964a0ae_0003b5eaa4925647a394a9c18cef6ded_2ed64eacb2ec22879526134543a16f88" @@ -55,8 +98,9 @@ "./data/narms/measure_info.json": "e291b7268d145b83c6a8a78319580e03", "./data/narms/standard/data_animal_pathogen.csv.gz": "c9fc74405cf50ceff73f4424f10f707f", "./data/narms/standard/data_food_animals.csv.gz": "0cdf8ab10824c64e6875f619c6afa240", - "./data/narms/standard/data_resistance_agent.csv.gz": "8fd624a40bca25bd64f6c3e20d85381e", - "./data/narms/standard/data_resistance_pattern.csv.gz": "e9c0ecfa3904b4f63f6af6868fed6cb5", + "./data/narms/standard/data_resistance_agent.csv.gz": "0ef8592f464334ca47eab5a4253c0a53", + "./data/narms/standard/data_resistance_pattern.csv.gz": "6e3dd3bd1fc66a99ed25dfa276387f2a", + "./data/narms/standard/data_retail_meats.csv": "faa9ddbe9cb6486656147fd582a0f448", "./data/narms/standard/data_retail_meats.csv.gz": "77808546890d5dcf669f74883a194424" } } diff --git a/data/narms/raw/cvm-narms-retail-meats.xlsx b/data/narms/raw/narms-retail-meats.xlsx similarity index 100% rename from data/narms/raw/cvm-narms-retail-meats.xlsx rename to data/narms/raw/narms-retail-meats.xlsx diff --git a/data/narms/raw/narms_now_agent.csv.gz b/data/narms/raw/narms_now_agent.csv.gz index 3c13df118..dba3c13ee 100644 Binary files a/data/narms/raw/narms_now_agent.csv.gz and b/data/narms/raw/narms_now_agent.csv.gz differ diff --git a/data/narms/raw/narms_now_pattern.csv.gz b/data/narms/raw/narms_now_pattern.csv.gz index 9c76ee5b0..383a0590d 100644 Binary files a/data/narms/raw/narms_now_pattern.csv.gz and b/data/narms/raw/narms_now_pattern.csv.gz differ diff --git a/data/narms/standard/data_animal_pathogen.csv.gz b/data/narms/standard/data_animal_pathogen.csv.gz index 21b5a525c..fe5619656 100644 Binary files a/data/narms/standard/data_animal_pathogen.csv.gz and b/data/narms/standard/data_animal_pathogen.csv.gz differ diff --git a/data/narms/standard/data_food_animals.csv.gz b/data/narms/standard/data_food_animals.csv.gz index dd1291078..08a6d45c8 100644 Binary files a/data/narms/standard/data_food_animals.csv.gz and b/data/narms/standard/data_food_animals.csv.gz differ diff --git a/data/narms/standard/data_resistance_agent.csv.gz b/data/narms/standard/data_resistance_agent.csv.gz index 5860938cc..1ab942ae9 100644 Binary files a/data/narms/standard/data_resistance_agent.csv.gz and b/data/narms/standard/data_resistance_agent.csv.gz differ diff --git a/data/narms/standard/data_resistance_pattern.csv.gz b/data/narms/standard/data_resistance_pattern.csv.gz index 4e30142e6..fe9f073da 100644 Binary files a/data/narms/standard/data_resistance_pattern.csv.gz and b/data/narms/standard/data_resistance_pattern.csv.gz differ diff --git a/data/narms/standard/data_retail_meats.csv.gz b/data/narms/standard/data_retail_meats.csv.gz index 1aff8fd65..3e281c016 100644 Binary files a/data/narms/standard/data_retail_meats.csv.gz and b/data/narms/standard/data_retail_meats.csv.gz differ