From 8e24042d043a25435f1cb4bf70c2b8e450f79bdc Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Thu, 6 Aug 2026 12:41:43 +0200 Subject: [PATCH 1/7] build_priogrid: now uses a parquet hive format instead of monolithic file --- DESCRIPTION | 4 +- R/build_priogrid.R | 330 +++++++++++++++++++-------- R/utility.R | 26 ++- tests/testthat/test-build_priogrid.R | 126 ++++++++++ 4 files changed, 381 insertions(+), 105 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 17e8996..3d4622d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,7 +39,6 @@ Suggests: terra, sf, exactextractr, - arrow, countrycode Config/testthat/edition: 3 Imports: @@ -59,7 +58,8 @@ Imports: digest, rlang, RefManageR, - nanoparquet, + arrow, + jsonlite, stats, utils Depends: diff --git a/R/build_priogrid.R b/R/build_priogrid.R index c480ee8..3229859 100644 --- a/R/build_priogrid.R +++ b/R/build_priogrid.R @@ -277,6 +277,166 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { invisible(NULL) } +# Internal: memory-bounded hive-partitioned builder for time-varying data. +# Writes one parquet partition per temporal slice following Apache Hive nested layout: +# year=YYYY/ for yearly; year=YYYY/=VV/ for sub-yearly. Also writes a gzipped CSV +# bundle and a pg_config.json manifest. Checksums all outputs. +# @keywords internal +.pg_build_timevarying <- function(base_path, config, version = NULL, type = NULL, overwrite = FALSE) { + rlang::check_installed("terra", reason = "to build PRIO-GRID data from individual variables") + + hive_dir <- file.path(base_path, "timevarying") + if (dir.exists(hive_dir)) { + if (!overwrite) { + message("Hive dataset already exists at '", hive_dir, "'. Use overwrite = TRUE to rebuild.") + return(invisible(hive_dir)) + } + unlink(hive_dir, recursive = TRUE) + stale_csv <- file.path(base_path, "pg_timevarying.csv.gz") + if (file.exists(stale_csv)) file.remove(stale_csv) + } + + unit <- pg_temporal_unit(config) + n_cells <- length(create_pg_indices(config)) + tmp_dir <- file.path(pg_rawfolder(), "tmp", basename(tempfile())) + dir.create(tmp_dir, recursive = TRUE) + on.exit(unlink(tmp_dir, recursive = TRUE), add = TRUE) + + # ----- Phase 1: per-variable long parquet (date-sorted) ----- + tv <- dplyr::filter(pgvariables, !static)$name + present <- tv[file.exists(file.path(base_path, paste0(tv, ".rds")))] + if (length(present) == 0L) + stop("No time-varying .rds rasters found in ", base_path) + + for (v in present) { + r <- terra::unwrap(readRDS(file.path(base_path, paste0(v, ".rds")))) + dt <- rast_to_df(r, static = FALSE, varname = v, config = config) + data.table::setorder(dt, measurement_date) + arrow::write_parquet(dt, file.path(tmp_dir, paste0(v, ".parquet")), + chunk_size = n_cells) + rm(r, dt); gc() + } + + # ----- Phase 2: per-slice wide assembly → hive partition + CSV rows ----- + dates <- pg_dates(config) + tmp_csv <- file.path(tmp_dir, "_all.csv") + partitions <- character(0L) + + for (d in dates) { + d <- as.Date(d, origin = "1970-01-01") + parts <- list() + for (v in present) { + sub <- dplyr::collect( + dplyr::select( + dplyr::filter(arrow::open_dataset(file.path(tmp_dir, paste0(v, ".parquet"))), + measurement_date == d), + pgid, dplyr::all_of(v))) + if (nrow(sub) > 0L) parts[[v]] <- data.table::setDT(sub) + } + if (length(parts) == 0L) next + + slice <- Reduce(function(a, b) merge(a, b, by = "pgid", all = TRUE), parts) + for (v in setdiff(present, names(slice))) slice[, (v) := NA_real_] + slice[, measurement_date := d] + data.table::setcolorder(slice, c("pgid", "measurement_date", present)) + + # Nested Hive partition subpath: year=YYYY for yearly; year=YYYY/=VV for sub-yearly + part_subpath <- switch(unit, + year = paste0("year=", lubridate::year(d)), + quarter = file.path(paste0("year=", lubridate::year(d)), + paste0("quarter=", lubridate::quarter(d))), + month = file.path(paste0("year=", lubridate::year(d)), + paste0("month=", sprintf("%02d", lubridate::month(d)))), + week = file.path(paste0("year=", lubridate::isoyear(d)), + paste0("week=", sprintf("%02d", lubridate::isoweek(d))))) + + part_dir <- file.path(hive_dir, part_subpath) + dir.create(part_dir, recursive = TRUE, showWarnings = FALSE) + pf <- file.path(part_dir, "part-0.parquet") + arrow::write_parquet(slice, pf) + .pg_record_checksum(pf, + file.path("timevarying", part_subpath, "part-0.parquet"), + base_path) + data.table::fwrite(slice, tmp_csv, append = length(partitions) > 0L) + partitions <- c(partitions, part_subpath) + } + + # ----- Phase 3: CSV bundle, manifest, cleanup ----- + csv_gz <- file.path(base_path, "pg_timevarying.csv.gz") + R.utils::gzip(tmp_csv, destname = csv_gz, overwrite = TRUE, remove = TRUE) + .pg_record_checksum(csv_gz, "pg_timevarying.csv.gz", base_path) + + static_path <- file.path(base_path, "pg_static.parquet") + static_vars <- if (file.exists(static_path)) + setdiff(names(arrow::open_dataset(static_path)), "pgid") + else character(0L) + + meta <- list( + priogrid_version = version %||% as.character(utils::packageVersion("priogrid")), + type = type %||% "custom", + generated_at = as.character(Sys.Date()), + grid = list( + nrow = config$nrow, + ncol = config$ncol, + crs = config$crs, + extent = list( + xmin = unname(config$extent["xmin"]), + xmax = unname(config$extent["xmax"]), + ymin = unname(config$extent["ymin"]), + ymax = unname(config$extent["ymax"]))), + temporal = list( + resolution = config$temporal_resolution, + partition_key = if (unit == "year") "year" else c("year", unit), + start_date = as.character(config$start_date), + end_date = as.character(config$end_date)), + pgid_to_coordinates = list( + description = "pgid is 1-based; cell 1 is the south-west corner (xmin, ymin), increasing eastward then northward.", + col_from_west = "(pgid - 1) %% ncol", + row_from_south = "(pgid - 1) %/% ncol", + lon_center = "xmin + (col_from_west + 0.5) * (xmax - xmin) / ncol", + lat_center = "ymin + (row_from_south + 0.5) * (ymax - ymin) / nrow"), + dataset_root = paste(version %||% as.character(utils::packageVersion("priogrid")), + type %||% "custom", sep = "/"), + layout = list( + timevarying = "timevarying//part-0.parquet", + static = "pg_static.parquet", + csv = list(timevarying = "pg_timevarying.csv.gz", + static = "pg_static.csv.gz")), + static_variables = static_vars, + timevarying_variables = present, + partitions = partitions) + + json_path <- file.path(base_path, "pg_config.json") + jsonlite::write_json(meta, json_path, auto_unbox = TRUE, pretty = TRUE) + .pg_record_checksum(json_path, "pg_config.json", base_path) + + return(invisible(hive_dir)) +} + +# Internal: verify MD5 checksums for hive partition files. +# Reads _checksums.csv from base_path and warns on mismatch for any +# entry whose varname starts with "timevarying/". +# @keywords internal +.pg_verify_hive <- function(base_path) { + cs_file <- file.path(base_path, "_checksums.csv") + if (!file.exists(cs_file)) return(invisible(NULL)) + cs_df <- utils::read.csv(cs_file, stringsAsFactors = FALSE) + hive_rows <- cs_df[startsWith(cs_df$varname, "timevarying/"), ] + for (i in seq_len(nrow(hive_rows))) { + fpath <- file.path(base_path, hive_rows$varname[i]) + if (!file.exists(fpath)) { + warning("Hive partition missing: ", hive_rows$varname[i], call. = FALSE) + next + } + actual <- tools::md5sum(fpath) + if (actual != hive_rows$md5[i]) + warning("Checksum mismatch for hive partition '", hive_rows$varname[i], + "': the file may be corrupted.", call. = FALSE) + } + invisible(NULL) +} + + # Internal admin helper: bulk-bootstrap _checksums.csv for all .rds/.parquet # files in a resolved output folder. Useful after files have been placed without # going through save_pgvariable() / read_pg_static() / read_pg_timevarying(). @@ -645,7 +805,7 @@ read_pg_static <- function(config = NULL, } } } - return(nanoparquet::read_parquet(fname)) + return(arrow::read_parquet(fname)) } # From here on, terra is required @@ -695,9 +855,11 @@ read_pg_static <- function(config = NULL, df <- rast_to_df(terra::rast(rasters), static = TRUE, config = data_config) # Save to cache - rlang::check_installed("arrow", reason = "to write parquet files") arrow::write_parquet(df, fname) .pg_record_checksum(fname, "pg_static.parquet", cfg$base_path) + static_csv <- file.path(cfg$base_path, "pg_static.csv.gz") + data.table::fwrite(df, static_csv, compress = "gzip") + .pg_record_checksum(static_csv, "pg_static.csv.gz", cfg$base_path) return(df) } @@ -753,111 +915,92 @@ read_pg_timevarying <- function(config = NULL, verify_checksums = FALSE) { cfg <- resolve_pg_mode(config, version, type, spatial_hash, temporal_hash, overwrite) + if (!is.null(cfg$warning)) warning(cfg$warning) + if (cfg$mode == "release") download_priogrid(version = cfg$version, type = cfg$type) - if (!is.null(cfg$warning)) { - warning(cfg$warning) - } - - if (cfg$mode == "release") { - download_priogrid(version = cfg$version, type = cfg$type) - } - - fname <- file.path(cfg$base_path, "pg_timevarying.parquet") + hive_dir <- file.path(cfg$base_path, "timevarying") + legacy <- file.path(cfg$base_path, "pg_timevarying.parquet") - # Return cached if available (lightweight path — no terra needed) - if (!as_raster && !test && file.exists(fname) && !cfg$overwrite) { + if (!as_raster && !test && !cfg$overwrite && (dir.exists(hive_dir) || file.exists(legacy))) { + if (dir.exists(hive_dir)) { + if (isTRUE(verify_checksums)) .pg_verify_hive(cfg$base_path) + return(data.table::setDT(dplyr::collect(arrow::open_dataset(hive_dir)))) + } if (isTRUE(verify_checksums)) { cs_file <- file.path(cfg$base_path, "_checksums.csv") if (file.exists(cs_file)) { - cs_df <- utils::read.csv(cs_file, stringsAsFactors = FALSE) + cs_df <- utils::read.csv(cs_file, stringsAsFactors = FALSE) expected <- cs_df[cs_df$varname == "pg_timevarying.parquet", ] - if (nrow(expected) == 1) { - actual <- tools::md5sum(fname) - if (actual != expected$md5) { - warning( - "Checksum mismatch for cached 'pg_timevarying.parquet': the file may be corrupted.\n", - " Expected MD5: ", expected$md5, "\n", - " Actual MD5: ", actual, "\n", - " Rebuild with: read_pg_timevarying(overwrite = TRUE)", - call. = FALSE) - } - } + if (nrow(expected) == 1 && tools::md5sum(legacy) != expected$md5) + warning("Checksum mismatch for cached 'pg_timevarying.parquet': the file may be corrupted.", + call. = FALSE) } } - return(nanoparquet::read_parquet(fname)) + return(arrow::read_parquet(legacy)) } - # From here on, terra is required rlang::check_installed("terra", reason = "to build PRIO-GRID data from individual variables") - - # Load individual variables timevarying <- pgvariables |> dplyr::filter(!static) - - if (cfg$mode == "release") { - rasters <- lapply(timevarying$name, load_pgvariable, - version = cfg$version, type = cfg$type) - } else { - if (is.null(cfg$config)) { - stop("config is required when using explicit spatial_hash/temporal_hash with read_pg_timevarying()") - } - rasters <- lapply(timevarying$name, load_pgvariable, - spatial_hash = cfg$spatial_hash, - temporal_hash = cfg$temporal_hash) - } - - names(rasters) <- timevarying$name - rasters <- rasters[!is.na(rasters)] - - # data_config is the resolved config (release params or user config) data_config <- cfg$config - # Test coverage if requested - if (test) { - pg_idx <- length(create_pg_indices(data_config)) - coverage_test <- dplyr::bind_rows( - lapply(rasters, function(x) terra::freq(is.na(x))), - .id = "variable" - ) |> - dplyr::filter(value == 0) |> # non-missing - dplyr::mutate( - coverage_pct = round(count / pg_idx, 4), - measurement_date = lapply(rasters, names) |> unlist() + if (test || as_raster) { + if (cfg$mode == "release") { + rasters <- lapply(timevarying$name, load_pgvariable, + version = cfg$version, type = cfg$type) + } else { + if (is.null(cfg$config)) + stop("config is required when using explicit spatial_hash/temporal_hash with read_pg_timevarying()") + rasters <- lapply(timevarying$name, load_pgvariable, + spatial_hash = cfg$spatial_hash, temporal_hash = cfg$temporal_hash) + } + names(rasters) <- timevarying$name + rasters <- rasters[!is.na(rasters)] + + if (test) { + pg_idx <- length(create_pg_indices(data_config)) + coverage_test <- dplyr::bind_rows( + lapply(rasters, function(x) terra::freq(is.na(x))), + .id = "variable" ) |> - dplyr::select(variable, measurement_date, coverage_n = count, coverage_pct) - - return(coverage_test) - } - - # Return rasters if requested - if (as_raster) { + dplyr::filter(value == 0) |> + dplyr::mutate( + coverage_pct = round(count / pg_idx, 4), + measurement_date = lapply(rasters, names) |> unlist() + ) |> + dplyr::select(variable, measurement_date, coverage_n = count, coverage_pct) + return(coverage_test) + } return(rasters) } - # Build data.table - timevarying_lst <- mapply(rast_to_df, rasters, names(rasters), - MoreArgs = list(static = FALSE, config = data_config), - SIMPLIFY = FALSE) - - # Only return dates that are within the data scope - min_date <- do.call(min, lapply(timevarying_lst, function(x) min(x$measurement_date))) - my_dates <- pg_dates(data_config) - my_dates <- my_dates[my_dates >= min_date] - - df <- expand.grid(pgid = create_pg_indices(data_config), measurement_date = my_dates) - df <- data.table::setDT(df, key = c("pgid", "measurement_date")) - - for (sdt in timevarying_lst) { - df <- merge(df, sdt, all.x = TRUE) - } - - # Save to cache - data.table::fwrite(df, paste0(tools::file_path_sans_ext(fname), ".csv.gz"), - compress = "gzip") - rlang::check_installed("arrow", reason = "to write parquet files") - arrow::write_parquet(df, fname) - .pg_record_checksum(fname, "pg_timevarying.parquet", cfg$base_path) + .pg_build_timevarying(base_path = cfg$base_path, config = data_config, + version = cfg$version, type = cfg$type, overwrite = TRUE) + data.table::setDT(dplyr::collect(arrow::open_dataset(hive_dir))) +} - return(df) +#' Build a PRIO-GRID hive-partitioned dataset +#' +#' Memory-safe alternative to `read_pg_timevarying()` that writes the hive +#' partitioned parquet dataset and CSV bundle without loading the full table. +#' Useful for manual rebuild and re-upload workflows. +#' +#' @param config A `pg_config` object for custom data, or NULL (default) for the +#' official release. +#' @param version Character string specifying PRIOGRID version (release mode only). +#' @param type Character string specifying release type. Default: "05deg_yearly". +#' @param spatial_hash 6-character MD5 hash of spatial options (advanced use). +#' @param temporal_hash 6-character MD5 hash of temporal options (advanced use). +#' @param overwrite Logical. If TRUE, rebuilds even if output already exists. +#' +#' @return Invisibly returns the base output path. +#' @export +build_pg_dataset <- function(config = NULL, version = NULL, type = "05deg_yearly", + spatial_hash = NULL, temporal_hash = NULL, + overwrite = FALSE) { + cfg <- resolve_pg_mode(config, version, type, spatial_hash, temporal_hash) + .pg_build_timevarying(base_path = cfg$base_path, config = cfg$config, + version = cfg$version, type = cfg$type, overwrite = overwrite) + invisible(cfg$base_path) } #' Build an official PRIO-GRID release @@ -909,7 +1052,8 @@ build_release <- function(version, type, # Calculate calc_pg(overwrite = TRUE, config = config) read_pg_static(config = config, overwrite = TRUE) - read_pg_timevarying(config = config, overwrite = TRUE) + .pg_build_timevarying(base_path = pgout_path(config = config), config = config, + version = version, type = type, overwrite = TRUE) # Promote to release from_path <- pgout_path(config = config) @@ -922,8 +1066,12 @@ build_release <- function(version, type, current_wd <- getwd() setwd(file.path(pg_rawfolder(), "priogrid")) safe_version <- stringr::str_replace_all(version, "\\.", "_") - zip(zipfile = paste0(paste("priogrid", safe_version, type, sep = "_"), ".zip"), - files = list.files(file.path("releases", version, type), full.names = TRUE)) + rel_dir <- file.path("releases", version, type) + all_top <- list.files(rel_dir, full.names = TRUE) + csv_files <- list.files(rel_dir, pattern = "\\.csv\\.gz$", full.names = TRUE) + base_zip <- paste("priogrid", safe_version, type, sep = "_") + zip(zipfile = paste0(base_zip, ".zip"), files = setdiff(all_top, csv_files)) + zip(zipfile = paste0(base_zip, "_csv.zip"), files = csv_files) setwd(current_wd) message("Release built at: ", to_path) diff --git a/R/utility.R b/R/utility.R index 66cad0d..398a6ef 100644 --- a/R/utility.R +++ b/R/utility.R @@ -54,6 +54,18 @@ prio_blank_grid <- function(config = pg_current_config()){ return(pg) } +#' Temporal unit for a config's resolution +#' @keywords internal +pg_temporal_unit <- function(config = pg_current_config()) { + tr <- config$temporal_resolution + dplyr::case_when( + grepl("month", tr) ~ "month", + grepl("week", tr) ~ "week", + grepl("quarter", tr) ~ "quarter", + grepl("year", tr) ~ "year" + ) +} + #' Get a sequence of dates #' #' This is a wrapper of [base::seq.Date], only with defaults that can be set in the config. @@ -70,12 +82,7 @@ pg_dates <- function(config = pg_current_config()){ end_date <- config$end_date temporal_resolution <- config$temporal_resolution - unit <- dplyr::case_when( - grepl("month", temporal_resolution) ~ "month", - grepl("week", temporal_resolution) ~ "week", - grepl("quarter", temporal_resolution) ~ "quarter", - grepl("year", temporal_resolution) ~ "year" - ) + unit <- pg_temporal_unit(config) is_end_of_month <- start_date == (lubridate::ceiling_date(start_date, "month") - lubridate::days(1)) @@ -109,12 +116,7 @@ pg_date_intervals <- function(config = pg_current_config()){ end_date <- config$end_date temporal_resolution <- config$temporal_resolution - unit <- dplyr::case_when( - grepl("month", temporal_resolution) ~ "month", - grepl("week", temporal_resolution) ~ "week", - grepl("quarter", temporal_resolution) ~ "quarter", - grepl("year", temporal_resolution) ~ "year" - ) + unit <- pg_temporal_unit(config) is_end_of_month <- start_date == (lubridate::ceiling_date(start_date, "month") - lubridate::days(1)) diff --git a/tests/testthat/test-build_priogrid.R b/tests/testthat/test-build_priogrid.R index 8c1bf80..f42af76 100644 --- a/tests/testthat/test-build_priogrid.R +++ b/tests/testthat/test-build_priogrid.R @@ -166,3 +166,129 @@ test_that(".pg_bootstrap_checksums skips existing entries by default", { expect_equal(result2$n_added, 0L) expect_equal(result2$n_updated, 0L) }) + + +# ---- Hive-partitioned builder tests ---- + +test_that(".pg_build_timevarying writes hive partitions, CSV bundle, and manifest", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + tmp_raw <- tempfile() + dir.create(tmp_raw) + on.exit(unlink(tmp_raw, recursive = TRUE), add = TRUE) + + pg_set_rawfolder(tmp_raw) + cfg <- test_config() + base <- pgout_path(config = cfg) + dir.create(base, recursive = TRUE) + + # Fabricate two time-varying .rds rasters + tv <- pgvariables$name[!pgvariables$static][1:2] + dates <- pg_dates(cfg) + + for (i in seq_along(tv)) { + r <- prio_blank_grid(cfg) + for (j in seq_along(dates)[-1]) terra::add(r) <- prio_blank_grid(cfg) + terra::values(r) <- runif(terra::ncell(r) * terra::nlyr(r)) + names(r) <- as.character(dates) + # First variable: set 2011-12-31 layer all-NA + if (i == 1L) terra::values(r[[which(dates == as.Date("2011-12-31"))]]) <- NA_real_ + save_pgvariable(r, tv[i], save_to = base) + } + + build_pg_dataset(config = cfg) + + # Hive partitions exist + for (yr in c("year=2010", "year=2011", "year=2012")) { + expect_true(file.exists(file.path(base, "timevarying", yr, "part-0.parquet"))) + } + # CSV bundle and manifest exist + expect_true(file.exists(file.path(base, "pg_timevarying.csv.gz"))) + expect_true(file.exists(file.path(base, "pg_config.json"))) + # Monolith must be absent + expect_false(file.exists(file.path(base, "pg_timevarying.parquet"))) + + # Read hive dataset and check schema / content + d <- data.table::setDT(dplyr::collect(arrow::open_dataset(file.path(base, "timevarying")))) + expect_true("year" %in% names(d)) + expect_true(all(tv %in% names(d))) + expect_true("pgid" %in% names(d)) + expect_true("measurement_date" %in% names(d)) + expect_equal(sort(unique(d$year)), 2010:2012) + + # First variable all-NA in 2011; second populated + d2011 <- d[d$year == 2011, ] + expect_true(all(is.na(d2011[[tv[1]]]))) + expect_true(any(!is.na(d2011[[tv[2]]]))) + + # Manifest assertions + m <- jsonlite::read_json(file.path(base, "pg_config.json"), simplifyVector = TRUE) + expect_equal(m$grid$nrow, 5L) + expect_equal(m$grid$ncol, 10L) + expect_equal(m$temporal$partition_key, "year") + expect_equal(m$grid$extent$xmin, -180) + expect_true(setequal(m$timevarying_variables, tv)) + expect_true(setequal(m$partitions, c("year=2010", "year=2011", "year=2012"))) + + # CSV bundle row count matches hive + csv <- data.table::fread(file.path(base, "pg_timevarying.csv.gz")) + expect_equal(nrow(csv), nrow(d)) + expect_true(setequal(setdiff(names(d), "year"), names(csv))) +}) + +test_that("read_pg_timevarying reads cached hive dataset (no terra required)", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + tmp_raw <- tempfile() + dir.create(tmp_raw) + on.exit(unlink(tmp_raw, recursive = TRUE), add = TRUE) + + pg_set_rawfolder(tmp_raw) + cfg <- test_config() + base <- pgout_path(config = cfg) + dir.create(base, recursive = TRUE) + + tv <- pgvariables$name[!pgvariables$static][1:2] + dates <- pg_dates(cfg) + for (i in seq_along(tv)) { + r <- prio_blank_grid(cfg) + for (j in seq_along(dates)[-1]) terra::add(r) <- prio_blank_grid(cfg) + terra::values(r) <- runif(terra::ncell(r) * terra::nlyr(r)) + names(r) <- as.character(dates) + save_pgvariable(r, tv[i], save_to = base) + } + + build_pg_dataset(config = cfg) + d_ref <- data.table::setDT(dplyr::collect(arrow::open_dataset(file.path(base, "timevarying")))) + + x <- read_pg_timevarying(config = cfg) + expect_s3_class(x, "data.table") + expect_equal(nrow(x), nrow(d_ref)) +}) + +test_that("read_pg_timevarying falls back to legacy pg_timevarying.parquet", { + skip_if_not_installed("arrow") + + tmp_raw <- tempfile() + dir.create(tmp_raw) + on.exit(unlink(tmp_raw, recursive = TRUE), add = TRUE) + + pg_set_rawfolder(tmp_raw) + cfg2 <- pg_config(nrow = 5L, ncol = 10L, + start_date = as.Date("2000-12-31"), + end_date = as.Date("2000-12-31"), + temporal_resolution = "1 year") + base2 <- pgout_path(config = cfg2) + dir.create(base2, recursive = TRUE) + + legacy_df <- data.frame(pgid = 1:3, + measurement_date = as.Date("2000-12-31"), + x = 1:3) + arrow::write_parquet(legacy_df, file.path(base2, "pg_timevarying.parquet")) + + result <- read_pg_timevarying(config = cfg2) + expect_equal(nrow(result), 3L) + expect_equal(sort(result$pgid), 1:3) +}) \ No newline at end of file From af3b01f155f11c387ae7801ed92e13bfa68f15c5 Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Fri, 7 Aug 2026 11:03:15 +0200 Subject: [PATCH 2/7] Add subsetting to read_pg_timevarying --- NAMESPACE | 1 + R/build_priogrid.R | 110 ++++++++++++++++++- README.md | 12 ++- man/build_pg_dataset.Rd | 37 +++++++ man/pg_temporal_unit.Rd | 12 +++ man/read_pg_timevarying.Rd | 26 +++++ tests/testthat/test-build_priogrid.R | 156 +++++++++++++++++++++++++++ vignettes/custom-config.Rmd | 19 ++++ vignettes/getting-started.Rmd | 54 ++++++++-- 9 files changed, 417 insertions(+), 10 deletions(-) create mode 100644 man/build_pg_dataset.Rd create mode 100644 man/pg_temporal_unit.Rd diff --git a/NAMESPACE b/NAMESPACE index 4e0f548..b39b992 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,6 +4,7 @@ S3method(print,pg_config) export(bdist1) export(bdist2) export(bdist3) +export(build_pg_dataset) export(build_release) export(calc_pg) export(calc_traveltime) diff --git a/R/build_priogrid.R b/R/build_priogrid.R index 3229859..f3d2ba2 100644 --- a/R/build_priogrid.R +++ b/R/build_priogrid.R @@ -436,6 +436,76 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { invisible(NULL) } +# Internal: resolve a spatial extent to the set of pgids it covers, using the +# config's grid definition. `extent` is a length-4 numeric c(xmin, xmax, ymin, ymax) +# in lon/lat (EPSG:4326) — the same convention as config$extent. For non-4326 configs +# the extent is reprojected internally (consistent with prio_blank_grid). A terra +# SpatExtent is used as-is in the grid's native CRS. Returns integer(0) on no overlap. +# @keywords internal +.pg_extent_to_pgids <- function(extent, config) { + rlang::check_installed("terra", reason = "to subset PRIO-GRID data by spatial extent") + pg <- prio_blank_grid(config) + if (inherits(extent, "SpatExtent")) { + e <- extent + } else { + nums <- as.numeric(extent) + e <- if (config$crs != "epsg:4326") { + sf::st_transform( + sf::st_bbox(c(xmin = nums[1], xmax = nums[2], ymin = nums[3], ymax = nums[4]), crs = 4326), + crs = config$crs + ) + } else { + terra::ext(nums) + } + } + cropped <- tryCatch(terra::crop(pg, e), error = function(err) NULL) + if (is.null(cropped)) return(integer(0)) + vals <- terra::values(cropped) + as.integer(sort(unique(vals[!is.na(vals)]))) +} + +# Internal: apply optional temporal / spatial / column subsetting to an opened +# arrow time-varying dataset, then collect to a data.table. `has_year` marks whether +# the source exposes the hive `year` partition column (TRUE hive, FALSE legacy monolith). +# @keywords internal +.pg_collect_timevarying <- function(ds, has_year, years = NULL, start_date = NULL, + end_date = NULL, pgids = NULL, extent = NULL, + variables = NULL, config = NULL) { + if (!is.null(variables)) { + bad <- setdiff(variables, names(ds)) + if (length(bad) > 0) + stop("Unknown variable(s) requested: ", paste(bad, collapse = ", "), ".\n", + " Available: ", + paste(setdiff(names(ds), c("pgid", "measurement_date", "year")), collapse = ", "), + call. = FALSE) + } + + if (!is.null(years)) { + sel_years <- as.integer(years) + ds <- if (has_year) dplyr::filter(ds, year %in% sel_years) + else dplyr::filter(ds, lubridate::year(measurement_date) %in% sel_years) + } + if (!is.null(start_date)) { start_dt <- as.Date(start_date); ds <- dplyr::filter(ds, measurement_date >= start_dt) } + if (!is.null(end_date)) { end_dt <- as.Date(end_date); ds <- dplyr::filter(ds, measurement_date <= end_dt) } + + sel_pgids <- if (!is.null(pgids)) as.integer(pgids) else NULL + if (!is.null(extent)) { + if (is.null(config)) + stop("Spatial 'extent' subsetting requires a resolvable config (grid dimensions). ", + "Use release/version mode or pass config=.", call. = FALSE) + ext_pgids <- .pg_extent_to_pgids(extent, config) + sel_pgids <- if (is.null(sel_pgids)) ext_pgids else sort(unique(c(sel_pgids, ext_pgids))) + } + if (!is.null(sel_pgids)) ds <- dplyr::filter(ds, pgid %in% sel_pgids) + + if (!is.null(variables)) { + keep <- c("pgid", "measurement_date", variables, if (has_year) "year") + ds <- dplyr::select(ds, dplyr::any_of(keep)) + } + + data.table::setDT(dplyr::collect(ds)) +} + # Internal admin helper: bulk-bootstrap _checksums.csv for all .rds/.parquet # files in a resolved output folder. Useful after files have been placed without @@ -880,6 +950,17 @@ read_pg_static <- function(config = NULL, #' @param type Character string specifying release type. Default: "05deg_yearly". #' @param spatial_hash Character string with 6-character spatial hash (custom only). #' @param temporal_hash Character string with 6-character temporal hash (custom only). +#' @param years Integer vector of years to keep. NULL (default) keeps all. Prunes hive +#' partitions on the `year` column. +#' @param start_date,end_date Date (or Date-coercible) bounds on `measurement_date` +#' (inclusive). NULL (default) leaves the respective bound open. +#' @param pgids Integer vector of PRIO-GRID cell ids to keep. NULL (default) keeps all cells. +#' @param extent Numeric `c(xmin, xmax, ymin, ymax)` in lon/lat (EPSG:4326), or a terra +#' `SpatExtent` in the grid's native CRS; resolved to the pgids it covers and unioned +#' with `pgids`. For non-4326 configs the lon/lat box is reprojected automatically. +#' Requires `terra`. NULL (default) applies no spatial filter. +#' @param variables Character vector of time-varying variable columns to return (in addition +#' to `pgid`, `measurement_date`, `year`). NULL (default) returns all variables. #' @param as_raster Logical. If TRUE, returns list of SpatRasters. If FALSE #' (default), returns data.table. #' @param test Logical. If TRUE, returns coverage summary data.frame. @@ -903,12 +984,22 @@ read_pg_static <- function(config = NULL, #' # Load custom data #' cfg <- pg_config(nrow = 180, ncol = 360) #' pg_dt <- read_pg_timevarying(config = cfg) +#' +#' # Subset: two years, a bounding box, and one variable +#' pg_dt <- read_pg_timevarying(version = "3.0.1", years = c(2010, 2011), +#' extent = c(10, 12, 50, 52), variables = "cru_tmp") #' } read_pg_timevarying <- function(config = NULL, version = NULL, type = "05deg_yearly", spatial_hash = NULL, temporal_hash = NULL, + years = NULL, + start_date = NULL, + end_date = NULL, + pgids = NULL, + extent = NULL, + variables = NULL, as_raster = FALSE, test = FALSE, overwrite = FALSE, @@ -918,13 +1009,22 @@ read_pg_timevarying <- function(config = NULL, if (!is.null(cfg$warning)) warning(cfg$warning) if (cfg$mode == "release") download_priogrid(version = cfg$version, type = cfg$type) + subset_used <- !is.null(years) || !is.null(start_date) || !is.null(end_date) || + !is.null(pgids) || !is.null(extent) || !is.null(variables) + if (subset_used && (as_raster || test)) + stop("Subsetting arguments (years/start_date/end_date/pgids/extent/variables) are only ", + "supported for the default data.table read, not with as_raster=TRUE or test=TRUE.", + call. = FALSE) + hive_dir <- file.path(cfg$base_path, "timevarying") legacy <- file.path(cfg$base_path, "pg_timevarying.parquet") if (!as_raster && !test && !cfg$overwrite && (dir.exists(hive_dir) || file.exists(legacy))) { if (dir.exists(hive_dir)) { if (isTRUE(verify_checksums)) .pg_verify_hive(cfg$base_path) - return(data.table::setDT(dplyr::collect(arrow::open_dataset(hive_dir)))) + return(.pg_collect_timevarying(arrow::open_dataset(hive_dir), has_year = TRUE, + years = years, start_date = start_date, end_date = end_date, + pgids = pgids, extent = extent, variables = variables, config = cfg$config)) } if (isTRUE(verify_checksums)) { cs_file <- file.path(cfg$base_path, "_checksums.csv") @@ -936,7 +1036,9 @@ read_pg_timevarying <- function(config = NULL, call. = FALSE) } } - return(arrow::read_parquet(legacy)) + return(.pg_collect_timevarying(arrow::open_dataset(legacy), has_year = FALSE, + years = years, start_date = start_date, end_date = end_date, + pgids = pgids, extent = extent, variables = variables, config = cfg$config)) } rlang::check_installed("terra", reason = "to build PRIO-GRID data from individual variables") @@ -975,7 +1077,9 @@ read_pg_timevarying <- function(config = NULL, .pg_build_timevarying(base_path = cfg$base_path, config = data_config, version = cfg$version, type = cfg$type, overwrite = TRUE) - data.table::setDT(dplyr::collect(arrow::open_dataset(hive_dir))) + .pg_collect_timevarying(arrow::open_dataset(hive_dir), has_year = TRUE, + years = years, start_date = start_date, end_date = end_date, + pgids = pgids, extent = extent, variables = variables, config = data_config) } #' Build a PRIO-GRID hive-partitioned dataset diff --git a/README.md b/README.md index fd44a7f..b544aff 100644 --- a/README.md +++ b/README.md @@ -79,10 +79,20 @@ Download the official release and read it into R: ```r download_priogrid() -pg_static <- read_pg_static() +pg_static <- read_pg_static() pg_timevarying <- read_pg_timevarying() ``` +Load only the rows and columns you need — filters push down to Arrow before anything is collected: + +```r +pg_sub <- read_pg_timevarying( + years = 2010:2015, + extent = c(xmin = -20, xmax = 55, ymin = -35, ymax = 40), + variables = "cru_tmp" +) +``` + Browse available variables: ```r diff --git a/man/build_pg_dataset.Rd b/man/build_pg_dataset.Rd new file mode 100644 index 0000000..6f46d93 --- /dev/null +++ b/man/build_pg_dataset.Rd @@ -0,0 +1,37 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/build_priogrid.R +\name{build_pg_dataset} +\alias{build_pg_dataset} +\title{Build a PRIO-GRID hive-partitioned dataset} +\usage{ +build_pg_dataset( + config = NULL, + version = NULL, + type = "05deg_yearly", + spatial_hash = NULL, + temporal_hash = NULL, + overwrite = FALSE +) +} +\arguments{ +\item{config}{A \code{pg_config} object for custom data, or NULL (default) for the +official release.} + +\item{version}{Character string specifying PRIOGRID version (release mode only).} + +\item{type}{Character string specifying release type. Default: "05deg_yearly".} + +\item{spatial_hash}{6-character MD5 hash of spatial options (advanced use).} + +\item{temporal_hash}{6-character MD5 hash of temporal options (advanced use).} + +\item{overwrite}{Logical. If TRUE, rebuilds even if output already exists.} +} +\value{ +Invisibly returns the base output path. +} +\description{ +Memory-safe alternative to \code{read_pg_timevarying()} that writes the hive +partitioned parquet dataset and CSV bundle without loading the full table. +Useful for manual rebuild and re-upload workflows. +} diff --git a/man/pg_temporal_unit.Rd b/man/pg_temporal_unit.Rd new file mode 100644 index 0000000..52744dd --- /dev/null +++ b/man/pg_temporal_unit.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utility.R +\name{pg_temporal_unit} +\alias{pg_temporal_unit} +\title{Temporal unit for a config's resolution} +\usage{ +pg_temporal_unit(config = pg_current_config()) +} +\description{ +Temporal unit for a config's resolution +} +\keyword{internal} diff --git a/man/read_pg_timevarying.Rd b/man/read_pg_timevarying.Rd index c6f8851..9a23534 100644 --- a/man/read_pg_timevarying.Rd +++ b/man/read_pg_timevarying.Rd @@ -10,6 +10,12 @@ read_pg_timevarying( type = "05deg_yearly", spatial_hash = NULL, temporal_hash = NULL, + years = NULL, + start_date = NULL, + end_date = NULL, + pgids = NULL, + extent = NULL, + variables = NULL, as_raster = FALSE, test = FALSE, overwrite = FALSE, @@ -28,6 +34,22 @@ official release.} \item{temporal_hash}{Character string with 6-character temporal hash (custom only).} +\item{years}{Integer vector of years to keep. NULL (default) keeps all. Prunes hive +partitions on the \code{year} column.} + +\item{start_date, end_date}{Date (or Date-coercible) bounds on \code{measurement_date} +(inclusive). NULL (default) leaves the respective bound open.} + +\item{pgids}{Integer vector of PRIO-GRID cell ids to keep. NULL (default) keeps all cells.} + +\item{extent}{Numeric \code{c(xmin, xmax, ymin, ymax)} in lon/lat (EPSG:4326), or a terra +\code{SpatExtent} in the grid's native CRS; resolved to the pgids it covers and unioned +with \code{pgids}. For non-4326 configs the lon/lat box is reprojected automatically. +Requires \code{terra}. NULL (default) applies no spatial filter.} + +\item{variables}{Character vector of time-varying variable columns to return (in addition +to \code{pgid}, \code{measurement_date}, \code{year}). NULL (default) returns all variables.} + \item{as_raster}{Logical. If TRUE, returns list of SpatRasters. If FALSE (default), returns data.table.} @@ -65,5 +87,9 @@ The mode is determined by the \code{config} argument: # Load custom data cfg <- pg_config(nrow = 180, ncol = 360) pg_dt <- read_pg_timevarying(config = cfg) + + # Subset: two years, a bounding box, and one variable + pg_dt <- read_pg_timevarying(version = "3.0.1", years = c(2010, 2011), + extent = c(10, 12, 50, 52), variables = "cru_tmp") } } diff --git a/tests/testthat/test-build_priogrid.R b/tests/testthat/test-build_priogrid.R index f42af76..0f6e9f8 100644 --- a/tests/testthat/test-build_priogrid.R +++ b/tests/testthat/test-build_priogrid.R @@ -291,4 +291,160 @@ test_that("read_pg_timevarying falls back to legacy pg_timevarying.parquet", { result <- read_pg_timevarying(config = cfg2) expect_equal(nrow(result), 3L) expect_equal(sort(result$pgid), 1:3) +}) + +# --------------------------------------------------------------------------- +# read_pg_timevarying subsetting tests +# --------------------------------------------------------------------------- + +.make_timevarying_fixture <- function() { + tmp_raw <- tempfile() + dir.create(tmp_raw) + pg_set_rawfolder(tmp_raw) + cfg <- test_config() + base <- pgout_path(config = cfg) + dir.create(base, recursive = TRUE) + + tv <- pgvariables$name[!pgvariables$static][1:2] + dates <- pg_dates(cfg) + for (i in seq_along(tv)) { + r <- prio_blank_grid(cfg) + for (j in seq_along(dates)[-1]) terra::add(r) <- prio_blank_grid(cfg) + terra::values(r) <- runif(terra::ncell(r) * terra::nlyr(r)) + names(r) <- as.character(dates) + save_pgvariable(r, tv[i], save_to = base) + } + build_pg_dataset(config = cfg) + list(cfg = cfg, base = base, tv = tv, tmp_raw = tmp_raw) +} + +test_that("read_pg_timevarying years filter returns only requested years", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + x <- read_pg_timevarying(config = fx$cfg, years = 2011) + expect_s3_class(x, "data.table") + expect_equal(unique(x$year), 2011L) + full <- read_pg_timevarying(config = fx$cfg) + expect_true(nrow(x) < nrow(full)) +}) + +test_that("read_pg_timevarying date range filter keeps only matching rows", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + x <- read_pg_timevarying(config = fx$cfg, + start_date = as.Date("2011-01-01"), + end_date = as.Date("2011-12-31")) + expect_true(all(lubridate::year(x$measurement_date) == 2011L)) +}) + +test_that("read_pg_timevarying pgids filter keeps only requested cells", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + x <- read_pg_timevarying(config = fx$cfg, pgids = c(1L, 2L, 3L)) + expect_true(all(x$pgid %in% 1:3)) +}) + +test_that("read_pg_timevarying extent filter returns non-empty subset of valid pgids", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + ext <- c(-180, -90, -90, 0) + x <- read_pg_timevarying(config = fx$cfg, extent = ext) + expected_pgids <- priogrid:::.pg_extent_to_pgids(ext, fx$cfg) + expect_true(nrow(x) > 0) + expect_true(all(x$pgid %in% expected_pgids)) +}) + +test_that("read_pg_timevarying variables filter projects to requested columns", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + x <- read_pg_timevarying(config = fx$cfg, variables = fx$tv[1]) + expect_setequal(names(x), c("pgid", "measurement_date", "year", fx$tv[1])) +}) + +test_that("read_pg_timevarying combined filters all hold simultaneously", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + x <- read_pg_timevarying(config = fx$cfg, + years = 2011, pgids = 1:3, variables = fx$tv[1]) + expect_true(all(x$pgid %in% 1:3)) + expect_equal(unique(x$year), 2011L) + expect_setequal(names(x), c("pgid", "measurement_date", "year", fx$tv[1])) +}) + +test_that("read_pg_timevarying errors when subsetting with as_raster=TRUE", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + expect_error( + read_pg_timevarying(config = fx$cfg, years = 2011, as_raster = TRUE), + "only" + ) +}) + +test_that("read_pg_timevarying errors on unknown variable name", { + skip_if_not_installed("arrow") + skip_if_not_installed("terra") + + fx <- .make_timevarying_fixture() + on.exit(unlink(fx$tmp_raw, recursive = TRUE), add = TRUE) + + expect_error( + read_pg_timevarying(config = fx$cfg, variables = "nope"), + "Unknown variable" + ) +}) + +test_that(".pg_extent_to_pgids reprojects lon/lat extent for non-4326 configs", { + skip_if_not_installed("terra") + skip_if_not_installed("sf") + + # UTM zone 32N over a small regional extent. Resolution is defined in 4326; + # terra determines the projected raster's actual dimensions, which may differ + # from nrow/ncol — that is expected behaviour. + cfg_proj <- pg_config( + nrow = 10L, ncol = 20L, + crs = "epsg:32632", + extent = c(xmin = 0, xmax = 10, ymin = 44, ymax = 48) + ) + + # Find which pgids exist in the projected grid. + pg <- prio_blank_grid(cfg_proj) + valid_pgids <- as.integer(sort(unique(terra::values(pg)[!is.na(terra::values(pg))]))) + skip_if(length(valid_pgids) == 0L, "projection yielded no cells") + + # A lon/lat box covering the full config extent. + # Before the fix: raw degree values (0..10, 44..48) were passed to terra::crop + # as UTM metre coordinates — a ~10 m × ~10 m box near the origin — returning + # integer(0). After the fix the lon/lat box is reprojected to UTM metres first. + result <- priogrid:::.pg_extent_to_pgids(c(0, 10, 44, 48), cfg_proj) + expect_type(result, "integer") + expect_true(length(result) > 0L) + expect_true(all(result %in% valid_pgids)) }) \ No newline at end of file diff --git a/vignettes/custom-config.Rmd b/vignettes/custom-config.Rmd index 2cb579e..a767eb3 100644 --- a/vignettes/custom-config.Rmd +++ b/vignettes/custom-config.Rmd @@ -204,6 +204,25 @@ pg_static_custom <- read_pg_static(config = cfg) pg_tv_custom <- read_pg_timevarying(config = cfg) ``` +All `read_pg_timevarying()` subsetting arguments (`years`, `start_date`, `end_date`, `pgids`, +`extent`, `variables`) work identically with custom configs. `extent` is always expressed in +lon/lat (EPSG:4326), matching the `config$extent` convention — reprojection is handled +automatically for non-4326 configs: + +```{r read-custom-subset, eval=FALSE} +cfg <- pg_config(nrow = 180, ncol = 360) +# calc_pg(varnames = c("cru_tmp", "ucdp_ged"), config = cfg) + +# This will only work if you have run the calc_pg function above (it should take some minutes to complete). +# The first time read_pg_timevarying runs, it will pull all variables and build a Parquet Hive. This also takes some time to finish. +# Subsequent calls to the function will be fast. +pg_tv_sub <- read_pg_timevarying( + config = cfg, + years = 2015, + variables = c("cru_tmp", "ucdp_ged") +) +``` + ## Release vs. Custom Mode Summary | Feature | Release mode (`config = NULL`) | Custom mode (`config = cfg`) | diff --git a/vignettes/getting-started.Rmd b/vignettes/getting-started.Rmd index de80be2..bd22d21 100644 --- a/vignettes/getting-started.Rmd +++ b/vignettes/getting-started.Rmd @@ -100,16 +100,58 @@ head(pg_tv) These tables can be joined together by `pgid`, or merged with your own data. -### Example: subsetting and merging +### Subsetting at read time -```{r subset-example, eval=FALSE} +All six filter arguments default to `NULL` (no filter — full dataset). When supplied, filters are +pushed down to Arrow before any data is collected, so only the requested rows and columns touch +memory. Temporal and spatial constraints compose as AND; `pgids` and `extent` compose as union. + +| Argument | Type | Effect | +|---|---|---| +| `years` | integer vector | Keep only the listed calendar years (prunes hive partitions) | +| `start_date` / `end_date` | Date | Inclusive bounds on `measurement_date` | +| `pgids` | integer vector | Keep specific PRIO-GRID cell IDs | +| `extent` | `c(xmin, xmax, ymin, ymax)` in lon/lat | Bounding box resolved to cell IDs (requires `terra`) | +| `variables` | character vector | Return only the named variable columns | + +```{r read-subsets, eval=FALSE} +# A single year +pg_2020 <- read_pg_timevarying(years = 2020) + +# Inclusive date range +pg_decade <- read_pg_timevarying( + start_date = as.Date("2010-01-01"), + end_date = as.Date("2019-12-31") +) + +# Spatial bounding box (Sub-Saharan Africa) +pg_africa <- read_pg_timevarying( + extent = c(xmin = -20, xmax = 55, ymin = -35, ymax = 40) +) + +# One variable column only +pg_tmp <- read_pg_timevarying(variables = "cru_tmp") + +# Combined: two years, one region, two variables +pg_sub <- read_pg_timevarying( + years = c(2010, 2015), + extent = c(xmin = -20, xmax = 55, ymin = -35, ymax = 40), + variables = c("cru_tmp", "ucdp_ged") +) +``` + +### Example: merging static and time-varying data + +```{r merge-example, eval=FALSE} library(data.table) -# Subset to a specific year -pg_2020 <- pg_tv[measurement_date == as.Date("2020-12-31")] +# Load a specific year and region with push-down, then merge with static data +pg_tv <- read_pg_timevarying( + years = 2020, + extent = c(xmin = -20, xmax = 55, ymin = -35, ymax = 40) +) -# Join static and time-varying on pgid -pg_merged <- merge(pg_static, pg_2020, by = "pgid") +pg_merged <- merge(pg_static, pg_tv, by = "pgid") ``` ## Browsing Available Variables From 8def0b147bbb8dbb2bd395a073d96eddffe0fdec Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Fri, 7 Aug 2026 14:46:03 +0200 Subject: [PATCH 3/7] use COG GeoTIFF with proper metadata instead of RDS for all variable results. --- R/build_priogrid.R | 283 ++++++++++++++++++++++----- R/data.R | 9 +- data/pgvariables.rda | Bin 1394 -> 1660 bytes data_raw/variables.csv | 78 ++++---- tests/testthat/test-build_priogrid.R | 20 +- tests/testthat/test-save-load.R | 6 +- 6 files changed, 300 insertions(+), 96 deletions(-) diff --git a/R/build_priogrid.R b/R/build_priogrid.R index f3d2ba2..94c8327 100644 --- a/R/build_priogrid.R +++ b/R/build_priogrid.R @@ -96,7 +96,7 @@ pgout_path <- function(version = NULL, #' #' Calculates PRIO-GRID variables based on current config and saves them #' to disk. Each variable is computed by calling its corresponding gen_*() function -#' and saved as an .rds file in the custom data folder. +#' and saved as a Cloud-Optimized GeoTIFF (COG) under the \code{cog/} sub-directory. #' #' @param varnames Character vector with variable names from [pgvariables]. #' If NULL (default), calculates all available variables. @@ -165,7 +165,8 @@ calc_pg <- function(varnames = NULL, overwrite = FALSE, config = pg_current_conf ), config_file) } - existing_files <- list.files(save_to) |> tools::file_path_sans_ext() + existing_files <- list.files(file.path(save_to, "cog"), pattern = "\\.tif$") |> + tools::file_path_sans_ext() if (!overwrite) { valid_varnames <- valid_varnames[!valid_varnames %in% existing_files] @@ -221,15 +222,40 @@ calc_pg <- function(varnames = NULL, overwrite = FALSE, config = pg_current_conf invisible(NULL) } +# Internal: build the named pg_* GDAL metatag vector for a variable from the +# current pgvariables row. Drop NA/empty fields so callers can call +# terra::metags(r) <- .pg_build_metatags(varname) directly. +# @keywords internal +.pg_build_metatags <- function(varname) { + meta <- pgvariables[pgvariables$name == varname, ] + if (nrow(meta) == 0L) stop("varname '", varname, "' not found in pgvariables.") + c( + pg_name = meta$name, + pg_static = as.character(meta$static), + if (!is.na(meta$label) && nzchar(meta$label)) c(pg_label = meta$label) else NULL, + if (!is.na(meta$unit) && nzchar(meta$unit)) c(pg_unit = meta$unit) else NULL, + if (!is.na(meta$transform) && nzchar(meta$transform)) c(pg_transform = meta$transform) else NULL, + if (!is.na(meta$palette) && nzchar(meta$palette)) c(pg_palette = meta$palette) else NULL, + if (!is.na(meta$plot_type) && nzchar(meta$plot_type)) c(pg_plot_type = meta$plot_type) else NULL, + if (!is.na(meta$value_min)) c(pg_value_min = as.character(meta$value_min)) else NULL, + if (!is.na(meta$value_max)) c(pg_value_max = as.character(meta$value_max)) else NULL + ) +} + #' Save a PRIO-GRID variable #' -#' Saves a terra SpatRaster as a wrapped .rds file. The variable must be -#' listed in [pgvariables]. +#' Saves a terra SpatRaster as a Cloud-Optimized GeoTIFF (COG) under the +#' \code{cog/} sub-directory of the dataset root. The variable must be listed +#' in [pgvariables]. Layer names, \code{time()}, and \code{units()} are +#' preserved natively inside the \code{.tif}. Plot metadata from [pgvariables] +#' (\code{label}, \code{unit}, \code{transform}, \code{palette}, +#' \code{plot_type}, \code{value_min}, \code{value_max}) is stamped as +#' \code{pg_*} GDAL metatags so the COG is self-describing. #' -#' @param rast Terra SpatRaster object from a gen_*() function -#' @param varname Character string with the variable name (must exist in pgvariables) -#' @param save_to Character string with folder path. Defaults to current custom -#' data location based on config. +#' @param rast Terra SpatRaster object from a gen_*() function. +#' @param varname Character string with the variable name (must exist in pgvariables). +#' @param save_to Character string with the dataset root path. Defaults to current +#' custom data location based on config. #' #' @return NULL (invisibly). Called for side effects (saving file). #' @export @@ -249,14 +275,42 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { stop("Output directory does not exist: ", save_to) } - filepath <- file.path(save_to, paste0(varname, ".rds")) + cog_dir <- file.path(save_to, "cog") + if (!dir.exists(cog_dir)) dir.create(cog_dir, recursive = TRUE) - if (!inherits(rast, "PackedSpatRaster")) { - rast <- terra::wrap(rast) + filepath <- file.path(cog_dir, paste0(varname, ".tif")) + + meta <- pgvariables[pgvariables$name == varname, ] + + # Stamp time() for time-varying rasters whose layer names are ISO dates + if (!isTRUE(meta$static) && all(is.na(terra::time(rast)))) { + parsed <- tryCatch(as.Date(names(rast)), error = function(e) NULL) + if (!is.null(parsed) && !any(is.na(parsed))) + terra::time(rast) <- parsed } - saveRDS(rast, filepath) + # Stamp units from pgvariables + if (!is.na(meta$unit) && nzchar(meta$unit)) + terra::units(rast) <- meta$unit + + terra::metags(rast) <- .pg_build_metatags(varname) + + terra::writeRaster(rast, filepath, filetype = "COG", overwrite = TRUE, + gdal = c("COMPRESS=DEFLATE", "NUM_THREADS=ALL_CPUS")) + + # Remove redundant sidecar (COG is self-describing) + aux <- paste0(filepath, ".aux.json") + if (file.exists(aux)) file.remove(aux) + .pg_record_checksum(filepath, varname, save_to) + + # Defensive: checksum categorical .aux.xml if terra wrote one + if (any(terra::is.factor(rast))) { + aux_xml <- paste0(filepath, ".aux.xml") + if (file.exists(aux_xml)) + .pg_record_checksum(aux_xml, file.path("cog", paste0(varname, ".tif.aux.xml")), save_to) + } + invisible(NULL) } @@ -277,6 +331,48 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { invisible(NULL) } +# Internal: re-stamp pg_* GDAL metatags (and terra units) on an existing COG +# from the current pgvariables row, without recomputing the raster data. +# Use this after editing data_raw/variables.csv + regenerating pgvariables.rda +# to propagate display metadata (label, palette, transform, …) into built COGs. +# @param varname Variable name; must exist in pgvariables and in the cog/ dir. +# @param save_to Dataset root (same convention as save_pgvariable). +# @return Path to the restamped COG, invisibly. +# @keywords internal +.pg_restamp_metatags <- function(varname, save_to = pgout_path()) { + rlang::check_installed("terra", reason = "to restamp COG metatags") + if (!varname %in% pgvariables$name) { + stop("varname '", varname, "' not found in pgvariables.") + } + filepath <- file.path(save_to, "cog", paste0(varname, ".tif")) + if (!file.exists(filepath)) { + stop("COG not found at: ", filepath, "\n Run calc_pg() first.") + } + + r <- terra::rast(filepath) + + # Sync terra units: set if non-empty, clear otherwise + meta <- pgvariables[pgvariables$name == varname, ] + terra::units(r) <- if (!is.na(meta$unit) && nzchar(meta$unit)) meta$unit else "" + + terra::metags(r) <- .pg_build_metatags(varname) + + # Write to a sibling temp file (same filesystem → rename is atomic), then + # replace the original. terra cannot write a lazy raster back to its own path. + tmp_path <- tempfile(tmpdir = dirname(filepath), fileext = ".tif") + on.exit(if (file.exists(tmp_path)) file.remove(tmp_path), add = TRUE) + terra::writeRaster(r, tmp_path, filetype = "COG", overwrite = TRUE, + gdal = c("COMPRESS=DEFLATE", "NUM_THREADS=ALL_CPUS")) + file.rename(tmp_path, filepath) + + aux <- paste0(filepath, ".aux.json") + if (file.exists(aux)) file.remove(aux) + + .pg_record_checksum(filepath, varname, save_to) + + invisible(filepath) +} + # Internal: memory-bounded hive-partitioned builder for time-varying data. # Writes one parquet partition per temporal slice following Apache Hive nested layout: # year=YYYY/ for yearly; year=YYYY/=VV/ for sub-yearly. Also writes a gzipped CSV @@ -304,12 +400,12 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { # ----- Phase 1: per-variable long parquet (date-sorted) ----- tv <- dplyr::filter(pgvariables, !static)$name - present <- tv[file.exists(file.path(base_path, paste0(tv, ".rds")))] + present <- tv[file.exists(file.path(base_path, "cog", paste0(tv, ".tif")))] if (length(present) == 0L) - stop("No time-varying .rds rasters found in ", base_path) + stop("No time-varying COGs found in ", file.path(base_path, "cog")) for (v in present) { - r <- terra::unwrap(readRDS(file.path(base_path, paste0(v, ".rds")))) + r <- terra::rast(file.path(base_path, "cog", paste0(v, ".tif"))) dt <- rast_to_df(r, static = FALSE, varname = v, config = config) data.table::setorder(dt, measurement_date) arrow::write_parquet(dt, file.path(tmp_dir, paste0(v, ".parquet")), @@ -371,6 +467,15 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { setdiff(names(arrow::open_dataset(static_path)), "pgid") else character(0L) + all_vars <- c(static_vars, present) + variables_meta <- lapply(all_vars, function(vn) { + row <- pgvariables[pgvariables$name == vn, ] + if (nrow(row) == 0L) return(list(name = vn)) + as.list(row[1L, c("name", "static", "source_ids", "label", "unit", + "transform", "palette", "plot_type", "value_min", "value_max")]) + }) + names(variables_meta) <- all_vars + meta <- list( priogrid_version = version %||% as.character(utils::packageVersion("priogrid")), type = type %||% "custom", @@ -398,12 +503,14 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { dataset_root = paste(version %||% as.character(utils::packageVersion("priogrid")), type %||% "custom", sep = "/"), layout = list( + cog = "cog/.tif", timevarying = "timevarying//part-0.parquet", static = "pg_static.parquet", csv = list(timevarying = "pg_timevarying.csv.gz", static = "pg_static.csv.gz")), static_variables = static_vars, timevarying_variables = present, + variables = variables_meta, partitions = partitions) json_path <- file.path(base_path, "pg_config.json") @@ -507,7 +614,7 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { } -# Internal admin helper: bulk-bootstrap _checksums.csv for all .rds/.parquet +# Internal admin helper: bulk-bootstrap _checksums.csv for all .tif/.parquet # files in a resolved output folder. Useful after files have been placed without # going through save_pgvariable() / read_pg_static() / read_pg_timevarying(). # @@ -530,10 +637,32 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { stop("Output directory does not exist: ", base_path, call. = FALSE) } - target_files <- list.files(base_path, pattern = "\\.(rds|parquet)$", full.names = FALSE) + # Root-level parquet files (label = filename, e.g. "pg_static.parquet") + parquet_files <- list.files(base_path, pattern = "\\.parquet$", full.names = FALSE) + parquet_entries <- if (length(parquet_files) > 0L) + data.frame(fname = parquet_files, + label = parquet_files, + stringsAsFactors = FALSE) + else + data.frame(fname = character(0), label = character(0), stringsAsFactors = FALSE) + + # COG .tif files under cog/ (label = varname without extension) + cog_dir <- file.path(base_path, "cog") + cog_files <- if (dir.exists(cog_dir)) + list.files(cog_dir, pattern = "\\.tif$", full.names = FALSE) + else + character(0) + cog_entries <- if (length(cog_files) > 0L) + data.frame(fname = file.path("cog", cog_files), + label = tools::file_path_sans_ext(cog_files), + stringsAsFactors = FALSE) + else + data.frame(fname = character(0), label = character(0), stringsAsFactors = FALSE) + + entries <- rbind(parquet_entries, cog_entries) - if (length(target_files) == 0) { - message("No .rds or .parquet files found in: ", base_path) + if (nrow(entries) == 0L) { + message("No .tif or .parquet files found in: ", base_path) return(invisible(list(n_added = 0L, n_updated = 0L))) } @@ -547,8 +676,9 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { n_added <- 0L n_updated <- 0L - for (fname in target_files) { - label <- if (grepl("\\.rds$", fname)) tools::file_path_sans_ext(fname) else fname + for (i in seq_len(nrow(entries))) { + fname <- entries$fname[i] + label <- entries$label[i] already_exists <- label %in% existing_labels if (already_exists && !overwrite_existing) next @@ -567,7 +697,7 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { "Bootstrap complete for: ", base_path, "\n", " Added: ", n_added, " new checksum(s)\n", " Updated: ", n_updated, " existing checksum(s)\n", - " Skipped: ", length(target_files) - n_added - n_updated, + " Skipped: ", nrow(entries) - n_added - n_updated, " already-present checksum(s) (use overwrite_existing=TRUE to force)" ) @@ -577,15 +707,20 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { #' Load a PRIO-GRID variable #' -#' Loads a PRIO-GRID variable from disk and returns it as a terra SpatRaster. -#' The mode is determined by the `config` argument: +#' Loads a PRIO-GRID variable from disk and returns it as a lazy, file-backed +#' terra SpatRaster (no data is loaded into memory until the raster is explicitly +#' materialized). Supports optional lazy subsetting by layers and spatial windowing, +#' enabling a larger-than-memory workflow. #' -#' - `config = NULL` (default): loads from the official release (downloads if needed). -#' - `config = pg_config(...)`: loads from custom data built with that config. -#' - `spatial_hash` + `temporal_hash`: loads from a specific custom folder by hash. +#' Mode is determined by the \code{config} argument: +#' \itemize{ +#' \item \code{config = NULL} (default): loads from the official release (downloads if needed). +#' \item \code{config = pg_config(...)}: loads from custom data built with that config. +#' \item \code{spatial_hash} + \code{temporal_hash}: loads from a specific custom folder by hash. +#' } #' #' @param varname Character string with the variable name. -#' @param config A `pg_config` object for custom data, or NULL (default) for the +#' @param config A \code{pg_config} object for custom data, or NULL (default) for the #' official release. #' @param version Character string specifying PRIOGRID version (e.g., "3.0.1"). #' Only used in release mode (config = NULL). Defaults to current package version. @@ -597,8 +732,17 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { #' spatial_hash. #' @param verify_checksums Logical. If TRUE, verifies the file's MD5 checksum #' against stored values. Default FALSE. -#' -#' @return Terra SpatRaster object +#' @param extent Optional spatial window. Either a \code{SpatExtent} (used as-is in +#' the raster's CRS) or a length-4 numeric vector \code{c(xmin, xmax, ymin, ymax)} +#' in EPSG:4326. Applied lazily via \code{terra::window()} — no data is loaded +#' until the raster is materialized. Enables efficient reading of spatial subsets +#' from large COGs (larger-than-memory workflow). +#' @param layers Optional layer selector. A character vector of layer names or an +#' integer vector of layer indices. Applied lazily via \code{terra::subset()}. +#' +#' @return A lazy, file-backed \code{SpatRaster}. Data is not loaded into memory +#' until explicitly materialized (e.g., via \code{terra::values()} or +#' \code{terra::crop()}). #' @export #' #' @examples @@ -617,6 +761,10 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { #' r <- load_pgvariable("cshapes_gwcode", #' spatial_hash = "ecf4dd", #' temporal_hash = "727cca") +#' +#' # Windowed / lazy load (larger-than-memory) +#' r <- load_pgvariable("cru_tmp", extent = c(-30, 60, 35, 72), layers = 1:12) +#' terra::inMemory(r) # FALSE #' } load_pgvariable <- function(varname, config = NULL, @@ -624,10 +772,12 @@ load_pgvariable <- function(varname, type = "05deg_yearly", spatial_hash = NULL, temporal_hash = NULL, - verify_checksums = FALSE) { + verify_checksums = FALSE, + extent = NULL, + layers = NULL) { rlang::check_installed("terra", reason = "to load PRIO-GRID variable rasters") - has_hashes <- !is.null(spatial_hash) && !is.null(temporal_hash) + has_hashes <- !is.null(spatial_hash) && !is.null(temporal_hash) has_one_hash <- xor(!is.null(spatial_hash), !is.null(temporal_hash)) if (has_one_hash) { @@ -641,36 +791,26 @@ load_pgvariable <- function(varname, } if (has_hashes) { - # Custom mode: explicit hashes - filepath <- file.path( - pgout_path(spatial_hash = spatial_hash, temporal_hash = temporal_hash), - paste0(varname, ".rds") - ) + base_dir <- pgout_path(spatial_hash = spatial_hash, temporal_hash = temporal_hash) } else if (!is.null(config)) { - # Custom mode: config-driven if (isTRUE(config$verbose)) { message("Loading custom data for config (", get_spatial_hash(config), "/", get_temporal_hash(config), ")") } - filepath <- file.path( - pgout_path(spatial_hash = get_spatial_hash(config), temporal_hash = get_temporal_hash(config)), - paste0(varname, ".rds") - ) + base_dir <- pgout_path(spatial_hash = get_spatial_hash(config), temporal_hash = get_temporal_hash(config)) } else { - # Release mode resolved_version <- version %||% as.character(packageVersion("priogrid")) download_priogrid(version = resolved_version, type = type) - filepath <- file.path( - pgout_path(version = resolved_version, type = type), - paste0(varname, ".rds") - ) + base_dir <- pgout_path(version = resolved_version, type = type) } + filepath <- file.path(base_dir, "cog", paste0(varname, ".tif")) + if (!file.exists(filepath)) { stop("Variable '", varname, "' not found at: ", filepath) } if (isTRUE(verify_checksums)) { - cs_file <- file.path(dirname(filepath), "_checksums.csv") + cs_file <- file.path(base_dir, "_checksums.csv") if (file.exists(cs_file)) { cs_df <- utils::read.csv(cs_file, stringsAsFactors = FALSE) expected <- cs_df[cs_df$varname == varname, ] @@ -689,7 +829,50 @@ load_pgvariable <- function(varname, } } - terra::unwrap(readRDS(filepath)) + r <- terra::rast(filepath) + + # time() fallback for hand-placed COGs whose layer names are ISO dates + # (save_pgvariable already stamps time(); this covers external files) + static_flag <- pgvariables$static[pgvariables$name == varname] + if (length(static_flag) == 1L && !isTRUE(static_flag) && all(is.na(terra::time(r)))) { + parsed <- tryCatch(as.Date(names(r)), error = function(e) NULL) + if (!is.null(parsed) && !any(is.na(parsed))) + terra::time(r) <- parsed + } + + # Lazy layer subsetting + if (!is.null(layers)) { + if (is.character(layers)) { + bad <- setdiff(layers, names(r)) + if (length(bad) > 0L) + stop("Layer name(s) not found in '", varname, "': ", paste(bad, collapse = ", ")) + } else { + if (any(layers < 1L | layers > terra::nlyr(r))) + stop("Layer index out of range for '", varname, "' (1:", terra::nlyr(r), ")") + } + r <- terra::subset(r, layers) + } + + # Lazy spatial windowing + if (!is.null(extent)) { + if (inherits(extent, "SpatExtent")) { + e <- extent + } else { + nums <- as.numeric(extent) + e <- if (!terra::is.lonlat(r)) { + rlang::check_installed("sf", reason = "to reproject spatial extent for non-4326 rasters") + bbox <- sf::st_bbox(c(xmin = nums[1], xmax = nums[2], + ymin = nums[3], ymax = nums[4]), crs = 4326) + bbox_t <- as.numeric(sf::st_transform(bbox, crs = terra::crs(r))) + terra::ext(bbox_t[1], bbox_t[3], bbox_t[2], bbox_t[4]) + } else { + terra::ext(nums) + } + } + terra::window(r) <- e + } + + r } #' Resolve PRIO-GRID data mode and paths @@ -1231,7 +1414,7 @@ pg_list_custom <- function() { ) cfg <- env$cfg - n_vars <- length(list.files(folder, pattern = "\\.rds$")) + n_vars <- length(list.files(file.path(folder, "cog"), pattern = "\\.tif$")) created <- format(file.info(config_files[i])$mtime, "%Y-%m-%d") cat(sprintf("[%d] %s/%s: nrow=%d, ncol=%d, %s to %s, %d vars, created %s\n", diff --git a/R/data.R b/R/data.R index 863bff5..d9d1da4 100644 --- a/R/data.R +++ b/R/data.R @@ -43,11 +43,18 @@ #' for the data depending on whether the data is static or with a temporal-dimension. #' #' @format ## `pgvariables` -#' A data frame with 4 rows and 5 columns: +#' A data frame with 38 rows and 10 columns: #' \describe{ #' \item{name}{Full name of the variable.} #' \item{static}{Boolean. Whether the variable varies over time or not.} #' \item{source_ids}{Comma-separated string. The Unique Universal Identifier (UUID) of the sources used to build the variable.} +#' \item{label}{Display title for the variable; defaults to the variable name.} +#' \item{unit}{Measurement unit (e.g., "°C", "mm"); \code{NA} if not applicable.} +#' \item{transform}{Display transform applied before plotting; one of \code{identity}, \code{log1p}, \code{log10}, or \code{sqrt}.} +#' \item{palette}{Color palette name for visualization. Default: \code{viridis}.} +#' \item{plot_type}{Plot type hint; one of \code{continuous}, \code{categorical}, or \code{diverging}.} +#' \item{value_min}{Suggested color-scale minimum; \code{NA} for data-driven scaling.} +#' \item{value_max}{Suggested color-scale maximum; \code{NA} for data-driven scaling.} #' } "pgvariables" diff --git a/data/pgvariables.rda b/data/pgvariables.rda index d4d4e6df886ca89fa32cdd75d1a9b268dd37103f..a070601cfe3340b359bf7e7bd27c272597843bf3 100644 GIT binary patch literal 1660 zcmV-?27~!RT4*^jL0KkKS-NCJ(*Ov||HA+O|HMTPf8Yfe7C^u6-{3$101yBH;0M2b z51;@6^a6kY030;aBN2qt0Sp3ZgkYGN4Kg$|2MsjH#9=hRLjam#7$zn|OpOfyl&PVi zrh`U+00001paG_U4NX(>kZ1q^00w{=0}ucJGE{niXaE{C02%-QXaS%!1xhA@VlqI~ z!5RvBo|7okH4jK+(-UPql-i~LPRSHqXPpkbWhpJj=@p63E4IG0Nqqv8fRI$%XzAcl zBo!e9Irx`o3M2MhQ_x66B!O&FlRK$I+DQ}h_J2Rp(H-0RCH9WeNv;E(LH>t*rUXC4#L@S?^Lbds=<2IzE;J;=I zm_#wd!MgSgz4lp}_6sbci=EmK7%&JBQG^CI0tN%%f{TT;HcC^ZQ=TL!{u0>U<`-3s zjQwNRNfJ)Dk_sde;Y309(tlsNsJx0KM0Dv`q9{gGOHlzWLO&WWqDj_KFcRWPg4`1M9!ZdgkV znc&f?mK$F>+anmg%M7?{AJ%qHa@c1WJo3pIZ?Y{i z&Bvx~^lQV1G;rq~+-6GS$z*3|7|LAAHBD(6QuJ48aw|=!;-s!MX@awQL!~9+mZ31v zZGUXRMzNz7$lf(_FWW1& z%J((#$swc7o!nyc5Svn-dTN*6uu#l{ctqx5K_BIOia@Ur$x(>JEm-`*;ZZ^JJ|+#Yef zTZPHXIz{($wN>{$@$N?^f{4jRY8)H~Bx~0HZ-uYRxjOD&MyRuyL)7|}?>HNyiDmB) z{T0c9iyT&Ue#d(DOd1_3eh&2gc5&y_@xYaCi=}pWM}6L?@M~X{yfEFJ4;DzdA9{1z zGu_V57X2r)3>umuvHKlye`?${(`!Xz3IdCw9YS@em+_Z4ptn$>NE`qrFF0U%Lw~dqaVz5JGa;I8(J{(#;OUpD*^hi}9 z$?;(OLb)F!-oa&wA7Vo16p@n24#Sfp(sFeBm0;=Q@C=QTI4aEh)acbCb47E!(K0mb zWV&2bRYPVTD=$$LQAYyjr$3|oFk*b(E1NT3RH{Qng_0jrZJV!ll@kP_LiOv)v2{9< zKg{6B=+z-q%kX(WieB=6t0vFQSlQND?zumL=UR__j8u_iQb%H@NxO;H*yzDh!xb2f z88|M<+>p7BPS-6MGD|1bt^BEnZF?Op5WfhAxy6a_a8=F4gNf9hkJNlwxW9u5hkwb< zuPP|2d{;R-?9qb7Q&*{t&&P)z_j9c=bas_&aY$7tkja8ad$rWFn-gbJLd_G&syuFZ zGU)9$NPp9?$)S4X5*az0FrzZ`_9Q8i^%h>42=tOf`OnMibRIG{r7m6n;_gVN3KADg Gh`L}WBQzrb literal 1394 zcmV-&1&#VbT4*^jL0KkKS*@TqJpcsUf5QL&|HMTPf8Yfe7C^u6-@rfs01yBH&;{Oh zu;Pk}IZ`4GI7%QuA)_jOnrWoTk*T#Y28N9Q9+PAsl8L6!n3_U>&;S4dpa1{?DXMu! zs5CUmrc9a!AO?nw0j7Y^(V@*UG8r&KMiT^SqedWUlOq5D1d#;O6wOT~KPE)f%|Lp9 z41tghs0|X}4+FJ>X{Kd&2?jDyT1nUmFos}WERo{L9?$tfhHWB(43HF|NHYuh?fi0r z^;q^%2+=?%YA1s<9Q|1(r96PvVFD8{z-p!%WCee!Kb$wT{=mwt?DxlED$9K`9t^j^ zT+2!%iTa%Ap9eWr>eGjGhllM>N;W(fao(A{QrDU(RLL7LYZ@NCpH1zeSgR1GHaD6R?krxb#6l^10(gA&`8ozFi-|Xq*)k57AA&4G6qqJ zN6L{#QxS!PS}y*&kxB1;Rm&GNGr8~hWle7t05vHSLZhGtxvl1+85x(1r&N4b1=NIS zhH0x;N7LWID32_Wk;n&)POY`LjwFptZXj0i97W%9(J3(xYLLnY(T3l-*rKh*0%A3t z&IqBdSINauw>_CrZqbVfr6nN}#-bplLfont(5&aMiR_L|l!vv*2&4-lOCm_9s^nqo3c_PLOszGc_V)c{} zIA$U?Oou9Bgd&SeMD|0VG343jI)}sh+uXa zg)XX)`MQ8vLr@`b2!S;+j}YWKQq0)R7dCN}He(dY&NNM|3X=jfz~F8yYS#x&BR3y; zuE^wyKv&HWHmRf$S0S`SXa@{%16tlCoQxR{t^`Q46{Uy)nJ58$YH%uyBSNU!Pj(>a zbZ#Z<@#b}R?D%)pxavC9Y`?6pufdOcx;WkpR_m#-;=VI1V^v$R&vHwr3Tw0m2x{4f zNZAX}#Dk=0m?cym+FZe$rnGm6(69`OHCW8d4Fq56_hB5kn?REm5gaO55fmX|<#H#B zE_nz>?0c;r940JblumY&;-t+0PHG|v^f{xh9Ffes{1N9pYR5_lfRD*<)fL|Zp|2I) z)iUbB9Ucb+OWGry?W|DfoWPvG$r7lrF$wD{f)eR!6%@@(G?$w^%d~!eVfa70{!)qD zkKcgIAvn2KoO+<)rjRZ!aPrq9-i;8}hLuKMbViOwjLVvSijF(FTsY|qcdZR5sWH^`FbW+RZ+Fi_~>)5C*UWK8=gnDo8VB?mM2 z0LYEJuyc`<1~K{)TGiECU=b4m8A;f+Oa%)giBefnNCO4H{H_t|v5hb34E7*~a|V*0 zQj2ZqruI-@=v~|QJ!^KU-fD@MrkJHPa+>6yw1EqGvQ0v(6JXkL%?6A~-ot!T=A?yr z1`~>qco^`t#L*^r(=%*^$98fOjvET6QYQ)_Lwx1j7_uAbIRE1ANT&)C6|@HDpfxpl A`~Uy| diff --git a/data_raw/variables.csv b/data_raw/variables.csv index 0aacc97..ab38600 100644 --- a/data_raw/variables.csv +++ b/data_raw/variables.csv @@ -1,39 +1,39 @@ -name static source_ids -cru_tmp FALSE ac037134-3567-49d9-a3ba-64f37c1ee698 -cru_pre FALSE 00575260-ad1c-4e87-a575-3922bc151f50 -cru_pet FALSE 95399c70-7db4-47f0-95e5-2e279b6b2054 -cshapes_cover_share FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b -cshapes_gwcode FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b -geoepr_reg_excluded FALSE 287bfdf7-2f4f-402a-88df-5fe1f8b7046b, 3900b527-a728-4c26-b0ab-f4441d3ee2e8 -bdist1 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b -bdist2 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b -bdist3 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b -ghsl_population_grid FALSE ae6a7612-4bef-452f-acd6-d2212cf9a7c5 -hilda_cropland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_forest FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_grassland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_ocean FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_pasture FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_sparse FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_urban FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -hilda_water FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 -linight_mean FALSE d99fbea7-2a01-4221-b900-29a58d33f591 -naturalearth_cover TRUE 92da9800-4520-4e87-a855-b28255452189 -naturalearth_cover_share TRUE 92da9800-4520-4e87-a855-b28255452189 -ruggedterrain_elevation_mean TRUE 8c8192eb-cc29-4598-8f8a-ec190ba35c2d -traveltime_mean TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 -traveltime_min TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 -geopko_troops_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 -geopko_operations_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 -ne_disputed_area_share TRUE 920663ad-d7e7-4528-b36d-4b7266def2b1 -speibase6_mean FALSE 14839384-623a-4cf7-9241-6166a8ac465b, 95399c70-7db4-47f0-95e5-2e279b6b2054, 00575260-ad1c-4e87-a575-3922bc151f50 -ghs_wup_degurba_urban FALSE 7f1f60a3-6664-4427-b086-b5359ebf45b7 -ucdp_ged FALSE 49f79d96-4e4d-4812-9dd1-862bacfca577 -shdi FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c -msch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c -esch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c -lifexp FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c -gnic FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c -side_excluded FALSE e42b30e3-75da-4dd4-a375-0d6557087804 -side_included FALSE e42b30e3-75da-4dd4-a375-0d6557087804 -side_irrelevant FALSE e42b30e3-75da-4dd4-a375-0d6557087804 \ No newline at end of file +name static source_ids label unit transform palette plot_type value_min value_max +cru_tmp FALSE ac037134-3567-49d9-a3ba-64f37c1ee698 cru_tmp identity viridis continuous +cru_pre FALSE 00575260-ad1c-4e87-a575-3922bc151f50 cru_pre identity viridis continuous +cru_pet FALSE 95399c70-7db4-47f0-95e5-2e279b6b2054 cru_pet identity viridis continuous +cshapes_cover_share FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b cshapes_cover_share identity viridis continuous +cshapes_gwcode FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b cshapes_gwcode identity viridis continuous +geoepr_reg_excluded FALSE 287bfdf7-2f4f-402a-88df-5fe1f8b7046b, 3900b527-a728-4c26-b0ab-f4441d3ee2e8 geoepr_reg_excluded identity viridis continuous +bdist1 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b bdist1 identity viridis continuous +bdist2 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b bdist2 identity viridis continuous +bdist3 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b bdist3 identity viridis continuous +ghsl_population_grid FALSE ae6a7612-4bef-452f-acd6-d2212cf9a7c5 ghsl_population_grid identity viridis continuous +hilda_cropland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_cropland identity viridis continuous +hilda_forest FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_forest identity viridis continuous +hilda_grassland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_grassland identity viridis continuous +hilda_ocean FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_ocean identity viridis continuous +hilda_pasture FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_pasture identity viridis continuous +hilda_sparse FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_sparse identity viridis continuous +hilda_urban FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_urban identity viridis continuous +hilda_water FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_water identity viridis continuous +linight_mean FALSE d99fbea7-2a01-4221-b900-29a58d33f591 linight_mean identity viridis continuous +naturalearth_cover TRUE 92da9800-4520-4e87-a855-b28255452189 naturalearth_cover identity viridis continuous +naturalearth_cover_share TRUE 92da9800-4520-4e87-a855-b28255452189 naturalearth_cover_share identity viridis continuous +ruggedterrain_elevation_mean TRUE 8c8192eb-cc29-4598-8f8a-ec190ba35c2d ruggedterrain_elevation_mean identity viridis continuous +traveltime_mean TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 traveltime_mean identity viridis continuous +traveltime_min TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 traveltime_min identity viridis continuous +geopko_troops_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 geopko_troops_count identity viridis continuous +geopko_operations_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 geopko_operations_count identity viridis continuous +ne_disputed_area_share TRUE 920663ad-d7e7-4528-b36d-4b7266def2b1 ne_disputed_area_share identity viridis continuous +speibase6_mean FALSE 14839384-623a-4cf7-9241-6166a8ac465b, 95399c70-7db4-47f0-95e5-2e279b6b2054, 00575260-ad1c-4e87-a575-3922bc151f50 speibase6_mean identity viridis continuous +ghs_wup_degurba_urban FALSE 7f1f60a3-6664-4427-b086-b5359ebf45b7 ghs_wup_degurba_urban identity viridis continuous +ucdp_ged FALSE 49f79d96-4e4d-4812-9dd1-862bacfca577 ucdp_ged identity viridis continuous +shdi FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c shdi identity viridis continuous +msch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c msch identity viridis continuous +esch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c esch identity viridis continuous +lifexp FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c lifexp identity viridis continuous +gnic FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c gnic identity viridis continuous +side_excluded FALSE e42b30e3-75da-4dd4-a375-0d6557087804 side_excluded identity viridis continuous +side_included FALSE e42b30e3-75da-4dd4-a375-0d6557087804 side_included identity viridis continuous +side_irrelevant FALSE e42b30e3-75da-4dd4-a375-0d6557087804 side_irrelevant identity viridis continuous diff --git a/tests/testthat/test-build_priogrid.R b/tests/testthat/test-build_priogrid.R index 0f6e9f8..709de01 100644 --- a/tests/testthat/test-build_priogrid.R +++ b/tests/testthat/test-build_priogrid.R @@ -135,7 +135,10 @@ test_that(".pg_bootstrap_checksums adds entries for existing files", { out_path <- pgout_path(spatial_hash = s_hash, temporal_hash = t_hash) dir.create(out_path, recursive = TRUE) - saveRDS(list(x = 1), file.path(out_path, "fake_var.rds")) + dir.create(file.path(out_path, "cog")) + terra::writeRaster(terra::rast(nrows = 2L, ncols = 2L, vals = 1:4), + file.path(out_path, "cog", "fake_var.tif"), + filetype = "COG", overwrite = TRUE) result <- priogrid:::.pg_bootstrap_checksums(config = cfg) @@ -159,7 +162,10 @@ test_that(".pg_bootstrap_checksums skips existing entries by default", { out_path <- pgout_path(spatial_hash = s_hash, temporal_hash = t_hash) dir.create(out_path, recursive = TRUE) - saveRDS(list(x = 1), file.path(out_path, "fake_var.rds")) + dir.create(file.path(out_path, "cog")) + terra::writeRaster(terra::rast(nrows = 2L, ncols = 2L, vals = 1:4), + file.path(out_path, "cog", "fake_var.tif"), + filetype = "COG", overwrite = TRUE) priogrid:::.pg_bootstrap_checksums(config = cfg) result2 <- priogrid:::.pg_bootstrap_checksums(config = cfg) @@ -183,7 +189,7 @@ test_that(".pg_build_timevarying writes hive partitions, CSV bundle, and manifes base <- pgout_path(config = cfg) dir.create(base, recursive = TRUE) - # Fabricate two time-varying .rds rasters + # Fabricate two time-varying COG rasters tv <- pgvariables$name[!pgvariables$static][1:2] dates <- pg_dates(cfg) @@ -231,6 +237,14 @@ test_that(".pg_build_timevarying writes hive partitions, CSV bundle, and manifes expect_true(setequal(m$timevarying_variables, tv)) expect_true(setequal(m$partitions, c("year=2010", "year=2011", "year=2012"))) + # COG files written by save_pgvariable + for (vn in tv) { + expect_true(file.exists(file.path(base, "cog", paste0(vn, ".tif")))) + } + # Manifest carries variables metadata + expect_true("variables" %in% names(m)) + expect_true(all(tv %in% names(m$variables))) + # CSV bundle row count matches hive csv <- data.table::fread(file.path(base, "pg_timevarying.csv.gz")) expect_equal(nrow(csv), nrow(d)) diff --git a/tests/testthat/test-save-load.R b/tests/testthat/test-save-load.R index d2c66f7..4f2c3e0 100644 --- a/tests/testthat/test-save-load.R +++ b/tests/testthat/test-save-load.R @@ -12,11 +12,11 @@ test_that("save_pgvariable and load_pgvariable round-trip works", { on.exit(unlink(tmp, recursive = TRUE), add = TRUE) save_pgvariable(r, varname, save_to = tmp) - expect_true(file.exists(file.path(tmp, paste0(varname, ".rds")))) + expect_true(file.exists(file.path(tmp, "cog", paste0(varname, ".tif")))) - loaded <- terra::unwrap(readRDS(file.path(tmp, paste0(varname, ".rds")))) + loaded <- terra::rast(file.path(tmp, "cog", paste0(varname, ".tif"))) expect_s4_class(loaded, "SpatRaster") - expect_equal(terra::values(loaded), terra::values(r)) + expect_equal(terra::values(loaded), terra::values(r), tolerance = 1e-5) }) test_that("save_pgvariable errors for unknown varname", { From 13e6b0a4dad528062bddf3800a1ae1f2d2e0b2c5 Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Fri, 7 Aug 2026 14:46:39 +0200 Subject: [PATCH 4/7] vignette: update contributing info with description of internal helpers++ --- vignettes/contributing.Rmd | 363 ++++++++++++++++++++++++++++++++++--- 1 file changed, 333 insertions(+), 30 deletions(-) diff --git a/vignettes/contributing.Rmd b/vignettes/contributing.Rmd index 417ce86..7fcc1a7 100644 --- a/vignettes/contributing.Rmd +++ b/vignettes/contributing.Rmd @@ -20,20 +20,21 @@ This tutorial walks through the process of adding a new data source and variable priogrid/ ├── R/ │ ├── data_{source}.R # read_*() and gen_*() functions for each source -│ ├── config.R # pg_config(), pg_set_config(), pg_set_rawfolder() +│ ├── config.R # pg_config(), pg_set_config(), pg_set_rawfolder(), .pg_release_specs │ ├── build_priogrid.R # calc_pg(), load_pgvariable(), read_pg_static(), etc. │ ├── utility.R # robust_transformation(), rast_to_df(), prio_blank_grid() │ ├── references.R # pgcitations(), get_bibliography() │ ├── source_class.R # Source R6 class (dev-only: used in devtools::load_all()) │ └── download_data.R # pgsearch(), pg_rawfiles(), check_pgsourcefiles() ├── data_raw/ -│ ├── sources.csv # Add new source here (54 entries) -│ ├── variables.csv # Add new variable here (32 entries) +│ ├── sources.csv # Add new source here (tab-separated) +│ ├── variables.csv # Add new variable here (tab-separated) │ ├── pgsources.R # Regenerates data/pgsources.rda from sources.csv │ └── pgvariables.R # Regenerates data/pgvariables.rda from variables.csv ├── data/ │ ├── pgsources.rda # Compiled metadata (auto-generated, do not edit) -│ └── pgvariables.rda # Compiled metadata (auto-generated, do not edit) +│ ├── pgvariables.rda # Compiled metadata (auto-generated, do not edit) +│ └── pgchecksum.rda # MD5 reference checksums (auto-generated) ├── inst/ │ ├── REFERENCES.bib # Full bibliography — add new BibTeX entries here │ └── extdata/urls/ # Multi-file URL lists (one .txt per source) @@ -45,7 +46,11 @@ priogrid/ ### Add to `data_raw/sources.csv` -Every data source needs a row in `data_raw/sources.csv`. Required fields: +`sources.csv` is **tab-separated** with 18 columns. The intended path is not hand-editing but +the `Source` class + `add_source()` dev workflow (see below), which fills `id`, `created_at`, +and the three `*_exists` booleans automatically. + +Required fields: | Column | Description | Example | |---|---|---| @@ -60,7 +65,9 @@ Every data source needs a row in `data_raw/sources.csv`. Required fields: Optional fields: `aws_bucket`, `aws_region`, `prio_mirror`, `tags`, `reference_keys`. -Generate a UUID for the `id` column using `uuid::UUIDgenerate()`: +The following columns are auto-generated by `Source$to_tibble()` and should **not** be +hand-written: `id`, `download_url_exists`, `website_url_exists`, `prio_mirror_exists`, +`created_at`. If you are writing a row by hand (not via `add_source()`), generate a UUID first: ```{r gen-uuid, eval=FALSE} uuid::UUIDgenerate() @@ -69,7 +76,8 @@ uuid::UUIDgenerate() ### Use the `Source` Class (dev mode) -In development mode (`devtools::load_all()`), use the `Source` R6 class to validate a new source before adding it to the CSV: +In development mode (`devtools::load_all()`), use the `Source` R6 class to validate a new +source before adding it to the CSV: ```{r source-class, eval=FALSE} new_source <- Source$new( @@ -86,7 +94,23 @@ new_source <- Source$new( new_source # prints validation report; warns if citation key not in REFERENCES.bib ``` -The `Source` class validates required fields, checks that `spatial_extent` and `temporal_resolution` use the allowed vocabulary, and verifies that citation keys exist in `inst/REFERENCES.bib`. +The `Source` class validates required fields, checks that `spatial_extent` and +`temporal_resolution` use the allowed vocabulary, and verifies that citation keys exist in +`inst/REFERENCES.bib`. Use `new_source$get_existing_tags()` and +`new_source$get_existing_licenses()` to inspect vocabulary already in use and keep tags and +licenses consistent across sources. + +Once validated, write the row to `sources.csv` with `add_source()`: + +```{r add-source, eval=FALSE} +add_source(new_source) +# Appends a tab-separated row to data_raw/sources.csv and writes +# inst/extdata/urls/{id}.txt if download_url or prio_mirror points at a URL list. +``` + +`add_source(source, csv_file = "data_raw/sources.csv")` is dev-only (requires +`devtools::load_all()`). It appends the row and calls `new_source$save_url_files()` +automatically. After running it, regenerate `pgsources.rda` as below. ### Add the Citation to `inst/REFERENCES.bib` @@ -106,7 +130,7 @@ Add a BibTeX entry to `inst/REFERENCES.bib` for each `citation_keys` value: ### Regenerate `pgsources.rda` -After editing `sources.csv`, regenerate the `.rda` file: +After adding a source, regenerate the `.rda` file: ```{r regen-sources, eval=FALSE} source("data_raw/pgsources.R") @@ -118,13 +142,36 @@ Verify the result: pgsources[pgsources$source_name == "My New Dataset", ] ``` +### Download the Raw Files + +Before writing `read_*()`/`gen_*()`, fetch the source's files. Set the raw-data folder first +if you haven't already: + +```{r set-rawfolder, eval=FALSE} +pg_set_rawfolder("/path/to/rawdata") +``` + +Then download and verify: + +```{r download-rawdata, eval=FALSE} +files <- pg_rawfiles() |> dplyr::filter(id == "a1b2c3d4-...") +download_pg_rawdata(file_info = files) +pg_data_availability() # confirm n_present == n_files +``` + +`download_pg_rawdata()` is resumable (`.part` files), batched (`batch_size = 20`, +`max_concurrent = 4`), retries with backoff (`max_retry = 10`), and verifies MD5 against +`pgchecksum` advisorily. See the [metadata vignette](metadata.html) for the discovery helpers +(`pgsearch`, `pg_rawfiles`, `pg_data_availability`). + ## Step 2: Write `read_*()` and `gen_*()` Functions Create a new file `R/data_mynewsource.R`. ### `read_*()` — Load Raw Data -The read function downloads (via `get_pgfile()`) and returns the raw data as an `sf` or `SpatRaster` object: +The read function downloads (via `get_pgfile()`) and returns the raw data as an `sf` or +`SpatRaster` object: ```{r read-function, eval=FALSE} #' Read My New Dataset @@ -145,11 +192,13 @@ read_mynewsource <- function() { } ``` -`get_pgfile()` automatically downloads the file if it is not present locally (when `automatic_download = TRUE` in the config). +`get_pgfile()` automatically downloads the file if it is not present locally (when +`automatic_download = TRUE` in the config). ### `gen_*()` — Generate the PRIOGRID Variable -The generate function transforms raw data to the PRIOGRID grid and returns a `SpatRaster`. The signature must accept `config`: +The generate function transforms raw data to the PRIOGRID grid and returns a `SpatRaster`. The +signature must accept `config`: ```{r gen-function, eval=FALSE} #' My New Variable @@ -183,7 +232,8 @@ gen_my_new_variable <- function(config = pg_current_config()) { ### Using `robust_transformation()` -`robust_transformation()` handles reprojection, cropping, aggregation, disaggregation, and final resampling in a single call. It is the standard way to re-grid any raster: +`robust_transformation()` handles reprojection, cropping, aggregation, disaggregation, and +final resampling in a single call. It is the standard way to re-grid any raster: ```{r robust-transform, eval=FALSE} robust_transformation( @@ -215,17 +265,31 @@ gen_my_yearly_variable <- function(config = pg_current_config()) { ## Step 3: Register the Variable -Add a row to `data_raw/variables.csv`: +### Add to `data_raw/variables.csv` + +`variables.csv` is **tab-separated** with 10 columns. Add one row per variable: + +| Column | Description / Allowed values | +|---|---| +| `name` | Must match the string set by `names(r)` in `gen_*()` and equal `gen_{name}` without the prefix | +| `static` | `TRUE` for variables without a time dimension, `FALSE` for time-varying | +| `source_ids` | Comma-separated source UUIDs from `sources.csv` | +| `label` | Display title shown in plots; set equal to `name` (no code default exists) | +| `unit` | Unit string (e.g. `°C`, `mm`); leave empty if not applicable | +| `transform` | One of: `identity`, `log1p`, `log10`, `sqrt` | +| `palette` | Palette name (e.g. `viridis`); default `viridis` | +| `plot_type` | One of: `continuous`, `categorical`, `diverging` | +| `value_min` | Numeric lower display bound, or empty for data-driven scaling | +| `value_max` | Numeric upper display bound, or empty for data-driven scaling | + +A hand-written row must include all 10 tab-separated fields; empty fields still need their tab +delimiter. Example (tabs shown as `→`): ``` -name,static,source_ids -my_new_variable,FALSE,a1b2c3d4-e5f6-7890-abcd-ef1234567890 +name→static→source_ids→label→unit→transform→palette→plot_type→value_min→value_max +my_new_variable→FALSE→a1b2c3d4-e5f6-7890-abcd-ef1234567890→my_new_variable→→identity→viridis→continuous→→ ``` -- `name` must match the string passed to `names(r)` in `gen_*()` and equal `gen_{name}` without the prefix. -- `static` is `TRUE` for variables without a time dimension, `FALSE` for time-varying. -- `source_ids` is the UUID from `sources.csv`. Multiple sources are comma-separated. - Regenerate `pgvariables.rda`: ```{r regen-vars, eval=FALSE} @@ -238,6 +302,18 @@ Verify: pgvariables[pgvariables$name == "my_new_variable", ] ``` +### Display Metadata and COG Metatags + +The seven display columns (`label`, `unit`, `transform`, `palette`, `plot_type`, `value_min`, +`value_max`) are stamped into each built COG as `pg_*` GDAL metatags by `save_pgvariable()` +via `.pg_build_metatags()`, making the `.tif` self-describing. + +After editing display columns for an **already-built** variable, re-stamp without recomputing: + +```{r restamp, eval=FALSE} +priogrid:::.pg_restamp_metatags("my_new_variable") +``` + ## Step 4: Document with roxygen2 PRIOGRID uses roxygen2 with Markdown support and Rdpack for citation references. @@ -256,23 +332,25 @@ devtools::document() ## Step 5: Add Tests -Add tests to `tests/testthat/`. For a gen_* function, the minimal test: +Add tests to `tests/testthat/`. For a `gen_*` function, use the shared test helpers from +`tests/testthat/helper-priogrid.R`: `test_config()` (nrow=5, ncol=10, 2010–2012 yearly) and +`skip_if_no_rawdata()`. A `gen_*` test needs downloaded raw data, so it must call +`skip_if_no_rawdata()`: ```{r test-example, eval=FALSE} # tests/testthat/test-data_mynewsource.R test_that("gen_my_new_variable returns a SpatRaster", { skip_if_not_installed("terra") - cfg <- pg_config( - nrow = 36, ncol = 72, # 5-degree resolution for fast tests - start_date = as.Date("2010-12-31"), - end_date = as.Date("2010-12-31") - ) - r <- gen_my_new_variable(config = cfg) + skip_if_no_rawdata() + r <- gen_my_new_variable(config = test_config()) expect_s4_class(r, "SpatRaster") expect_equal(names(r), "my_new_variable") }) ``` +Internal accessors (hashes, `resolve_pg_mode`) are tested via `priogrid:::` — see +`tests/testthat/test-build_priogrid.R` for examples. + Run tests: ```{r run-tests, eval=FALSE} @@ -301,14 +379,239 @@ pg_tv <- read_pg_timevarying(config = cfg) pgcitations("my_new_variable") ``` +## Multi-file Sources and URL Lists + +When a source serves multiple files — or when URLs do not carry a usable filename (e.g. +FigShare download endpoints) — place a plain-text URL list at `inst/extdata/urls/{id}.txt`. +Each line is one URL; an optional **tab-separated second column** names the local file: + +``` +https://example.com/files/data_part1.nc +https://ndownloader.figshare.com/files/17626052 my_local_name.nc +``` + +The second column is only needed when the server does not expose the filename via +Content-Disposition or the post-redirect URL. Helpers: `pg_read_url_list()`, +`pg_format_url_list()`, `pg_default_filename()`. `add_source()` calls +`new_source$save_url_files()` automatically and writes the file when `download_url` or +`prio_mirror` is a local list path. + +**Resolving opaque filenames.** `priogrid:::pg_resolve_filenames(id, write = FALSE)` queries +each server (Content-Disposition header, then post-redirect URL) and prints a proposed filename +column, flagging suspicious names (no extension, login endpoints). Re-run with `write = TRUE` +to commit the names, then rebuild and run `pg_update_checksums()`. + +```{r resolve-filenames, eval=FALSE} +priogrid:::pg_resolve_filenames("a1b2c3d4-e5f6-7890-abcd-ef1234567890", write = FALSE) +# Review proposed names, then commit: +priogrid:::pg_resolve_filenames("a1b2c3d4-e5f6-7890-abcd-ef1234567890", write = TRUE) +``` + +**Renaming already-downloaded files.** When a source gains an explicit filename column, use +`priogrid:::pg_migrate_rawfiles()` to rename already-downloaded files in place so they are not +re-downloaded: + +```{r migrate-rawfiles, eval=FALSE} +priogrid:::pg_migrate_rawfiles(dry_run = TRUE) # preview renames +priogrid:::pg_migrate_rawfiles(dry_run = FALSE) # apply +``` + +## The Build Pipeline: From gen_*() to a Dataset + +### `calc_pg()` and output layout + +`calc_pg(varnames = NULL, overwrite = FALSE, config)` calls each `gen_(config=)` and +passes the result to `save_pgvariable()`. Outputs land under: + +``` +/priogrid/custom//// +``` + +The folder is created on first build and includes a `_config.R` script (a reproducible +`pg_config()` call). Skips already-built variables unless `overwrite = TRUE`; per-variable +failures are collected and reported, never fatal. + +After `calc_pg()`, the folder contains: + +- `cog/.tif` — Cloud Optimised GeoTIFF (COG, DEFLATE) with `time()`/`units()` fields + and `pg_*` GDAL metatags +- `_checksums.csv` — per-file MD5s +- `_config.R` — reproducible config script + +**Spatial/temporal hashing.** `pgout_path()` keys output folders on 6-character MD5 hashes +derived from grid dimensions/CRS/extent (`get_spatial_hash()`) and from the date range +(`get_temporal_hash()`). The temporal hash floors `end_date` so small date changes do not fork +folders into new locations. + +### `load_pgvariable()` + +```{r load-pgvariable, eval=FALSE} +r <- load_pgvariable( + varname = "my_new_variable", + config = cfg, # NULL => release mode (auto-download_priogrid()) + extent = c(-20, 50, -10, 40), # optional: windowed spatial read + layers = 1:5 # optional: subset time layers +) +``` + +Returns a **lazy, file-backed** `SpatRaster`; `extent`/`layers` enable windowed +larger-than-memory reads. + +### Table readers + +```{r table-readers, eval=FALSE} +# Static variables → pg_static.parquet + pg_static.csv.gz +static <- read_pg_static(config = cfg) + +# Time-varying → hive-partitioned parquet + pg_timevarying.csv.gz + pg_config.json manifest +# Supports lazy subsetting: +tv <- read_pg_timevarying( + config = cfg, + years = 2000:2010, + variables = c("my_new_variable", "another_var"), + extent = c(-20, 50, -10, 40) +) +``` + +`build_pg_dataset()` writes the hive dataset memory-safely without collecting the full table +into RAM. + +### Listing custom builds + +```{r list-custom, eval=FALSE} +customs <- pg_list_custom() +# Prints a summary of each custom folder; returns an indexed list of pg_config objects. +read_pg_static(config = customs[[1]]) +calc_pg("new_var", config = customs[[2]]) +``` + +### Memory management for large rasters + +`pg_configure_terra_memory()` (called inside `robust_transformation()`) auto-enables `terra` +on-disk processing when an estimated raster exceeds ~4 GB. This is relevant when developing +high-resolution `gen_*()` functions. + +## File Integrity and Checksums + +`pgchecksum` is a bundled data object (`data/pgchecksum.rda`) with reference MD5s for the +exact raw files used to build the official release. Columns: `source_name`, `source_version`, +`id`, `filename`, `md5`. + +**Runtime verification.** `check_pgsourcefiles()` compares local raw files to `pgchecksum` and +warns for files with no reference entry (sources added since the last tested build). Enable +per-read verification with: + +```{r verify-checksums, eval=FALSE} +cfg <- pg_config(verify_checksums = TRUE) +# or per file: +get_pgfile(source_name = "My New Dataset", source_version = "1.0", + id = "a1b2c3d4-...", verify_checksums = TRUE) +``` + +**Regenerating checksums (maintainer).** After a fully verified, clean download set, recompute +and rewrite `pgchecksum.rda`: + +```{r update-checksums, eval=FALSE} +priogrid:::pg_update_checksums(only_present = TRUE) +``` + +New sources have no checksum entry until PRIO-GRID has been built from them once and this is +run. + +## Cutting an Official Release + +A release is registered in two places, both keyed `"_"`: + +**1. Grid spec** — add an entry to `.pg_release_specs` in `R/config.R`: + +```{r release-spec, eval=FALSE} +# Inside .pg_release_specs in R/config.R: +"3.0.2_05deg_yearly" = list( + nrow = 360L, + ncol = 720L, + crs = "epsg:4326", + extent = c(xmin = -180, xmax = 180, ymin = -90, ymax = 90), + temporal_resolution = "1 year", + start_date = as.Date("1850-12-31"), + end_date = as.Date("2026-08-26") +) +``` + +**2. CDN URL** — add the corresponding entry to the `releases` list inside +`download_priogrid()` in `R/build_priogrid.R`: + +```{r release-url, eval=FALSE} +# Inside the releases list in download_priogrid() in R/build_priogrid.R: +"3.0.2_05deg_yearly" = "https://cdn.cloud.prio.org/files/" +``` + +Then build the release: + +```{r build-release, eval=FALSE} +build_release( + version = "3.0.2", + type = "05deg_yearly", + nrow = 360, + ncol = 720, + crs = "epsg:4326", + extent = c(xmin = -180, xmax = 180, ymin = -90, ymax = 90), + temporal_resolution = "1 year", + start_date = as.Date("1850-12-31"), + end_date = as.Date("2026-08-26") +) +``` + +`build_release()` calculates all variables to the custom location, builds static and hive +time-varying tables, copies the output to `releases///`, and writes two zip +archives: `priogrid__.zip` and `priogrid___csv.zip`. + +Retrieve a release config programmatically, or list all published releases: + +```{r release-config, eval=FALSE} +cfg <- pg_release_config("3.0.2") +download_priogrid(list_releases = TRUE) +``` + +## Developer Utilities Reference + +| Function | Purpose | Location | +|---|---|---| +| `Source` | R6 class: validate and serialise a new source | `R/source_class.R` | +| `add_source()` | Append validated source to `sources.csv`; write URL list file | `R/utility.R` | +| `priogrid:::pg_resolve_filenames()` | Query servers for filenames of opaque URLs | `R/download_data.R` | +| `priogrid:::pg_migrate_rawfiles()` | Rename downloaded files after explicit filename column is added | `R/download_data.R` | +| `priogrid:::pg_update_checksums()` | Recompute MD5s and rewrite `pgchecksum.rda` | `R/download_data.R` | +| `priogrid:::.pg_restamp_metatags()` | Re-stamp GDAL `pg_*` metatags without recomputing a variable | `R/build_priogrid.R` | +| `priogrid:::.pg_build_metatags()` | Build the metatag list from `pgvariables` for `save_pgvariable()` | `R/build_priogrid.R` | +| `get_spatial_hash()` / `get_temporal_hash()` | 6-char MD5 hashes that key output folders | `R/build_priogrid.R` | +| `resolve_pg_mode()` | Determine release vs. custom output path | `R/build_priogrid.R` | +| `pg_configure_terra_memory()` | Auto-enable on-disk processing for large rasters | `R/config.R` | +| `create_pg_indices()` | Create PRIOGRID cell indices | `R/utility.R` | +| `rast_to_df()` | Convert a SpatRaster to a data frame | `R/utility.R` | +| `pg_dates()` | Sequence of dates for a config | `R/utility.R` | +| `pg_date_intervals()` | Date intervals for a config | `R/utility.R` | + +Functions marked `priogrid:::` lack `@export`; call them with the triple-colon operator in dev +mode (`devtools::load_all()`). + ## Summary Checklist -- [ ] Add row to `data_raw/sources.csv` with UUID, license, URLs, citation keys +### Contributor Checklist + +- [ ] Use `Source$new()` to validate the source, then `add_source()` to write the row to `data_raw/sources.csv` - [ ] Add BibTeX entry to `inst/REFERENCES.bib` - [ ] Run `source("data_raw/pgsources.R")` to regenerate `pgsources.rda` +- [ ] Run `download_pg_rawdata()`, then `pg_data_availability()` to confirm all files are present - [ ] Create `R/data_{source}.R` with `read_*()` and `gen_*()` functions -- [ ] Add row to `data_raw/variables.csv` +- [ ] Add row to `data_raw/variables.csv` (tab-separated, all 10 columns) - [ ] Run `source("data_raw/pgvariables.R")` to regenerate `pgvariables.rda` - [ ] Run `devtools::document()` to regenerate `NAMESPACE` and `man/` -- [ ] Add tests in `tests/testthat/` +- [ ] Add tests in `tests/testthat/`; call `skip_if_no_rawdata()` in `gen_*` tests - [ ] Verify with `calc_pg()` → `load_pgvariable()` → `pgcitations()` + +### Maintainer Checklist + +- [ ] After a verified clean download, regenerate checksums: `priogrid:::pg_update_checksums()` +- [ ] For opaque-URL sources, resolve filenames: `priogrid:::pg_resolve_filenames(id, write = TRUE)` +- [ ] Register `.pg_release_specs` entry in `R/config.R` and CDN URL in `download_priogrid()` in `R/build_priogrid.R` +- [ ] Cut the release: `build_release(version, type, nrow, ncol, crs, extent, temporal_resolution, start_date, end_date)` From 3c06f4032be4f7a24338913e0536ae67f0b3559d Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Mon, 10 Aug 2026 08:50:17 +0200 Subject: [PATCH 5/7] Update vignettes --- README.md | 3 ++- vignettes/custom-config.Rmd | 1 + vignettes/getting-started.Rmd | 21 ++++++++++++++------- vignettes/metadata.Rmd | 9 +++++++++ vignettes/working-with-rasters.Rmd | 2 +- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b544aff..6ea3ef7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ An R-package for collecting and standardizing open spatial data into a common gr - **R, not SQL** — More researchers know R, and the package leverages excellent spatial-data infrastructure with `sf`, `terra`, and `exactextractr`. - **Flexible spatio-temporal configuration** — Change resolution, extent, and projection to test the modifiable areal unit problem or create tailored datasets (e.g., area-equal projections for polar regions). - **PRIOGRID is a research tool, not just a dataset.** +- **Efficient, self-describing outputs** — Variables ship as Cloud-Optimized GeoTIFFs and time-varying tables as Hive-partitioned Parquet, so `read_pg_timevarying()` pushes year/date/cell/variable filters down to Arrow before loading. ## Installation @@ -29,7 +30,7 @@ renv::install("prio-data/priogrid") ### Optional R Packages -`terra`, `sf`, `exactextractr`, and `arrow` are listed in `Suggests` and are **not installed automatically**. They are only required for specific functionality (working with rasters) and will be requested the first time you use a function that needs them. +`terra`, `sf`, and `exactextractr` are listed in `Suggests` and are **not installed automatically**. They are only required for spatial functionality (working with rasters, and `extent`-based subsetting) and will be requested the first time you use a function that needs them. ### Troubleshooting Installation diff --git a/vignettes/custom-config.Rmd b/vignettes/custom-config.Rmd index a767eb3..5b65aae 100644 --- a/vignettes/custom-config.Rmd +++ b/vignettes/custom-config.Rmd @@ -46,6 +46,7 @@ cfg | `start_date` / `end_date` | Date | First and last measurement dates | | `verbose` | logical | Print progress messages | | `automatic_download` | logical | Auto-download missing source files | +| `verify_checksums` | logical | Verify raw-file MD5s against `pgchecksum` on read (default `FALSE`) | > **Date convention:** Use the last day of each temporal increment. For yearly data with December measurement dates, use `1990-12-31`. For monthly data with January measurement dates, use `1990-01-31`. diff --git a/vignettes/getting-started.Rmd b/vignettes/getting-started.Rmd index bd22d21..c291057 100644 --- a/vignettes/getting-started.Rmd +++ b/vignettes/getting-started.Rmd @@ -179,18 +179,25 @@ After downloading, your data folder has this structure: ``` {rawfolder}/ ├── priogrid/ -│ ├── priogrid_3_0_1_05deg_yearly.zip # Downloaded archive +│ ├── priogrid_3_0_1_05deg_yearly.zip # Downloaded archive │ └── releases/ │ └── 3.0.1/ │ └── 05deg_yearly/ -│ ├── pg_static.parquet # Static data (cached) -│ ├── pg_timevarying.parquet # Time-varying data (cached) -│ └── {varname}.rds # Individual variable rasters -├── {source_name}/{version}/{id}/ # Raw source files -└── tmp/ # Temporary processing files +│ ├── cog/ +│ │ └── {varname}.tif # Individual variables as Cloud-Optimized GeoTIFFs +│ ├── timevarying/ +│ │ └── year=YYYY/ +│ │ └── part-0.parquet # Hive-partitioned time-varying data +│ ├── pg_static.parquet # Static data (wide table) +│ ├── pg_static.csv.gz # Static data (CSV bundle) +│ ├── pg_timevarying.csv.gz # Time-varying data (CSV bundle) +│ ├── pg_config.json # Dataset manifest (grid, layout, variables) +│ └── _checksums.csv # Per-file MD5s +├── {source_name}/{version}/{id}/ # Raw source files +└── tmp/ # Temporary processing files ``` -The `.parquet` files are what `read_pg_static()` and `read_pg_timevarying()` read by default. The `.rds` files contain individual variables as rasters (used by `load_pgvariable()`). +`read_pg_static()` reads `pg_static.parquet`; `read_pg_timevarying()` reads the hive-partitioned `timevarying/` dataset, pushing any filters down to Arrow first. The `cog/*.tif` files hold individual variables as Cloud-Optimized GeoTIFFs and are read by `load_pgvariable()`. The `*.csv.gz` bundles provide the same tables for non-R users, and `pg_config.json` records the grid and layout. ## Next Steps diff --git a/vignettes/metadata.Rmd b/vignettes/metadata.Rmd index 1971799..fb1842e 100644 --- a/vignettes/metadata.Rmd +++ b/vignettes/metadata.Rmd @@ -27,6 +27,15 @@ pgvariables$name | `name` | Variable name used in `load_pgvariable()`, `calc_pg()`, etc. | | `static` | `TRUE` = no temporal dimension; `FALSE` = time-varying | | `source_ids` | Comma-separated UUIDs of the data sources feeding this variable | +| `label` | Display title shown in plots | +| `unit` | Unit string (e.g. `°C`, `mm`); empty if not applicable | +| `transform` | Display transform: `identity`, `log1p`, `log10`, or `sqrt` | +| `palette` | Colour palette name (e.g. `viridis`) | +| `plot_type` | `continuous`, `categorical`, or `diverging` | +| `value_min` | Numeric lower display bound (empty = data-driven) | +| `value_max` | Numeric upper display bound (empty = data-driven) | + +The seven display columns (`label`…`value_max`) are also stamped into each built COG as `pg_*` GDAL metatags, making the `.tif` self-describing (see the [Contributing](contributing.html) vignette). Static and time-varying variables: diff --git a/vignettes/working-with-rasters.Rmd b/vignettes/working-with-rasters.Rmd index 920d44b..fa80e3f 100644 --- a/vignettes/working-with-rasters.Rmd +++ b/vignettes/working-with-rasters.Rmd @@ -148,7 +148,7 @@ terra::plot(c(elev_africa, log1p(pop_africa)), ## Saving and Reusing Custom Results -`save_pgvariable()` saves any `SpatRaster` back to the PRIOGRID data folder as a wrapped `.rds` file: +`save_pgvariable()` writes any `SpatRaster` to the dataset's `cog/` sub-folder as a Cloud-Optimized GeoTIFF (`{varname}.tif`), stamping layer dates via `time()`, `units()`, and the `pg_*` display metadata from `pgvariables` so the file is self-describing: ```{r save-var, eval=FALSE} r <- gen_ruggedterrain_elevation_mean() From 9d600f8d8a092b8009fe5507c0e0a7cab9ea99e1 Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Mon, 10 Aug 2026 12:23:42 +0200 Subject: [PATCH 6/7] improve plotting metadata and add plot_pgvariable --- NAMESPACE | 1 + R/build_priogrid.R | 115 ++++++++++++++++--- R/data.R | 14 +-- R/plot.R | 135 +++++++++++++++++++++++ data/pgvariables.rda | Bin 1660 -> 2200 bytes data_raw/variables.csv | 78 ++++++------- man/calc_pg.Rd | 2 +- man/load_pgvariable.Rd | 28 ++++- man/pgvariables.Rd | 9 +- man/plot_pgvariable.Rd | 72 ++++++++++++ man/save_pgvariable.Rd | 24 +++- tests/testthat/test-metadata-integrity.R | 16 +++ tests/testthat/test-plot.R | 39 +++++++ tests/testthat/test-save-load.R | 44 ++++++++ vignettes/contributing.Rmd | 27 +++-- vignettes/metadata.Rmd | 15 +-- 16 files changed, 525 insertions(+), 94 deletions(-) create mode 100644 R/plot.R create mode 100644 man/plot_pgvariable.Rd create mode 100644 tests/testthat/test-plot.R diff --git a/NAMESPACE b/NAMESPACE index b39b992..7dbf30c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -75,6 +75,7 @@ export(pg_set_rawfolder) export(pgcitations) export(pgout_path) export(pgsearch) +export(plot_pgvariable) export(prio_blank_grid) export(rast_to_df) export(read_cru_pet) diff --git a/R/build_priogrid.R b/R/build_priogrid.R index 94c8327..2fb1853 100644 --- a/R/build_priogrid.R +++ b/R/build_priogrid.R @@ -222,23 +222,75 @@ calc_pg <- function(varnames = NULL, overwrite = FALSE, config = pg_current_conf invisible(NULL) } -# Internal: build the named pg_* GDAL metatag vector for a variable from the -# current pgvariables row. Drop NA/empty fields so callers can call -# terra::metags(r) <- .pg_build_metatags(varname) directly. +# Internal: data-driven display stats from a built raster. +# value_min/value_max: overall min/max across layers (NA-robust). +# value_mean/value_std: per-layer mean/sd averaged across layers (display hint). +# nunique/class_values: distinct non-NA values, only for plot_type == "discrete". # @keywords internal -.pg_build_metatags <- function(varname) { +.pg_variable_stats <- function(rast, plot_type = NA_character_) { + empty <- list(value_min = NA_real_, value_max = NA_real_, + value_mean = NA_real_, value_std = NA_real_, + nunique = NA_integer_, class_values = NA_character_) + if (terra::nlyr(rast) == 0L || terra::ncell(rast) == 0L) return(empty) + + g <- terra::global(rast, fun = c("min", "max", "mean", "sd"), na.rm = TRUE) + fin <- function(x) if (length(x) && is.finite(x)) x else NA_real_ + vmin <- fin(suppressWarnings(min(g$min, na.rm = TRUE))) + vmax <- fin(suppressWarnings(max(g$max, na.rm = TRUE))) + vmean <- fin(suppressWarnings(mean(g$mean, na.rm = TRUE))) + vstd <- fin(suppressWarnings(mean(g$sd, na.rm = TRUE))) + + nuniq <- NA_integer_; cls <- NA_character_ + if (!is.na(plot_type) && plot_type == "discrete") { + f <- terra::freq(rast, bylayer = FALSE) + if (nrow(f) > 0L) { + vals <- sort(unique(f$value)) + nuniq <- length(vals) + cls <- paste(vals, collapse = ",") + } + } + list(value_min = vmin, value_max = vmax, value_mean = vmean, + value_std = vstd, nunique = nuniq, class_values = cls) +} + +# Internal: recommended rio-tiler/matplotlib colormap for a plot_type. +# @keywords internal +.pg_colormap <- function(plot_type) { + switch(as.character(plot_type), + continuous = "rdbu_r", # diverging (temperature, elevation, SPEI) + positive_real = "viridis", # sequential magnitude + count = "inferno", # sequential hotspot + share = "viridis", # sequential 0-1 + discrete = "tab20", # qualitative + "viridis") # default +} + +# Internal: build the named pg_* GDAL metatag vector for a variable. +# Drop NA/empty fields so callers can assign terra::metags(r) <- ... directly. +# Accepts optional computed stats from .pg_variable_stats(); pg_colormap is +# derived from plot_type, never stored in pgvariables. +# @keywords internal +.pg_build_metatags <- function(varname, stats = NULL) { meta <- pgvariables[pgvariables$name == varname, ] if (nrow(meta) == 0L) stop("varname '", varname, "' not found in pgvariables.") + + cmap <- if (!is.na(meta$plot_type) && nzchar(meta$plot_type)) .pg_colormap(meta$plot_type) else NULL + st <- function(k) if (!is.null(stats) && !is.na(stats[[k]])) stats[[k]] else NULL + c( pg_name = meta$name, pg_static = as.character(meta$static), if (!is.na(meta$label) && nzchar(meta$label)) c(pg_label = meta$label) else NULL, if (!is.na(meta$unit) && nzchar(meta$unit)) c(pg_unit = meta$unit) else NULL, if (!is.na(meta$transform) && nzchar(meta$transform)) c(pg_transform = meta$transform) else NULL, - if (!is.na(meta$palette) && nzchar(meta$palette)) c(pg_palette = meta$palette) else NULL, if (!is.na(meta$plot_type) && nzchar(meta$plot_type)) c(pg_plot_type = meta$plot_type) else NULL, - if (!is.na(meta$value_min)) c(pg_value_min = as.character(meta$value_min)) else NULL, - if (!is.na(meta$value_max)) c(pg_value_max = as.character(meta$value_max)) else NULL + if (!is.null(cmap)) c(pg_colormap = cmap) else NULL, + if (!is.null(st("value_min"))) c(pg_value_min = as.character(stats$value_min)) else NULL, + if (!is.null(st("value_max"))) c(pg_value_max = as.character(stats$value_max)) else NULL, + if (!is.null(st("value_mean"))) c(pg_value_mean = as.character(stats$value_mean)) else NULL, + if (!is.null(st("value_std"))) c(pg_value_std = as.character(stats$value_std)) else NULL, + if (!is.null(st("nunique"))) c(pg_nunique = as.character(stats$nunique)) else NULL, + if (!is.null(st("class_values"))) c(pg_class_values = stats$class_values) else NULL ) } @@ -248,9 +300,16 @@ calc_pg <- function(varnames = NULL, overwrite = FALSE, config = pg_current_conf #' \code{cog/} sub-directory of the dataset root. The variable must be listed #' in [pgvariables]. Layer names, \code{time()}, and \code{units()} are #' preserved natively inside the \code{.tif}. Plot metadata from [pgvariables] -#' (\code{label}, \code{unit}, \code{transform}, \code{palette}, -#' \code{plot_type}, \code{value_min}, \code{value_max}) is stamped as -#' \code{pg_*} GDAL metatags so the COG is self-describing. +#' (\code{label}, \code{unit}, \code{transform}, \code{plot_type}) is stamped as +#' \code{pg_*} GDAL metatags so the COG is self-describing. In addition, +#' \code{pg_colormap} (derived from \code{plot_type}), \code{pg_value_min}, +#' \code{pg_value_max}, \code{pg_value_mean}, \code{pg_value_std} (computed +#' from the raster at save time), and — for discrete variables — +#' \code{pg_nunique} and \code{pg_class_values} are also stamped. +#' \code{value_*}, \code{nunique}, and \code{class_values} are computed from +#' the raster at save time; \code{colormap} is derived from \code{plot_type}. +#' These map directly onto rio-tiler/TiTiler \code{rescale} and +#' \code{colormap_name}. #' #' @param rast Terra SpatRaster object from a gen_*() function. #' @param varname Character string with the variable name (must exist in pgvariables). @@ -293,7 +352,8 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { if (!is.na(meta$unit) && nzchar(meta$unit)) terra::units(rast) <- meta$unit - terra::metags(rast) <- .pg_build_metatags(varname) + stats <- .pg_variable_stats(rast, plot_type = meta$plot_type) + terra::metags(rast) <- .pg_build_metatags(varname, stats = stats) terra::writeRaster(rast, filepath, filetype = "COG", overwrite = TRUE, gdal = c("COMPRESS=DEFLATE", "NUM_THREADS=ALL_CPUS")) @@ -334,7 +394,7 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { # Internal: re-stamp pg_* GDAL metatags (and terra units) on an existing COG # from the current pgvariables row, without recomputing the raster data. # Use this after editing data_raw/variables.csv + regenerating pgvariables.rda -# to propagate display metadata (label, palette, transform, …) into built COGs. +# to propagate display metadata (label, unit, transform, plot_type, colormap, …) into built COGs. # @param varname Variable name; must exist in pgvariables and in the cog/ dir. # @param save_to Dataset root (same convention as save_pgvariable). # @return Path to the restamped COG, invisibly. @@ -355,7 +415,8 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { meta <- pgvariables[pgvariables$name == varname, ] terra::units(r) <- if (!is.na(meta$unit) && nzchar(meta$unit)) meta$unit else "" - terra::metags(r) <- .pg_build_metatags(varname) + stats <- .pg_variable_stats(r, plot_type = meta$plot_type) + terra::metags(r) <- .pg_build_metatags(varname, stats = stats) # Write to a sibling temp file (same filesystem → rename is atomic), then # replace the original. terra cannot write a lazy raster back to its own path. @@ -471,8 +532,32 @@ save_pgvariable <- function(rast, varname, save_to = pgout_path()) { variables_meta <- lapply(all_vars, function(vn) { row <- pgvariables[pgvariables$name == vn, ] if (nrow(row) == 0L) return(list(name = vn)) - as.list(row[1L, c("name", "static", "source_ids", "label", "unit", - "transform", "palette", "plot_type", "value_min", "value_max")]) + base <- as.list(row[1L, c("name", "static", "source_ids", "label", "unit", + "transform", "plot_type")]) + tag_val <- function(tags, key) { + v <- tags$value[tags$name == key] + if (length(v) == 0L) NA else v[[1]] + } + cog <- file.path(base_path, "cog", paste0(vn, ".tif")) + if (file.exists(cog)) { + tags <- terra::metags(terra::rast(cog)) + base$colormap <- tag_val(tags, "pg_colormap") + base$value_min <- suppressWarnings(as.numeric(tag_val(tags, "pg_value_min"))) + base$value_max <- suppressWarnings(as.numeric(tag_val(tags, "pg_value_max"))) + base$value_mean <- suppressWarnings(as.numeric(tag_val(tags, "pg_value_mean"))) + base$value_std <- suppressWarnings(as.numeric(tag_val(tags, "pg_value_std"))) + base$nunique <- suppressWarnings(as.integer(tag_val(tags, "pg_nunique"))) + base$class_values <- tag_val(tags, "pg_class_values") + } else { + base$colormap <- if (!is.na(row$plot_type) && nzchar(row$plot_type)) .pg_colormap(row$plot_type) else NA + base$value_min <- NA + base$value_max <- NA + base$value_mean <- NA + base$value_std <- NA + base$nunique <- NA + base$class_values <- NA + } + base }) names(variables_meta) <- all_vars diff --git a/R/data.R b/R/data.R index d9d1da4..e7821b6 100644 --- a/R/data.R +++ b/R/data.R @@ -43,19 +43,19 @@ #' for the data depending on whether the data is static or with a temporal-dimension. #' #' @format ## `pgvariables` -#' A data frame with 38 rows and 10 columns: +#' A data frame with 38 rows and 7 columns: #' \describe{ #' \item{name}{Full name of the variable.} #' \item{static}{Boolean. Whether the variable varies over time or not.} #' \item{source_ids}{Comma-separated string. The Unique Universal Identifier (UUID) of the sources used to build the variable.} -#' \item{label}{Display title for the variable; defaults to the variable name.} -#' \item{unit}{Measurement unit (e.g., "°C", "mm"); \code{NA} if not applicable.} +#' \item{label}{Human-readable display title for plots/legends.} +#' \item{unit}{Measurement unit for colorbar labels; empty if dimensionless.} #' \item{transform}{Display transform applied before plotting; one of \code{identity}, \code{log1p}, \code{log10}, or \code{sqrt}.} -#' \item{palette}{Color palette name for visualization. Default: \code{viridis}.} -#' \item{plot_type}{Plot type hint; one of \code{continuous}, \code{categorical}, or \code{diverging}.} -#' \item{value_min}{Suggested color-scale minimum; \code{NA} for data-driven scaling.} -#' \item{value_max}{Suggested color-scale maximum; \code{NA} for data-driven scaling.} +#' \item{plot_type}{Plot type hint; one of \code{continuous}, \code{positive_real}, \code{count}, \code{share}, or \code{discrete}.} #' } +#' Raster statistics (\code{value_min}/\code{value_max}/\code{value_mean}/\code{value_std}/\code{nunique}/\code{class_values}) +#' and the derived \code{colormap} are not stored here; they are computed at build and stamped into +#' each COG as \code{pg_*} metatags. "pgvariables" diff --git a/R/plot.R b/R/plot.R new file mode 100644 index 0000000..3baf2e2 --- /dev/null +++ b/R/plot.R @@ -0,0 +1,135 @@ +# Internal: base-R colour vector for a pg_colormap string. +# Maps the pg_colormap vocabulary to verified grDevices::hcl.pals() palette names. +# @keywords internal +.pg_plot_palette <- function(colormap, n = 256L) { + pal <- switch(as.character(colormap), + viridis = "Viridis", + inferno = "Inferno", + rdbu_r = "Blue-Red 3", # diverging: low = blue, high = red + tab20 = "Dark 3", # qualitative + "Viridis") + grDevices::hcl.colors(n, pal) +} + +# Internal: resolve display metadata (metatags first, pgvariables fallback). +# Prefers embedded pg_* metatags (terra::metags() -> data.frame), then the +# matching pgvariables row, then safe defaults. Derives colormap via +# .pg_colormap(plot_type) when not stamped. +# @keywords internal +.pg_plot_meta <- function(x, varname = NULL) { + tags <- tryCatch(terra::metags(x), error = function(e) NULL) + tv <- function(key) { + if (is.null(tags) || !nrow(tags)) return(NA_character_) + v <- tags$value[tags$name == key] + if (length(v) == 0L) NA_character_ else v[[1]] + } + vn <- varname %||% { m <- tv("pg_name"); if (is.na(m)) names(x)[1] else m } + row <- pgvariables[pgvariables$name == vn, ] + pick <- function(key, col) { + v <- tv(key) + if (!is.na(v) && nzchar(v)) return(v) + if (nrow(row) == 1L && !is.na(row[[col]]) && nzchar(as.character(row[[col]]))) + return(as.character(row[[col]])) + NA_character_ + } + label <- pick("pg_label", "label"); if (is.na(label)) label <- vn + unit <- pick("pg_unit", "unit"); if (is.na(unit)) unit <- "" + transform <- pick("pg_transform", "transform"); if (is.na(transform)) transform <- "identity" + plot_type <- pick("pg_plot_type", "plot_type"); if (is.na(plot_type)) plot_type <- "continuous" + colormap <- tv("pg_colormap") + if (is.na(colormap) || !nzchar(colormap)) colormap <- .pg_colormap(plot_type) + num <- function(key) { v <- tv(key); if (is.na(v)) NA_real_ else suppressWarnings(as.numeric(v)) } + list(varname = vn, label = label, unit = unit, transform = transform, + plot_type = plot_type, colormap = colormap, + value_min = num("pg_value_min"), value_max = num("pg_value_max")) +} + +#' Plot a PRIO-GRID variable +#' +#' Renders a PRIO-GRID raster using display metadata from its embedded +#' \code{pg_*} GDAL metatags (written by \code{\link{save_pgvariable}()}), +#' falling back to the \code{\link{pgvariables}} table and then to safe +#' defaults. The backend is \code{terra::plot()}: native C++ rendering with +#' \code{maxcell} downsampling and no materialisation of the raster into a +#' data frame. No new package dependency — colours come from base +#' \code{grDevices::hcl.colors()}. +#' +#' @param x A \code{SpatRaster} (e.g. from \code{\link{load_pgvariable}()}) +#' or a character variable name (loaded via \code{load_pgvariable()}). +#' @param layer Integer. Which layer to draw for multi-layer rasters. Default 1. +#' Can also be a string with the name of the layer. For timevarying rasters, the +#' layer name is a date with the format "YYYY-MM-DD". +#' @param config A \code{\link{pg_config}} object. Passed to +#' \code{\link{load_pgvariable}()} when \code{x} is a variable name string; +#' ignored when \code{x} is already a \code{SpatRaster}. +#' \code{NULL} (default) resolves via the release path or current config. +#' @param version Character. Release version string (e.g. \code{"3.0.1"}); +#' passed to \code{load_pgvariable()}. Cannot be combined with \code{config}. +#' @param type Character. Release type (e.g. \code{"05deg_yearly"}); +#' passed to \code{load_pgvariable()}. Default \code{"05deg_yearly"}. +#' @param spatial_hash,temporal_hash Six-character MD5 hashes that key a custom +#' output folder; passed to \code{load_pgvariable()}. Must be supplied +#' together. Cannot be combined with \code{version}. +#' @param ... Additional arguments forwarded to \code{terra::plot()}, overriding +#' any metadata-derived default (e.g. \code{range}, \code{main}, \code{col}). +#' +#' @return The drawn \code{SpatRaster} layer, invisibly. +#' @export +#' +#' @examples +#' \dontrun{ +#' # Plot directly from a loaded raster: +#' r <- load_pgvariable("ucdp_ged") +#' plot_pgvariable(r) +#' +#' # Plot by name from a custom data directory: +#' cfg <- pg_config(nrow = 180, ncol = 360) +#' plot_pgvariable("ucdp_ged", config = cfg) +#' +#' # Plot a specific release: +#' plot_pgvariable("ucdp_ged", version = "3.0.1") +#' +#' # Override the title: +#' plot_pgvariable(r, main = "Custom title") +#' } +plot_pgvariable <- function(x, layer = 1, + config = NULL, + version = NULL, + type = "05deg_yearly", + spatial_hash = NULL, + temporal_hash = NULL, + ...) { + rlang::check_installed("terra", reason = "to plot PRIO-GRID variables") + varname <- NULL + if (is.character(x)) { + varname <- x + x <- load_pgvariable(x, config = config, version = version, type = type, + spatial_hash = spatial_hash, temporal_hash = temporal_hash) + } + + meta <- .pg_plot_meta(x, varname = varname) + r <- if (terra::nlyr(x) > 1L) x[[layer]] else x + + if (identical(meta$plot_type, "discrete")) { + rf <- terra::as.factor(r) + ncl <- tryCatch(nrow(terra::cats(rf)[[1]]), error = function(e) 256L) + if (is.null(ncl) || is.na(ncl) || ncl < 1L) ncl <- 256L + args <- list(x = rf, type = "classes", main = meta$label, + col = .pg_plot_palette(meta$colormap, ncl), + legend = ncl <= 7L) + } else { + rr <- switch(meta$transform, + log1p = log1p(r), log10 = log10(r), sqrt = sqrt(r), r) + main <- if (!identical(meta$transform, "identity")) + paste0(meta$label, " (", meta$transform, " scale)") else meta$label + args <- list(x = rr, main = main, col = .pg_plot_palette(meta$colormap, 256L)) + if (nzchar(meta$unit)) args$plg <- list(title = meta$unit) + if (identical(meta$transform, "identity") && + is.finite(meta$value_min) && is.finite(meta$value_max)) + args$range <- c(meta$value_min, meta$value_max) + } + + args <- utils::modifyList(args, list(...)) + do.call(terra::plot, args) + invisible(args$x) +} diff --git a/data/pgvariables.rda b/data/pgvariables.rda index a070601cfe3340b359bf7e7bd27c272597843bf3..3b23e1f7489281582f248890f1100a51f8a89b94 100644 GIT binary patch literal 2200 zcmV;J2xs>~T4*^jL0KkKS%A^MBmf8!f6o8^|IgiT|KLCW-oU^A-|#>HKmZT`01yBH z;0Yg7(y>qg1@7P+AV^YxAtFeprX<2oO;hzYDd9~$0Z&shO)vmP(@1SYO{hIjLs0ad zp~DfTnhgMG0001JG7U5h02(v_h-eI&dV%UP4FEI%0MGz5000^TH2?q&PfB`8qB0v( z)G}fNKm$ga0io(WKxhEa$Y=mKVl>l1pbY>301XB~rh%XXMu0I50FodkKtwc|Hd1XS znnTl5)Osc-rlF7k4^wIy8e|@(NpAt+^@tm_gMAY+$qod-eXAWJI|!2?M8t_nAus9@ z{kA{3AesC$grvbPn`%ua=)FaJ)eU?jSXxKQ@qMHEV*`LJekylK`Qh2{8a|w1Fj&f)9hn#FF(iP$I+v4{l64`IcxSCJuHi z&Q2qU3gh|FRvFB+B8>WUI})uq#xc3;nlsJ)RC+O5@rhQ4`z>+Jk7@t0m6zHmXKaQ> z4V-pDM-pR67pFlm?ry-N1kxzj(SZ))EbiO=K9z@Q#Cn(A2pZYGK#>BYSSzsz7csp+e(C4boeUL{S|xHKrEH9+km&V?FqywX)NX zqFHX1?{}D#N0pIda1R_#9Qm!;2-KPB+;7l{j&U4Duc@TV;34Um7)1G~Q=^%;S!0<% zFcGd9oDpM4X9h7w?kf>N!&*Q#EL< ziHu`uK@)}oBXr1e#lSZXU{7~G&5T1Iso4RIKKQ5GU1$kJOOO6WGG z6IT%pM`5a06=a9O)B@@n0SrJQ1`7htA&}Lqjl_|evk;EdHMALvs%aAw03%&T10v>) zh&pi?IQtrJM{yN^vB3~GSk4Huklhg(fxlhgX>4L$$ia~MY9dA0wQfKR2|x=r>N4pEKPNTM`=8zD`Eug(Q%@wvqvT|iqC>v*I~AuvASna}40>S%;4~R< zW)xE~$rJTlHjr@SHV+j0!|YtDX&uLgWUkqV^f>TZfq)kd_d&ol%sZBlXAkFM;UQzC7DRA>v$G;l+AcrDcTwjNnQ$2uYcYX<8HmSu z^7!*Q=#=)4>%}HIf(^4v)%nqMQidJhqA*6POd26icF-x-l}~aH21-N{TuS z{<)d+KZSS%#oI>j8K%dcZf;OFh4SbAJ4&@;(cB^!hBDl8s=W72K`;xfW5n+{U||Bk z*rG&@a55PtCo#;Yok&K)HQJ2E{W=!Vi-zbJf6b7l>d;?XnD#~VV#k(&0f}NJFzs*b zO}M(a?bb_we@(QF=@1$iv%c8v&!olGBmqY1M-*{$2pIORWP1qjuG5qW7#D=q_jAv2rk674 zXV0MKnKEmada+t*!Qvn_TaOe{jTmGZ!O-c`6e|J;x#GwncgQF+9|g?L*+9(UtUa(G z2X~T{QQjrTqMg&aU0W}yOTLBRMrJDnUdg1}fAsw}%Qw9cw0iL9>BfVO#d%O2k_5Wx}g@~;V; z29zkAU|F%W=|0`wHU?OeFR+&B+(2AmDrQf4fXV{90&$#=U%EW=ceQw-YPdK_uurD6Do%n%J{bN zl<;+!N3#H^)>ZQfPO$S&W)+44NJ9)T92}sS=b6{my^k)mU(k_7=@_;tfnv%7A(#=M zmlpE*H7ZQx+QuX^zC}CSaOtMuut|q7*Ljip%b0im9d<3fh6fjzffJ>oHZygTPA>->WcibnS2u~s z*XX{GuTxkZ1q^00w{=0}ucJGE{niXaE{C02%-QXaS%!1xhA@VlqI~ z!5RvBo|7okH4jK+(-UPql-i~LPRSHqXPpkbWhpJj=@p63E4IG0Nqqv8fRI$%XzAcl zBo!e9Irx`o3M2MhQ_x66B!O&FlRK$I+DQ}h_J2Rp(H-0RCH9WeNv;E(LH>t*rUXC4#L@S?^Lbds=<2IzE;J;=I zm_#wd!MgSgz4lp}_6sbci=EmK7%&JBQG^CI0tN%%f{TT;HcC^ZQ=TL!{u0>U<`-3s zjQwNRNfJ)Dk_sde;Y309(tlsNsJx0KM0Dv`q9{gGOHlzWLO&WWqDj_KFcRWPg4`1M9!ZdgkV znc&f?mK$F>+anmg%M7?{AJ%qHa@c1WJo3pIZ?Y{i z&Bvx~^lQV1G;rq~+-6GS$z*3|7|LAAHBD(6QuJ48aw|=!;-s!MX@awQL!~9+mZ31v zZGUXRMzNz7$lf(_FWW1& z%J((#$swc7o!nyc5Svn-dTN*6uu#l{ctqx5K_BIOia@Ur$x(>JEm-`*;ZZ^JJ|+#Yef zTZPHXIz{($wN>{$@$N?^f{4jRY8)H~Bx~0HZ-uYRxjOD&MyRuyL)7|}?>HNyiDmB) z{T0c9iyT&Ue#d(DOd1_3eh&2gc5&y_@xYaCi=}pWM}6L?@M~X{yfEFJ4;DzdA9{1z zGu_V57X2r)3>umuvHKlye`?${(`!Xz3IdCw9YS@em+_Z4ptn$>NE`qrFF0U%Lw~dqaVz5JGa;I8(J{(#;OUpD*^hi}9 z$?;(OLb)F!-oa&wA7Vo16p@n24#Sfp(sFeBm0;=Q@C=QTI4aEh)acbCb47E!(K0mb zWV&2bRYPVTD=$$LQAYyjr$3|oFk*b(E1NT3RH{Qng_0jrZJV!ll@kP_LiOv)v2{9< zKg{6B=+z-q%kX(WieB=6t0vFQSlQND?zumL=UR__j8u_iQb%H@NxO;H*yzDh!xb2f z88|M<+>p7BPS-6MGD|1bt^BEnZF?Op5WfhAxy6a_a8=F4gNf9hkJNlwxW9u5hkwb< zuPP|2d{;R-?9qb7Q&*{t&&P)z_j9c=bas_&aY$7tkja8ad$rWFn-gbJLd_G&syuFZ zGU)9$NPp9?$)S4X5*az0FrzZ`_9Q8i^%h>42=tOf`OnMibRIG{r7m6n;_gVN3KADg Gh`L}WBQzrb diff --git a/data_raw/variables.csv b/data_raw/variables.csv index ab38600..1890c82 100644 --- a/data_raw/variables.csv +++ b/data_raw/variables.csv @@ -1,39 +1,39 @@ -name static source_ids label unit transform palette plot_type value_min value_max -cru_tmp FALSE ac037134-3567-49d9-a3ba-64f37c1ee698 cru_tmp identity viridis continuous -cru_pre FALSE 00575260-ad1c-4e87-a575-3922bc151f50 cru_pre identity viridis continuous -cru_pet FALSE 95399c70-7db4-47f0-95e5-2e279b6b2054 cru_pet identity viridis continuous -cshapes_cover_share FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b cshapes_cover_share identity viridis continuous -cshapes_gwcode FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b cshapes_gwcode identity viridis continuous -geoepr_reg_excluded FALSE 287bfdf7-2f4f-402a-88df-5fe1f8b7046b, 3900b527-a728-4c26-b0ab-f4441d3ee2e8 geoepr_reg_excluded identity viridis continuous -bdist1 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b bdist1 identity viridis continuous -bdist2 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b bdist2 identity viridis continuous -bdist3 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b bdist3 identity viridis continuous -ghsl_population_grid FALSE ae6a7612-4bef-452f-acd6-d2212cf9a7c5 ghsl_population_grid identity viridis continuous -hilda_cropland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_cropland identity viridis continuous -hilda_forest FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_forest identity viridis continuous -hilda_grassland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_grassland identity viridis continuous -hilda_ocean FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_ocean identity viridis continuous -hilda_pasture FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_pasture identity viridis continuous -hilda_sparse FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_sparse identity viridis continuous -hilda_urban FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_urban identity viridis continuous -hilda_water FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 hilda_water identity viridis continuous -linight_mean FALSE d99fbea7-2a01-4221-b900-29a58d33f591 linight_mean identity viridis continuous -naturalearth_cover TRUE 92da9800-4520-4e87-a855-b28255452189 naturalearth_cover identity viridis continuous -naturalearth_cover_share TRUE 92da9800-4520-4e87-a855-b28255452189 naturalearth_cover_share identity viridis continuous -ruggedterrain_elevation_mean TRUE 8c8192eb-cc29-4598-8f8a-ec190ba35c2d ruggedterrain_elevation_mean identity viridis continuous -traveltime_mean TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 traveltime_mean identity viridis continuous -traveltime_min TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 traveltime_min identity viridis continuous -geopko_troops_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 geopko_troops_count identity viridis continuous -geopko_operations_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 geopko_operations_count identity viridis continuous -ne_disputed_area_share TRUE 920663ad-d7e7-4528-b36d-4b7266def2b1 ne_disputed_area_share identity viridis continuous -speibase6_mean FALSE 14839384-623a-4cf7-9241-6166a8ac465b, 95399c70-7db4-47f0-95e5-2e279b6b2054, 00575260-ad1c-4e87-a575-3922bc151f50 speibase6_mean identity viridis continuous -ghs_wup_degurba_urban FALSE 7f1f60a3-6664-4427-b086-b5359ebf45b7 ghs_wup_degurba_urban identity viridis continuous -ucdp_ged FALSE 49f79d96-4e4d-4812-9dd1-862bacfca577 ucdp_ged identity viridis continuous -shdi FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c shdi identity viridis continuous -msch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c msch identity viridis continuous -esch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c esch identity viridis continuous -lifexp FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c lifexp identity viridis continuous -gnic FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c gnic identity viridis continuous -side_excluded FALSE e42b30e3-75da-4dd4-a375-0d6557087804 side_excluded identity viridis continuous -side_included FALSE e42b30e3-75da-4dd4-a375-0d6557087804 side_included identity viridis continuous -side_irrelevant FALSE e42b30e3-75da-4dd4-a375-0d6557087804 side_irrelevant identity viridis continuous +name static source_ids label unit transform plot_type +cru_tmp FALSE ac037134-3567-49d9-a3ba-64f37c1ee698 Mean temperature °C identity continuous +cru_pre FALSE 00575260-ad1c-4e87-a575-3922bc151f50 Precipitation mm identity positive_real +cru_pet FALSE 95399c70-7db4-47f0-95e5-2e279b6b2054 Potential evapotranspiration mm/day identity positive_real +cshapes_cover_share FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b State territory coverage (share) identity share +cshapes_gwcode FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b Country (Gleditsch-Ward code) identity discrete +geoepr_reg_excluded FALSE 287bfdf7-2f4f-402a-88df-5fe1f8b7046b, 3900b527-a728-4c26-b0ab-f4441d3ee2e8 Excluded ethnic groups (regional, share) identity share +bdist1 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b Distance to nearest land border m identity positive_real +bdist2 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b Distance to nearest international border m identity positive_real +bdist3 FALSE ec3eea2e-6bec-40d5-a09c-e9c6ff2f8b6b Distance to own country border m identity positive_real +ghsl_population_grid FALSE ae6a7612-4bef-452f-acd6-d2212cf9a7c5 Population count persons log10 positive_real +hilda_cropland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Cropland cover (share) identity share +hilda_forest FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Forest cover (share) identity share +hilda_grassland FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Grassland cover (share) identity share +hilda_ocean FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Ocean cover (share) identity share +hilda_pasture FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Pasture cover (share) identity share +hilda_sparse FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Sparse vegetation cover (share) identity share +hilda_urban FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Urban cover (share) identity share +hilda_water FALSE 82bc4c6f-9904-484f-aa9a-77771d076690 Water cover (share) identity share +linight_mean FALSE d99fbea7-2a01-4221-b900-29a58d33f591 Nighttime light (harmonized, mean) identity positive_real +naturalearth_cover TRUE 92da9800-4520-4e87-a855-b28255452189 Land cover mask (share) identity share +naturalearth_cover_share TRUE 92da9800-4520-4e87-a855-b28255452189 Land cover (share) identity share +ruggedterrain_elevation_mean TRUE 8c8192eb-cc29-4598-8f8a-ec190ba35c2d Mean elevation m identity continuous +traveltime_mean TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 Travel time to major city (mean) minutes identity positive_real +traveltime_min TRUE 9aa052f6-4d04-4ed1-9eed-e47e08828d38 Travel time to major city (minimum) minutes identity positive_real +geopko_troops_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 UN peacekeeping troops (count) personnel log1p count +geopko_operations_count FALSE 7dcbfbfb-9667-4684-af34-85f69fa8d0a0 UN peacekeeping operations (count) log1p count +ne_disputed_area_share TRUE 920663ad-d7e7-4528-b36d-4b7266def2b1 Disputed area coverage (share) identity share +speibase6_mean FALSE 14839384-623a-4cf7-9241-6166a8ac465b, 95399c70-7db4-47f0-95e5-2e279b6b2054, 00575260-ad1c-4e87-a575-3922bc151f50 SPEI drought index (6-month, mean) identity continuous +ghs_wup_degurba_urban FALSE 7f1f60a3-6664-4427-b086-b5359ebf45b7 Urban area (DEGURBA, share) identity share +ucdp_ged FALSE 49f79d96-4e4d-4812-9dd1-862bacfca577 Battle-related deaths (UCDP GED) deaths log1p count +shdi FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c Subnational Human Development Index identity positive_real +msch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c Mean years of schooling years identity positive_real +esch FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c Expected years of schooling years identity positive_real +lifexp FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c Life expectancy at birth years identity positive_real +gnic FALSE 8aaf6b27-6372-43da-87a9-d4235095bb2c, a8e35e36-9f7e-4194-9cc4-ce8ca59f7b51, 8aaf6b27-6372-43da-87a9-d4235095bb2c GNI per capita 1000 USD (2011 PPP) identity positive_real +side_excluded FALSE e42b30e3-75da-4dd4-a375-0d6557087804 Excluded ethnic population (share) identity share +side_included FALSE e42b30e3-75da-4dd4-a375-0d6557087804 Included ethnic population (share) identity share +side_irrelevant FALSE e42b30e3-75da-4dd4-a375-0d6557087804 Politically irrelevant ethnic population (share) identity share diff --git a/man/calc_pg.Rd b/man/calc_pg.Rd index 77112e8..c1e56b0 100644 --- a/man/calc_pg.Rd +++ b/man/calc_pg.Rd @@ -21,7 +21,7 @@ NULL (invisibly). Called for side effects (saving files). \description{ Calculates PRIO-GRID variables based on current config and saves them to disk. Each variable is computed by calling its corresponding gen_*() function -and saved as an .rds file in the custom data folder. +and saved as a Cloud-Optimized GeoTIFF (COG) under the \code{cog/} sub-directory. } \examples{ \dontrun{ diff --git a/man/load_pgvariable.Rd b/man/load_pgvariable.Rd index 855b9e0..b743ea7 100644 --- a/man/load_pgvariable.Rd +++ b/man/load_pgvariable.Rd @@ -11,7 +11,9 @@ load_pgvariable( type = "05deg_yearly", spatial_hash = NULL, temporal_hash = NULL, - verify_checksums = FALSE + verify_checksums = FALSE, + extent = NULL, + layers = NULL ) } \arguments{ @@ -34,15 +36,29 @@ spatial_hash.} \item{verify_checksums}{Logical. If TRUE, verifies the file's MD5 checksum against stored values. Default FALSE.} + +\item{extent}{Optional spatial window. Either a \code{SpatExtent} (used as-is in +the raster's CRS) or a length-4 numeric vector \code{c(xmin, xmax, ymin, ymax)} +in EPSG:4326. Applied lazily via \code{terra::window()} — no data is loaded +until the raster is materialized. Enables efficient reading of spatial subsets +from large COGs (larger-than-memory workflow).} + +\item{layers}{Optional layer selector. A character vector of layer names or an +integer vector of layer indices. Applied lazily via \code{terra::subset()}.} } \value{ -Terra SpatRaster object +A lazy, file-backed \code{SpatRaster}. Data is not loaded into memory +until explicitly materialized (e.g., via \code{terra::values()} or +\code{terra::crop()}). } \description{ -Loads a PRIO-GRID variable from disk and returns it as a terra SpatRaster. -The mode is determined by the \code{config} argument: +Loads a PRIO-GRID variable from disk and returns it as a lazy, file-backed +terra SpatRaster (no data is loaded into memory until the raster is explicitly +materialized). Supports optional lazy subsetting by layers and spatial windowing, +enabling a larger-than-memory workflow. } \details{ +Mode is determined by the \code{config} argument: \itemize{ \item \code{config = NULL} (default): loads from the official release (downloads if needed). \item \code{config = pg_config(...)}: loads from custom data built with that config. @@ -65,5 +81,9 @@ The mode is determined by the \code{config} argument: r <- load_pgvariable("cshapes_gwcode", spatial_hash = "ecf4dd", temporal_hash = "727cca") + + # Windowed / lazy load (larger-than-memory) + r <- load_pgvariable("cru_tmp", extent = c(-30, 60, 35, 72), layers = 1:12) + terra::inMemory(r) # FALSE } } diff --git a/man/pgvariables.Rd b/man/pgvariables.Rd index c12925e..2a1bcfa 100644 --- a/man/pgvariables.Rd +++ b/man/pgvariables.Rd @@ -7,12 +7,19 @@ \format{ \subsection{\code{pgvariables}}{ -A data frame with 4 rows and 5 columns: +A data frame with 38 rows and 7 columns: \describe{ \item{name}{Full name of the variable.} \item{static}{Boolean. Whether the variable varies over time or not.} \item{source_ids}{Comma-separated string. The Unique Universal Identifier (UUID) of the sources used to build the variable.} +\item{label}{Human-readable display title for plots/legends.} +\item{unit}{Measurement unit for colorbar labels; empty if dimensionless.} +\item{transform}{Display transform applied before plotting; one of \code{identity}, \code{log1p}, \code{log10}, or \code{sqrt}.} +\item{plot_type}{Plot type hint; one of \code{continuous}, \code{positive_real}, \code{count}, \code{share}, or \code{discrete}.} } +Raster statistics (\code{value_min}/\code{value_max}/\code{value_mean}/\code{value_std}/\code{nunique}/\code{class_values}) +and the derived \code{colormap} are not stored here; they are computed at build and stamped into +each COG as \code{pg_*} metatags. } } \usage{ diff --git a/man/plot_pgvariable.Rd b/man/plot_pgvariable.Rd new file mode 100644 index 0000000..c616c78 --- /dev/null +++ b/man/plot_pgvariable.Rd @@ -0,0 +1,72 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plot.R +\name{plot_pgvariable} +\alias{plot_pgvariable} +\title{Plot a PRIO-GRID variable} +\usage{ +plot_pgvariable( + x, + layer = 1, + config = NULL, + version = NULL, + type = "05deg_yearly", + spatial_hash = NULL, + temporal_hash = NULL, + ... +) +} +\arguments{ +\item{x}{A \code{SpatRaster} (e.g. from \code{\link{load_pgvariable}()}) +or a character variable name (loaded via \code{load_pgvariable()}).} + +\item{layer}{Integer. Which layer to draw for multi-layer rasters. Default 1. +Can also be a string with the name of the layer. For timevarying rasters, the +layer name is a date with the format "YYYY-MM-DD".} + +\item{config}{A \code{\link{pg_config}} object. Passed to +\code{\link{load_pgvariable}()} when \code{x} is a variable name string; +ignored when \code{x} is already a \code{SpatRaster}. +\code{NULL} (default) resolves via the release path or current config.} + +\item{version}{Character. Release version string (e.g. \code{"3.0.1"}); +passed to \code{load_pgvariable()}. Cannot be combined with \code{config}.} + +\item{type}{Character. Release type (e.g. \code{"05deg_yearly"}); +passed to \code{load_pgvariable()}. Default \code{"05deg_yearly"}.} + +\item{spatial_hash, temporal_hash}{Six-character MD5 hashes that key a custom +output folder; passed to \code{load_pgvariable()}. Must be supplied +together. Cannot be combined with \code{version}.} + +\item{...}{Additional arguments forwarded to \code{terra::plot()}, overriding +any metadata-derived default (e.g. \code{range}, \code{main}, \code{col}).} +} +\value{ +The drawn \code{SpatRaster} layer, invisibly. +} +\description{ +Renders a PRIO-GRID raster using display metadata from its embedded +\code{pg_*} GDAL metatags (written by \code{\link{save_pgvariable}()}), +falling back to the \code{\link{pgvariables}} table and then to safe +defaults. The backend is \code{terra::plot()}: native C++ rendering with +\code{maxcell} downsampling and no materialisation of the raster into a +data frame. No new package dependency — colours come from base +\code{grDevices::hcl.colors()}. +} +\examples{ +\dontrun{ + # Plot directly from a loaded raster: + r <- load_pgvariable("ucdp_ged") + plot_pgvariable(r) + + # Plot by name from a custom data directory: + cfg <- pg_config(nrow = 180, ncol = 360) + plot_pgvariable("ucdp_ged", config = cfg) + + # Plot a specific release: + plot_pgvariable("ucdp_ged", version = "3.0.1") + + # Override the title: + plot_pgvariable(r, main = "Custom title") +} +} diff --git a/man/save_pgvariable.Rd b/man/save_pgvariable.Rd index 7597824..2449931 100644 --- a/man/save_pgvariable.Rd +++ b/man/save_pgvariable.Rd @@ -7,19 +7,31 @@ save_pgvariable(rast, varname, save_to = pgout_path()) } \arguments{ -\item{rast}{Terra SpatRaster object from a gen_*() function} +\item{rast}{Terra SpatRaster object from a gen_*() function.} -\item{varname}{Character string with the variable name (must exist in pgvariables)} +\item{varname}{Character string with the variable name (must exist in pgvariables).} -\item{save_to}{Character string with folder path. Defaults to current custom -data location based on config.} +\item{save_to}{Character string with the dataset root path. Defaults to current +custom data location based on config.} } \value{ NULL (invisibly). Called for side effects (saving file). } \description{ -Saves a terra SpatRaster as a wrapped .rds file. The variable must be -listed in \link{pgvariables}. +Saves a terra SpatRaster as a Cloud-Optimized GeoTIFF (COG) under the +\code{cog/} sub-directory of the dataset root. The variable must be listed +in \link{pgvariables}. Layer names, \code{time()}, and \code{units()} are +preserved natively inside the \code{.tif}. Plot metadata from \link{pgvariables} +(\code{label}, \code{unit}, \code{transform}, \code{plot_type}) is stamped as +\code{pg_*} GDAL metatags so the COG is self-describing. In addition, +\code{pg_colormap} (derived from \code{plot_type}), \code{pg_value_min}, +\code{pg_value_max}, \code{pg_value_mean}, \code{pg_value_std} (computed +from the raster at save time), and — for discrete variables — +\code{pg_nunique} and \code{pg_class_values} are also stamped. +\code{value_*}, \code{nunique}, and \code{class_values} are computed from +the raster at save time; \code{colormap} is derived from \code{plot_type}. +These map directly onto rio-tiler/TiTiler \code{rescale} and +\code{colormap_name}. } \examples{ \dontrun{ diff --git a/tests/testthat/test-metadata-integrity.R b/tests/testthat/test-metadata-integrity.R index 5e85465..7ca2bc4 100644 --- a/tests/testthat/test-metadata-integrity.R +++ b/tests/testthat/test-metadata-integrity.R @@ -25,3 +25,19 @@ test_that("pgvariables names are unique", { test_that("pgchecksum has required columns", { expect_true(all(c("source_name", "source_version", "id", "filename", "md5") %in% names(pgchecksum))) }) + + +test_that("pgvariables has only authored columns (no palette, no statistics)", { + expect_setequal(names(pgvariables), + c("name", "static", "source_ids", "label", "unit", "transform", "plot_type")) + expect_false(any(c("palette","value_min","value_max","nunique","value_mean", + "value_std","class_values") %in% names(pgvariables))) +}) + +test_that("pgvariables plot_type/transform vocabularies and non-empty labels", { + expect_true(all(pgvariables$plot_type %in% + c("continuous", "positive_real", "count", "share", "discrete"))) + expect_true(all(pgvariables$transform %in% + c("identity", "log1p", "log10", "sqrt"))) + expect_true(all(nzchar(pgvariables$label)) && !any(is.na(pgvariables$label))) +}) \ No newline at end of file diff --git a/tests/testthat/test-plot.R b/tests/testthat/test-plot.R new file mode 100644 index 0000000..de2265c --- /dev/null +++ b/tests/testthat/test-plot.R @@ -0,0 +1,39 @@ +test_that(".pg_plot_palette maps colormap strings to distinct colour vectors", { + expect_length(priogrid:::.pg_plot_palette("viridis", 4), 4) + expect_false(identical(priogrid:::.pg_plot_palette("viridis", 8), + priogrid:::.pg_plot_palette("inferno", 8))) +}) + +test_that("plot_pgvariable draws a count variable from its metatags", { + skip_if_not_installed("terra") + cfg <- test_config() + r <- prio_blank_grid(cfg) + terra::values(r) <- rep(c(0, 1, 5, NA), length.out = terra::ncell(r)) + names(r) <- "ucdp_ged" + tmp <- tempfile(); dir.create(tmp); on.exit(unlink(tmp, recursive = TRUE), add = TRUE) + save_pgvariable(r, "ucdp_ged", save_to = tmp) + rr <- terra::rast(file.path(tmp, "cog", "ucdp_ged.tif")) + + png_path <- tempfile(fileext = ".png"); on.exit(unlink(png_path), add = TRUE) + grDevices::png(png_path) + expect_no_error(plot_pgvariable(rr)) + grDevices::dev.off() + expect_true(file.exists(png_path) && file.info(png_path)$size > 0) +}) + +test_that("plot_pgvariable draws a discrete variable as classes", { + skip_if_not_installed("terra") + cfg <- test_config() + r <- prio_blank_grid(cfg) + terra::values(r) <- rep(c(2, 20, 200, NA), length.out = terra::ncell(r)) + names(r) <- "cshapes_gwcode" + tmp <- tempfile(); dir.create(tmp); on.exit(unlink(tmp, recursive = TRUE), add = TRUE) + save_pgvariable(r, "cshapes_gwcode", save_to = tmp) + rr <- terra::rast(file.path(tmp, "cog", "cshapes_gwcode.tif")) + + png_path <- tempfile(fileext = ".png"); on.exit(unlink(png_path), add = TRUE) + grDevices::png(png_path) + expect_no_error(plot_pgvariable(rr)) + grDevices::dev.off() + expect_true(file.exists(png_path)) +}) diff --git a/tests/testthat/test-save-load.R b/tests/testthat/test-save-load.R index 4f2c3e0..0c3c5a3 100644 --- a/tests/testthat/test-save-load.R +++ b/tests/testthat/test-save-load.R @@ -28,3 +28,47 @@ test_that("save_pgvariable errors for unknown varname", { "not found in pgvariables" ) }) + + +test_that("save_pgvariable stamps plotting metatags for a discrete variable", { + skip_if_not_installed("terra") + cfg <- test_config() + varname <- "cshapes_gwcode" # plot_type = discrete + r <- prio_blank_grid(cfg) + terra::values(r) <- rep(c(2, 20, 20, 200, NA), length.out = terra::ncell(r)) + names(r) <- varname + + tmp <- tempfile(); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE), add = TRUE) + save_pgvariable(r, varname, save_to = tmp) + + tags <- terra::metags(terra::rast(file.path(tmp, "cog", paste0(varname, ".tif")))) + get <- function(k) tags$value[tags$name == k] + expect_equal(get("pg_plot_type"), "discrete") + expect_equal(get("pg_colormap"), "tab20") + expect_equal(as.numeric(get("pg_value_min")), 2) + expect_equal(as.numeric(get("pg_value_max")), 200) + expect_equal(as.integer(get("pg_nunique")), 3L) # {2, 20, 200} + expect_equal(get("pg_class_values"), "2,20,200") + expect_true("pg_value_mean" %in% tags$name) + expect_false("pg_palette" %in% tags$name) +}) + +test_that("save_pgvariable derives colormap/transform for a count variable", { + skip_if_not_installed("terra") + cfg <- test_config() + varname <- "ucdp_ged" # plot_type = count, transform = log1p + r <- prio_blank_grid(cfg) + terra::values(r) <- rep(c(0, 1, 5, NA), length.out = terra::ncell(r)) + names(r) <- varname + + tmp <- tempfile(); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE), add = TRUE) + save_pgvariable(r, varname, save_to = tmp) + + tags <- terra::metags(terra::rast(file.path(tmp, "cog", paste0(varname, ".tif")))) + get <- function(k) tags$value[tags$name == k] + expect_equal(get("pg_transform"), "log1p") + expect_equal(get("pg_colormap"), "inferno") + expect_false("pg_class_values" %in% tags$name) # not discrete +}) \ No newline at end of file diff --git a/vignettes/contributing.Rmd b/vignettes/contributing.Rmd index 7fcc1a7..10c1c6f 100644 --- a/vignettes/contributing.Rmd +++ b/vignettes/contributing.Rmd @@ -267,27 +267,24 @@ gen_my_yearly_variable <- function(config = pg_current_config()) { ### Add to `data_raw/variables.csv` -`variables.csv` is **tab-separated** with 10 columns. Add one row per variable: +`variables.csv` is **tab-separated** with 7 columns. Add one row per variable: | Column | Description / Allowed values | |---|---| | `name` | Must match the string set by `names(r)` in `gen_*()` and equal `gen_{name}` without the prefix | | `static` | `TRUE` for variables without a time dimension, `FALSE` for time-varying | | `source_ids` | Comma-separated source UUIDs from `sources.csv` | -| `label` | Display title shown in plots; set equal to `name` (no code default exists) | -| `unit` | Unit string (e.g. `°C`, `mm`); leave empty if not applicable | +| `label` | Human-readable display title for plots/legends | +| `unit` | Unit string (e.g. `°C`, `mm`); leave empty if dimensionless | | `transform` | One of: `identity`, `log1p`, `log10`, `sqrt` | -| `palette` | Palette name (e.g. `viridis`); default `viridis` | -| `plot_type` | One of: `continuous`, `categorical`, `diverging` | -| `value_min` | Numeric lower display bound, or empty for data-driven scaling | -| `value_max` | Numeric upper display bound, or empty for data-driven scaling | +| `plot_type` | One of: `continuous`, `positive_real`, `count`, `share`, `discrete` | -A hand-written row must include all 10 tab-separated fields; empty fields still need their tab +A hand-written row must include all 7 tab-separated fields; empty fields still need their tab delimiter. Example (tabs shown as `→`): ``` -name→static→source_ids→label→unit→transform→palette→plot_type→value_min→value_max -my_new_variable→FALSE→a1b2c3d4-e5f6-7890-abcd-ef1234567890→my_new_variable→→identity→viridis→continuous→→ +name→static→source_ids→label→unit→transform→plot_type +my_new_variable→FALSE→a1b2c3d4-e5f6-7890-abcd-ef1234567890→My new variable→→identity→continuous ``` Regenerate `pgvariables.rda`: @@ -304,9 +301,11 @@ pgvariables[pgvariables$name == "my_new_variable", ] ### Display Metadata and COG Metatags -The seven display columns (`label`, `unit`, `transform`, `palette`, `plot_type`, `value_min`, -`value_max`) are stamped into each built COG as `pg_*` GDAL metatags by `save_pgvariable()` -via `.pg_build_metatags()`, making the `.tif` self-describing. +The four authored display columns (`label`, `unit`, `transform`, `plot_type`) are stamped into +each built COG as `pg_*` GDAL metatags by `save_pgvariable()` via `.pg_build_metatags()`, +making the `.tif` self-describing. Built COGs additionally carry the derived `pg_colormap` and +computed `pg_value_min`/`pg_value_max`/`pg_value_mean`/`pg_value_std`/`pg_nunique`/`pg_class_values` +metatags — these are not authored columns. After editing display columns for an **already-built** variable, re-stamp without recomputing: @@ -603,7 +602,7 @@ mode (`devtools::load_all()`). - [ ] Run `source("data_raw/pgsources.R")` to regenerate `pgsources.rda` - [ ] Run `download_pg_rawdata()`, then `pg_data_availability()` to confirm all files are present - [ ] Create `R/data_{source}.R` with `read_*()` and `gen_*()` functions -- [ ] Add row to `data_raw/variables.csv` (tab-separated, all 10 columns) +- [ ] Add row to `data_raw/variables.csv` (tab-separated, all 7 columns) - [ ] Run `source("data_raw/pgvariables.R")` to regenerate `pgvariables.rda` - [ ] Run `devtools::document()` to regenerate `NAMESPACE` and `man/` - [ ] Add tests in `tests/testthat/`; call `skip_if_no_rawdata()` in `gen_*` tests diff --git a/vignettes/metadata.Rmd b/vignettes/metadata.Rmd index fb1842e..297cea1 100644 --- a/vignettes/metadata.Rmd +++ b/vignettes/metadata.Rmd @@ -27,15 +27,16 @@ pgvariables$name | `name` | Variable name used in `load_pgvariable()`, `calc_pg()`, etc. | | `static` | `TRUE` = no temporal dimension; `FALSE` = time-varying | | `source_ids` | Comma-separated UUIDs of the data sources feeding this variable | -| `label` | Display title shown in plots | -| `unit` | Unit string (e.g. `°C`, `mm`); empty if not applicable | +| `label` | Human-readable display title for plots/legends | +| `unit` | Unit string (e.g. `°C`, `mm`); empty if dimensionless | | `transform` | Display transform: `identity`, `log1p`, `log10`, or `sqrt` | -| `palette` | Colour palette name (e.g. `viridis`) | -| `plot_type` | `continuous`, `categorical`, or `diverging` | -| `value_min` | Numeric lower display bound (empty = data-driven) | -| `value_max` | Numeric upper display bound (empty = data-driven) | +| `plot_type` | One of: `continuous`, `positive_real`, `count`, `share`, `discrete` | -The seven display columns (`label`…`value_max`) are also stamped into each built COG as `pg_*` GDAL metatags, making the `.tif` self-describing (see the [Contributing](contributing.html) vignette). +The four authored display columns (`label`, `unit`, `transform`, `plot_type`) are stamped into +each built COG as `pg_*` GDAL metatags by `save_pgvariable()`. Built COGs additionally carry +computed `pg_colormap`/`pg_value_min`/`pg_value_max`/`pg_value_mean`/`pg_value_std`/`pg_nunique`/`pg_class_values` +(not authored columns). These map onto rio-tiler `rescale`=`pg_value_min`,`pg_value_max` and +`colormap_name`=`pg_colormap`. Static and time-varying variables: From 35b42f95896d681594d31680fb039fcc771d3962 Mon Sep 17 00:00:00 2001 From: Jonas Vestby Date: Tue, 11 Aug 2026 09:28:35 +0200 Subject: [PATCH 7/7] plot: improved by adding borders, citation, date, etc --- NAMESPACE | 1 - R/plot.R | 206 +++++++++++++++++++++++++++++++++++++++-- man/plot_pgvariable.Rd | 49 ++++++++-- 3 files changed, 241 insertions(+), 15 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 7dbf30c..b39b992 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -75,7 +75,6 @@ export(pg_set_rawfolder) export(pgcitations) export(pgout_path) export(pgsearch) -export(plot_pgvariable) export(prio_blank_grid) export(rast_to_df) export(read_cru_pet) diff --git a/R/plot.R b/R/plot.R index 3baf2e2..1dd049c 100644 --- a/R/plot.R +++ b/R/plot.R @@ -1,3 +1,86 @@ +# Named continent bounding boxes: c(xmin, xmax, ymin, ymax) in WGS84 lon/lat. +# Keys are lowercase, space/underscore-stripped for flexible matching. +# @keywords internal +.pg_continent_extents <- list( + africa = c( -25, 52, -35, 38), + antarctica = c(-180, 180, -90, -60), + asia = c( 25, 180, -11, 77), + europe = c( -25, 65, 34, 72), + northamerica = c(-170, -52, 7, 84), + oceania = c( 110, 180, -50, 10), + southamerica = c( -82, -34, -56, 13) +) + +# Internal: resolve extent to a numeric vector. +# Accepts numeric (passed through) or a continent name string (case-insensitive; +# spaces/underscores stripped). Errors with the list of valid names on mismatch. +# @keywords internal +.pg_resolve_extent <- function(extent) { + if (is.numeric(extent)) return(extent) + key <- gsub("[ _]", "", tolower(trimws(extent))) + bbox <- .pg_continent_extents[[key]] + if (is.null(bbox)) { + valid <- paste(c("africa", "antarctica", "asia", "europe", + "north america", "oceania", "south america"), + collapse = ", ") + stop("Unknown extent '", extent, "'. Valid continent names: ", valid, ".", + call. = FALSE) + } + bbox +} + +# Internal: compact "Author et al. (Year); ..." citation string for a variable. +# Follows the pgvariables -> pgsources -> REFERENCES.bib chain. +# @keywords internal +.pg_citation_string <- function(varname) { + row <- pgvariables[pgvariables$name == varname, ] + if (nrow(row) == 0L) return(NULL) + + source_ids <- trimws(unlist(strsplit(row$source_ids[[1]], ","))) + src_rows <- pgsources[pgsources$id %in% source_ids, ] + + # "Family et al. (Year)" from a person list + year string. + inline_cite <- function(author, year) { + if (is.null(author) || length(author) == 0L) return(NULL) + family <- author[[1L]]$family + if (is.null(family) || !nzchar(family)) return(NULL) + suffix <- if (length(author) > 1L) " et al." else "" + paste0(family, suffix, " (", if (nzchar(year %||% "")) year else "n.d.", ")") + } + + cites <- character(0L) + + if (nrow(src_rows) > 0L) { + bibkeys <- unique(trimws(unlist(strsplit(src_rows$citation_keys, ";")))) + bibkeys <- bibkeys[nzchar(bibkeys)] + if (length(bibkeys) > 0L) { + bib <- tryCatch(get_bibliography(bibkeys), error = function(e) NULL) + if (!is.null(bib) && length(bib) > 0L) { + cites <- vapply(seq_along(bib), function(i) { + raw <- unclass(bib[[i]])[[1L]] + inline_cite(raw$author, raw$year) %||% "" + }, character(1L)) + cites <- cites[nzchar(cites)] + } + } + } + + # priogrid package citation — base-R bibentry; same person/year structure. + pkg_cites <- tryCatch({ + pkg <- citation("priogrid") + refs <- vapply(seq_along(pkg), function(i) { + entry <- unclass(pkg)[[i]] + inline_cite(entry$author, entry$year) %||% "" + }, character(1L)) + refs[nzchar(refs)] + }, error = function(e) character(0L)) + + all_cites <- unique(c(cites, pkg_cites)) + if (length(all_cites) == 0L) return(NULL) + paste(all_cites, collapse = "; ") +} + + # Internal: base-R colour vector for a pg_colormap string. # Maps the pg_colormap vocabulary to verified grDevices::hcl.pals() palette names. # @keywords internal @@ -70,11 +153,31 @@ #' @param spatial_hash,temporal_hash Six-character MD5 hashes that key a custom #' output folder; passed to \code{load_pgvariable()}. Must be supplied #' together. Cannot be combined with \code{version}. +#' @param extent Numeric vector \code{c(xmin, xmax, ymin, ymax)} in lon/lat, +#' or a continent name string: \code{"Africa"}, \code{"Antarctica"}, +#' \code{"Asia"}, \code{"Europe"}, \code{"North America"}, \code{"Oceania"}, +#' \code{"South America"} (case-insensitive; spaces/underscores optional). +#' When \code{x} is a variable name string, passed to +#' \code{load_pgvariable()} for a windowed COG read — only the blocks +#' covering the requested region are fetched. Ignored with a warning when +#' \code{x} is already a \code{SpatRaster}; crop it before calling if needed. +#' @param add_borders Logical. If \code{TRUE}, overlays country borders from +#' \code{\link{read_cshapes}()} (requires the cShapes raw data). When the +#' selected layer name is a parseable ISO date (e.g. \code{"1999-12-31"}), +#' the historically correct borders for that date are used; otherwise the +#' most recent snapshot (\code{max(gwsdate)}) is used as a fallback. +#' @param add_ne Logical. If \code{TRUE}, overlays land outlines from +#' \code{\link{read_naturalearth_10m_land}()} (requires the Natural Earth +#' raw data). Drawn beneath \code{add_borders} when both are \code{TRUE}. #' @param ... Additional arguments forwarded to \code{terra::plot()}, overriding #' any metadata-derived default (e.g. \code{range}, \code{main}, \code{col}). #' -#' @return The drawn \code{SpatRaster} layer, invisibly. -#' @export +#' @param add_citation Logical. If \code{TRUE}, renders a compact +#' \dQuote{Author et al. (Year)} citation string in the bottom margin, +#' sourced from \code{\link{pgvariables}}, \code{\link{pgsources}}, and +#' \code{inst/REFERENCES.bib}. Variables with multiple data sources produce +#' a semicolon-separated list. The bottom margin is expanded automatically +#' to accommodate the text. #' #' @examples #' \dontrun{ @@ -82,14 +185,24 @@ #' r <- load_pgvariable("ucdp_ged") #' plot_pgvariable(r) #' -#' # Plot by name from a custom data directory: +#' # Windowed read by continent name: +#' plot_pgvariable("cru_tmp", extent = "Africa") +#' +#' # Windowed read by numeric bbox (xmin, xmax, ymin, ymax): +#' plot_pgvariable("cru_tmp", extent = c(-20, 20, 5, 20)) +#' +#' # Historical borders + land outline overlaid: +#' plot_pgvariable("ucdp_ged", layer = "1999-12-31", +#' extent = "Africa", add_borders = TRUE, add_ne = TRUE) +#' +#' # Custom data directory: #' cfg <- pg_config(nrow = 180, ncol = 360) #' plot_pgvariable("ucdp_ged", config = cfg) #' -#' # Plot a specific release: +#' # Specific release: #' plot_pgvariable("ucdp_ged", version = "3.0.1") #' -#' # Override the title: +#' # Override terra::plot() args: #' plot_pgvariable(r, main = "Custom title") #' } plot_pgvariable <- function(x, layer = 1, @@ -98,17 +211,27 @@ plot_pgvariable <- function(x, layer = 1, type = "05deg_yearly", spatial_hash = NULL, temporal_hash = NULL, + extent = NULL, + add_borders = FALSE, + add_ne = FALSE, + add_citation = FALSE, ...) { rlang::check_installed("terra", reason = "to plot PRIO-GRID variables") + if (!is.null(extent)) extent <- .pg_resolve_extent(extent) varname <- NULL if (is.character(x)) { varname <- x x <- load_pgvariable(x, config = config, version = version, type = type, - spatial_hash = spatial_hash, temporal_hash = temporal_hash) + spatial_hash = spatial_hash, temporal_hash = temporal_hash, + extent = extent) + } else if (!is.null(extent)) { + warning("'extent' is ignored when 'x' is a SpatRaster; ", + "crop before calling plot_pgvariable() if needed.", call. = FALSE) } meta <- .pg_plot_meta(x, varname = varname) - r <- if (terra::nlyr(x) > 1L) x[[layer]] else x + r <- if (terra::nlyr(x) > 1L) x[[layer]] else x + layer_date <- tryCatch(as.Date(names(r)), error = function(e) as.Date(NA_character_)) if (identical(meta$plot_type, "discrete")) { rf <- terra::as.factor(r) @@ -123,13 +246,80 @@ plot_pgvariable <- function(x, layer = 1, main <- if (!identical(meta$transform, "identity")) paste0(meta$label, " (", meta$transform, " scale)") else meta$label args <- list(x = rr, main = main, col = .pg_plot_palette(meta$colormap, 256L)) - if (nzchar(meta$unit)) args$plg <- list(title = meta$unit) if (identical(meta$transform, "identity") && is.finite(meta$value_min) && is.finite(meta$value_max)) args$range <- c(meta$value_min, meta$value_max) } + if (!is.na(layer_date)) + args$main <- paste0(args$main, " \u2014 ", format(layer_date)) args <- utils::modifyList(args, list(...)) + + # Pre-compute citation lines before plotting so margin can be sized correctly. + cit_lines <- NULL + if (add_citation) { + old_mar <- graphics::par("mar") + on.exit(graphics::par(mar = old_mar), add = TRUE) + cit_raw <- tryCatch(.pg_citation_string(meta$varname), + error = function(e) { + warning("add_citation: ", conditionMessage(e), call. = FALSE) + NULL + }) + if (!is.null(cit_raw) && nzchar(cit_raw)) { + # Wrap each per-reference line independently, then flatten. + cit_lines <- unlist(lapply( + strsplit(cit_raw, "\n")[[1L]], + function(l) strwrap(l, width = 80L) + )) + cit_lines <- cit_lines[nzchar(trimws(cit_lines))] + } + n_cit <- max(length(cit_lines), 1L) + new_mar <- old_mar + new_mar[1] <- max(old_mar[1], 4.5 + n_cit) + graphics::par(mar = new_mar) + } + do.call(terra::plot, args) + + if (add_ne || add_borders) { + crop_to_extent <- function(sf_obj) { + if (is.null(extent)) return(sf_obj) + bbox <- sf::st_bbox(c(xmin = extent[1], xmax = extent[2], + ymin = extent[3], ymax = extent[4]), + crs = sf::st_crs(4326)) + suppressWarnings(sf::st_crop(sf_obj, bbox)) + } + + if (add_ne) { + ne <- tryCatch(crop_to_extent(read_naturalearth_10m_land()), + error = function(e) { + warning("add_ne: ", conditionMessage(e), call. = FALSE); NULL + }) + if (!is.null(ne)) + plot(sf::st_geometry(ne), add = TRUE, border = "grey40", col = NA, lwd = 0.4) + } + + if (add_borders) { + cs <- tryCatch(read_cshapes(), error = function(e) { + warning("add_borders: ", conditionMessage(e), call. = FALSE); NULL + }) + if (!is.null(cs)) { + d <- if (!is.na(layer_date)) layer_date else max(cs$gwsdate, na.rm = TRUE) + borders <- dplyr::filter(cs, d %within% date_interval) + borders <- tryCatch(crop_to_extent(borders), error = function(e) borders) + plot(sf::st_geometry(borders), add = TRUE, border = "grey20", col = NA, lwd = 0.5) + } + } + } + + if (!is.null(cit_lines) && length(cit_lines) > 0L) { + n <- length(cit_lines) + for (i in seq_along(cit_lines)) { + # Stack lines bottom-to-top: last line sits at 4, each prior line 1 step higher. + graphics::mtext(cit_lines[i], side = 1, line = 3.5 + (n - i), + cex = 0.55, col = "grey30", adj = 1) + } + } + invisible(args$x) } diff --git a/man/plot_pgvariable.Rd b/man/plot_pgvariable.Rd index c616c78..0db1dea 100644 --- a/man/plot_pgvariable.Rd +++ b/man/plot_pgvariable.Rd @@ -12,6 +12,10 @@ plot_pgvariable( type = "05deg_yearly", spatial_hash = NULL, temporal_hash = NULL, + extent = NULL, + add_borders = FALSE, + add_ne = FALSE, + add_citation = FALSE, ... ) } @@ -38,12 +42,35 @@ passed to \code{load_pgvariable()}. Default \code{"05deg_yearly"}.} output folder; passed to \code{load_pgvariable()}. Must be supplied together. Cannot be combined with \code{version}.} +\item{extent}{Numeric vector \code{c(xmin, xmax, ymin, ymax)} in lon/lat, +or a continent name string: \code{"Africa"}, \code{"Antarctica"}, +\code{"Asia"}, \code{"Europe"}, \code{"North America"}, \code{"Oceania"}, +\code{"South America"} (case-insensitive; spaces/underscores optional). +When \code{x} is a variable name string, passed to +\code{load_pgvariable()} for a windowed COG read — only the blocks +covering the requested region are fetched. Ignored with a warning when +\code{x} is already a \code{SpatRaster}; crop it before calling if needed.} + +\item{add_borders}{Logical. If \code{TRUE}, overlays country borders from +\code{\link{read_cshapes}()} (requires the cShapes raw data). When the +selected layer name is a parseable ISO date (e.g. \code{"1999-12-31"}), +the historically correct borders for that date are used; otherwise the +most recent snapshot (\code{max(gwsdate)}) is used as a fallback.} + +\item{add_ne}{Logical. If \code{TRUE}, overlays land outlines from +\code{\link{read_naturalearth_10m_land}()} (requires the Natural Earth +raw data). Drawn beneath \code{add_borders} when both are \code{TRUE}.} + +\item{add_citation}{Logical. If \code{TRUE}, renders a compact +\dQuote{Author et al. (Year)} citation string in the bottom margin, +sourced from \code{\link{pgvariables}}, \code{\link{pgsources}}, and +\code{inst/REFERENCES.bib}. Variables with multiple data sources produce +a semicolon-separated list. The bottom margin is expanded automatically +to accommodate the text.} + \item{...}{Additional arguments forwarded to \code{terra::plot()}, overriding any metadata-derived default (e.g. \code{range}, \code{main}, \code{col}).} } -\value{ -The drawn \code{SpatRaster} layer, invisibly. -} \description{ Renders a PRIO-GRID raster using display metadata from its embedded \code{pg_*} GDAL metatags (written by \code{\link{save_pgvariable}()}), @@ -59,14 +86,24 @@ data frame. No new package dependency — colours come from base r <- load_pgvariable("ucdp_ged") plot_pgvariable(r) - # Plot by name from a custom data directory: + # Windowed read by continent name: + plot_pgvariable("cru_tmp", extent = "Africa") + + # Windowed read by numeric bbox (xmin, xmax, ymin, ymax): + plot_pgvariable("cru_tmp", extent = c(-20, 20, 5, 20)) + + # Historical borders + land outline overlaid: + plot_pgvariable("ucdp_ged", layer = "1999-12-31", + extent = "Africa", add_borders = TRUE, add_ne = TRUE) + + # Custom data directory: cfg <- pg_config(nrow = 180, ncol = 360) plot_pgvariable("ucdp_ged", config = cfg) - # Plot a specific release: + # Specific release: plot_pgvariable("ucdp_ged", version = "3.0.1") - # Override the title: + # Override terra::plot() args: plot_pgvariable(r, main = "Custom title") } }