From 2459a4d33662487bec16fb463a8ac93168e48b46 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Mon, 25 May 2026 22:50:25 -0400 Subject: [PATCH 1/4] drought fix - live tests fix --- .github/workflows/check-standard.yaml | 4 +-- .github/workflows/test-coverage-local.yaml | 4 +-- .github/workflows/test-live.yaml | 8 ++--- DESCRIPTION | 2 +- R/download.R | 37 ++++++++++++++-------- tests/testthat/test-modis-live.R | 36 +++++++++++++-------- 6 files changed, 54 insertions(+), 37 deletions(-) diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml index e9171039..704dcf0e 100644 --- a/.github/workflows/check-standard.yaml +++ b/.github/workflows/check-standard.yaml @@ -27,8 +27,8 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} - NASA_EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} + EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN || secrets.NASA_EARTHDATA_TOKEN }} + NASA_EARTHDATA_TOKEN: ${{ secrets.NASA_EARTHDATA_TOKEN || secrets.EARTHDATA_TOKEN }} R_KEEP_PKG_SOURCE: yes steps: diff --git a/.github/workflows/test-coverage-local.yaml b/.github/workflows/test-coverage-local.yaml index f547be09..8a3264d7 100644 --- a/.github/workflows/test-coverage-local.yaml +++ b/.github/workflows/test-coverage-local.yaml @@ -18,8 +18,8 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} - NASA_EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} + EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN || secrets.NASA_EARTHDATA_TOKEN }} + NASA_EARTHDATA_TOKEN: ${{ secrets.NASA_EARTHDATA_TOKEN || secrets.EARTHDATA_TOKEN }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-live.yaml b/.github/workflows/test-live.yaml index 7807ac0f..f81b99cb 100644 --- a/.github/workflows/test-live.yaml +++ b/.github/workflows/test-live.yaml @@ -1,13 +1,13 @@ # Scheduled live-API test workflow. # -# Runs every Monday at 06:00 UTC, plus on-demand (with optional `filter` +# Runs nightly at 06:00 UTC, plus on-demand (with optional `filter` # input). Sets AMADEUS_LIVE_TESTS=true so that `skip_if_no_live_tests()` # does not skip the `test-*-live.R` files. On failure, an issue is # auto-opened (label: live-test-failure) so repo watchers receive an # email notification. on: schedule: - - cron: '0 6 * * 1' + - cron: '0 6 * * *' workflow_dispatch: inputs: filter: @@ -25,8 +25,8 @@ jobs: issues: write env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} - NASA_EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} + EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN || secrets.NASA_EARTHDATA_TOKEN }} + NASA_EARTHDATA_TOKEN: ${{ secrets.NASA_EARTHDATA_TOKEN || secrets.EARTHDATA_TOKEN }} AMADEUS_LIVE_TESTS: "true" NOT_CRAN: "true" diff --git a/DESCRIPTION b/DESCRIPTION index 70bbe0f3..a2971d62 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: amadeus Title: Accessing and Analyzing Large-Scale Environmental Data -Version: 2.0.0 +Version: 2.0.0.0001 Authors@R: c( person(given = "Mitchell", family = "Manware", role = c("aut", "ctb"), comment = c(ORCID = "0009-0003-6440-6106")), person(given = "Insang", family = "Song", role = c("aut", "ctb"), comment = c(ORCID = "0000-0001-8732-3256")), diff --git a/R/download.R b/R/download.R index c382b63a..62c7f39b 100644 --- a/R/download.R +++ b/R/download.R @@ -5330,30 +5330,39 @@ download_drought <- function( return(invisible(list(success = 0, failed = 0, skipped = 1))) } - url <- NULL + download_result <- NULL + selected_url <- NULL + for (candidate_url in spei_url_candidates) { - if (amadeus::check_url_status(candidate_url)) { - url <- candidate_url + download_result <- amadeus::download_run_method( + urls = candidate_url, + destfiles = destfile, + token = NULL, + show_progress = show_progress, + max_tries = max_tries, + rate_limit = rate_limit + ) + + if ( + isTRUE(download_result$success > 0) && + file.exists(destfile) && + file.info(destfile)$size > 0 + ) { + selected_url <- candidate_url break } } - if (is.null(url)) { + if (is.null(selected_url)) { stop(sprintf( - "SPEI timescale %s returned HTTP 404. Check `timescale` parameter.\n", + paste0( + "SPEI timescale %s was unavailable from all known source URLs. ", + "Check upstream SPEI service status and `timescale` parameter.\n" + ), ts_str )) } - download_result <- amadeus::download_run_method( - urls = url, - destfiles = destfile, - token = NULL, - show_progress = show_progress, - max_tries = max_tries, - rate_limit = rate_limit - ) - if (hash) { return(amadeus::download_hash(hash = TRUE, directory_to_save)) } diff --git a/tests/testthat/test-modis-live.R b/tests/testthat/test-modis-live.R index 45220955..6cf19d3a 100644 --- a/tests/testthat/test-modis-live.R +++ b/tests/testthat/test-modis-live.R @@ -9,13 +9,15 @@ testthat::test_that( ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MOD09GA", version = "061", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), date = c("2024-01-01", "2024-01-01"), extent = c(-79, 35, -78, 36), directory_to_save = dir, @@ -34,13 +36,15 @@ testthat::test_that( ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MOD11A1", version = "061", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), date = c("2022-01-01", "2022-01-01"), extent = c(-79, 35, -78, 36), directory_to_save = dir, @@ -59,13 +63,15 @@ testthat::test_that( ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MCD19A2", version = "061", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), date = c("2022-01-01", "2022-01-01"), extent = c(-79, 35, -78, 36), directory_to_save = dir, @@ -79,18 +85,20 @@ testthat::test_that( testthat::test_that( paste0( - "download_modis(product='MOD06_L2', date=, extent=): ", + "download_modis(product='MOD06_L2', date=, extent=): ", "downloads cloud product file" ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MOD06_L2", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), - date = c("2022-01-01", "2022-01-01"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), + date = c("2022-01-01", "2022-01-31"), extent = c(-79, 35, -78, 36), directory_to_save = dir, acknowledgement = TRUE From 21f330c707636b1a0104c8da6f2df244d2692b85 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Mon, 25 May 2026 23:06:19 -0400 Subject: [PATCH 2/4] check_url update --- R/download.R | 37 ++++++++++++++----------------------- R/download_auxiliary.R | 35 +++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/R/download.R b/R/download.R index 62c7f39b..c382b63a 100644 --- a/R/download.R +++ b/R/download.R @@ -5330,39 +5330,30 @@ download_drought <- function( return(invisible(list(success = 0, failed = 0, skipped = 1))) } - download_result <- NULL - selected_url <- NULL - + url <- NULL for (candidate_url in spei_url_candidates) { - download_result <- amadeus::download_run_method( - urls = candidate_url, - destfiles = destfile, - token = NULL, - show_progress = show_progress, - max_tries = max_tries, - rate_limit = rate_limit - ) - - if ( - isTRUE(download_result$success > 0) && - file.exists(destfile) && - file.info(destfile)$size > 0 - ) { - selected_url <- candidate_url + if (amadeus::check_url_status(candidate_url)) { + url <- candidate_url break } } - if (is.null(selected_url)) { + if (is.null(url)) { stop(sprintf( - paste0( - "SPEI timescale %s was unavailable from all known source URLs. ", - "Check upstream SPEI service status and `timescale` parameter.\n" - ), + "SPEI timescale %s returned HTTP 404. Check `timescale` parameter.\n", ts_str )) } + download_result <- amadeus::download_run_method( + urls = url, + destfiles = destfile, + token = NULL, + show_progress = show_progress, + max_tries = max_tries, + rate_limit = rate_limit + ) + if (hash) { return(amadeus::download_hash(hash = TRUE, directory_to_save)) } diff --git a/R/download_auxiliary.R b/R/download_auxiliary.R index 0e55f45a..596f0151 100644 --- a/R/download_auxiliary.R +++ b/R/download_auxiliary.R @@ -842,9 +842,9 @@ check_url_status <- function( ) { http_status_ok <- c(200, 206) - tryCatch( + status_head <- tryCatch( { - status <- url |> + url |> httr2::request() |> httr2::req_method("HEAD") |> httr2::req_error(is_error = \(resp) FALSE) |> @@ -854,15 +854,34 @@ check_url_status <- function( ) |> httr2::req_perform() |> httr2::resp_status() + }, + error = function(e) NA_integer_ + ) + + if (!is.na(status_head) && status_head %in% http_status_ok) { + Sys.sleep(1) + return(TRUE) + } - Sys.sleep(1) - return(status %in% http_status_ok) + # Some hosts reject/flake on HEAD; probe with a tiny ranged GET. + status_get <- tryCatch( + { + url |> + httr2::request() |> + httr2::req_headers(Range = "bytes=0-0") |> + httr2::req_error(is_error = \(resp) FALSE) |> + httr2::req_retry( + max_tries = max_tries, + retry_on_failure = TRUE + ) |> + httr2::req_perform() |> + httr2::resp_status() }, - error = function(e) { - # Return FALSE for any errors (network, DNS, SSL, etc.) - return(FALSE) - } + error = function(e) NA_integer_ ) + + Sys.sleep(1) + !is.na(status_get) && status_get %in% http_status_ok } #' Import download commands From f85dba01723df3f7dbc392b7ec93404b2041b503 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Mon, 25 May 2026 23:07:00 -0400 Subject: [PATCH 3/4] live-edgar update --- tests/testthat/test-edgar-live.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-edgar-live.R b/tests/testthat/test-edgar-live.R index 3c1d6061..5bcf2e66 100644 --- a/tests/testthat/test-edgar-live.R +++ b/tests/testthat/test-edgar-live.R @@ -61,7 +61,7 @@ testthat::test_that( dir <- withr::local_tempdir() amadeus::download_edgar( version = "8.1_voc", - voc = "01", + voc = "1", format = "nc", output = "emi", directory_to_save = dir, From f9a6c9031dfa1823036db81e9c4d7f21492669d1 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Mon, 25 May 2026 23:18:56 -0400 Subject: [PATCH 4/4] proper patch increment --- DESCRIPTION | 2 +- tests/testthat/test-edgar-live.R | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index a2971d62..2e024913 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: amadeus Title: Accessing and Analyzing Large-Scale Environmental Data -Version: 2.0.0.0001 +Version: 2.0.1 Authors@R: c( person(given = "Mitchell", family = "Manware", role = c("aut", "ctb"), comment = c(ORCID = "0009-0003-6440-6106")), person(given = "Insang", family = "Song", role = c("aut", "ctb"), comment = c(ORCID = "0000-0001-8732-3256")), diff --git a/tests/testthat/test-edgar-live.R b/tests/testthat/test-edgar-live.R index 5bcf2e66..e4511004 100644 --- a/tests/testthat/test-edgar-live.R +++ b/tests/testthat/test-edgar-live.R @@ -10,7 +10,7 @@ testthat::test_that( { skip_if_no_live_tests() dir <- withr::local_tempdir() -amadeus::download_edgar( + amadeus::download_edgar( species = "CO", version = "8.1", temp_res = "yearly", @@ -35,7 +35,7 @@ testthat::test_that( { skip_if_no_live_tests() dir <- withr::local_tempdir() -amadeus::download_edgar( + amadeus::download_edgar( species = "SO2", version = "8.1", temp_res = "monthly", @@ -53,13 +53,13 @@ amadeus::download_edgar( testthat::test_that( paste0( - "download_edgar(version='8.1_voc', voc='01'): ", + "download_edgar(version='8.1_voc', voc='1'): ", "downloads VOC speciation file" ), { skip_if_no_live_tests() dir <- withr::local_tempdir() -amadeus::download_edgar( + amadeus::download_edgar( version = "8.1_voc", voc = "1", format = "nc",