diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml index e9171039..704dcf0e 100644 --- a/.github/workflows/check-standard.yaml +++ b/.github/workflows/check-standard.yaml @@ -27,8 +27,8 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} - NASA_EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} + EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN || secrets.NASA_EARTHDATA_TOKEN }} + NASA_EARTHDATA_TOKEN: ${{ secrets.NASA_EARTHDATA_TOKEN || secrets.EARTHDATA_TOKEN }} R_KEEP_PKG_SOURCE: yes steps: diff --git a/.github/workflows/test-coverage-local.yaml b/.github/workflows/test-coverage-local.yaml index f547be09..8a3264d7 100644 --- a/.github/workflows/test-coverage-local.yaml +++ b/.github/workflows/test-coverage-local.yaml @@ -18,8 +18,8 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} - NASA_EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} + EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN || secrets.NASA_EARTHDATA_TOKEN }} + NASA_EARTHDATA_TOKEN: ${{ secrets.NASA_EARTHDATA_TOKEN || secrets.EARTHDATA_TOKEN }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-live.yaml b/.github/workflows/test-live.yaml index 7807ac0f..f81b99cb 100644 --- a/.github/workflows/test-live.yaml +++ b/.github/workflows/test-live.yaml @@ -1,13 +1,13 @@ # Scheduled live-API test workflow. # -# Runs every Monday at 06:00 UTC, plus on-demand (with optional `filter` +# Runs nightly at 06:00 UTC, plus on-demand (with optional `filter` # input). Sets AMADEUS_LIVE_TESTS=true so that `skip_if_no_live_tests()` # does not skip the `test-*-live.R` files. On failure, an issue is # auto-opened (label: live-test-failure) so repo watchers receive an # email notification. on: schedule: - - cron: '0 6 * * 1' + - cron: '0 6 * * *' workflow_dispatch: inputs: filter: @@ -25,8 +25,8 @@ jobs: issues: write env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} - NASA_EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }} + EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN || secrets.NASA_EARTHDATA_TOKEN }} + NASA_EARTHDATA_TOKEN: ${{ secrets.NASA_EARTHDATA_TOKEN || secrets.EARTHDATA_TOKEN }} AMADEUS_LIVE_TESTS: "true" NOT_CRAN: "true" diff --git a/DESCRIPTION b/DESCRIPTION index 70bbe0f3..2e024913 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: amadeus Title: Accessing and Analyzing Large-Scale Environmental Data -Version: 2.0.0 +Version: 2.0.1 Authors@R: c( person(given = "Mitchell", family = "Manware", role = c("aut", "ctb"), comment = c(ORCID = "0009-0003-6440-6106")), person(given = "Insang", family = "Song", role = c("aut", "ctb"), comment = c(ORCID = "0000-0001-8732-3256")), diff --git a/R/download_auxiliary.R b/R/download_auxiliary.R index 0e55f45a..596f0151 100644 --- a/R/download_auxiliary.R +++ b/R/download_auxiliary.R @@ -842,9 +842,9 @@ check_url_status <- function( ) { http_status_ok <- c(200, 206) - tryCatch( + status_head <- tryCatch( { - status <- url |> + url |> httr2::request() |> httr2::req_method("HEAD") |> httr2::req_error(is_error = \(resp) FALSE) |> @@ -854,15 +854,34 @@ check_url_status <- function( ) |> httr2::req_perform() |> httr2::resp_status() + }, + error = function(e) NA_integer_ + ) + + if (!is.na(status_head) && status_head %in% http_status_ok) { + Sys.sleep(1) + return(TRUE) + } - Sys.sleep(1) - return(status %in% http_status_ok) + # Some hosts reject/flake on HEAD; probe with a tiny ranged GET. + status_get <- tryCatch( + { + url |> + httr2::request() |> + httr2::req_headers(Range = "bytes=0-0") |> + httr2::req_error(is_error = \(resp) FALSE) |> + httr2::req_retry( + max_tries = max_tries, + retry_on_failure = TRUE + ) |> + httr2::req_perform() |> + httr2::resp_status() }, - error = function(e) { - # Return FALSE for any errors (network, DNS, SSL, etc.) - return(FALSE) - } + error = function(e) NA_integer_ ) + + Sys.sleep(1) + !is.na(status_get) && status_get %in% http_status_ok } #' Import download commands diff --git a/tests/testthat/test-edgar-live.R b/tests/testthat/test-edgar-live.R index 3c1d6061..e4511004 100644 --- a/tests/testthat/test-edgar-live.R +++ b/tests/testthat/test-edgar-live.R @@ -10,7 +10,7 @@ testthat::test_that( { skip_if_no_live_tests() dir <- withr::local_tempdir() -amadeus::download_edgar( + amadeus::download_edgar( species = "CO", version = "8.1", temp_res = "yearly", @@ -35,7 +35,7 @@ testthat::test_that( { skip_if_no_live_tests() dir <- withr::local_tempdir() -amadeus::download_edgar( + amadeus::download_edgar( species = "SO2", version = "8.1", temp_res = "monthly", @@ -53,15 +53,15 @@ amadeus::download_edgar( testthat::test_that( paste0( - "download_edgar(version='8.1_voc', voc='01'): ", + "download_edgar(version='8.1_voc', voc='1'): ", "downloads VOC speciation file" ), { skip_if_no_live_tests() dir <- withr::local_tempdir() -amadeus::download_edgar( + amadeus::download_edgar( version = "8.1_voc", - voc = "01", + voc = "1", format = "nc", output = "emi", directory_to_save = dir, diff --git a/tests/testthat/test-modis-live.R b/tests/testthat/test-modis-live.R index 45220955..6cf19d3a 100644 --- a/tests/testthat/test-modis-live.R +++ b/tests/testthat/test-modis-live.R @@ -9,13 +9,15 @@ testthat::test_that( ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MOD09GA", version = "061", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), date = c("2024-01-01", "2024-01-01"), extent = c(-79, 35, -78, 36), directory_to_save = dir, @@ -34,13 +36,15 @@ testthat::test_that( ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MOD11A1", version = "061", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), date = c("2022-01-01", "2022-01-01"), extent = c(-79, 35, -78, 36), directory_to_save = dir, @@ -59,13 +63,15 @@ testthat::test_that( ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MCD19A2", version = "061", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), date = c("2022-01-01", "2022-01-01"), extent = c(-79, 35, -78, 36), directory_to_save = dir, @@ -79,18 +85,20 @@ testthat::test_that( testthat::test_that( paste0( - "download_modis(product='MOD06_L2', date=, extent=): ", + "download_modis(product='MOD06_L2', date=, extent=): ", "downloads cloud product file" ), { skip_if_no_live_tests() - testthat::skip_if(!nzchar(Sys.getenv("EARTHDATA_TOKEN")), - "no Earthdata token") + testthat::skip_if( + !nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN")), + "no Earthdata token" + ) dir <- withr::local_tempdir() amadeus::download_modis( product = "MOD06_L2", - nasa_earth_data_token = Sys.getenv("EARTHDATA_TOKEN"), - date = c("2022-01-01", "2022-01-01"), + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), + date = c("2022-01-01", "2022-01-31"), extent = c(-79, 35, -78, 36), directory_to_save = dir, acknowledgement = TRUE