Merge pull request DOI-USGS#867 from ldecicco-USGS/develop

Getting ready for CRAN submission

Author: ldecicco-USGS

.Rbuildignore

@@ -2,16 +2,12 @@
 ^Temp$
 ^doc$
 .Rhistory
-vignettes/figures
 .gitignore
-dataRetrieval.Rproj
-dataRetrieval-concordance.tex
 README.md
 ^.*\.Rproj$
 ^\.Rproj\.user$
 ^\.Rproj\.gitignore
 ^\.Rproj\.RData
-^\.Rproj\dataRetrieval.Rproj
 ^\.Rproj\.Rhistory
 CONDUCT.md
 DISCLAIMER.md
@@ -21,15 +17,11 @@ pkgdown
 _pkgdown.yml
 vignettes/earthquake.Rmd
 vignettes/statsServiceMap.Rmd
-vignettes/usMaps.Rmd
-vignettes/usMaps_cache
 vignettes/movingAverages.Rmd
 vignettes/tutorial_slides.Rmd
 vignettes/nldi*
 vignettes/phWI.rds
 vignettes/az_data.rds
-vignettes/atascadero-creek-nldi.png
-vignettes/ucsb.png
 man/figures/highrezlogo-01.png
 man/figures/USGS_R.png
 ^CRAN-RELEASE$
@@ -46,46 +38,10 @@ vignettes/phosData.rds
 vignettes/waterservices.png
 vignettes/wqp_large_pull_targets.Rmd
 vignettes/wqp_large_pull_script.Rmd
-vignettes/full_pipeline.png
-vignettes/pipeline_snap_char_names.png
-vignettes/map_original_grid_cells.png
-vignettes/log_file.png
-vignettes/pipeline_snap_check_inventory.png
-vignettes/map_updated_grid_cells.png
-vignettes/pipeline_snap_update_inventory.png
 .gitlab-ci.yml
-deploy_simple.R
 .lintr
-vignettes/back_to_main.png
-vignettes/delete_branch.png
-vignettes/pipeline.png
-vignettes/merge_request.png
-vignettes/create_merge.png
-vignettes/doc_all.png
-vignettes/commit2.png
-vignettes/commit1.png
-vignettes/test.png
-vignettes/check.png
-vignettes/new_branch.png
-vignettes/pull.png
-vignettes/roxygen.png
-vignettes/install.png
-vignettes/git_repository.png
-vignettes/https_clone.png
-vignettes/new_project.png
 vignettes/Contributing.Rmd
-vignettes/no_manual.png
-vignettes/install_error.png
-vignettes/git_login.png
-vignettes/login.png
-vignettes/git_repository.png
-vignettes/https_clone.png
-vignettes/test_included.png
 vignettes/troubleshooting.Rmd
-vignettes/pat.png
-vignettes/github_code.png
-vignettes/save_pat.png
-vignettes/more_tests.png
 vignettes/stylesSlides.css
 vignettes/Status.Rmd
 vignettes/long_to_wide.Rmd
@@ -94,7 +50,7 @@ vignettes/wqx3_development_plan.Rmd
 vignettes/read_waterdata_functions.Rmd
 vignettes/dataretrieval_discrete_changes_images/*
 vignettes/Wide_data_example.xlsx
-vignettes/messyData.png
+^vignettes/images$
 ^ci$
 ^public$
 ^tutorials$
@@ -108,3 +64,6 @@ vignettes/wqpData.rds
 _quarto.yml
 vignettes/basic_slides.Rmd
 vignettes/changes_slides.Rmd
+vignettes/daily_data_statistics.Rmd
+vignettes/continuous_pr.Rmd
+vignettes/quick_slides.Rmd

.github/workflows/R-CMD-check.yaml

@@ -34,7 +34,7 @@ jobs:
 
       - uses: r-lib/actions/setup-pandoc@6012817847b5f064d0882d67a7b5e2ca6639afb2
 
-      - uses: r-lib/actions/setup-r@473c68190595b311a74f208fba61a8d8c0d4c247
+      - uses: r-lib/actions/setup-r@cdfe59f7d9ed0bc33451d5e5d5d9dde04246fa59
         with:
           r-version: ${{ matrix.config.r }}
           http-user-agent: ${{ matrix.config.http-user-agent }}

.gitignore

@@ -17,3 +17,5 @@ vignettes/*.R
 
 
 /.quarto/
+
+**/*.quarto_ipynb

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: dataRetrieval
 Type: Package
 Title: Retrieval Functions for USGS and EPA Hydrology and Water Quality Data
-Version: 2.7.22
+Version: 2.7.23
 Authors@R: c(
     person("Laura", "DeCicco", role = c("aut","cre"),
             email = "ldecicco@usgs.gov",
@@ -26,7 +26,7 @@ Authors@R: c(
             comment=c(ORCID = "0000-0003-2521-5043")),
     person("Lee", "Stanish", role="ctb",
             comment=c(ORCID = "0000-0002-9775-6861")),
-    person("Joeseph", "Zemmels", role="ctb",
+    person("Joseph", "Zemmels", role="aut",
             email = "jzemmels@usgs.gov",
             comment=c(ORCID = "0009-0008-1463-6313")),
     person("Elise", "Hinman", role="aut",
@@ -55,7 +55,8 @@ Imports:
     jsonlite,
     httr2,
     whisker,
-    sf
+    sf,
+    data.table
 Suggests:
     covr,
     dplyr,

NAMESPACE

@@ -32,8 +32,6 @@ export(readNGWMNlevels)
 export(readNGWMNsites)
 export(readNWISdata)
 export(readNWISdv)
-export(readNWISgwl)
-export(readNWISmeas)
 export(readNWISpCode)
 export(readNWISpeak)
 export(readNWISrating)
@@ -46,15 +44,20 @@ export(readWQPqw)
 export(readWQPsummary)
 export(read_USGS_samples)
 export(read_waterdata)
+export(read_waterdata_channel)
+export(read_waterdata_combined_meta)
 export(read_waterdata_continuous)
 export(read_waterdata_daily)
 export(read_waterdata_field_measurements)
+export(read_waterdata_field_meta)
 export(read_waterdata_latest_continuous)
 export(read_waterdata_latest_daily)
 export(read_waterdata_metadata)
 export(read_waterdata_monitoring_location)
 export(read_waterdata_parameter_codes)
 export(read_waterdata_samples)
+export(read_waterdata_stats_daterange)
+export(read_waterdata_stats_por)
 export(read_waterdata_ts_meta)
 export(renameNWISColumns)
 export(setAccess)
@@ -70,3 +73,12 @@ export(whatWQPsamples)
 export(whatWQPsites)
 export(wqp_check_status)
 export(zeroPad)
+importFrom(data.table,":=")
+importFrom(data.table,.BY)
+importFrom(data.table,.EACHI)
+importFrom(data.table,.GRP)
+importFrom(data.table,.I)
+importFrom(data.table,.N)
+importFrom(data.table,.NGRP)
+importFrom(data.table,.SD)
+importFrom(data.table,data.table)

NEWS

@@ -1,3 +1,26 @@
+dataRetrieval 2.7.23
+===================
+* Added data.table to Imports
+* Added read_waterdata_stats_por and read_waterdata_stats_daterange to access USGS daily data statistics.
+* Updated vignettes, examples, and README to reflect new stats functions.
+* Added tests for read_waterdata_stats functions
+* Updated next_req_url to allow paging through /statistics API output
+* Added deprecation message to readNWISstat function
+* Specify "UTC" attribute for returned time
+* Added options("dataRetrieval.attach_request" = TRUE) as default to attach the 
+request object as an attribute to the returned data frames. Setting 
+options("dataRetrieval.attach_request" = FALSE) will return a data frame without
+the request attribute
+* Improved time documentation
+* Improved error handling in WQP functions
+* Added read_waterdata_field_meta, read_waterdata_combine_meta, 
+and read_waterdata_channel
+* Removed readNWISgwl and readNWISmeas as services have been turned off
+* Updated CQL2 templates to allow HUC queries specifically to use a wildcard
+to get multiple inclusive HUCs.
+* Add deprecation message to readNGWMN functions. read_ngwmn will be coming soon.
+
+
 dataRetrieval 2.7.22
 ===================
 * Added read_waterdata_latest_daily to access latest daily USGS water data.

R/AAA.R

@@ -4,12 +4,15 @@ pkg.env <- new.env()
   suppressMessages(setAccess("public"))
   pkg.env$nldi_base <- "https://api.water.usgs.gov/nldi/linked-data/"
   pkg.env$local_sf <- requireNamespace("sf", quietly = TRUE)
-  options("dataRetrieval" = list("api_version" = "v0"))
+  options("dataRetrieval.api_version" = "v0")
+  options("dataRetrieval.api_version_stat" = "v0")
+  options("dataRetrieval.attach_request" = TRUE)
 
   services <- c("server", "daily", "time-series-metadata",
                "monitoring-locations", "latest-continuous",
                "field-measurements", "latest-daily",
-               "continuous")
+               "continuous", "field-measurements-metadata",
+               "combined-metadata", "channel-measurements")
   collections <- c("parameter-codes", "agency-codes", "altitude-datums", "aquifer-codes",
                    "aquifer-types", "coordinate-accuracy-codes", "coordinate-datum-codes",
                    "coordinate-method-codes", "medium-codes", "counties",

R/citations.R

@@ -13,7 +13,9 @@
 #'
 #' @examples
 #' \donttest{
-#' nwisData <- readNWISdv("04085427", "00060", "2012-01-01", "2012-06-30")
+#' nwisData <- read_waterdata_daily(monitoring_location_id = "USGS-04085427", 
+#'                                  parameter_code = "00060", 
+#'                                  time = c("2012-01-01", "2012-06-30"))
 #' nwis_citation <- create_NWIS_bib(nwisData)
 #' nwis_citation
 #' 
@@ -24,16 +26,14 @@ create_NWIS_bib <- function(x){
 
   textVersion <- paste0("U.S. Geological Survey, ",
                         format(attr(x, "queryTime"), "%Y"),
-                        ", National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed ",
+                        ", USGS Water Data for the Nation: U.S. Geological Survey National Water Information System database, accessed ",
                         format(attr(x, "queryTime"), "%b %d, %Y"),
-                        ", at ",
-                        attr(x, "url"),
-                        ", https://dx.doi.org/10.5066/F7P55KJN")
+                        ", at https://dx.doi.org/10.5066/F7P55KJN")
 
   ref <- utils::bibentry(
     bibtype = "Manual",
     textVersion = textVersion,
-    title = "National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)",
+    title = "USGS Water Data for the Nation: U.S. Geological Survey National Water Information System database",
     author = utils::person("U.S. Geological Survey"),
     doi = "10.5066/F7P55KJN",
     note = paste("Accessed", format(attr(x, "queryTime"), "%b %d, %Y")),

R/constructNWISURL.R

@@ -11,10 +11,8 @@
 #' @param statCd string or vector USGS statistic code only used for daily value service.
 #' This is usually 5 digits.  Daily mean (00003) is the default.
 #' @param service string USGS service to call. Possible values are "dv" (daily values),
-#' "uv" (unit/instantaneous values),
-#' "gwlevels" (groundwater),and "rating" (rating curve),
-#' "peak", "meas" (discrete streamflow measurements),
-#' "stat" (statistics web service BETA).
+#' "uv" (unit/instantaneous values), and "rating" (rating curve),
+#' "peak", "stat" (statistics web service BETA).
 #' @param format string, can be "tsv" or "xml", and is only applicable for daily
 #' and unit value requests.  "tsv" returns results faster, but there is a possibility
 #' that an incomplete file is returned without warning. XML is slower,
@@ -55,9 +53,6 @@
 #' )
 #' url_rating <- constructNWISURL(site_id, service = "rating", ratingType = "base")
 #' url_peak <- constructNWISURL(site_id, service = "peak")
-#' url_meas <- constructNWISURL(site_id, service = "meas")
-#' url_gwl <- constructNWISURL(site_id, service = "gwlevels",
-#'                             startDate = "2024-05-01", endDate = "2024-05-30")
 constructNWISURL <- function(siteNumbers,
                              parameterCd = "00060",
                              startDate = "",
@@ -71,8 +66,8 @@ constructNWISURL <- function(siteNumbers,
                              statType = "mean") {
 
   service <- match.arg(service, c(
-    "dv", "uv", "iv", "iv_recent", "gwlevels",
-    "rating", "peak", "meas", "stat"))
+    "dv", "uv", "iv", "iv_recent", 
+    "rating", "peak", "stat"))
 
   service[service == "meas"] <- "measurements"
   service[service == "uv"] <- "iv"
@@ -212,31 +207,7 @@ constructNWISURL <- function(siteNumbers,
                                 missingData = "off")
            }
          },
-         gwlevels = {
-           url <- get_or_post(baseURL,
-                              POST = POST, 
-                              site_no = siteNumbers,
-                              .multi = "comma")
-           url <- get_or_post(url,
-                              POST = POST, 
-                              format = "rdb")
-           if (nzchar(startDate)) {
-             url <- get_or_post(url,
-                                POST = POST, 
-                                begin_date = startDate)
-           }
-           if (nzchar(endDate)) {
-             url <- get_or_post(url,
-                                POST = POST,  
-                                end_date = endDate)
-           }
-           url <- get_or_post(url,
-                              POST = POST, 
-                              group_key = "NONE",
-                              date_format = "YYYY-MM-DD",
-                              rdb_compression = "value")
-         },
-         { # this will be either dv, uv, groundwater
+         { # this will be either dv, uv
 
            format <- match.arg(format, c("xml", "tsv", "wml1", "wml2", "rdb"))