Merge pull request #364 from dblodgett-usgs/main

3DHP Service

.Rbuildignore

@@ -1,6 +1,6 @@
 ^.*\.Rproj$
 ^\.Rproj\.user$
-^.*cache
+^.*cache$
 LICENSE
 .travis.yml
 .lintr

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: nhdplusTools
 Type: Package
 Title: NHDPlus Tools
-Version: 1.0.1
+Version: 1.1.0
 Authors@R: c(person(given = "David", 
                     family = "Blodgett", 
                     role = c("aut", "cre"), 
@@ -28,7 +28,7 @@ URL: https://doi-usgs.github.io/nhdplusTools/ https://github.com/doi-usgs/nhdplu
 BugReports: https://github.com/doi-usgs/nhdplusTools/issues/
 Depends:
     R (>= 4.0)
-Imports: hydroloom, dataRetrieval, dplyr, sf, units, magrittr, jsonlite, httr, xml2, R.utils, utils, tidyr, methods, maptiles, mapsf, fst, arrow, tools, zip, pbapply
+Imports: hydroloom, dataRetrieval, dplyr, sf, units, magrittr, jsonlite, httr, xml2, R.utils, utils, tidyr, methods, maptiles, mapsf, fst, arrow, tools, zip, pbapply, memoise, digest
 Suggests: testthat, knitr, rmarkdown, ggmap, ggplot2, lwgeom, gifski, leaflet, httptest, future, future.apply
 License: CC0
 Encoding: UTF-8

NAMESPACE

@@ -14,6 +14,7 @@ export(download_rf1)
 export(download_vaa)
 export(download_wbd)
 export(fix_flowdir)
+export(get_3dhp)
 export(get_DD)
 export(get_DM)
 export(get_UM)
@@ -26,8 +27,6 @@ export(get_flowline_index)
 export(get_gagesII)
 export(get_hr_data)
 export(get_huc)
-export(get_huc12)
-export(get_huc8)
 export(get_hydro_location)
 export(get_levelpaths)
 export(get_nhdarea)
@@ -64,6 +63,7 @@ export(make_standalone)
 export(map_nhdplus)
 export(navigate_network)
 export(navigate_nldi)
+export(nhdplusTools_cache_settings)
 export(nhdplusTools_data_dir)
 export(nhdplus_path)
 export(plot_nhdplus)
@@ -77,6 +77,7 @@ export(subset_rpu)
 export(subset_vpu)
 importFrom(arrow,open_dataset)
 importFrom(arrow,s3_bucket)
+importFrom(digest,digest)
 importFrom(dplyr,across)
 importFrom(dplyr,all_of)
 importFrom(dplyr,any_of)
@@ -103,6 +104,7 @@ importFrom(fst,read.fst)
 importFrom(httr,GET)
 importFrom(httr,POST)
 importFrom(httr,RETRY)
+importFrom(httr,content)
 importFrom(httr,progress)
 importFrom(httr,write_disk)
 importFrom(hydroloom,accumulate_downstream)
@@ -130,13 +132,17 @@ importFrom(hydroloom,sort_network)
 importFrom(hydroloom,st_compatibalize)
 importFrom(jsonlite,fromJSON)
 importFrom(magrittr,`%>%`)
+importFrom(memoise,cache_filesystem)
+importFrom(memoise,cache_memory)
+importFrom(memoise,memoise)
 importFrom(methods,as)
 importFrom(methods,is)
 importFrom(pbapply,pbapply)
 importFrom(pbapply,pblapply)
 importFrom(pbapply,pboptions)
 importFrom(sf,"st_geometry<-")
 importFrom(sf,read_sf)
+importFrom(sf,sf_use_s2)
 importFrom(sf,st_as_sf)
 importFrom(sf,st_as_sfc)
 importFrom(sf,st_bbox)
@@ -149,6 +155,7 @@ importFrom(sf,st_filter)
 importFrom(sf,st_geometry)
 importFrom(sf,st_geometry_type)
 importFrom(sf,st_layers)
+importFrom(sf,st_make_valid)
 importFrom(sf,st_multilinestring)
 importFrom(sf,st_nearest_feature)
 importFrom(sf,st_sf)

NEWS.md

@@ -1,9 +1,22 @@
-nhdplusTools 1.0.1
+nhdplusTools 1.1.0
 ==========
 
+This release has a significant consolidation of code to work with web services.
+It incorporates an ArcGIS REST service for the first time. It has very few 
+backward compatibility issues with previous versions and introduces new functions
+and behavior.
+
+A key change in behavior is the introduction of caching with the `memoise` package.
+See #366, pull request #364, and documentation in `nhdplusTools_cache_settings()`
+for more.
+
 - fixed nhd and nhdplushr urls #368
 - fix issue with nhdplusTools_data_dir() #365
 - fixed bug with large nhdplus downloads with empty tiles. #361
+- Added 3DHP_all service client. #363
+- Removed deprecated function `get_huc12` and `get_huc8`
+- Updated documentation of `get_huc()` and other web service functions.
+- added `nhdplusTools_cache_settings()` to control use of a `memoise` cache. #366
 
 nhdplusTools 1.0.0
 ==========

R/A_nhdplusTools.R

@@ -93,6 +93,9 @@ assign("nhdplus_attributes", nhdplus_attributes, envir = nhdplusTools_env)
 assign("geoserver_root", "https://labs.waterdata.usgs.gov/geoserver/",
        envir = nhdplusTools_env)
 
+assign("arcrest_root", "https://hydro.nationalmap.gov/arcgis/rest/services/",
+       envir = nhdplusTools_env)
+
 assign("split_flowlines_attributes",
        c("COMID", "toCOMID", "LENGTHKM"),
        envir = nhdplusTools_env)
@@ -350,6 +353,76 @@ nhdplus_path <- function(path = NULL, warn = FALSE) {
   }
 }
 
+#' @title nhdplusTools cache settings
+#' @description
+#' Provides an interface to adjust nhdplusTools `memoise` cache.
+#'
+#' Mode and timeout can also be set using environment variables.
+#' `NHDPLUSTOOLS_MEMOISE_CACHE` and `NHDPLUSTOOLS_MEMOISE_TIMEOUT` are
+#' used unless overriden with this function.
+#'
+#' @param mode character 'memory' or 'filesystem'
+#' @param timeout numeric number of seconds until caches invalidate
+#' @return list containing settings at time of calling. If inputs are
+#' NULL, current settings. If settings are altered, previous setting values.
+#' @export
+#'
+nhdplusTools_cache_settings <- function(mode = NULL, timeout = NULL) {
+  current_mode <- get("nhdpt_mem_cache", envir = nhdplusTools_env)
+  current_timeout <- get("nhdpt_cache_timeout", envir = nhdplusTools_env)
+
+  if(!is.null(mode) && mode %in% c("memory", "filesystem")) {
+    assign("nhdpt_mem_cache", mode, envir = nhdplusTools_env)
+  }
+
+  if(!is.null(timeout) && is.numeric(timeout)) {
+    assign("nhdpt_cache_timeout", timeout, envir = nhdplusTools_env)
+  }
+
+  return(invisible(list(mode = current_mode, timeout = current_timeout)))
+}
+
+#' @importFrom memoise memoise cache_memory cache_filesystem
+#' @importFrom digest digest
+nhdplusTools_memoise_cache <- function() {
+  sys_memo_cache <- Sys.getenv("NHDPLUSTOOLS_MEMOISE_CACHE")
+  ses_memo_cache <- try(get("nhdpt_mem_cache", envir = nhdplusTools_env), silent = TRUE)
+
+  # if it hasn't been set up yet, try to use the system env
+  if(!inherits(ses_memo_cache, "try-error")) {
+    return(ses_memo_cache)
+  } else {
+    if(sys_memo_cache == "memory") {
+      memoise::cache_memory()
+    } else {
+      dir.create(nhdplusTools_data_dir(), showWarnings = FALSE, recursive = TRUE)
+
+      memoise::cache_filesystem(nhdplusTools_data_dir())
+    }
+
+  }
+
+}
+
+nhdplusTools_memoise_timeout <- function() {
+  sys_timeout <- Sys.getenv("NHDPLUSTOOLS_MEMOISE_TIMEOUT")
+  ses_timeout <- try(get("nhdpt_cache_timeout", envir = nhdplusTools_env), silent = TRUE)
+
+  # if it hasn't been set up yet, try to use the system env
+  if(!inherits(ses_timeout, "try-error")) {
+    return(ses_timeout)
+  } else {
+    if(sys_timeout != "") {
+      as.numeric(sys_timeout)
+    } else {
+      # default to one day
+      oneday_seconds <- 60 * 60 * 24
+    }
+  }
+}
+
+assign("nhdpt_mem_cache", nhdplusTools_memoise_cache(), envir = nhdplusTools_env)
+assign("nhdpt_cache_timeout", nhdplusTools_memoise_timeout(), envir = nhdplusTools_env)
 
 #' @title Align NHD Dataset Names
 #' @description this function takes any NHDPlus dataset and aligns the attribute names with those used in nhdplusTools.