diff --git a/.Rbuildignore b/.Rbuildignore index 8e258972..abe2a3f6 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -29,5 +29,7 @@ CLAUDE.md dev .claude inst/build -vignettes/drainage_area_figures PLAN.md +^tests/testthat/fixtures$ +^vignettes/articles$ +^\.kiro$ \ No newline at end of file diff --git a/.gitignore b/.gitignore index a44ce848..b48dc8b0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,7 @@ CLAUDE.md dev vignettes/*.html vignettes/*_figures/ +vignettes/articles/*.html +vignettes/articles/*_figures/ +vignettes/articles/*_files/ +.kiro/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4c03a19..c811c3e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,28 +2,60 @@ variables: _R_CHECK_CRAN_INCOMING_: "false" _R_CHECK_FORCE_SUGGESTS_: "true" _R_CHECK_DONTTEST_EXAMPLES_: "false" - APT_PKGS: "p7zip-full" + _R_CHECK_RD_XREFS_: "false" -before_script: - - wget --no-check-certificate -O /usr/local/share/ca-certificates/DOIRootCA2.crt https://raw.githubusercontent.com/dblodgett-usgs/hydrogeoenv/master/linux/DOIRootCA2.cer - - chmod 644 /usr/local/share/ca-certificates/DOIRootCA2.crt && update-ca-certificates - - apt-get update - - apt-get install -y --no-install-recommends ${APT_PKGS} - - apt-get install -y --no-install-recommends qpdf pandoc - - export PATH="/usr/local/lib/R/site-library/littler/examples/:${PATH}" - - echo "options(Ncpus = $(nproc --all))" >> /usr/local/lib/R/etc/Rprofile.site - - install2.r devtools - - r -e 'devtools::install_dev_deps()' +stages: + - check + - build + - verify -test: - stage: test +workflow: + rules: + - if: $CI_COMMIT_BRANCH =~ /^rc\// + +.r-base: tags: - chs-shared image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/rocker/geospatial:latest + before_script: + - wget --no-check-certificate -O /usr/local/share/ca-certificates/DOIRootCA2.crt https://raw.githubusercontent.com/dblodgett-usgs/hydrogeoenv/master/linux/DOIRootCA2.cer + - chmod 644 /usr/local/share/ca-certificates/DOIRootCA2.crt && update-ca-certificates + - apt-get update + - apt-get install -y --no-install-recommends p7zip-full qpdf pandoc curl + - export PATH="/usr/local/lib/R/site-library/littler/examples/:${PATH}" + - echo "options(Ncpus = $(nproc --all))" >> /usr/local/lib/R/etc/Rprofile.site + - r -e 'pak::local_install_dev_deps()' + +check: + extends: .r-base + stage: check + script: + - R CMD build --no-build-vignettes --no-manual . + - R CMD check --no-tests --no-examples --no-vignettes --no-manual *.tar.gz + artifacts: + paths: + - "*.Rcheck" + when: always + +build: + extends: .r-base + stage: build + script: + - R CMD build . + - PKG_FILE=$(ls *.tar.gz) + - PKG_NAME=$(echo "$PKG_FILE" | sed 's/_.*//') + - PKG_VERSION=$(echo "$PKG_FILE" | sed 's/.*_\(.*\)\.tar\.gz/\1/') + - 'curl --silent --show-error --write-out "\nHTTP status: %{http_code}\n" --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "$PKG_FILE" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PKG_NAME}/${PKG_VERSION}/${PKG_FILE}"' + artifacts: + paths: + - "*.tar.gz" + +verify: + extends: .r-base + stage: verify script: - - r -e 'devtools::check(check_dir = ".")' - - r -e 'capture.output(print(covr::coverage_to_list(covr::package_coverage(type = "all")), width = 20), file = "covr.txt", split = TRUE)' + - R CMD check --as-cran --no-manual *.tar.gz artifacts: paths: - "*.Rcheck" - - "covr.txt" + when: always diff --git a/DESCRIPTION b/DESCRIPTION index c76290e8..ffdf5196 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ -Package: nhdplusTools +Package: hydrogeofetch Type: Package -Title: NHDPlus Tools -Version: 1.5.0 +Title: Hydrologic Geospatial Fabric Extraction Tool Chain +Version: 2.0.0 Authors@R: c(person(given = "David", family = "Blodgett", role = c("aut", "cre"), @@ -9,7 +9,7 @@ Authors@R: c(person(given = "David", comment = c(ORCID = "0000-0001-9489-1710")), person(given = "Mike", family = "Johnson", - role = "aut", + role = "ctb", comment = c(ORCID = "0000-0002-5288-8350")), person(given = "Marc", family = "Weber", @@ -23,18 +23,19 @@ Authors@R: c(person(given = "David", family = "Koenig", role = "ctb", comment = c(ORCID = "0000-0002-7790-330X"))) -Description: Tools for traversing and working with National Hydrography Dataset Plus (NHDPlus) data. All methods implemented in 'nhdplusTools' are available in the NHDPlus documentation available from the US Environmental Protection Agency . +Description: Tools for traversing and working with National Hydrography Dataset Plus (NHDPlus) data. All methods implemented in 'hydrogeofetch' are available in the NHDPlus documentation available from the US Environmental Protection Agency . Previously published as 'nhdplusTools'. URL: https://doi-usgs.github.io/nhdplusTools/ https://github.com/doi-usgs/nhdplusTools/ BugReports: https://github.com/doi-usgs/nhdplusTools/issues/ Depends: R (>= 4.1) -Imports: hydroloom, dataRetrieval, dplyr, sf, units, magrittr, jsonlite, httr, xml2, R.utils, utils, tidyr, methods, maptiles, mapsf, fst, arrow, tools, zip, pbapply, memoise, digest -Suggests: testthat, knitr, rmarkdown, ggmap, ggplot2, lwgeom, gifski, leaflet, httptest, future, future.apply, StreamCatTools, terra +Imports: hydroloom, dataRetrieval, dplyr, sf, units, jsonlite, httr2, xml2, utils, data.table, methods, arrow, tools, zip, memoise, digest +Suggests: testthat, knitr, rmarkdown, ggmap, ggplot2, lwgeom, gifski, leaflet, httptest2, StreamCatTools, terra, maptiles, mapsf License: CC0 Encoding: UTF-8 -RoxygenNote: 7.3.3 VignetteBuilder: knitr Config/testthat/parallel: true Config/testthat/edition: 3 LazyData: true +LazyDataCompression: xz Language: en-US +Config/roxygen2/version: 8.0.0 diff --git a/NAMESPACE b/NAMESPACE index 6ab39502..c4b3328f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,10 +1,8 @@ # Generated by roxygen2: do not edit by hand -export(add_plus_network_attributes) +export(add_mainstems) export(align_nhdplus_names) -export(calculate_arbolate_sum) -export(calculate_total_drainage_area) -export(disambiguate_flowline_indexes) +export(check_mainstems) export(discover_geoconnex_reference) export(discover_nhdplus_id) export(download_nhd) @@ -13,24 +11,16 @@ export(download_nhdplusv2) export(download_rf1) export(download_vaa) export(download_wbd) -export(fix_flowdir) export(get_3dhp) -export(get_DD) -export(get_DM) -export(get_UM) -export(get_UT) export(get_boundaries) export(get_catchment_characteristics) export(get_characteristics_metadata) export(get_drainage_area_estimates) export(get_elev_along_path) -export(get_flowline_index) export(get_gagesII) export(get_geoconnex_reference) export(get_hr_data) export(get_huc) -export(get_hydro_location) -export(get_levelpaths) export(get_nhdarea) export(get_nhdphr) export(get_nhdplus) @@ -38,47 +28,36 @@ export(get_nhdplushr) export(get_nldi_basin) export(get_nldi_feature) export(get_nldi_index) -export(get_node) export(get_nwis) -export(get_partial_length) -export(get_path_lengths) -export(get_path_members) -export(get_pathlength) -export(get_pfaf) export(get_raindrop_trace) -export(get_sorted) export(get_split_catchment) -export(get_streamlevel) -export(get_streamorder) -export(get_terminal) export(get_tocomid) export(get_vaa) export(get_vaa_names) export(get_vaa_path) export(get_waterbodies) -export(get_waterbody_index) export(get_wb_outlet) export(get_xs_point) export(get_xs_points) -export(make_node_topology) +export(hydrogeofetch_cache_settings) +export(hydrogeofetch_data_dir) export(make_standalone) export(map_nhdplus) -export(navigate_network) export(navigate_nldi) -export(nhdplusTools_cache_settings) -export(nhdplusTools_data_dir) export(nhdplus_path) export(plot_nhdplus) export(prepare_nhdplus) -export(rename_geometry) export(rescale_catchment_characteristics) -export(rescale_measures) -export(st_compatibalize) export(subset_nhdplus) export(subset_rpu) export(subset_vpu) +export(update_mainstems) importFrom(arrow,open_dataset) +importFrom(arrow,read_csv_arrow) importFrom(arrow,s3_bucket) +importFrom(arrow,write_parquet) +importFrom(data.table,as.data.table) +importFrom(data.table,dcast) importFrom(dataRetrieval,findNLDI) importFrom(digest,digest) importFrom(dplyr,across) @@ -87,6 +66,7 @@ importFrom(dplyr,any_of) importFrom(dplyr,arrange) importFrom(dplyr,bind_rows) importFrom(dplyr,collect) +importFrom(dplyr,contains) importFrom(dplyr,desc) importFrom(dplyr,distinct) importFrom(dplyr,everything) @@ -102,48 +82,25 @@ importFrom(dplyr,select) importFrom(dplyr,starts_with) importFrom(dplyr,summarize) importFrom(dplyr,ungroup) -importFrom(fst,metadata_fst) -importFrom(fst,read.fst) importFrom(graphics,par) -importFrom(httr,GET) -importFrom(httr,RETRY) -importFrom(httr,content) -importFrom(httr,progress) -importFrom(httr,write_disk) -importFrom(hydroloom,accumulate_downstream) -importFrom(hydroloom,add_levelpaths) -importFrom(hydroloom,add_pathlength) -importFrom(hydroloom,add_pfafstetter) -importFrom(hydroloom,add_streamlevel) -importFrom(hydroloom,add_streamorder) +importFrom(httr2,req_perform) +importFrom(httr2,request) importFrom(hydroloom,add_toids) importFrom(hydroloom,disambiguate_indexes) -importFrom(hydroloom,fix_flowdir) importFrom(hydroloom,get_hydro_location) importFrom(hydroloom,get_node) -importFrom(hydroloom,get_partial_length) -importFrom(hydroloom,hy) importFrom(hydroloom,index_points_to_lines) -importFrom(hydroloom,index_points_to_waterbodies) -importFrom(hydroloom,make_fromids) -importFrom(hydroloom,make_index_ids) -importFrom(hydroloom,navigate_connected_paths) importFrom(hydroloom,navigate_hydro_network) importFrom(hydroloom,navigate_network_dfs) importFrom(hydroloom,rename_geometry) importFrom(hydroloom,rescale_measures) importFrom(hydroloom,sort_network) -importFrom(hydroloom,st_compatibalize) importFrom(jsonlite,fromJSON) -importFrom(magrittr,`%>%`) importFrom(memoise,cache_filesystem) importFrom(memoise,cache_memory) importFrom(memoise,memoise) importFrom(methods,as) importFrom(methods,is) -importFrom(pbapply,pbapply) -importFrom(pbapply,pblapply) -importFrom(pbapply,pboptions) importFrom(sf,"st_geometry<-") importFrom(sf,read_sf) importFrom(sf,st_area) @@ -171,12 +128,8 @@ importFrom(sf,st_union) importFrom(sf,st_zm) importFrom(sf,write_sf) importFrom(stats,weighted.mean) -importFrom(tidyr,contains) -importFrom(tidyr,pivot_wider) -importFrom(tidyr,replace_na) importFrom(tools,R_user_dir) importFrom(units,set_units) -importFrom(utils,combn) importFrom(utils,download.file) importFrom(utils,read.delim) importFrom(utils,tail) diff --git a/NEWS.md b/NEWS.md index d46199f7..7a03450a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,47 @@ +hydrogeofetch 2.0.0 +========== + +`hydrogeofetch` is the renamed successor to `nhdplusTools`. The package scope has grown beyond NHDPlus, and the new name reflects its broader role as a hydrologic geospatial data access and network tool chain. `nhdplusTools` will remain on CRAN as a deprecation shim until October 2028. + +## New features + +- New `add_mainstems()` joins geoconnex mainstem identifiers onto a table containing NHDPlusV2 (comid/featureid) or NHDPlusHR (nhdplusid) identifiers, using a cached lookup table from the [ref_rivers](https://github.com/internetofwater/ref_rivers) release. #469 +- New `check_mainstems()` checks whether mainstem identifiers have been superseded by a newer reference release; new `update_mainstems()` re-indexes points with a known mainstem id onto its replacement using `hydroloom::index_points_to_lines()`. #470 + +## Removed functions (moved to hydroloom) + +Network analysis and spatial indexing functions that duplicated [hydroloom](https://doi-usgs.github.io/hydroloom/) have been removed. Users should call hydroloom directly. See the [hydroloom reference](https://doi-usgs.github.io/hydroloom/reference/) for full documentation, especially the [indexing](https://doi-usgs.github.io/hydroloom/reference/index.html#indexing-and-linear-referencing) and [navigation](https://doi-usgs.github.io/hydroloom/reference/index.html#network-navigation-and-accumulation) sections. + +- `get_flowline_index()` → `hydroloom::index_points_to_lines()` +- `get_waterbody_index()` → `hydroloom::index_points_to_waterbodies()` +- `disambiguate_flowline_indexes()` → `hydroloom::disambiguate_indexes()` +- `get_UT()`, `get_DD()` → `hydroloom::navigate_hydro_network(x, start, "UT")` / `"DD"` +- `get_UM()`, `get_DM()` → `hydroloom::navigate_hydro_network(x, start, "UM")` / `"DM"` (note: the old wrappers also ran `align_nhdplus_names`, filtered, and supported `sort`/`include` arguments) +- `navigate_network()` — workflow glue; use `navigate_nldi()` + `subset_nhdplus()` or hydroloom navigation directly +- `get_sorted()` → `hydroloom::sort_network()` +- `get_pathlength()` → `hydroloom::add_pathlength()` +- `get_streamorder()` → `hydroloom::add_streamorder()` +- `get_streamlevel()` → `hydroloom::add_streamlevel()` +- `get_levelpaths()` → `hydroloom::add_levelpaths()` +- `get_pfaf()` → `hydroloom::add_pfafstetter()` +- `get_terminal()` → `hydroloom::sort_network(x, split = TRUE)` +- `calculate_total_drainage_area()` → `hydroloom::accumulate_downstream(x, "area")` +- `calculate_arbolate_sum()` → `hydroloom::accumulate_downstream(x, "length")` +- `get_path_members()`, `get_path_lengths()` → `hydroloom::navigate_connected_paths()` +- `make_node_topology()` → `hydroloom::make_node_topology()` +- `add_plus_network_attributes()` → use hydroloom's `add_levelpaths()`, `add_pathlength()`, `accumulate_downstream()` etc. directly + +## Removed re-exports + +These hydroloom functions are no longer re-exported; call `hydroloom::fn()` directly: + +- `st_compatibalize()`, `rename_geometry()`, `get_node()`, `fix_flowdir()`, `rescale_measures()`, `get_hydro_location()`, `get_partial_length()` + +## Dependency changes + +- Removed `pbapply` from Imports (no longer needed) +- Removed `future`, `future.apply` from Suggests (no longer needed) + nhdplusTools 1.5.0 ========== diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 00000000..383b9938 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,412 @@ +# hydrogeofetch migration plan + +nhdplusTools 1.5.0 is the last feature release under the old name. This plan covers the rename to hydrogeofetch, the deprecation shim, and the eventual CRAN archive of nhdplusTools. Each milestone ends with a gate — review or communication work that should finish before moving on. + +## Status (as of 2026-06-22) + +**Done:** Milestones 1–4. `hydrogeofetch` v2.0.0 builds and checks clean under its new name, dependencies are modernized (httr2, no fst/tidyr/R.utils/magrittr), every web-service caller degrades gracefully, and the functions that duplicated hydroloom are gone — replaced with a migration vignette, an updated README, and a NEWS.md entry. A fresh reverse-dependency sweep of `amadeus`, `elfgen`, and `StreamCatTools` (2026-06-22) confirmed none of them call a removed function. + +**Outstanding:** +- **Milestone 5 (USGS release/DOI)** — not started. `inst/CITATION` has been pre-updated with the `hydrogeofetch` title/author/version text, but the DOI/URL fields are still `"TBD"`; `code.json` still describes `nhdplusTools` entirely; no DOI has been minted on code.usgs.gov yet. **This is the next concrete blocker** — milestones 6 and 7 both depend on identifiers/URLs that come out of this step. +- **Milestone 6 (CRAN submission)** — not started. No `rc/2.0.0` branch exists yet. +- **Milestone 7 (GitHub repo rename)** — not started. `origin` is still `doi-usgs/nhdplusTools`. +- **Milestone 8 (deprecation shim)** — not started. No shim branch exists. +- **Milestone 9 (archive nhdplusTools, Oct 2028)** — blocked on 8; not due for ~2 years. + + +## 1. Close out nhdplusTools 1.5.0 ✓ + +*Complete.* nhdplusTools v1.5.0 released on CRAN, tagged and released on both GitHub and GitLab (May 2026). pkgdown site live. README points to the rename plan. + + +## 2. Build hydrogeofetch v2.0.0 ✓ + +Rename all internal references so the package installs and loads as `hydrogeofetch`. Function names and signatures stay identical except where the package name is embedded in the function name itself (`nhdplusTools_data_dir`, `nhdplusTools_cache_settings`). New hex sticker is already in place. + +### 2a. Package identity ✓ + +- [x] DESCRIPTION: `Package: hydrogeofetch`, `Title: Hydrologic Geospatial Fabric Extraction Tool Chain`, `Version: 2.0.0`, updated `Description:` (leave `URL:` and `BugReports:` pointing at current repo until milestone 6) +- [x] NEWS.md: added hydrogeofetch 2.0.0 header; nhdplusTools history preserved below +- [x] README.Rmd: updated title, heading, narrative text. Badge URLs, `install.packages()`, `remotes::install_github()` left pointing at nhdplusTools. README.md re-knit. + +Deferred to later milestones: +- inst/CITATION: update after DOI is minted (milestone 4) +- code.json: update after USGS software release process begins (milestone 4) +- _pkgdown.yml `url:`: update after repo rename (milestone 6) +- DESCRIPTION `URL:`/`BugReports:`: update after repo rename (milestone 6) +- README badge URLs and install instructions: update after repo rename (milestone 6) and CRAN acceptance (milestone 5) + +### 2b. R source — package internals ✓ + +- [x] Renamed `R/A_nhdplusTools.R` → `R/A_hydrogeofetch.R` +- [x] Internal environment: `nhdplusTools_env` → `hydrogeofetch_env` (all 10 R files) +- [x] Debug env var: `debug_nhdplusTools` → `debug_hydrogeofetch` +- [x] Cache directory: `tools::R_user_dir("nhdplusTools")` → `tools::R_user_dir("hydrogeofetch")` +- [x] Renamed exported functions: `nhdplusTools_data_dir()` → `hydrogeofetch_data_dir()`, `nhdplusTools_cache_settings()` → `hydrogeofetch_cache_settings()` +- [x] Renamed internal functions: `nhdplusTools_memoise_cache()` → `hydrogeofetch_memoise_cache()`, `nhdplusTools_memoise_timeout()` → `hydrogeofetch_memoise_timeout()` + +### 2c. Environment variables (user-facing) ✓ + +Renamed to `HYDROGEOFETCH_MEMOISE_CACHE` and `HYDROGEOFETCH_MEMOISE_TIMEOUT`. Old `NHDPLUSTOOLS_*` names recognized as fallback with a one-time `packageStartupMessage()`. Remove the fallback in v2.0. + +### 2d. Mechanical find-replace ✓ + +- [x] `package = "nhdplusTools"` → `package = "hydrogeofetch"` (50 files) +- [x] `library(nhdplusTools)` → `library(hydrogeofetch)` (8 files) +- [x] `nhdplusTools::` → `hydrogeofetch::` (30 files) +- [x] Roxygen prose and deprecation warning strings updated +- [x] `test_check("nhdplusTools")` → `test_check("hydrogeofetch")` +- [x] Temp directory name strings in inst/extdata updated +- [x] Vignette narrative prose updated (URLs preserved) + +### 2e. File renames ✓ + +- [x] `vignettes/nhdplusTools.Rmd` → `vignettes/hydrogeofetch.Rmd` (VignetteIndexEntry was already generic) +- [x] `tests/testthat/test_nhdplusTools.R` → `tests/testthat/test_hydrogeofetch.R` +- [x] Deleted `man/nhdplusTools_*.Rd` (regenerated by document()) +- [x] Deleted `docs/` directory (302 files — pkgdown rebuilt after repo rename) + +### 2f. Rebuild and check ✓ + +- [x] `devtools::document()` — NAMESPACE and man pages regenerated clean +- [x] `devtools::build_readme()` — README.md re-knit +- [x] `devtools::check(vignettes = FALSE)` — 0 errors, 0 warnings, 0 notes (with `--no-examples` and `--no-build-vignettes` to avoid slow network-dependent examples) +- [ ] `pkgdown::build_site()` — deferred until repo rename (milestone 6) + +Additional fixes applied during check: +- Stripped UTF-8 BOM from 71 files (introduced by PowerShell bulk edits) +- Changed deprecation messages from `message()` to `packageStartupMessage()` so they are suppressible +- Added once-per-session guard so each deprecation message fires only once +- Disabled parallel testing (`Config/testthat/parallel: false`) — subprocess crashes under parallel fst/arrow usage +- Skipped `test_get_vaa.R` "vaa examples" and `test_02_subset_extras.R` "big rpu test" (R session crashes, deferred to milestone 3 fst → arrow migration) +- Added `skip_if(is.null(...))` to `test_01_get_nldi.R` for NLDI service unavailability +- Added TODO comments for geoconnex TLS cert failures in examples + +### 2g. Validate ✓ + +- [x] `grep -r "nhdplusTools"` in R/, tests/, vignettes/, man/, DESCRIPTION, NAMESPACE — only URLs, deferred install instructions, and DESCRIPTION `URL:`/`BugReports:` remain +- [x] `grep -r "NHDPLUSTOOLS_" R/` — only in env var fallback code +- [x] `library(hydrogeofetch)` loads without errors +- [x] `hydrogeofetch_data_dir()` returns new cache path (`R/data/R/hydrogeofetch`) +- [x] `hydrogeofetch_cache_settings()` returns mode/timeout correctly +- [x] R CMD check: 0 errors, 0 warnings, 0 notes +- [ ] `?hydrogeofetch_data_dir` / `?hydrogeofetch_cache_settings` help pages — verify manually after install +- [ ] `devtools::build_vignettes()` — deferred (slow, network-dependent) +- [ ] `example(get_nldi)` — NLDI pygeoapi returning server-side errors (not rename-related) +- [ ] `testthat::test_local()` in clean session — deferred; known crash issues in test_get_vaa and test_02_subset_extras tracked for milestone 3 + +**Done when:** ~~all items above checked off, R CMD check clean.~~ Complete. R CMD check passes clean. Remaining unchecked items are blocked by external services or deferred to milestone 3. + +**Gate:** Internal review of the renamed package. Share with a few colleagues to install and test before anything goes public. + + +## 3. Dependency and code modernization ✓ + +Clean up legacy dependencies and bring the codebase to a consistent modern style. + +### 3a. HTTP client ✓ + +- [x] httr → httr2 migration across all web-service callers +- [x] httptest2 adopted for unit tests (recorded fixtures via `with_mock_hgf`); live integration tests gated behind `skip_if_no_integration()` +- [x] Binary downloads (GeoPackage etc.) stay as live-only tests behind `skip_on_cran()` +- [x] `hgf_json` / `hgf_sf` wrappers at the HTTP boundary catch errors (including TLS failures) and return `NULL` with a warning instead of propagating + +### 3b. Dependency cleanup ✓ + +- [x] fst removed (commit `1a43313`) +- [x] R.utils dropped — single `gunzip(remove = FALSE, skip = TRUE)` call replaced by a local `gunzip_keep()` helper using base-R `gzfile()`/`readBin()`/`writeBin()` +- [x] tidyr dropped — four call sites swapped: + - `tidyr::unnest()` in `get_index_ids` → base-R `rep()` + `unlist()` against `make_index_ids()$to_list` + - two `replace_na()` calls (`topo_sort_network`, `add_plus_network_attributes`) → base-R `x[is.na(x)] <- 0` + - `pivot_wider()` in `rescale_catchment_characteristics` → `data.table::dcast` + - stale `tidyr::contains` import → dplyr re-export +- [x] data.table added to Imports (replaces tidyr); arrow boundary in [get_vaa.R](R/get_vaa.R) hardened to call `as.data.frame()` on `collect()` / `read_parquet()` output so `[` keeps data.frame semantics when arrow's R metadata roundtrip falls back to data.table (the "Potentially unsafe or invalid elements ... externalptr" warning case) +- [x] maptiles, mapsf moved to Suggests with `check_pkg()` guards at the basemap call in [plot_nhdplus.R](R/plot_nhdplus.R#L165) +- [x] arrow confirmed wired up for VAA caching (`open_dataset`, `read_parquet` in `R/get_vaa.R`) — stays in Imports + +### 3c. Code style ✓ + +- [x] All `%>%` converted to native `|>` — 51 occurrences across 10 R/ files, 32 across 8 test files, 6 in `vignettes/indexing.Rmd`, 7 in `inst/extdata/get_hr_data.R` +- [x] magrittr dropped from Imports; `@importFrom magrittr `%>%`` removed from `R/index_nhdplus.R` +- [x] Modernized formula-style lambdas in files we touched: `~ifelse(. == -9999, ...)` → `\(x) ifelse(x == -9999, ...)` in `R/get_vaa.R`; `~gsub("_rescaled", "", .)` and `~.x*.data$split_area_prop` → `\(x) ...` in `R/rescale_catchments.R` +- [x] One `.` placeholder in a magrittr pipe (`get_hydro.R` site-distance block) rewritten without the placeholder so the native pipe works without needing R 4.2's `_` +- [x] `R CMD check --no-tests` clean: 0 errors, 0 warnings, 0 notes. The pre-existing httptest2 fixture-path NOTE (61 paths >100 chars under `tests/testthat/fixtures/`) was resolved by bundling the fixture tree as `tests/testthat/fixtures.tar.gz` (1.3 MB) and `.Rbuildignore`-ing the loose directory. [helper.R](tests/testthat/helper.R#L59) handles both modes transparently: when the loose tree is present (dev), it is used directly and the tarball is auto-regenerated whenever any fixture is newer than it (so re-record runs never need a manual repack); when the loose tree is absent (R CMD check / CRAN unpack of the source tarball), the tarball is extracted to `tempdir()` on first use. Built source tarball confirmed to have zero paths over 100 chars. + +### 3d. Caller hardening (web-service robustness) + +Policy is set in CLAUDE.md and the `feedback_web_service_response_handling` memory: keep `\donttest` examples, but every web-service caller must validate response shape strictly and degrade gracefully so CRAN-run `\donttest` examples don't crash on degraded services. HTTP boundary now returns `NULL` on failure; the work left is making the downstream callers handle that `NULL` and unexpected payload shapes cleanly. + +- [x] `test_01_get_nldi.R get_nldi_index()` flakiness — resolved by httptest2 mocking (the `skip_if(is.null(...))` carryover at [test_01_get_nldi.R:212](tests/testthat/test_01_get_nldi.R#L212) is a different `xs` test gated by integration skips) +- [x] `get_geoconnex_reference` TLS crash at the HTTP boundary — `hgf_sf`/`hgf_json` now warn + return `NULL` instead of crashing +- [x] `discover_oafeat` / `get_oafeat` at [oafeat_tools.R:164-199](R/oafeat_tools.R#L164-L199) — both `mem_get_json` returns are now `NULL`-checked with a warning + early return; `get_oafeat` short-circuits on `NULL avail` instead of falling through to a misleading "Type must be in available ids: ." `stop()`. +- [x] Methodical audit of downstream callers complete. Concrete crash sites and shape-inconsistencies fixed: + - `hgf_download` ([downloading_tools.R:394-405](R/downloading_tools.R#L394-L405)) was the only HTTP-boundary helper without a tryCatch — now matches `hgf_json`/`hgf_sf` (warn + `NULL` on transport failure). Callers at [downloading_tools.R:127,186,240](R/downloading_tools.R#L127) and [get_vaa.R:188](R/get_vaa.R#L188) now guard with `if(is.null(hgf_download(...))) return(NULL)` so the next-step unzip/7z/`read_sf` doesn't run on a missing file. The [get_vaa.R:275](R/get_vaa.R#L275) site already has an enclosing `tryCatch` that converts the failure cleanly. + - `query_usgs_arcrest` ([arcrest_tools.R:132](R/arcrest_tools.R#L132)) — `all_ids` is now declared `NULL` before the `tryCatch` so a future change to `hgf_json` that re-introduces throwing can't leave the variable unbound. Inert `out <- NULL` in the handler removed. + - `discover_nhdplus_id` ([discover_nhdplus.R:54-77](R/discover_nhdplus.R#L54-L77)) — failure mode normalized from `integer(0)` (via `as.integer(NULL)`) to explicit `NULL`. Caller `is.null()` checks at [get_nhdplus.R:65](R/get_nhdplus.R#L65) and elsewhere now actually catch degraded service. + - `extract_comid_nwis` ([get_nhdplus.R:145-153](R/get_nhdplus.R#L145-L153)) — `hgf_json` returning `NULL` no longer silently disappears into the caller's `unlist`; now warns with the NWIS id so failed lookups are visible. (Memoise caching of `NULL` accepted as-is — short timeout makes natural re-attempt cheap.) + +### 3e. Test infrastructure ✓ + +- [x] `Config/testthat/parallel: true` re-enabled at [DESCRIPTION:36](DESCRIPTION#L36). Full `devtools::test()` run completed clean — no R subprocess crashes, no test failures, only pre-existing warnings (tidyselect bare-variable deprecation in `arrow::read_parquet(col_select = include_names)` inside `R/get_vaa.R` — known, not parallel-related) and the two `skip_if(...)` skips from milestone 3a. Confirms fst removal resolved the parallel/fst+arrow segfault. + +### 3f. Vignettes ✓ + +- [x] `tools::buildVignettes(dir = '.')` run with `BUILD_VIGNETTES` unset — all 7 vignettes build clean (no errors, no warnings). This is the CRAN path: every vignette uses `eval = local` gated on `Sys.getenv("BUILD_VIGNETTES") == "TRUE"`, so on CRAN no chunk evaluates and no service is contacted. Confirms that flaky or unavailable services cannot break the CRAN vignette build. +- [x] Two live-build failures (`BUILD_VIGNETTES=TRUE`) surfaced and fixed: + - `get_nhdplus()` ([R/get_nhdplus.R:131](R/get_nhdplus.R#L131)) returned `list()` instead of `NULL` when every realization's `query_usgs_oafeat()` returned `NULL` (the milestone-3a hardened-HTTP signature). Normalized the return contract — empty result now returns `NULL`, matching the milestone-3d caller-hardening convention. `@return` doc updated. + - The `drainage_area_estimation.Rmd` chunk `[french_broad_boundaries]` failure was a direct downstream symptom: `all_catchments` was stored as a `list()` and `st_transform()` blew up. With the `NULL` contract in place, the existing `if(!is.null(da$all_catchments))` guard in [plot_boundaries](vignettes/drainage_area_estimation.Rmd#L678) drops that layer cleanly. Live render of the vignette now succeeds end-to-end. (Stale cached `da_est_french_broad.rds` had to be patched to `all_catchments = NULL` to simulate the new contract; re-fetching would produce the same result naturally.) + - Hardened the two remaining unguarded `get_nhdplus()` call sites in [get_drainage_area_estimates.R](R/get_drainage_area_estimates.R): line 699 (`all_net`) now `stop()`s with a clear message when the upstream network fetch fails; line 1679 (`extra_cat`) now warns and falls back to an empty `sf` so scalar estimates still flow. + - `get_data_overview.Rmd` WBD chunk: fixed the `length(wbd_out == 0)` typo, removed the brittle `list.files(..., pattern = ".gdb")` re-discovery in favor of using `download_wbd()`'s return value, and added a graceful skip when the service is unavailable. + +**Done when:** No remaining httr, magrittr, R.utils, or tidyr usage. arrow wired up for VAA caching or removed. Tests pass in both mock and live modes. Vignettes build cleanly. Every web-service caller validates response shape strictly and degrades gracefully so `\donttest` examples remain CRAN-safe under failing services. R CMD check clean. Parallel testing re-enabled with no subprocess crashes. + +**Gate:** Run the full test suite live to confirm service compatibility. Review httptest2 fixture sizes — if any are too large to check in, simplify or move those tests to live-only. + +### Deferred: Lazy OGC API discovery + +`discover_oafeat` ([oafeat_tools.R:164](R/oafeat_tools.R#L164)) eagerly fetches queryables for every collection (~30 requests) even though each caller needs only one. Make queryables fetching lazy — fetch only for the collection actually requested. Cuts ~30 HTTP round-trips to 3 (landing, collections, one queryables), shrinks httptest2 fixture directories from ~35 files to ~5, and reduces fragility when the API adds or renames collections. + + +## 4. Reduce hydroloom duplication ✓ + +hydrogeofetch should be a data-fetching and NHD-specific preparation package. Network analysis and spatial indexing belong in hydroloom. Functions that are thin wrappers over hydroloom — adding only column renames or trivial glue — should be removed so users call hydroloom directly. + +### 4a. Drop re-exports ✓ + +Remove the 7 pure re-export *exports* from `R/A_hydrogeofetch.R` (the `@export` + `hydroloom::fn` lines, ~585–602). These pass through hydroloom functions with zero added logic: + +- `st_compatibalize` +- `rename_geometry` +- `get_node` +- `fix_flowdir` +- `rescale_measures` +- `get_hydro_location` +- `get_partial_length` + +Users who need these call `hydroloom::fn()` or `library(hydroloom)`. + +**Keep four as plain imports, not exports.** Retained internal code calls these as bare names, and the re-export block is currently the only thing that imports them into the namespace: + +- `get_hydro_location` — `get_drainage_area_estimates.R` (lines 386, 625) +- `rescale_measures` — `get_drainage_area_estimates.R` (line 593) +- `rename_geometry` — `get_drainage_area_estimates.R` (lines 895, 951) +- `get_node` — `get_nhdplus.R` (line 118) + +When the re-export block is deleted, replace it with a plain roxygen import (no `@export`) so these still resolve: + +```r +#' @importFrom hydroloom get_node get_hydro_location rescale_measures rename_geometry +NULL +``` + +Without this, `R CMD check` raises "no visible global function definition" and the functions error at runtime. `st_compatibalize`, `fix_flowdir`, and `get_partial_length` have no internal callers and can be dropped completely. + +### 4b. Drop indexing functions ✓ + +Remove `R/index_nhdplus.R` entirely: + +- `get_flowline_index` — wraps `hydroloom::index_points_to_lines` with optional auto-download and column rename +- `disambiguate_flowline_indexes` — renames `id` ↔ `point_id` around `hydroloom::disambiguate_indexes` +- `get_waterbody_index` — literally just `index_points_to_waterbodies(...)` + +The only internal caller is `navigate_network` (see 4d below). + +### 4c. Drop navigation shorthands ✓ + +Remove `get_UT`, `get_UM`, `get_DM`, `get_DD` from `R/get_network.R`. `get_UT` and `get_DD` are true one-liners over `hydroloom::navigate_hydro_network`. `get_UM` and `get_DM` are *not* — they call `align_nhdplus_names`, filter the network, add `sort`/`include` arguments, and return the `COMID` column rather than the raw navigation result. Users who only want the COMID set call hydroloom directly with the mode string; the `sort`/`include` behavior has to be reproduced by hand (and is preserved in the shim — see 4i). + +Removing all four empties `R/get_network.R` (it also holds `navigate_network`, removed in 4d), so delete the file. + +Internal callers to fix: +- `plot_nhdplus.R` (2 sites: lines 418, 475) — replace `get_UT(align_nhdplus_names(flowline), x)` with `navigate_hydro_network(align_nhdplus_names(flowline), x, "UT")`. The `navigate_hydro_network` import survives the `get_network.R` deletion because `get_drainage_area_estimates.R` also declares it. + +### 4d. Drop `navigate_network` ✓ + +Remove the `navigate_network` function. It stitches together NLDI calls, navigation, indexing, and trimming — workflow glue that belongs in a vignette example, not a function. It's the main internal consumer of both indexing and navigation shorthands. + +### 4e. Drop remaining thin wrappers ✓ + +Remove these exported functions that add only column renames around a single hydroloom call: + +- `get_sorted` — rename cols → `sort_network` → rename back +- `get_streamorder` — `check_names` + `add_streamorder()$stream_order` +- `get_streamlevel` — `check_names` + `add_streamlevel()$stream_level` +- `get_pathlength` — rename cols → `add_pathlength` → rename back + +Internal callers to fix: +- `subset_nhdplus.R` line 683 — call `sort_network` directly. `sort_network` takes `id`/`toid`, the call site passes `fline[c("comid", "tocomid")]`, and the next line reads `network$comid`. `get_sorted` did the rename round-trip internally, so the replacement must rename in and read the renamed column out: + + ```r + network <- sort_network( + stats::setNames(fline[c("comid", "tocomid")], c("id", "toid")), + split = FALSE, outlets = outlets$comid) + network$id + ``` + + Add `#' @importFrom hydroloom sort_network` to `subset_nhdplus.R` — that tag currently lives only in `get_paths.R` (line 98), which is deleted below. + +Note: `topo_sort_network`, `get_index_ids`, and `get_fromids` (all internal, `@noRd`, in `get_paths.R`) have **no callers** anywhere in `R/`, `tests/`, or `vignettes/`. They are dead helpers, so there is nothing to rewrite — they are removed along with `get_sorted`. With `get_levelpaths`/`get_terminal` (4f) also gone, `R/get_paths.R` is empty and should be deleted. Their hydroloom imports (`make_index_ids`, `make_fromids`) drop out with them. (Confirm these aren't intentional scaffolding for planned work before deleting.) + +### 4f. Drop all already-deprecated wrappers ✓ + +Remove these functions that already emit deprecation warnings and just forward to hydroloom: + +- `get_levelpaths` → `hydroloom::add_levelpaths` +- `get_pfaf` → `hydroloom::add_pfafstetter` +- `get_terminal` → `sort_network` (via `get_sorted`) +- `calculate_total_drainage_area` → `hydroloom::accumulate_downstream` +- `calculate_arbolate_sum` → `hydroloom::accumulate_downstream` +- `get_path_members` / `get_path_lengths` → `hydroloom::navigate_connected_paths` +- `make_node_topology` → `hydroloom::make_node_topology` +- `add_plus_network_attributes` → multiple hydroloom fns + +This removes `R/calc_network.R`, `R/get_path_lengths.R`, `R/rebuild_topology.R`, and `R/run_plus_attributes.R`. Combined with the thin-wrapper and dead-helper removals in 4e, `R/get_paths.R` and `R/get_codes.R` are also emptied and deleted (`get_paths.R`: `get_levelpaths`/`get_levelpaths_internal`, `get_terminal`, `get_sorted`, dead helpers; `get_codes.R`: `get_pfaf`, `get_streamorder`, `get_streamlevel`). `get_levelpaths_internal` and `get_paths_internal` are removed here too — their only callers (`add_plus_network_attributes`, `get_levelpaths`, `get_path_members`/`get_path_lengths`) are all in this set. + +### 4g. Fix the `:::` call ✓ + +`hydroloom:::check_search_radius` in `index_nhdplus.R` uses an unexported hydroloom function. Since we're removing `get_flowline_index`, this goes away automatically. If any remaining code needs this logic, copy the few lines of unit-conversion rather than depending on an unexported function. + +### 4h. What stays + +Functions that have real NHD-specific logic beyond what hydroloom provides: + +- **`get_tocomid`** — NHD-specific coastal filtering, divergence-to-headwater handling, terminalpa grouping around `add_toids` +- **`prepare_nhdplus`** — substantial NHD-specific filtering (network size, path length, dendritic purging), calls `get_tocomid` +- **`align_nhdplus_names`** — NHD column name translation (bridge between NHD names and hydroloom's expected names) + +Internal hydroloom imports that stay (used as implementation details by functions above): +- `add_toids` (via `get_tocomid`) +- `navigate_hydro_network` (via `plot_nhdplus` after rewrite, and `get_drainage_area_estimates`) +- `navigate_network_dfs` (via `get_nhdplushr`, `get_drainage_area_estimates`) +- `index_points_to_lines` (via `get_drainage_area_estimates` — direct call, no wrapper) +- `accumulate_downstream` (if needed internally by retained code) +- `sort_network` (via `subset_nhdplus` after rewrite) +- `check_valid` (via `subset_nhdplus`, `oafeat_tools`, `arcrest_tools`) +- `get_node`, `get_hydro_location`, `rescale_measures`, `rename_geometry` (formerly re-exported; now plain imports — see 4a — used by `get_nhdplus` and `get_drainage_area_estimates`) + +`make_fromids` and `make_index_ids` do **not** stay — their only users (`get_fromids`, `get_index_ids`) are dead helpers removed in 4e. + +### 4i. Shim implications + +The nhdplusTools shim (milestone 8) needs to forward the removed functions somewhere. For functions that mapped directly to hydroloom, the shim forwards to `hydroloom::*` directly rather than routing through hydrogeofetch. The shim's `get_UT` calls `hydroloom::navigate_hydro_network(..., "UT")`, etc. This removes the two-level indirection problem. + +Exception: `get_UM` and `get_DM` are not bare forwards (see 4c). Their shim versions must carry the wrapper body — `align_nhdplus_names`, the network filter, and the `sort`/`include` arguments returning `$COMID` — so existing `get_UM(net, comid, sort = TRUE)` calls keep working. Copy the current function bodies into the shim rather than forwarding. + +### 4j. Verification ✓ + +1. `devtools::document()` — NAMESPACE regenerates without the removed exports +2. `devtools::check()` — 0 errors, 0 warnings +3. `devtools::test()` — remove or relocate tests for dropped functions: + - **Delete** (dedicated to removed functions): `test_get_network.R`, `test_index.R`, `test_navigate.R`, `test_calc_network.R`, `test_get_path.R`, `test_get_path_lengths.R`, `test_rebuild_topology.R`, `test_run_plus_attributes.R`, `test_wb_index.R`, `test_get_codes.R`, `test_get_geometry.R` (re-exports). + - **Rewrite, don't delete**: `test_02_subset.R`, `test_get_nhdplushr.R`, `test_00_plot_nhdplus.R`, `test_00_plot_local_nhdplus.R`, `test_get_drainage_area_estimates.R` use `get_UT` only to build input COMID lists — swap to `navigate_hydro_network(..., "UT")`. +4. Grep: `grep -r "get_UT\|get_UM\|get_DM\|get_DD\|get_flowline_index\|get_waterbody_index\|get_sorted\|get_streamorder\|get_streamlevel" R/` returns zero hits outside of retained internal uses +5. NAMESPACE contains no `hydroloom::` re-exports (but retains the plain `importFrom` for `get_node`/`get_hydro_location`/`rescale_measures`/`rename_geometry`) +6. No `:::` calls remain +7. `R CMD check`'s "checking R code for possible problems" pass shows no "no visible global function definition" notes — the canary for a dropped import (4a) +8. Prune `_pkgdown.yml` reference sections: ~28 entries point at the removed functions and will be dangling man topics. `pkgdown::build_site()` (deferred to milestone 6) errors on missing topics, so either prune now or carry an explicit note into milestone 6's gate. + +### 4k. Sequencing ✓ + +1. **Commit 1**: Remove the re-export *exports* but keep the four plain imports (4a). Remove `R/index_nhdplus.R` and `navigate_network` (4b, 4d) — `navigate_network` is the consumer of the indexing and navigation shorthands, so it goes before them. +2. **Commit 2**: Remove navigation shorthands and delete `R/get_network.R`. Fix `plot_nhdplus.R` internal calls (4c). +3. **Commit 3**: Remove all deprecated wrappers and their source files (`calc_network.R`, `get_path_lengths.R`, `rebuild_topology.R`, `run_plus_attributes.R`, plus `get_pfaf`/`get_terminal`/`get_levelpaths` and the internal helpers) (4f). **This must precede the thin-wrapper removal** — `add_plus_network_attributes` depends on `get_sorted`, `get_pathlength`, and `get_levelpaths_internal`, so removing those first would break the intermediate build. +4. **Commit 4**: Remove the remaining thin wrappers `get_sorted`, `get_streamorder`, `get_streamlevel`, `get_pathlength`; delete the now-empty `get_paths.R`/`get_codes.R`. Rewrite the `subset_nhdplus.R` caller to use `sort_network` directly (4e). +5. **Commit 5**: Update tests (4j step 3), scale back vignettes (4l, 4m), examples, NEWS.md, and the `_pkgdown.yml` reference index. NEWS: add a new entry listing the removed functions and their hydroloom replacements; do not rewrite the historical nhdplusTools entries (they document shipped versions). + +Each commit leaves R CMD check clean. (Commits 3 and 4 are deliberately ordered deprecated-before-thin so no intermediate state references a removed function.) + +### 4l. Migration vignette ✓ + +Write `vignettes/articles/migrating_from_nhdplusTools.Rmd` — a concise guide for users switching from nhdplusTools. + +Contents: + +1. **What moved where** — a lookup table grouped by category (indexing, navigation, network attributes, re-exports). Two columns: old nhdplusTools call → replacement call (`hydroloom::fn()` or `hydrogeofetch::fn()`). +2. **Why the split** — one paragraph. hydrogeofetch fetches and prepares NHD data; hydroloom does network analysis and spatial indexing. Users who did both now load both. +3. **The three migration patterns** — before/after code blocks: + - Functions that moved to hydroloom with a changed signature (e.g. `get_UT(net, 101)` → `hydroloom::navigate_hydro_network(net, 101, "UT")`). For `get_UM`/`get_DM`, note the extra behavior: the old wrappers ran `align_nhdplus_names`, filtered, and returned the sorted/included `COMID` vector, so a bare `navigate_hydro_network(..., "UM")` is not equivalent — show how to reproduce `sort`/`include` if a user relied on them. + - Pure re-exports that just need the package qualifier (e.g. `st_compatibalize` → `hydroloom::st_compatibalize`) + - Thin wrappers that had column-name magic — show the hydroloom call plus `align_nhdplus_names()` if needed +4. **The deprecation shim** — `install.packages("nhdplusTools")` still works during the sunset period, gives deprecation warnings naming the replacement, and forwards to the right place. +5. **Environment variables** — `NHDPLUSTOOLS_*` still recognized as fallback; `HYDROGEOFETCH_*` is the new name. + +Skip internal refactoring details, exhaustive hydroloom API docs (link to hydroloom's site), and history of the rename decision. + +### 4m. Scale back existing vignettes ✓ + +Four existing article vignettes document functionality that moved to hydroloom and will fail to build once the functions are gone. Scale them back: remove the moved-function material, and where a retained example still needs that behavior, call the hydroloom function directly rather than a re-export. + +- `vignettes/articles/indexing.Rmd` — the entire vignette is built on `get_flowline_index`, `get_waterbody_index`, `disambiguate_flowline_indexes`, `get_path_lengths`, and `get_partial_length`. This is now hydroloom's territory. Cut it down to a short pointer that indexing lives in hydroloom (link to hydroloom's indexing vignette), keeping only any hydrogeofetch-specific framing (e.g. fetching the flowlines to index with `get_nhdplus()`, then handing off to `hydroloom::index_points_to_lines()`). Do not maintain a parallel indexing tutorial here. +- `vignettes/hydrogeofetch.Rmd` (main intro) — the navigation walk-through uses `navigate_network`, `get_UM`/`get_UT`/`get_DM`/`get_DD`, and `get_flowline_index`. Replace the navigation section with the fetch-then-hand-off pattern: use hydrogeofetch to get/prepare the network, then `hydroloom::navigate_hydro_network()` for navigation and `hydroloom::index_points_to_lines()` for indexing. Drop the `navigate_network()` mention. +- `vignettes/articles/get_data_overview.Rmd` — uses `navigate_network`, `get_DM`, `get_flowline_index`, `disambiguate_flowline_indexes`. Replace the navigation/indexing snippets with direct hydroloom calls; keep the data-overview/fetching content that is the point of the vignette. +- `vignettes/articles/nhdplushr.Rmd` — uses `get_flowline_index` and `get_UT`. Swap to `hydroloom::index_points_to_lines()` and `hydroloom::navigate_hydro_network(..., "UT")`. + +Where these examples currently lean on re-exported helpers (`get_node`, `get_hydro_location`, `rescale_measures`, `rename_geometry`, etc.), call `hydroloom::fn()` directly so the vignettes don't depend on imports that are no longer user-facing. + +**Done when:** ~~hydrogeofetch exports only data-fetching, NHD-specific preparation, and visualization functions. No thin hydroloom wrappers remain. R CMD check clean.~~ Complete. Verified directly against the repo: all target files and functions removed, zero `:::` calls remain, NAMESPACE carries no `hydroloom::` re-exports (only the plain `importFrom`), and `_pkgdown.yml` has zero dangling references to removed functions. + +**Gate:** ~~Update vignettes and README to show users how to combine hydrogeofetch (for fetching) with hydroloom (for network analysis/indexing). Confirm reverse-dep packages don't import the removed functions from hydrogeofetch directly.~~ Complete. README.Rmd and `vignettes/hydrogeofetch.Rmd`/`migrating_from_nhdplusTools.Rmd` document the fetch-then-hand-off pattern. Reverse-dependency sweep (2026-06-22, cloned each repo and grepped its source): `amadeus` uses only `get_huc`, `elfgen` uses only `get_nhdplus`, `StreamCatTools` uses `get_waterbodies`/`navigate_nldi`/`get_nldi_basin`/`get_nhdplus`/`discover_nhdplus_id` — none call a function removed in this milestone. + + +## 5. USGS software release + +Create a new entry on code.usgs.gov for hydrogeofetch. Mint a new DOI. The existing nhdplusTools DOI stays pointed at a historical release page. + +**Status:** Not started — no DOI minted yet. `inst/CITATION` title/author/version/year text has already been updated to `hydrogeofetch` 2.0.0, but `url`/`textVersion` are placeholder `"TBD"` pending the DOI. `code.json` still describes `nhdplusTools` (name, version history) and has not been touched. **This is the current blocker** — milestones 6 and 7 depend on identifiers and URLs that come out of this step. + +**Done when:** DOI assigned for hydrogeofetch. + +**Gate:** Update the items deferred from milestone 2: +- inst/CITATION: `url`/`textVersion` DOI fields (title/author/version/year already done) +- code.json: `name`, URLs, version entries +- DESCRIPTION: DOI in any relevant fields +Confirm the plan for the old nhdplusTools DOI redirect with the DOI coordinator. + + +## 6. hydrogeofetch on CRAN + +Submit hydrogeofetch v1.0 to CRAN. The repo is still named nhdplusTools at this point — that's fine, CRAN only cares about the package tarball. + +**Pre-submission checklist:** +- `R CMD check --as-cran` clean (0 errors, 0 warnings, ideally 0 notes) +- `devtools::build_vignettes()` succeeds — vignettes render without network failures +- All `\donttest` examples fail gracefully under degraded service conditions (CRAN runs `\donttest` examples) — strict response-type checking in every web-service caller, not `\dontrun` conversions +- Help pages for renamed functions (`?hydrogeofetch_data_dir`, `?hydrogeofetch_cache_settings`) render correctly + +**Release workflow:** Push an `rc/2.0.0` branch to code.usgs.gov. The GitLab CI pipeline runs a lightweight structural check, builds the source tarball, uploads it to the GitLab package registry, and runs `R CMD check --as-cran` on that tarball. Download the verified tarball from the registry and submit it to CRAN. See `.gitlab-ci.yml` and the README "Build and release" section for details. + +**Done when:** Package accepted and available on CRAN. + +**Gate:** Update README.Rmd `install.packages()` line to `install.packages("hydrogeofetch")`. Announce hydrogeofetch v1.0. Notify reverse-dependency maintainers (amadeus, elfgen, StreamCatTools) that the deprecation shim is coming and they should plan to switch their imports. + + +## 7. Rename the GitHub repo + +Rename doi-usgs/nhdplusTools to doi-usgs/hydrogeofetch. GitHub maintains redirects from the old URL. Requires org admin access — this milestone can move earlier if access becomes available. + +**Done when:** The repo is accessible at doi-usgs/hydrogeofetch and old nhdplusTools URLs redirect. + +**Gate:** Verify redirects work for clone URLs, issue links, and pkgdown. Then update the items deferred from milestone 2: +- DESCRIPTION `URL:` and `BugReports:` → new repo URL +- _pkgdown.yml `url:` → new GitHub Pages URL +- README.Rmd badge URLs and `remotes::install_github()` → new repo name +- CI config and any other hardcoded URLs that GitHub won't redirect (e.g., raw.githubusercontent.com paths) +- Re-knit README.md, deploy pkgdown site + + +## 8. nhdplusTools deprecation shim + +Build a shim version of nhdplusTools (maintained as a branch in the hydrogeofetch repo). Every exported function becomes a `.Deprecated()` wrapper that forwards to `hydrogeofetch::*` or `hydroloom::*` as appropriate. Functions that were removed from hydrogeofetch because they duplicated hydroloom (indexing, navigation shorthands, network analysis wrappers) forward directly to hydroloom. Functions that remain in hydrogeofetch forward there. Submit to CRAN. + +**Done when:** Shim accepted on CRAN. Installing nhdplusTools gives deprecation warnings that name hydrogeofetch or hydroloom as the replacement. + +**Gate:** Confirm reverse-dep maintainers have a migration path and timeline. Announce the deprecation and the October 2028 archive date. + + +## 9. Archive nhdplusTools — October 2028 + +Request CRAN archive after the two-year sunset. Archived packages remain installable from the CRAN archive, so existing code pinned to nhdplusTools keeps working. + +**Done when:** nhdplusTools archived on CRAN. + +**Gate:** Verify no remaining reverse dependencies on CRAN. Post a final announcement. Confirm the archived package installs cleanly from the archive. diff --git a/R/A_nhdplusTools.R b/R/A_hydrogeofetch.R similarity index 60% rename from R/A_nhdplusTools.R rename to R/A_hydrogeofetch.R index 83346a68..1b65521b 100644 --- a/R/A_nhdplusTools.R +++ b/R/A_hydrogeofetch.R @@ -1,11 +1,11 @@ # Primary HydroShare Data Resource vaa_hydroshare <- - 'https://www.hydroshare.org/resource/6092c8a62fac45be97a09bfd0b0bf726/data/contents/nhdplusVAA.fst' + 'https://www.hydroshare.org/resource/6092c8a62fac45be97a09bfd0b0bf726/data/contents/nhdplusVAA.parquet' vaa_sciencebase <- - 'https://www.sciencebase.gov/catalog/file/get/63cb311ed34e06fef14f40a3?name=enhd_nhdplusatts.fst' + 'https://www.sciencebase.gov/catalog/file/get/63cb311ed34e06fef14f40a3?name=enhd_nhdplusatts.parquet' -nhdplusTools_env <- new.env() +hydrogeofetch_env <- new.env() # NHDPlus Attributes COMID <- "COMID" @@ -90,143 +90,78 @@ nhdplus_attributes <- list( .data <- . <- NULL -assign("nhdplus_attributes", nhdplus_attributes, envir = nhdplusTools_env) +assign("nhdplus_attributes", nhdplus_attributes, envir = hydrogeofetch_env) assign("arcrest_root", "https://hydro.nationalmap.gov/arcgis/rest/services/", - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("gocnx_ref_base_url", "https://reference.geoconnex.us/", - envir = nhdplusTools_env) + envir = hydrogeofetch_env) + +assign("ref_rivers_release", "v3.2", envir = hydrogeofetch_env) + +assign("ref_rivers_base_url", + "https://github.com/internetofwater/ref_rivers/releases/download/", + envir = hydrogeofetch_env) assign("usgs_water_root", "https://api.water.usgs.gov/fabric/pygeoapi/", - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("split_flowlines_attributes", c("COMID", "toCOMID", "LENGTHKM"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("collapse_flowlines_attributes", c("COMID", "toCOMID", "LENGTHKM", "LevelPathI", "Hydroseq"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("reconcile_collapsed_flowlines_attributes", c("COMID", "toCOMID", "LENGTHKM", "LevelPathI", "Hydroseq"), - envir = nhdplusTools_env) - -assign("get_UT_attributes", - c("COMID", "Pathlength", "LENGTHKM", "Hydroseq", - "LevelPathI", "DnHydroseq"), - envir = nhdplusTools_env) - -assign("get_UM_attributes", - c("COMID", "Pathlength", "LevelPathI", - "Hydroseq"), - envir = nhdplusTools_env) - -assign("get_DM_attributes", - c("COMID", "Pathlength", "LENGTHKM", - "LevelPathI", "DnLevelPat", - "DnHydroseq", "Hydroseq"), - envir = nhdplusTools_env) - -assign("get_DM_nolength_attributes", - c("COMID", - "LevelPathI", "DnLevelPat", - "DnHydroseq", "Hydroseq"), - envir = nhdplusTools_env) - -assign("get_DD_attributes", - c("COMID", "Pathlength", "LENGTHKM", - "LevelPathI", "DnLevelPat", - "DnHydroseq", "Hydroseq", "DnMinorHyd"), - envir = nhdplusTools_env) - -assign("get_flowline_index_attributes", - c("COMID", "REACHCODE", "ToMeas", "FromMeas"), - envir = nhdplusTools_env) - -assign("get_levelpaths_attributes", - c("ID", "toID", "nameID", "weight"), - envir = nhdplusTools_env) - -assign("get_streamorder_attributes", - c("ID", "toID"), - envir = nhdplusTools_env) - -assign("get_streamlevel_attributes", - c("LevelPathI", "DnLevelPat"), - envir = nhdplusTools_env) - -assign("get_pfaf_attributes", - c("ID", "toID", "totda", "outletID", "topo_sort", "levelpath"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("make_standalone_tonode_attributes", c("COMID", "ToNode", "FromNode", "TerminalFl", "Hydroseq", "TerminalPa", "LevelPathI", "FCODE"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("make_standalone_tocomid_attributes", c("COMID", "toCOMID", "Hydroseq", "TerminalPa", - "LevelPathI", "FCODE"), envir = nhdplusTools_env) - -assign("get_waterbody_index_waterbodies_attributes", - c("COMID"), envir = nhdplusTools_env) - -assign("get_waterbody_index_flines_attributes", - c("COMID", "WBAREACOMI", "Hydroseq"), - envir = nhdplusTools_env) - -assign("disambiguate_flowline_indexes_attributes", - c("id", "COMID", "REACHCODE", "REACH_meas", "offset"), - envir = nhdplusTools_env) - -assign("add_plus_network_attributes_attributes", - c("comid", "tocomid", "nameID", "lengthkm"), - envir = nhdplusTools_env) + "LevelPathI", "FCODE"), envir = hydrogeofetch_env) assign("subset_rpu_attributes", c("COMID", "Pathlength", "LENGTHKM", "Hydroseq", "LevelPathI", "DnLevelPat", "RPUID", "ArbolateSu", "TerminalPa"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("subset_vpu_attributes", c(get("subset_rpu_attributes", - envir = nhdplusTools_env), + envir = hydrogeofetch_env), "VPUID"), - envir = nhdplusTools_env) - -assign("fix_flowdir_attributes", - c("COMID", "toCOMID"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("get_hydro_location_attributes", c("COMID", "ToMeas", "FromMeas"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("get_wb_outlet_mres_attributes", c("COMID", "Hydroseq", "WBAREACOMI"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("get_wb_outlet_hires_attributes", c("WBArea_Permanent_Identifier", "Hydroseq"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("on_off_network_attributes", c("COMID", "WBAREACOMI"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) assign("get_tocomid_attributes", c("COMID", "ToNode", "FromNode"), - envir = nhdplusTools_env) - -assign("get_partial_length_attributes", - c("REACHCODE", "FromNode", "ToNode", "LENGTHKM"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) # assigned here for record keeping. Used as a status counter in apply functions. -assign("cur_count", 0, envir = nhdplusTools_env) +assign("cur_count", 0, envir = hydrogeofetch_env) check_names <- function(x, function_name, align = TRUE, tolower = FALSE) { if(align) { @@ -234,7 +169,7 @@ check_names <- function(x, function_name, align = TRUE, tolower = FALSE) { } names_x <- names(x) expect_names <- get(paste0(function_name, "_attributes"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) if ( !all(expect_names %in% names_x)) { stop(paste0("Missing some required attributes in call to: ", function_name, ". Expected: ", @@ -255,34 +190,35 @@ check_names <- function(x, function_name, align = TRUE, tolower = FALSE) { default_nhdplus_path <- "../NHDPlusV21_National_Seamless.gdb" -assign("default_nhdplus_path", default_nhdplus_path, envir = nhdplusTools_env) +assign("default_nhdplus_path", default_nhdplus_path, envir = hydrogeofetch_env) nhd_bucket <- "https://prd-tnm.s3.amazonaws.com/" nhdhr_file_list <- "?prefix=StagedProducts/Hydrography/NHDPlusHR/VPU/Current/GDB/" archive_nhdhr_file_list <- "?prefix=StagedProducts/Hydrography/NHDPlusHR/VPU/Archive/GDB/" nhd_file_list <- "?prefix=StagedProducts/Hydrography/NHD/HU4/GDB/" -assign("nhd_bucket", nhd_bucket, envir = nhdplusTools_env) -assign("nhdhr_file_list", nhdhr_file_list, envir = nhdplusTools_env) -assign("archive_nhdhr_file_list", archive_nhdhr_file_list, envir = nhdplusTools_env) +assign("nhd_bucket", nhd_bucket, envir = hydrogeofetch_env) +assign("nhdhr_file_list", nhdhr_file_list, envir = hydrogeofetch_env) +assign("archive_nhdhr_file_list", archive_nhdhr_file_list, envir = hydrogeofetch_env) -assign("nldi_tier", "prod", - envir = nhdplusTools_env) +assign("api_water_tier", "prod", + envir = hydrogeofetch_env) nhdplus_debug <- function() { - Sys.getenv("debug_nhdplusTools") == "true" + Sys.getenv("debug_hydrogeofetch") == "true" } #' @noRd -get_nldi_url <- function(pygeo = FALSE) { +get_api_water_tier <- function() { + tier_env <- Sys.getenv("API_WATER_TIER") + if(tier_env != "") tier_env + else get("api_water_tier", envir = hydrogeofetch_env) +} - tier_env <- Sys.getenv("NLDI_TIER") +#' @noRd +get_nldi_url <- function(pygeo = FALSE) { - tier <- if(tier_env != "") { - tier_env - } else { - get("nldi_tier", envir = nhdplusTools_env) - } + tier <- get_api_water_tier() if(pygeo) { if (tier == "prod") { @@ -303,7 +239,19 @@ get_nldi_url <- function(pygeo = FALSE) { } } -#' Get or set nhdplusTools data directory +#' @noRd +get_water_url <- function() { + tier <- get_api_water_tier() + if (tier == "prod") { + "https://api.water.usgs.gov/fabric/pygeoapi/" + } else if (tier == "test") { + "https://labs-beta.waterdata.usgs.gov/api/fabric/pygeoapi/" + } else { + stop("only prod or test allowed.") + } +} + +#' Get or set hydrogeofetch data directory #' @description if left unset, will return the user data dir #' as returned by `tools::R_user_dir` for this package. #' @param dir path of desired data directory @@ -311,32 +259,32 @@ get_nldi_url <- function(pygeo = FALSE) { #' @importFrom tools R_user_dir #' @export #' @examples -#' nhdplusTools_data_dir() +#' hydrogeofetch_data_dir() #' -#' nhdplusTools_data_dir("demo") +#' hydrogeofetch_data_dir("demo") #' -#' nhdplusTools_data_dir(tools::R_user_dir("nhdplusTools")) +#' hydrogeofetch_data_dir(tools::R_user_dir("hydrogeofetch")) #' -nhdplusTools_data_dir <- function(dir = NULL) { +hydrogeofetch_data_dir <- function(dir = NULL) { if(is.null(dir)) { - nhdpt_dat_dir <- try(get("nhdpt_dat_dir", envir = nhdplusTools_env), silent = TRUE) + nhdpt_dat_dir <- try(get("nhdpt_dat_dir", envir = hydrogeofetch_env), silent = TRUE) if(inherits(nhdpt_dat_dir, "try-error") || grepl("CRAN", nhdpt_dat_dir)) { assign("nhdpt_dat_dir", - tools::R_user_dir("nhdplusTools"), - envir = nhdplusTools_env) + tools::R_user_dir("hydrogeofetch"), + envir = hydrogeofetch_env) } - return(get("nhdpt_dat_dir", envir = nhdplusTools_env)) + return(get("nhdpt_dat_dir", envir = hydrogeofetch_env)) } else { assign("nhdpt_dat_dir", dir, - envir = nhdplusTools_env) - return(invisible(get("nhdpt_dat_dir", envir = nhdplusTools_env))) + envir = hydrogeofetch_env) + return(invisible(get("nhdpt_dat_dir", envir = hydrogeofetch_env))) } } @@ -362,7 +310,7 @@ nhdplusTools_data_dir <- function(dir = NULL) { nhdplus_path <- function(path = NULL, warn = FALSE) { if (!is.null(path)) { - assign("nhdplus_data", path, envir = nhdplusTools_env) + assign("nhdplus_data", path, envir = hydrogeofetch_env) if (warn) { warning("Path does not exist.") @@ -372,17 +320,18 @@ nhdplus_path <- function(path = NULL, warn = FALSE) { invisible(0) } } else { - return(get("nhdplus_data", envir = nhdplusTools_env)) + return(get("nhdplus_data", envir = hydrogeofetch_env)) } } -#' @title nhdplusTools cache settings +#' @title hydrogeofetch cache settings #' @description -#' Provides an interface to adjust nhdplusTools `memoise` cache. +#' Provides an interface to adjust hydrogeofetch `memoise` cache. #' #' Mode and timeout can also be set using environment variables. -#' `NHDPLUSTOOLS_MEMOISE_CACHE` and `NHDPLUSTOOLS_MEMOISE_TIMEOUT` are -#' used unless overriden with this function. +#' `HYDROGEOFETCH_MEMOISE_CACHE` and `HYDROGEOFETCH_MEMOISE_TIMEOUT` are +#' used unless overridden with this function. The old `NHDPLUSTOOLS_*` +#' names are still recognized as a fallback. #' #' @param mode character 'memory' or 'filesystem' #' @param timeout numeric number of seconds until caches invalidate @@ -390,18 +339,18 @@ nhdplus_path <- function(path = NULL, warn = FALSE) { #' NULL, current settings. If settings are altered, previous setting values. #' @export #' -nhdplusTools_cache_settings <- function(mode = NULL, timeout = NULL) { - current_mode <- tryCatch(get("nhdpt_mem_cache", envir = nhdplusTools_env), +hydrogeofetch_cache_settings <- function(mode = NULL, timeout = NULL) { + current_mode <- tryCatch(get("nhdpt_mem_cache", envir = hydrogeofetch_env), error = \(e) "filesystem") # default to filesystem - current_timeout <- tryCatch(get("nhdpt_cache_timeout", envir = nhdplusTools_env), - error = \(e) nhdplusTools_memoise_timeout()) + current_timeout <- tryCatch(get("nhdpt_cache_timeout", envir = hydrogeofetch_env), + error = \(e) hydrogeofetch_memoise_timeout()) if(!is.null(mode) && mode %in% c("memory", "filesystem")) { - assign("nhdpt_mem_cache", mode, envir = nhdplusTools_env) + assign("nhdpt_mem_cache", mode, envir = hydrogeofetch_env) } if(!is.null(timeout) && is.numeric(timeout)) { - assign("nhdpt_cache_timeout", timeout, envir = nhdplusTools_env) + assign("nhdpt_cache_timeout", timeout, envir = hydrogeofetch_env) } return(invisible(list(mode = current_mode, timeout = current_timeout))) @@ -409,62 +358,82 @@ nhdplusTools_cache_settings <- function(mode = NULL, timeout = NULL) { #' @importFrom memoise memoise cache_memory cache_filesystem #' @importFrom digest digest -nhdplusTools_memoise_cache <- function() { - sys_memo_cache <- Sys.getenv("NHDPLUSTOOLS_MEMOISE_CACHE") - ses_memo_cache <- try(get("nhdpt_mem_cache", envir = nhdplusTools_env), silent = TRUE) +hydrogeofetch_memoise_cache <- function() { + ses_memo_cache <- try(get("nhdpt_mem_cache", envir = hydrogeofetch_env), silent = TRUE) - # if it hasn't been set up yet, try to use the system env if(!inherits(ses_memo_cache, "try-error")) { return(ses_memo_cache) - } else { - if(sys_memo_cache == "memory") { - memoise::cache_memory() - } else { - dir.create(nhdplusTools_data_dir(), showWarnings = FALSE, recursive = TRUE) + } - memoise::cache_filesystem(nhdplusTools_data_dir()) + sys_memo_cache <- Sys.getenv("HYDROGEOFETCH_MEMOISE_CACHE") + if(sys_memo_cache == "") { + old <- Sys.getenv("NHDPLUSTOOLS_MEMOISE_CACHE") + if(old != "") { + warned <- try(get("warned_cache_env", envir = hydrogeofetch_env), silent = TRUE) + if(inherits(warned, "try-error")) { + packageStartupMessage("NHDPLUSTOOLS_MEMOISE_CACHE is deprecated. ", + "Set HYDROGEOFETCH_MEMOISE_CACHE instead.") + assign("warned_cache_env", TRUE, envir = hydrogeofetch_env) + } + sys_memo_cache <- old } - } + if(sys_memo_cache == "memory") { + memoise::cache_memory() + } else { + dir.create(hydrogeofetch_data_dir(), showWarnings = FALSE, recursive = TRUE) + memoise::cache_filesystem(hydrogeofetch_data_dir()) + } } -nhdplusTools_memoise_timeout <- function() { - sys_timeout <- Sys.getenv("NHDPLUSTOOLS_MEMOISE_TIMEOUT") - ses_timeout <- try(get("nhdpt_cache_timeout", envir = nhdplusTools_env), silent = TRUE) +hydrogeofetch_memoise_timeout <- function() { + ses_timeout <- try(get("nhdpt_cache_timeout", envir = hydrogeofetch_env), silent = TRUE) - # if it hasn't been set up yet, try to use the system env if(!inherits(ses_timeout, "try-error")) { return(ses_timeout) - } else { - if(sys_timeout != "") { - as.numeric(sys_timeout) - } else { - # default to one day - oneday_seconds <- 60 * 60 * 24 + } + + sys_timeout <- Sys.getenv("HYDROGEOFETCH_MEMOISE_TIMEOUT") + if(sys_timeout == "") { + old <- Sys.getenv("NHDPLUSTOOLS_MEMOISE_TIMEOUT") + if(old != "") { + warned <- try(get("warned_timeout_env", envir = hydrogeofetch_env), silent = TRUE) + if(inherits(warned, "try-error")) { + packageStartupMessage("NHDPLUSTOOLS_MEMOISE_TIMEOUT is deprecated. ", + "Set HYDROGEOFETCH_MEMOISE_TIMEOUT instead.") + assign("warned_timeout_env", TRUE, envir = hydrogeofetch_env) + } + sys_timeout <- old } } + + if(sys_timeout != "") { + as.numeric(sys_timeout) + } else { + 60 * 60 * 24 + } } .onLoad <- function(libname, pkgname) { query_usgs_arcrest <<- memoise::memoise(query_usgs_arcrest, - ~memoise::timeout(nhdplusTools_memoise_timeout()), - cache = nhdplusTools_memoise_cache()) + ~memoise::timeout(hydrogeofetch_memoise_timeout()), + cache = hydrogeofetch_memoise_cache()) query_usgs_oafeat <<- memoise::memoise(query_usgs_oafeat, - ~memoise::timeout(nhdplusTools_memoise_timeout()), - cache = nhdplusTools_memoise_cache()) + ~memoise::timeout(hydrogeofetch_memoise_timeout()), + cache = hydrogeofetch_memoise_cache()) query_nldi <<- memoise::memoise(query_nldi, - ~memoise::timeout(nhdplusTools_memoise_timeout()), - cache = nhdplusTools_memoise_cache()) + ~memoise::timeout(hydrogeofetch_memoise_timeout()), + cache = hydrogeofetch_memoise_cache()) } #' @title Align NHD Dataset Names -#' @description this function takes any NHDPlus dataset and aligns the attribute names with those used in nhdplusTools. +#' @description this function takes any NHDPlus dataset and aligns the attribute names with those used in hydrogeofetch. #' @param x a \code{sf} object of nhdplus flowlines #' @return data.frame renamed \code{sf} object #' @export #' @examples -#' source(system.file("extdata/new_hope_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/new_hope_data.R", package = "hydrogeofetch")) #' #' names(new_hope_flowline) #' @@ -476,7 +445,7 @@ nhdplusTools_memoise_timeout <- function() { #' align_nhdplus_names <- function(x){ - attribute_names <- get("nhdplus_attributes", envir = nhdplusTools_env) + attribute_names <- get("nhdplus_attributes", envir = hydrogeofetch_env) # get into correct case good_names <- unique(unlist(do.call(rbind, attribute_names))[,1]) @@ -544,30 +513,5 @@ tc <- function(x) { Filter(Negate(is.null), x) } -#' @importFrom hydroloom st_compatibalize -#' @export -hydroloom::st_compatibalize - -#' @importFrom hydroloom rename_geometry -#' @export -hydroloom::rename_geometry - -#' @importFrom hydroloom get_node -#' @export -hydroloom::get_node - -#' @importFrom hydroloom fix_flowdir -#' @export -hydroloom::fix_flowdir - -#' @importFrom hydroloom rescale_measures -#' @export -hydroloom::rescale_measures - -#' @importFrom hydroloom get_hydro_location -#' @export -hydroloom::get_hydro_location - -#' @importFrom hydroloom get_partial_length -#' @export -hydroloom::get_partial_length +#' @importFrom hydroloom get_node get_hydro_location rescale_measures rename_geometry +NULL diff --git a/R/arcrest_tools.R b/R/arcrest_tools.R index f419984d..e1c953ad 100644 --- a/R/arcrest_tools.R +++ b/R/arcrest_tools.R @@ -2,11 +2,11 @@ get_arcrest_service_info <- memoise::memoise(function(service = "3DHP_all") { stopifnot(service %in% c("3DHP_all", "NHDPlus_HR")) - url_base <- paste0(get("arcrest_root", envir = nhdplusTools_env), + url_base <- paste0(get("arcrest_root", envir = hydrogeofetch_env), service, "/MapServer/") - all_layers <- jsonlite::read_json(paste0(url_base, "?f=json")) + all_layers <- hgf_json(paste0(url_base, "?f=json"), simplifyVector = FALSE) id_name <- "id3dhp" if(service == "NHDPlus_HR") id_name <- "nhdplusid" @@ -46,7 +46,6 @@ get_arcrest_service_info <- memoise::memoise(function(service = "3DHP_all") { #' @return a simple features (sf) object or valid types if no type supplied #' @keywords internal #' @importFrom sf st_crs st_geometry_type st_buffer st_transform st_zm read_sf st_bbox st_as_sfc -#' @importFrom httr RETRY content #' @importFrom dplyr filter #' @importFrom methods as query_usgs_arcrest <- function(AOI = NULL, ids = NULL, @@ -70,16 +69,23 @@ query_usgs_arcrest <- function(AOI = NULL, ids = NULL, return(source) } - if(!type %in% source$user_call) { + exact_match <- type %in% source$user_call + # Allow a prefix to act as a virtual group, e.g. "hydrolocation" expands to + # all user_call entries of the form "hydrolocation - ...". + prefix_match <- !exact_match & startsWith(source$user_call, paste0(type, " - ")) + + if(!exact_match && !any(prefix_match)) { warning("\"", type, "\" not in `type` input. Must be one of: \n\t\"", paste(source$user_call, collapse = "\"\n\t\""), "\"") return(NULL) } - if(length(group_layers) > 0 && - grepl(paste(sapply(group_layers, \(x) x$name), - collapse = "|"), - type, ignore.case = TRUE)) { + if(any(prefix_match)) { + need_layers <- as.integer(source$layer[prefix_match]) + } else if(length(group_layers) > 0 && + grepl(paste(sapply(group_layers, \(x) x$name), + collapse = "|"), + type, ignore.case = TRUE)) { layer_id <- filter(source, .data$user_call == !!type)$layer group_layer <- group_layers[[sapply(group_layers, \(x) x$id == layer_id)]] @@ -130,21 +136,14 @@ query_usgs_arcrest <- function(AOI = NULL, ids = NULL, post_body) } + all_ids <- NULL tryCatch({ - if(nhdplus_debug()) { - message(paste(URL, "\n")) - message(post_body) - } - - all_ids <- content(RETRY("POST", - URL, - body = post_body, - encode = "form")) - all_ids <- unlist(all_ids$objectIds) + resp <- hgf_json(URL, body = post_body, encode = "form", + simplifyVector = FALSE) + all_ids <- unlist(resp$objectIds) }, error = function(e) { warning("Something went wrong trying to access a service.") - out <- NULL }) length_ids <- length(all_ids) @@ -171,26 +170,7 @@ query_usgs_arcrest <- function(AOI = NULL, ids = NULL, outFields = "*", f = "geojson") - tryCatch({ - if(nhdplus_debug()) { - message(paste(URL, "\n")) - message(post_body) - } - - out[[i]] <- rawToChar(httr::RETRY("POST", - URL, - body = post_body, - encode = "form", - pause_base = 2, - times = 3)$content) - }, error = function(e) { - warning("Something went wrong trying to access a service.") - out <- NULL - }) - - out[[i]] <- tryCatch({ - sf::st_zm(sf::read_sf(out[[i]]))}, - error = function(e) NULL) + out[i] <- list(hgf_sf(URL, body = post_body, encode = "form")) } if(inherits(out[[1]], "data.frame")) { @@ -238,7 +218,7 @@ query_usgs_arcrest <- function(AOI = NULL, ids = NULL, } -assign("bb_break_size", value = 2, nhdplusTools_env) +assign("bb_break_size", value = 2, hydrogeofetch_env) #' @title Construct a BBOX spatial filter for geoservers #' @description From an 'area of intferest' object (sf POINT or POLYGON), diff --git a/R/boundary_datasets.R b/R/boundary_datasets.R index 4ad54cd0..6356bd84 100644 --- a/R/boundary_datasets.R +++ b/R/boundary_datasets.R @@ -21,9 +21,9 @@ get_boundaries = function(type = "vpu"){ if(tolower(type) == "vpu"){ - nhdplusTools::vpu_boundaries + hydrogeofetch::vpu_boundaries } else if(tolower(type) == "rpu"){ - nhdplusTools::rpu_boundaries + hydrogeofetch::rpu_boundaries } else { stop("Type must be either: rpu or vpu", call. = FALSE) } diff --git a/R/calc_network.R b/R/calc_network.R deleted file mode 100644 index 42cbfe87..00000000 --- a/R/calc_network.R +++ /dev/null @@ -1,64 +0,0 @@ -#' DEPRECATED: Total Drainage Area -#' @description Calculates total drainage area given a dendritic -#' network and incremental areas. -#' @param x data.frame with ID, toID, and area columns. -#' @return numeric with total area. -#' @importFrom dplyr select left_join -#' @importFrom hydroloom accumulate_downstream -#' @export -#' @examples -#' library(dplyr) -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' catchment_area <- select(walker_flowline, COMID, AreaSqKM) %>% -#' right_join(prepare_nhdplus(walker_flowline, 0, 0, -#' purge_non_dendritic = FALSE, warn = FALSE), by = "COMID") %>% -#' select(ID = COMID, toID = toCOMID, area = AreaSqKM) -#' -#' new_da <- calculate_total_drainage_area(catchment_area) -#' -#' catchment_area$totda <- new_da -#' catchment_area$nhdptotda <- walker_flowline$TotDASqKM -#' -#' mean(abs(catchment_area$totda - catchment_area$nhdptotda)) -#' max(abs(catchment_area$totda - catchment_area$nhdptotda)) -#' - -calculate_total_drainage_area <- function(x) { - - warning("calculate_total_drainage_area() is deprecated. Please switch to hydroloom equivalent.") - - return(accumulate_downstream(x, "area")) - -} - -#' DEPRECATED: Calculate Arbolate Sum -#' @description Calculates arbolate sum given a dendritic -#' network and incremental lengths. Arbolate sum is the total length -#' of all upstream flowlines. -#' @param x data.frame with ID, toID, and length columns. -#' @return numeric with arbolate sum. -#' @export -#' @examples -#' library(dplyr) -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' catchment_length <- select(walker_flowline, COMID, AreaSqKM) %>% -#' right_join(prepare_nhdplus(walker_flowline, 0, 0, -#' purge_non_dendritic = FALSE, warn = FALSE), by = "COMID") %>% -#' select(ID = COMID, toID = toCOMID, length = LENGTHKM) -#' -#' arb_sum <- calculate_arbolate_sum(catchment_length) -#' -#' catchment_length$arb_sum <- arb_sum -#' catchment_length$nhd_arb_sum <- walker_flowline$ArbolateSu -#' -#' mean(abs(catchment_length$arb_sum - catchment_length$nhd_arb_sum)) -#' max(abs(catchment_length$arb_sum - catchment_length$nhd_arb_sum)) -#' - -calculate_arbolate_sum <- function(x) { - - warning("calculate_arbolate_sum is deprecated. Please switch to hydroloom equivalent.") - - return(accumulate_downstream(x, "length")) - -} diff --git a/R/discover_nhdplus.R b/R/discover_nhdplus.R index 7744f593..c32980d2 100644 --- a/R/discover_nhdplus.R +++ b/R/discover_nhdplus.R @@ -53,14 +53,18 @@ discover_nhdplus_id <- function(point = NULL, nldi_feature = NULL, raindrop = FA comid <- tryCatch({ - URL <- paste0(get("usgs_water_root", envir = nhdplusTools_env), + URL <- paste0(get_water_url(), "collections/catchmentsp/items", "?bbox=", coords[1], ",", coords[2], ",", coords[1], ",", coords[2], "&properties=featureid&skipGeometry=true") - d <- jsonlite::fromJSON(rawToChar(RETRY("GET", utils::URLencode(URL))$content)) + d <- hgf_json(utils::URLencode(URL)) + if(is.null(d)) return(NULL) - as.integer(d$features$properties$featureid) + featureid <- d$features$properties$featureid + if(is.null(featureid) || length(featureid) == 0) return(NULL) + + as.integer(featureid) }, error = function(e) NULL) @@ -68,6 +72,7 @@ discover_nhdplus_id <- function(point = NULL, nldi_feature = NULL, raindrop = FA } else if (!is.null(nldi_feature)) { nldi <- get_nldi_feature(nldi_feature) + if(is.null(nldi)) return(NULL) return(as.integer(nldi$comid)) diff --git a/R/downloading_tools.R b/R/downloading_tools.R index d8d3124f..06c806bb 100644 --- a/R/downloading_tools.R +++ b/R/downloading_tools.R @@ -31,11 +31,11 @@ #' } download_nhdplushr <- function(nhd_dir, hu_list, download_files = TRUE, archive = FALSE) { - list_source <- get("nhdhr_file_list", envir = nhdplusTools_env) + list_source <- get("nhdhr_file_list", envir = hydrogeofetch_env) - if(archive) list_source <- get("archive_nhdhr_file_list", envir = nhdplusTools_env) + if(archive) list_source <- get("archive_nhdhr_file_list", envir = hydrogeofetch_env) - download_nhd_internal(get("nhd_bucket", envir = nhdplusTools_env), + download_nhd_internal(get("nhd_bucket", envir = hydrogeofetch_env), list_source, "NHDPLUS_H_", nhd_dir, hu_list, download_files) } @@ -56,8 +56,8 @@ download_nhdplushr <- function(nhd_dir, hu_list, download_files = TRUE, archive #' } download_nhd <- function(nhd_dir, hu_list, download_files = TRUE) { - download_nhd_internal(get("nhd_bucket", envir = nhdplusTools_env), - get("nhd_file_list", envir = nhdplusTools_env), + download_nhd_internal(get("nhd_bucket", envir = hydrogeofetch_env), + get("nhd_file_list", envir = hydrogeofetch_env), "NHD_H_", nhd_dir, hu_list, download_files) } @@ -86,9 +86,9 @@ download_nhd_internal <- function(bucket, file_list_snip, prefix, nhd_dir, hu_li file_list <- tryCatch({ read_xml(paste0(bucket, file_list_snip, - prefix, hu02)) %>% - xml_ns_strip() %>% - xml_find_all(xpath = "//Key") %>% + prefix, hu02)) |> + xml_ns_strip() |> + xml_find_all(xpath = "//Key") |> xml_text() }, error= function(e) { NULL @@ -124,7 +124,7 @@ download_nhd_internal <- function(bucket, file_list_snip, prefix, nhd_dir, hu_li unlink(out_file) } - httr::RETRY("GET", url, httr::write_disk(out_file), httr::progress()) + if(is.null(hgf_download(url, out_file))) return(NULL) tryCatch({zip::unzip(out_file, exdir = out[length(out)])}, error = function(e) { @@ -179,7 +179,12 @@ download_nhdplusv2 <- function(outdir, progress = TRUE) { tryCatch({ - file <- downloader(outdir, url, "nhdplusV2", progress) + if(!dir.exists(outdir)) dir.create(outdir, recursive = TRUE) + file <- file.path(outdir, basename(url)) + if(!file.exists(file)) { + message("Downloading ", basename(url)) + if(is.null(hgf_download(url, file, progress))) return(NULL) + } if(!any(grepl("gdb", list.dirs(outdir)))) { @@ -228,7 +233,12 @@ download_wbd <- function(outdir, progress = TRUE) { tryCatch({ - file <- downloader(outdir, url, "WBD", progress) + if(!dir.exists(outdir)) dir.create(outdir, recursive = TRUE) + file <- file.path(outdir, basename(url)) + if(!file.exists(file)) { + message("Downloading ", basename(url)) + if(is.null(hgf_download(url, file, progress))) return(NULL) + } message("Extracting data ...") @@ -246,6 +256,25 @@ download_wbd <- function(outdir, }) } +#' @noRd +#' @description gunzip a file in place. Leaves the .gz file intact. Skips if +#' the destination already exists. Replaces R.utils::gunzip(remove = FALSE, +#' skip = TRUE) so we don't carry R.utils as a dependency for one call. +gunzip_keep <- function(file) { + out <- sub("\\.gz$", "", file) + if(file.exists(out)) return(invisible(out)) + con_in <- gzfile(file, "rb") + on.exit(close(con_in), add = TRUE) + con_out <- file(out, "wb") + on.exit(close(con_out), add = TRUE) + repeat { + chunk <- readBin(con_in, what = "raw", n = 1e6) + if(length(chunk) == 0) break + writeBin(chunk, con_out) + } + invisible(out) +} + #' @title Download the seamless Reach File (RF1) Database #' @description This function downloads and decompresses staged RF1 data. #' See: https://water.usgs.gov/GIS/metadata/usgswrd/XML/erf1_2.xml for metadata. @@ -261,11 +290,16 @@ download_rf1 <- function(outdir, url = "https://water.usgs.gov/GIS/dsdl/erf1_2.e00.gz", progress = TRUE){ tryCatch({ - file <- downloader(outdir, url, "RF1", progress) + if(!dir.exists(outdir)) dir.create(outdir, recursive = TRUE) + file <- file.path(outdir, basename(url)) + if(!file.exists(file)) { + message("Downloading ", basename(url)) + if(is.null(hgf_download(url, file, progress))) return(NULL) + } message("Extracting data ...") - R.utils::gunzip(file, remove = FALSE, skip = TRUE) + gunzip_keep(file) path <- list.files(outdir, full.names = TRUE)[!grepl("gz", list.files(outdir))] path <- path[grepl("rf1", path)] @@ -280,96 +314,101 @@ download_rf1 <- function(outdir, } -#' @title Function to download data from URL to out directory using `httr`. -#' @description General downloader -#' @param dir path to output directory -#' @param url the location of the online resource -#' @param type the type of data being downloaded -#' @return the downloaded file path -#' @importFrom httr GET write_disk progress +#' @title Utility to see in 7z is local +#' @description Checks if 7z is on system. If not, provides an informative error #' @noRd -downloader <- function(dir, url, type, progress = TRUE){ - - if (!dir.exists(dir)) { - dir.create(dir, recursive = TRUE) - } +check7z <- function() { - file <- file.path(dir, basename(url)) + tryCatch({ + system("7z", intern = TRUE) + }, error = function(e) { + # 7z not on PATH; check default Windows install location + win_path <- "C:/Program Files/7-Zip/7z.exe" + if(.Platform$OS.type == "windows" && file.exists(win_path)) { + Sys.setenv(PATH = paste(Sys.getenv("PATH"), dirname(win_path), sep = ";")) + return(invisible(TRUE)) + } + stop(simpleError( + "Please Install 7zip (Windows) or p7zip (MacOS/Unix). Choose accordingly: + Windows: https://www.7-zip.org/download.html + Mac: 'brew install p7zip' or 'sudo port install p7zip' + Linux: https://sourceforge.net/projects/p7zip/" + )) + }) - if(grepl("name=", basename(url))) { - file <- file.path(dir, tail(strsplit(basename(url), "=")[[1]], 1)) - } +} - if (!file.exists(file)) { +################################################################# +# httr2 helpers — all HTTP in the package flows through these # +################################################################# - message("Downloading ", basename(url)) +#' @importFrom httr2 request req_perform +#' @noRd +build_hgf_req <- function(url, body = NULL, content_type = NULL, encode = NULL) { + req <- httr2::request(url) |> + httr2::req_user_agent( + paste0("hydrogeofetch/", utils::packageVersion("hydrogeofetch")) + ) |> + httr2::req_retry(max_tries = 3) + + if(nhdplus_debug()) { + message(if(is.null(body)) "GET " else "POST ", url) + if(!is.null(body) && is.character(body)) message(body) + } - if(progress) { - resp <- httr::GET(url, - httr::write_disk(file, overwrite = TRUE), - httr::progress()) + if(!is.null(body)) { + if(identical(encode, "form")) { + req <- do.call(httr2::req_body_form, c(list(req), body)) } else { - resp <- httr::GET(url, - httr::write_disk(file, overwrite = TRUE)) + ct <- content_type %||% "application/octet-stream" + req <- httr2::req_body_raw(req, charToRaw(body), type = ct) } - - if (resp$status_code != 200) { - stop("Download unsuccessfull :(") - } - - } else { - message("Compressed ", toupper(type), " file already exists ...") } - return(file) - + req } -#' @title Utility to see in 7z is local -#' @description Checks if 7z is on system. If not, provides an informative error #' @noRd -check7z <- function() { - +hgf_json <- function(url, body = NULL, content_type = NULL, encode = NULL, + simplifyVector = TRUE, ...) { tryCatch({ - system("7z", intern = TRUE) + resp <- httr2::req_perform(build_hgf_req(url, body, content_type, encode)) + httr2::resp_body_json(resp, simplifyVector = simplifyVector, ...) }, error = function(e) { - stop( simpleError( - - "Please Install 7zip (Windows) or p7zip (MacOS/Unix). Choose accordingly: - Windows: https://www.7-zip.org/download.html - Mac: 'brew install p7zip' or 'sudo port install p7zip' - Linux: https://sourceforge.net/projects/p7zip/" - - ) - ) + warning("Failed to get JSON from ", url, ": ", conditionMessage(e), + call. = FALSE) + NULL }) - } -#' memoise get json -#' @description -#' attempts to get a url as JSON and return the content. -#' -#' Will return NULL if anything fails -#' -#' @param url character url to get -#' @return list containing parsed json on success, NULL otherwise #' @noRd -mem_get_json <- memoise::memoise(\(url) { +hgf_sf <- function(url, body = NULL, content_type = NULL, encode = NULL) { tryCatch({ - retn <- httr::GET(url, httr::accept_json()) + resp <- httr2::req_perform(build_hgf_req(url, body, content_type, encode)) + sf::st_zm(sf::read_sf(httr2::resp_body_string(resp))) + }, error = function(e) { + warning("Failed to get features from ", url, ": ", conditionMessage(e), + call. = FALSE) + NULL + }) +} - if(retn$status_code == 200 & grepl("json", retn$headers$`content-type`)) { - return(httr::content(retn, simplifyVector = FALSE, type = "application/json")) - } else { - warning("Can't access json from ", url) - return(NULL) - } +#' @noRd +hgf_download <- function(url, path, progress = TRUE) { + tryCatch({ + req <- build_hgf_req(url) + if(progress && interactive()) req <- httr2::req_progress(req) + httr2::req_perform(req, path = path) + invisible(path) }, error = function(e) { - warning("Error accessing ", url, "\n\n", e) - return(NULL) + warning("Failed to download from ", url, ": ", conditionMessage(e), + call. = FALSE) + NULL }) -}) +} + +#' @noRd +mem_get_json <- memoise::memoise(\(url) hgf_json(url, simplifyVector = FALSE)) #' @importFrom sf st_make_valid st_as_sfc st_bbox st_buffer st_transform st_crs check_query_params <- function(AOI, ids, type, where, source, t_srs, buffer) { diff --git a/R/get_codes.R b/R/get_codes.R deleted file mode 100644 index 734d085b..00000000 --- a/R/get_codes.R +++ /dev/null @@ -1,171 +0,0 @@ -#' @title Get Streamorder -#' @description Applies a topological sort and calculates strahler stream order. -#' Algorithm: If more than one upstream flowpath has an order equal to the -#' maximum upstream order then the downstream flowpath is assigned the maximum -#' upstream order plus one. Otherwise it is assigned the max upstream order. -#' @param x data.frame with dendritic ID and toID columns. -#' @param status logical show progress update messages? -#' @return numeric stream order in same order as input -#' @importFrom dplyr left_join select -#' @importFrom hydroloom add_streamorder -#' @export -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' test_flowline <- prepare_nhdplus(walker_flowline, 0, 0, FALSE) -#' -#' test_flowline <- data.frame( -#' ID = test_flowline$COMID, -#' toID = test_flowline$toCOMID) -#' -#' (order <- get_streamorder(test_flowline)) -#' -#' walker_flowline$order <- order -#' -#' plot(sf::st_geometry(walker_flowline), lwd = walker_flowline$order, col = "blue") -#' -get_streamorder <- function(x, status = TRUE) { - check_names(x, "get_streamorder") - - add_streamorder(x, status)$stream_order - -} - -#' @title Get Streamlevel -#' @description Applies a topological sort and calculates stream level. -#' Algorithm: Terminal level paths are assigned level 1 (see note 1). -#' Paths that terminate at a level 1 are assigned level 2. This pattern is -#' repeated until no paths remain. -#' -#' If a TRUE/FALSE coastal attribute is included, coastal terminal paths -#' begin at 1 and internal terminal paths begin at 4 as is implemented by -#' the NHD stream leveling rules. -#' -#' @param x data.frame with levelpathi, dnlevelpat, and optionally a -#' coastal flag. If no coastal flag is included, all terminal paths are -#' assumed to be coastal. -#' -#' @return numeric stream order in same order as input -#' @importFrom hydroloom add_streamlevel -#' @export -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' test_flowline <- data.frame( -#' levelpathi = walker_flowline$LevelPathI, -#' dnlevelpat = walker_flowline$DnLevelPat) -#' -#' test_flowline$dnlevelpat[1] <- 0 -#' -#' (level <- get_streamlevel(test_flowline)) -#' -#' walker_flowline$level <- level -#' -#' plot(sf::st_geometry(walker_flowline), lwd = walker_flowline$level, col = "blue") -#' -#' test_flowline$coastal <- rep(FALSE, nrow(test_flowline)) -#' (level <- get_streamlevel(test_flowline)) -#' -#' test_flowline$coastal[!test_flowline$dnlevelpat %in% test_flowline$levelpathi] <- TRUE -#' (level <- get_streamlevel(test_flowline)) -#' -get_streamlevel <- function(x) { - - check_names(x, "get_streamlevel") - - coastal <- NULL - coastal <- if("coastal" %in% names(x)) "coastal" - - # TODO: drop suppressWarnings once hydroloom's hy() stops probing $id - # for streamlevel-only inputs (warns "Unknown column: id"). - suppressWarnings(add_streamlevel(x, coastal))$stream_level - -} - -#' @title Get Pfafstetter Codes (DEPRECATED) -#' @description Determines Pfafstetter codes for a dendritic network with -#' total drainage area, levelpath, and topo_sort attributes. -#' @param x sf data.frame with ID, toID, totda, outletID, topo_sort, -#' and levelpath attributes. -#' @param max_level integer number of pfaf levels to attempt to calculate. -#' If the network doesn't have resolution to support the desired level, -#' unexpected behavior may occur. -#' @param status boolean print status or not -#' @importFrom hydroloom add_pfafstetter -#' @return data.frame with ID and pfaf columns. -#' @export -#' @examples -#' \donttest{ -#' library(dplyr) -#' source(system.file("extdata/nhdplushr_data.R", package = "nhdplusTools")) -#' hr_flowline <- align_nhdplus_names(hr_data$NHDFlowline) -#' -#' fl <- select(hr_flowline, COMID, AreaSqKM) %>% -#' right_join(prepare_nhdplus(hr_flowline, 0, 0, -#' purge_non_dendritic = FALSE, -#' warn = FALSE), -#' by = "COMID") %>% -#' sf::st_sf() %>% -#' select(ID = COMID, toID = toCOMID, area = AreaSqKM) -#' -#' fl$nameID = "" -#' fl$totda <- calculate_total_drainage_area(sf::st_set_geometry(fl, NULL)) -#' fl <- left_join(fl, get_levelpaths(rename(sf::st_set_geometry(fl, NULL), -#' weight = totda)), by = "ID") -#' -#' pfaf <- get_pfaf(fl, max_level = 3) -#' -#' fl <- left_join(fl, pfaf, by = "ID") -#' -#' plot(fl["pf_level_3"], lwd = 2) -#' -#' pfaf <- get_pfaf(fl, max_level = 4) -#' -#' hr_catchment <- left_join(hr_data$NHDPlusCatchment, pfaf, by = c("FEATUREID" = "ID")) -#' -#' colors <- data.frame(pf_level_4 = unique(hr_catchment$pf_level_4), -#' color = sample(terrain.colors(length(unique(hr_catchment$pf_level_4)))), -#' stringsAsFactors = FALSE) -#' hr_catchment <- left_join(hr_catchment, colors, by = "pf_level_4") -#' plot(hr_catchment["color"], border = NA, reset = FALSE) -#' plot(sf::st_geometry(hr_flowline), col = "blue", add = TRUE) -#' -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' fl <- select(walker_flowline, COMID, AreaSqKM) %>% -#' right_join(prepare_nhdplus(walker_flowline, 0, 0, -#' purge_non_dendritic = FALSE, warn = FALSE), -#' by = "COMID") %>% -#' sf::st_sf() %>% -#' select(ID = COMID, toID = toCOMID, area = AreaSqKM) -#' -#' fl$nameID = "" -#' fl$totda <- calculate_total_drainage_area(sf::st_set_geometry(fl, NULL)) -#' fl <- left_join(fl, get_levelpaths(rename(sf::st_set_geometry(fl, NULL), -#' weight = totda)), by = "ID") -#' -#' pfaf <- get_pfaf(fl, max_level = 2) -#' -#' fl <- left_join(fl, pfaf, by = "ID") -#' -#' plot(fl["pf_level_2"], lwd = 2) -#' } -get_pfaf <- function(x, max_level = 2, status = FALSE) { - - warning("get_pfaf is deprecated, please use hydroloom") - - x <- st_drop_geometry(x) - - check_names(x, "get_pfaf") - - x <- select(x, all_of(get("get_pfaf_attributes", - envir = nhdplusTools_env))) - - x <- add_pfafstetter(x, max_level, status) - - x <- select(x, -all_of(c("toID", "totda", "outletID", "topo_sort", "levelpath"))) - - x <- filter(x, !is.na(.data$pf_level_1)) - - return(x) -} diff --git a/R/get_drainage_area_estimates.R b/R/get_drainage_area_estimates.R index afc93647..6533d553 100644 --- a/R/get_drainage_area_estimates.R +++ b/R/get_drainage_area_estimates.R @@ -435,7 +435,7 @@ resolve_comid_from_reachcode <- function(reachcode, measure, vaa = NULL) { if(nrow(rows) == 0) stop("No flowlines found in VAA for reachcode ", reachcode) } else { - base <- get("usgs_water_root", envir = nhdplusTools_env) + base <- get_water_url() out <- get_oafeat( base = base, AOI = NULL, type = "nhdflowline_network", filter = paste0("reachcode%20=%20%27", reachcode, "%27"), @@ -701,6 +701,11 @@ fetch_upstream_network <- function(outlet_comids, vaa = NULL, skip_geometry = TRUE ) + if(is.null(all_net) || nrow(all_net) == 0) + stop("Could not fetch flowline attributes for upstream network ", + "(NHDPlusV2 OGC API unavailable). Try again or supply local data.", + call. = FALSE) + if(local_outlets) huc12_outlets <- huc12_outlets[huc12_outlets$comid %in% upstream_comids, ] } @@ -1679,6 +1684,12 @@ compute_gap_area <- function(outlet_huc, all_net, nav_net = all_net, extra_cat <- get_nhdplus( comid = extra_net$comid, realization = "catchment" ) + if(is.null(extra_cat)) { + warning("Could not fetch extra catchment geometries ", + "(NHDPlusV2 OGC API unavailable); scalar estimates still ", + "valid but catchment polygons unavailable.", call. = FALSE) + extra_cat <- st_sf(geometry = st_sfc(crs = 5070)) + } } else { extra_cat <- gap_catchments[ gap_catchments$comid %in% extra_net$comid, ] diff --git a/R/get_geoconnex.R b/R/get_geoconnex.R index c137b238..93604648 100644 --- a/R/get_geoconnex.R +++ b/R/get_geoconnex.R @@ -12,7 +12,7 @@ #' discover_geoconnex_reference <- function() { - discover_oafeat(get("gocnx_ref_base_url", envir = nhdplusTools_env)) + discover_oafeat(get("gocnx_ref_base_url", envir = hydrogeofetch_env)) } @@ -29,7 +29,6 @@ discover_geoconnex_reference <- function() { #' @export #' @examples #' \donttest{ -#' #' dplyr::distinct(discover_geoconnex_reference()[c("id", "title")]) #' #' AOI <- sf::st_as_sfc(sf::st_bbox(c(xmin = -89.56684, ymin = 42.99816, @@ -52,7 +51,7 @@ get_geoconnex_reference <- function(AOI, buffer = 0.5, status = TRUE) { - base <- get("gocnx_ref_base_url", envir = nhdplusTools_env) + base <- get("gocnx_ref_base_url", envir = hydrogeofetch_env) get_oafeat(base, AOI, type = type, t_srs = t_srs, buffer = buffer, status = status) diff --git a/R/get_hydro.R b/R/get_hydro.R index 23b4c2e2..91726c5b 100644 --- a/R/get_hydro.R +++ b/R/get_hydro.R @@ -143,8 +143,8 @@ get_nwis <- function(AOI = NULL, t_srs = NULL, buffer = 20000){ if(AOI_type == "POINT"){ pt <- AOI - AOI <- sf::st_buffer(sf::st_transform(AOI, 5070), buffer) %>% - sf::st_bbox() %>% + AOI <- sf::st_buffer(sf::st_transform(AOI, 5070), buffer) |> + sf::st_bbox() |> sf::st_as_sfc() } @@ -193,14 +193,13 @@ get_nwis <- function(AOI = NULL, t_srs = NULL, buffer = 20000){ station_nm = xml2::xml_attr(sites, "sna"), site_type = xml2::xml_attr(sites, "cat"), lat = as.numeric(xml2::xml_attr(sites, "lat")), - lon = as.numeric(xml2::xml_attr(sites, "lng"))) %>% + lon = as.numeric(xml2::xml_attr(sites, "lng"))) |> st_as_sf(coords = c("lon", "lat"), crs = 4326) if(AOI_type == "POINT"){ - sites_sf <- sites_sf %>% - mutate(distance_m = st_distance(st_transform(., 5070), - st_transform(pt, 5070))) %>% - arrange(.data$distance_m) + sites_sf$distance_m <- st_distance(st_transform(sites_sf, 5070), + st_transform(pt, 5070)) + sites_sf <- arrange(sites_sf, .data$distance_m) } return(st_transform(sites_sf, t_srs)) @@ -240,17 +239,17 @@ get_nwis <- function(AOI = NULL, t_srs = NULL, buffer = 20000){ #' #' # given mainstem ids from any source, can query for them in ids. #' -#' CO <- get_3dhp(ids = "https://geoconnex.us/ref/mainstems/29559", +#' SU <- get_3dhp(ids = "https://geoconnex.us/ref/mainstems/194408", #' type = "flowline") #' -#' if(!is.null(CO)) -#' plot(sf::st_geometry(CO), col = "blue") +#' if(!is.null(SU)) +#' plot(sf::st_geometry(SU), col = "blue") #' -#' # get all the waterbodies along the CO river -#' CO_wb <- get_3dhp(ids = unique(CO$waterbodyid3dhp), type = "waterbody") +#' # get all the waterbodies along the Susquehanna river +#' SU_wb <- get_3dhp(ids = unique(SU$waterbodyid3dhp), type = "waterbody") #' -#' if(!is.null(CO_wb)) { -#' plot(sf::st_geometry(CO_wb[grepl("Powell", CO_wb$gnisidlabel),]), +#' if(!is.null(SU_wb)) { +#' plot(sf::st_geometry(SU_wb[grepl("Otsego", SU_wb$gnisidlabel),]), #' col = "blue", border = "NA") } #' #' # given a workunitid, can query for features in that work unit diff --git a/R/get_network.R b/R/get_network.R deleted file mode 100644 index 35e40851..00000000 --- a/R/get_network.R +++ /dev/null @@ -1,418 +0,0 @@ -#' @title Navigate Upstream with Tributaries -#' @description Traverse NHDPlus network upstream with tributaries -#' @param network data.frame NHDPlus flowlines including at a minimum: -#' COMID, Pathlength, LENGTHKM, and Hydroseq. -#' @param comid integer Identifier to start navigating from. -#' @param distance numeric distance in km to limit how many COMIDs are -#' returned. The COMID that exceeds the distance specified is returned. -#' @return integer vector of all COMIDs upstream with tributaries of the -#' starting COMID. -#' @importFrom dplyr filter select -#' @importFrom hydroloom navigate_hydro_network -#' @export -#' @examples -#' library(sf) -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) -#' plot(sample_flines$geom) -#' start_COMID <- 11690196 -#' UT_COMIDs <- get_UT(sample_flines, start_COMID) -#' plot(dplyr::filter(sample_flines, COMID %in% UT_COMIDs)$geom, -#' col = "red", add = TRUE) -#' -#' UT_COMIDs <- get_UT(sample_flines, start_COMID, distance = 50) -#' plot(dplyr::filter(sample_flines, COMID %in% UT_COMIDs)$geom, -#' col = "blue", add = TRUE) -#' -get_UT <- function(network, comid, distance = NULL) { - navigate_hydro_network(network, comid, "UT", distance) -} - -#' @title Navigate Upstream Mainstem -#' @description Traverse NHDPlus network upstream main stem -#' @param network data.frame NHDPlus flowlines including at a minimum: -#' COMID,Pathlength, LevelPathI, and Hydroseq. -#' @param comid integer identifier to start navigating from. -#' @param distance numeric distance in km to limit how many COMIDs are -#' @param sort if TRUE, the returned COMID vector will be sorted in order of distance from the input COMID (nearest to farthest) -#' @param include if TRUE, the input COMID will be included in the returned COMID vector -#' returned. The COMID that exceeds the distance specified is returned. -#' @return integer vector of all COMIDs upstream of the starting COMID -#' along the mainstem -#' @importFrom dplyr filter select arrange -#' @export -#' @examples -#' library(sf) -#' -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) -#' -#' plot(sample_flines$geom) -#' start_COMID <- 11690196 -#' UM_COMIDs <- get_UM(sample_flines, start_COMID) -#' plot(dplyr::filter(sample_flines, COMID %in% UM_COMIDs)$geom, -#' col = "red", add = TRUE, lwd = 3) -#' -#' UM_COMIDs <- get_UM(sample_flines, start_COMID, distance = 50) -#' plot(dplyr::filter(sample_flines, COMID %in% UM_COMIDs)$geom, -#' col = "blue", add = TRUE, lwd = 2) -#' -get_UM <- function(network, comid, distance = NULL, sort = FALSE, include = TRUE) { - - network <- align_nhdplus_names(network) - - main_us <- filter(network, .data$COMID %in% - navigate_hydro_network(network, comid, "UM", distance)) - - if(sort) { main_us <- arrange(main_us, .data$Hydroseq) } - if(!include) { main_us = filter(main_us, .data$COMID != comid) } - - return(main_us$COMID) -} - -#' @title Navigate Downstream Mainstem -#' @description Traverse NHDPlus network downstream main stem -#' @param network data.frame NHDPlus flowlines including at a minimum: -#' COMID, LENGTHKM, DnHydroseq, and Hydroseq. -#' @param comid integer identifier to start navigating from. -#' @param distance numeric distance in km to limit how many COMIDs are -#' returned. The COMID that exceeds the distance specified is returned. -#' @param sort if TRUE, the returned COMID vector will be sorted in order of distance from the input COMID (nearest to farthest) -#' @param include if TRUE, the input COMID will be included in the returned COMID vector -#' @return integer vector of all COMIDs downstream of the starting COMID -#' along the mainstem -#' @importFrom dplyr select filter arrange desc -#' @export -#' @examples -#' library(sf) -#' -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) -#' -#' plot(sample_flines$geom) -#' start_COMID <- 11690092 -#' DM_COMIDs <- get_DM(sample_flines, start_COMID) -#' plot(dplyr::filter(sample_flines, COMID %in% DM_COMIDs)$geom, -#' col = "red", add = TRUE, lwd = 3) -#' -#' DM_COMIDs <- get_DM(sample_flines, start_COMID, distance = 40) -#' plot(dplyr::filter(sample_flines, COMID %in% DM_COMIDs)$geom, -#' col = "blue", add = TRUE, lwd = 2) -#' -get_DM <- function(network, comid, distance = NULL, sort = FALSE, include = TRUE) { - - network <- align_nhdplus_names(network) - - main_ds <- filter(network, .data$COMID %in% - navigate_hydro_network(network, comid, "DM", distance)) - - if(sort){ main_ds <- arrange(main_ds, desc(.data$Hydroseq)) } - if(!include){ main_ds <- filter(main_ds, .data$COMID != comid) } - - return(main_ds$COMID) -} - -#' @title Navigate Downstream with Diversions -#' @description Traverse NHDPlus network downstream with diversions -#' NOTE: This algorithm may not scale well in large watersheds. -#' For reference, the lower Mississippi will take over a minute. -#' @param network data.frame NHDPlus flowlines including at a minimum: -#' COMID, DnMinorHyd, DnHydroseq, and Hydroseq. -#' @param comid integer identifier to start navigating from. -#' @param distance numeric distance in km to limit how many -#' COMIDs are returned. -#' The COMID that exceeds the distance specified is returned. -#' The longest of the diverted paths is used for limiting distance. -#' @return integer vector of all COMIDs downstream of the starting COMID -#' @importFrom dplyr filter -#' @export -#' @examples -#' library(sf) -#' start_COMID <- 11688818 -#' -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) -#' -#' DD_COMIDs <- get_DD(sample_flines, start_COMID, distance = 4) -#' plot(dplyr::filter(sample_flines, COMID %in% DD_COMIDs)$geom, -#' col = "red", lwd = 2) -#' -#' DM_COMIDs <- get_DM(sample_flines, start_COMID, distance = 4) -#' plot(dplyr::filter(sample_flines, COMID %in% DM_COMIDs)$geom, -#' col = "blue", add = TRUE, lwd = 2) -#' -get_DD <- function(network, comid, distance = NULL) { - - navigate_hydro_network(network, comid, "DD", distance) - -} - -#' Navigate Network -#' @description Provides a full feature network navigation function that -#' will work with local or web service data. Parameter details provide -#' context. -#' @param start list, integer, sf, or sfc if list must be a valid NLDI feature -#' if integer must be a valid comid. If sf, must contain a "comid" field. -#' @param mode character chosen from c(UM, DM, UT, or DD) -#' @param network sf should be compatible with network navigation functions -#' If NULL, network will be derived from requests to the NLDI -#' @param output character flowline or a valid NLDI data source -#' @param distance_km numeric distance to navigate in km -#' @param trim_start logical should start be trimmed or include entire catchment? -#' @param trim_stop logical should stop(s) be trimmed or include entire catchment(s)? -#' # Not supported -#' @param trim_tolerance numeric from 0 to 100 percent of flowline length. If amount -#' to trim is less than this tolerance, no trim will be applied. -#' @export -#' @examples -#' -#' \donttest{ -#' navigate_network(list(featureSource = "nwissite", featureID = "USGS-06287800"), -#' "UM", -#' output = "flowlines", -#' trim_start = TRUE) -#' } -#' -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' hydro_location <- list(comid = 5329339, -#' reachcode = "18050005000078", -#' reach_meas = 30) -#' -#' hydro_location <- sf::st_sf( -#' hydro_location, -#' geom = nhdplusTools::get_hydro_location(data.frame(hydro_location), -#' walker_flowline)) -#' -#' net <- navigate_network(hydro_location, -#' mode = "DM", network = walker_flowline, -#' trim_start = TRUE, distance_km = 20) -#' -#' plot(sf::st_geometry(walker_flowline)) -#' plot(sf::st_geometry(hydro_location), add = TRUE) -#' plot(sf::st_geometry(net), add = TRUE, col = "blue", lwd = 2) -#' -navigate_network <- function(start, mode = "UM", network = NULL, - output = "flowlines", distance_km = 10, - trim_start = FALSE, trim_stop = FALSE, - trim_tolerance = 5) { - - # Work with start to get a start_comid - if(!is.numeric(start)) { - if(inherits(start, "sfc")) { - - start_comid <- discover_nhdplus_id(point = start) - - } else if(inherits(start, "sf")) { - - start_comid <- start$comid - - } else if(is.list(start)) { - - start <- get_nldi_feature(start) - start_comid <- as.integer(start$comid) - - } else { - - stop("start must be integer, list, or sfc geometry") - - } - - } else { - - if(!start %% 1 == 0) { - stop("If start is numeric it must be a comid integer") - } - - start <- floor(start) - start_comid <- start - - } - - if(is.null(start) | length(start_comid) == 0) { - warning("something went wrong trying to find the start comid, this won't work") - return(NULL) - } - - # in the case that we don't have a network, we need to get it - # from the NLDI and web services. - if(is.null(network)) { - - network <- navigate_nldi(list(featureSource = "comid", - featureID = start_comid), - mode, "flowlines", distance_km) - - network <- network[names(network) != "origin"][[1]]$nhdplus_comid - - if(is.null(network)) { - warning("Something went wrong getting network data.") - return(NULL) - } - - network <- subset_nhdplus(as.integer(network), - nhdplus_data = "download", - status = TRUE, - flowline_only = TRUE) - - if(is.null(network)) return(NULL) - - network <- network$NHDFlowline_Network - - } else { - - # If we have a network, we need to filter it down to - # the desired navigation. - network <- check_names(network, paste0("get_", mode), - tolower = TRUE) - - if(!start_comid %in% network$comid) - stop("start comid not in network?") - - network <- dplyr::filter(network, .data$comid %in% - if(mode == "UM") { - get_UM(network, start_comid, distance_km, include = TRUE) - } else if(mode == "UT") { - get_UT(network, start_comid, distance_km) - } else if(mode == "DD") { - get_DD(network, start_comid, distance_km) - } else if(mode == "DM") { - get_DM(network, start_comid, distance_km, include = TRUE) - } - ) - - } - - - # we now have a network that matches our navigation and, if trim options are - # FALSE, can output the desired output data. - - # we need a precise measure for our point - if(trim_start) { - - if(!is.numeric(start)) { - - if(inherits(start, "sf") | inherits(start, "sfc")) { - start <- sf::st_transform(start, - sf::st_crs(network)) - } - - if(sf::st_is_longlat(start)) { - search_radius <- units::set_units(0.001, "degrees") - } else { - search_radius <- units::set_units(100, "m") - } - - event <- get_flowline_index(flines = network, - points = start, - search_radius = search_radius, - precision = 10) - - event <- sf::st_sf(event, - geom = get_hydro_location(event, network)) - - } - - } - - # get the output data if it's not flowlines - if(!output == "flowlines") { - - out_features <- navigate_nldi(list(featureSource = "comid", - featureID = start_comid), - mode, output, distance_km - )[[paste0(mode, "_", output)]] - - } else { - - out_features <- NULL - - } - - if(trim_stop) { - - stop("Trim Stop Not Supported") - - } - - # now trim start if requested - if(trim_start) { - - if(!(is.numeric(start) && start %% 1 == 0)) { - if(output == "flowlines") { - # trim event flowline to measure of event - - l <- network[network$comid == start_comid, ] - - rm <- as.numeric(event$REACH_meas) - lf <- l$frommeas - lt <- l$tomeas - - # the event measure can be outside the network? - if(!dplyr::between(rm, lf, lt)) { - df <- abs(c(lf, lt) - rm) - split <- c(lf, lt)[which(df == min(df))] - } else { - # Convert start to comid measure - split <- rescale_measures(as.numeric(event$REACH_meas), - l$frommeas, l$tomeas) - } - - if(grepl("UM|UT", mode)) { - - f <- 0 - t <- 1 - (split / 100) - - } else { - - f <- 1 - (split / 100) - t <- 1 - - } - - if(abs(f - t) < trim_tolerance / 100) { - - warning("No split applied, under tolerance.") - - } else { - - sf::st_geometry(network)[network$comid == start_comid] <- - suppressWarnings({ - - if(!requireNamespace("lwgeom")) { - stop("lwgeom required to trim flowlines to a specific measure.") - } - - lwgeom::st_linesubstring( - sf::st_cast(sf::st_geometry(l), "LINESTRING"), f, t) - - }) - } - - } else { - if("measure" %in% names(out_features)) { - # remove if reachcode is same as start and - # measure is greater for upstream - # measure is less for downstream - - out_features <- dplyr::filter( - out_features, - if(grepl("UM|UT", mode)) { - .data$reachcode != event$REACHCODE | - (.data$reachcode == event$REACHCODE & .data$measure > event$REACH_meas) - } else { - .data$reachcode != event$REACHCODE | - (.data$reachcode == event$REACHCODE & .data$measure < event$REACH_meas) - }) - - } - } - } else { - - warning("trim_start ignored for comid start") - - } - - } - - if(!is.null(out_features)) { - return(out_features) - } else { - return(network) - } - -} diff --git a/R/get_nhdplus.R b/R/get_nhdplus.R index 540b4932..8f7e0527 100644 --- a/R/get_nhdplus.R +++ b/R/get_nhdplus.R @@ -19,7 +19,8 @@ #' @param realization character. What realization to return. #' Default is flowline and options include: outlet, flowline, catchment, #' and all -#' @return sfc a single, or list, of simple feature objects +#' @return sf single, or named list of, simple feature objects. \code{NULL} if +#' no realization succeeded (e.g. degraded service). #' @examples #' \donttest{ #' point <- sf::st_sfc(sf::st_point(c(-119.845, 34.4146)), crs = 4326) @@ -88,7 +89,7 @@ get_nhdplus <- function(AOI = NULL, geoms = list() if(!is.null(nwis)){ - comid = c(unlist(lapply(nwis, extact_comid_nwis)), comid) + comid = c(unlist(lapply(nwis, extract_comid_nwis)), comid) } if("catchment" %in% realization){ @@ -128,6 +129,8 @@ get_nhdplus <- function(AOI = NULL, geoms = geoms[names(geoms) %in% realization] + if(length(geoms) == 0) return(NULL) + if(length(geoms) == 1){ geoms = geoms[[1]]} return(geoms) @@ -140,15 +143,14 @@ get_nhdplus <- function(AOI = NULL, #' @keywords internal #' @return a vector of COMIDs #' @noRd -#' @importFrom httr RETRY GET #' @importFrom jsonlite fromJSON -extact_comid_nwis <- memoise::memoise(function(nwis){ - # We could export this from dataRetrieval dataRetrieval:::pkg.env$nldi_base - #but currently its not... - baseURL <- paste0(get_nldi_url(), "/linked-data/") - url <- paste0(baseURL, "nwissite/USGS-", nwis) - c <- rawToChar(httr::RETRY("GET", url)$content) - f.comid <- jsonlite::fromJSON(c, simplifyVector = TRUE) +extract_comid_nwis <- memoise::memoise(function(nwis){ + url <- paste0(get_nldi_url(), "/linked-data/nwissite/USGS-", nwis) + f.comid <- hgf_json(url) + if(is.null(f.comid)) { + warning("NWIS lookup failed for USGS-", nwis, call. = FALSE) + return(NULL) + } f.comid$features$properties$comid }) diff --git a/R/get_nhdplushr.R b/R/get_nhdplushr.R index f42de5dd..90ed7c95 100644 --- a/R/get_nhdplushr.R +++ b/R/get_nhdplushr.R @@ -33,7 +33,7 @@ #' \dontrun{ #' # Note this will download a lot of data to a temp directory. #' # Change 'temp_dir' to your directory of choice. -#' temp_dir <- file.path(nhdplusTools_data_dir(), "temp_hr_cache") +#' temp_dir <- file.path(hydrogeofetch_data_dir(), "temp_hr_cache") #' #' download_dir <- download_nhdplushr(temp_dir, c("0302", "0303")) #' @@ -102,7 +102,7 @@ get_nhdplushr <- function(hr_dir, out_gpkg = NULL, if(grepl("flowline", layer, ignore.case = TRUE) & check_terminals) { expect_names <- get(paste0("make_standalone_tonode_attributes"), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) if(!all(expect_names %in% names(out))) { warning("check_terminals is true but attributes selected do not support the checks.") } else { @@ -132,7 +132,7 @@ get_nhdplushr <- function(hr_dir, out_gpkg = NULL, #' @param min_size_sqkm numeric minimum basin size to be included in the output #' @param proj a projection specification compatible with \link[sf]{st_crs} #' @param simp numeric simplification tolerance in units of projection -#' @param rename boolean if TRUE, nhdplusTools standard attribute values will +#' @param rename boolean if TRUE, hydrogeofetch standard attribute values will #' be applied. #' @export #' @return sf data.frame containing requested data @@ -217,7 +217,7 @@ cull_cols <- function(x, keep_cols) { #' \donttest{ #' library(dplyr) #' library(sf) -#' source(system.file("extdata/nhdplushr_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/nhdplushr_data.R", package = "hydrogeofetch")) #' #' (outlet <- filter(hr_data$NHDFlowline, Hydroseq == min(Hydroseq))) #' nrow(filter(hr_data$NHDFlowline, TerminalPa == outlet$Hydroseq)) @@ -227,7 +227,7 @@ cull_cols <- function(x, keep_cols) { #' (outlet <- filter(hr_data$NHDFlowline, Hydroseq == min(Hydroseq))) #' nrow(filter(hr_data$NHDFlowline, TerminalPa == outlet$Hydroseq)) #' -#' source(system.file("extdata/nhdplushr_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/nhdplushr_data.R", package = "hydrogeofetch")) #' #' # Remove mainstem and non-dendritic stuff. #' subset <- filter(hr_data$NHDFlowline, diff --git a/R/get_nldi.R b/R/get_nldi.R index 7aeb3efe..61cffd46 100644 --- a/R/get_nldi.R +++ b/R/get_nldi.R @@ -19,13 +19,13 @@ #' nldi_nwis <- list(featureSource = "nwissite", featureID = "USGS-05428500") #' #' navigate_nldi(nldi_feature = nldi_nwis, -#' mode = "upstreamTributaries")$UT %>% -#' st_geometry() %>% +#' mode = "upstreamTributaries")$UT |> +#' st_geometry() |> #' plot() #' #' navigate_nldi(nldi_feature = nldi_nwis, -#' mode = "UM")$UM %>% -#' st_geometry() %>% +#' mode = "UM")$UM |> +#' st_geometry() |> #' plot(col = "blue", add = TRUE) #' #' @@ -34,7 +34,7 @@ #' mode = "UT", #' data_source = "nwissite")$UT_nwissite #' -#' st_geometry(nwissite) %>% +#' st_geometry(nwissite) |> #' plot(col = "green", add = TRUE) #' #' nwissite @@ -123,19 +123,20 @@ get_nldi_basin <- function(nldi_feature, simplify = TRUE, split = FALSE) { tryCatch({ nldi_feature <- check_nldi_feature(nldi_feature, convert = FALSE) - o <- query_nldi( - paste0(nldi_feature[["featureSource"]], "/", - nldi_feature[["featureID"]], "/", - "basin?simplify=", - ifelse(simplify, "true", "false"), "&", - "splitCatchment=", - ifelse(split, "true", "false")), - parse_json = FALSE, - err_mess = "Are you sure your featureID exists in the NLDI?") + url <- utils::URLencode(paste0( + get_nldi_url(), "/linked-data/", + nldi_feature[["featureSource"]], "/", + nldi_feature[["featureID"]], + "/basin?simplify=", ifelse(simplify, "true", "false"), + "&splitCatchment=", ifelse(split, "true", "false"))) - if(is.null(o)) return(NULL) + out <- hgf_sf(url) - read_sf(o) + if(is.null(out)) + warning("Are you sure your featureID exists in the NLDI?", + call. = FALSE) + + out }, error = function(e) { warning(e) return(NULL) @@ -194,53 +195,34 @@ get_nldi_feature <- function(nldi_feature) { #' } get_nldi_index <- function(location) { - tryCatch({ - sf::read_sf(query_nldi(paste0("hydrolocation?coords=POINT(", - location[1],"%20", location[2],")"), - parse_json = FALSE, - err_mess = "Make sure your POINT is lon,lat and in the NHDPlusV2 domain.")) - }, error = function(e) { - warning(paste("Something went wrong querying the NLDI.\n", e)) - NULL - }) + url <- utils::URLencode(paste0( + get_nldi_url(), "/linked-data/hydrolocation?coords=POINT(", + location[1], "%20", location[2], ")")) + + out <- hgf_sf(url) + + if(is.null(out)) + warning("Make sure your POINT is lon,lat and in the NHDPlusV2 domain.", + call. = FALSE) + + out } -#' @importFrom httr GET #' @importFrom jsonlite fromJSON #' @noRd -query_nldi <- function(query, base_path = "/linked-data", parse_json = TRUE, err_mess = "") { +query_nldi <- function(query, base_path = "/linked-data", err_mess = "") { nldi_base_url <- paste0(get_nldi_url(), base_path) - url <- paste(nldi_base_url, query, - sep = "/") + url <- utils::URLencode(paste(nldi_base_url, query, sep = "/")) - tryCatch({ - if(nhdplus_debug()) { - message(url) - } + out <- hgf_json(url) - req_data <- rawToChar(httr::RETRY("GET", utils::URLencode(url))$content) + if(is.null(out) && nchar(err_mess) > 0) + warning("Something went wrong accessing the NLDI.\n", err_mess, + call. = FALSE) - if (nchar(req_data) == 0) { - NULL - } else { - if (parse_json) { - tryCatch(jsonlite::fromJSON(req_data, simplifyVector = TRUE), - error = function(e) { - message("Something went wrong accessing the NLDI.\n", e) - }, warning = function(w) { - message("Something went wrong accessing the NLDI.\n", w) - }) - } else { - req_data - } - } - }, error = function(e) { - warning("Something went wrong accessing the NLDI.\n", e, - "\n", err_mess) - NULL - }) + out } #' @noRd diff --git a/R/get_oaproc.R b/R/get_oaproc.R index 2799881d..0dbc30d9 100644 --- a/R/get_oaproc.R +++ b/R/get_oaproc.R @@ -18,7 +18,7 @@ #' if(inherits(trace, "sf")) { #' bbox <- sf::st_bbox(trace) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_sfc(point, crs = 4326), 3857), add = TRUE) #' plot(sf::st_transform(sf::st_geometry(trace)[1], 3857), add = TRUE, col = "red") @@ -40,7 +40,8 @@ get_raindrop_trace <- function(point, direction = "down") { stop(paste("direction must be in", paste(allowed_direction, collapse = ", "))) - return(sf_post(url, make_json_input_trace(point, direction = direction))) + return(hgf_sf(url, body = make_json_input_trace(point, direction = direction), + content_type = "application/json")) } @@ -82,7 +83,7 @@ get_raindrop_trace <- function(point, direction = "down") { #' if(inherits(catchment, "sf")) { #' bbox <- sf::st_bbox(catchment) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_geometry(catchment)[2], 3857), add = TRUE, col = "black") #' plot(sf::st_transform(sf::st_geometry(catchment)[1], 3857), add = TRUE, col = "red") @@ -95,7 +96,7 @@ get_raindrop_trace <- function(point, direction = "down") { #' if(inherits(catchment, "sf")) { #' bbox <- sf::st_bbox(catchment) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_geometry(catchment)[1], 3857), add = TRUE, col = "red") #' plot(sf::st_transform(sf::st_geometry(catchment)[2], 3857), add = TRUE, col = "black") @@ -109,7 +110,7 @@ get_raindrop_trace <- function(point, direction = "down") { #' if(inherits(catchment, "sf")) { #' bbox <- sf::st_bbox(catchment) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_geometry(catchment)[1], 3857), add = TRUE, col = "red") #' plot(sf::st_transform(sf::st_geometry(catchment)[2], 3857), add = TRUE, col = "black") @@ -127,10 +128,13 @@ get_split_catchment <- function(point, upstream = TRUE) { url <- paste0(url_base, "nldi-splitcatchment/execution") - out <- sf_post(url, make_json_input_split(point, upstream), - err_mess = paste("Ensure that the point you submitted is within\n the", - "coterminous US and consider trying get_raindrop_trace\ to ensure", - "your point is not too close to a catchment boundary.")) + out <- hgf_sf(url, body = make_json_input_split(point, upstream), + content_type = "application/json") + + if(is.null(out)) + message("Ensure that the point you submitted is within the ", + "coterminous US and consider trying get_raindrop_trace to ensure ", + "your point is not too close to a catchment boundary.") try({ if(!is.null(out)) { @@ -184,7 +188,7 @@ remove_shards <- function(g, thresh = 0.01) { #' #' bbox <- sf::st_bbox(xs) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_geometry(xs), 3857), pch = ".", add = TRUE, col = "red") #' plot(sf::st_transform(sf::st_sfc(point, crs = 4326), 3857), add = TRUE) @@ -227,7 +231,7 @@ get_xs_point <- function(point, width, num_pts) { #' #' bbox <- sf::st_bbox(xs) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_geometry(xs), 3857), pch = ".", add = TRUE, col = "red") #' plot(sf::st_transform(sf::st_sfc(point1, crs = 4326), 3857), add = TRUE) @@ -286,7 +290,7 @@ check_res <- function(res) { #' #' bbox <- sf::st_bbox(xs) + c(-0.005, -0.005, 0.005, 0.005) #' -#' nhdplusTools::plot_nhdplus(bbox = bbox, cache_data = FALSE) +#' hydrogeofetch::plot_nhdplus(bbox = bbox, cache_data = FALSE) #' #' plot(sf::st_transform(sf::st_geometry(xs), 3857), pch = ".", add = TRUE, col = "red") #' plot(sf::st_transform(sf::st_sfc(point1, crs = 4326), 3857), add = TRUE) @@ -364,7 +368,7 @@ get_elev <- function(url, fun, points, num_pts, res, status) { #' @importFrom dplyr rename get_xs <- function(url, fun, ...) { - sf <- sf_post(url, fun(...)) + sf <- hgf_sf(url, body = fun(...), content_type = "application/json") if(is.null(sf)) { return(NULL) @@ -375,32 +379,6 @@ get_xs <- function(url, fun, ...) { elevation_m = "elevation") } -sf_post <- function(url, json, err_mess = "") { - tryCatch({ - - if(nhdplus_debug()) { - message(paste(url, "\n")) - message(json) - } - - out <- httr::RETRY("POST", url, httr::accept_json(), - httr::content_type_json(), - body = json) - - if(out$status_code == 200) { - sf::read_sf(rawToChar(out$content)) - } else { - stop(rawToChar(out$content)) - } - - }, error = function(e) { - message("Error calling processing service. \n Original error: \n", e, - "\n", err_mess) - NULL - }) -} - - check_point <- function(p) { mess <- "Point must be of type sfc and have a CRS declared." @@ -453,16 +431,16 @@ make_json_input_xspt <- function(p, w, n) { jsonlite::toJSON(list(inputs = list(list(id = "lat", type = "text/plain", - value = p[2]), + value = as.character(p[2])), list(id = "lon", type = "text/plain", - value = p[1]), + value = as.character(p[1])), list(id = "width", type = "text/plain", - value = w), + value = as.character(w)), list(id = "numpts", type = "text/plain", - value = n))), + value = as.character(n)))), pretty = TRUE, auto_unbox = TRUE) } @@ -471,15 +449,15 @@ make_json_input_xspts <- function(p1, p2, n, r) { jsonlite::toJSON(list(inputs = list(list(id = "lat", type = "text/plain", - value = c(p1[2], p2[2])), + value = as.character(c(p1[2], p2[2]))), list(id = "lon", type = "text/plain", - value = c(p1[1], p2[1])), + value = as.character(c(p1[1], p2[1]))), list(id = "3dep_res", type = "text/plain", value = as.character(r)), list(id = "numpts", type = "text/plain", - value = n))), + value = as.character(n)))), pretty = TRUE, auto_unbox = TRUE) } diff --git a/R/get_path_lengths.R b/R/get_path_lengths.R deleted file mode 100644 index 0f17e6d9..00000000 --- a/R/get_path_lengths.R +++ /dev/null @@ -1,80 +0,0 @@ -#' Get Path Members (DEPRECATED) -#' @description Given a network and set of IDs, finds paths between all -#' identified flowpath outlets. This algorithm finds members between outlets -#' regardless of flow direction. -#' @inheritParams get_path_lengths -#' @return list of lists containing flowpath identifiers along path that connect -#' outlets. -#' @export -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' fline <- walker_flowline -#' -#' outlets <- c(5329303, 5329357, 5329317, 5329365, 5329435, 5329817) -#' -#' # Add toCOMID -#' fline <- nhdplusTools::get_tocomid(fline, add = TRUE) -#' -#' fl <- dplyr::select(fline, ID = comid, toID = tocomid, lengthkm) -#' -#' get_path_members(outlets, fl) -#' -get_path_members <- function(outlets, network, cores = 1, status = FALSE) { - - get_paths_internal(outlets, network, cores, status, lengths = FALSE) - -} - -#' Get Path Lengths (DEPRECATED) -#' @description Given a network and set of IDs, finds path lengths between all -#' identified flowpath outlets. This algorithm finds distance between outlets -#' regardless of flow direction. -#' @param outlets vector of IDs from data.frame -#' @param network data.frame with ID, toID, and lengthkm attributes. -#' @param cores integer number of cores to use for parallel computation. -#' @param status logical print status and progress bars? -#' @return data.frame containing the distance between pairs of network outlets. -#' For a network with one terminal outlet, the data.frame will have -#' `nrow(network)^2` rows. -#' @importFrom pbapply pboptions pbapply pblapply -#' @importFrom utils combn -#' @export -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' fline <- walker_flowline -#' -#' outlets <- c(5329303, 5329357, 5329317, 5329365, 5329435, 5329817) -#' -#' # Add toCOMID -#' fline <- nhdplusTools::get_tocomid(fline, add = TRUE) -#' -#' fl <- dplyr::select(fline, ID = comid, toID = tocomid, lengthkm) -#' -#' get_path_lengths(outlets, fl) -#' -get_path_lengths <- function(outlets, network, cores = 1, status = FALSE) { - - get_paths_internal(outlets, network, cores, status, lengths = TRUE) -} - -#' @importFrom hydroloom navigate_connected_paths -get_paths_internal <- function(outlets, network, cores = 1, status = FALSE, - lengths = FALSE) { - - warning("get_paths* is deprecated in favor of get_paths in the hydroloom package.") - - if(!is.null(cores) && cores != 1) { - message("the future plan is being modified and will be changed back on exit") - oplan <- future::plan(future::multisession, workers = cores) - on.exit(future::plan(oplan), add = TRUE) - } - - paths <- navigate_connected_paths(network, outlets) - - if(!lengths) { - - return(select(paths, -any_of("network_distance_km"))) - } - - select(paths, all_of(c(ID_1 = "id_1", ID_2 = "id_2", "network_distance_km"))) -} diff --git a/R/get_paths.R b/R/get_paths.R deleted file mode 100644 index 01b0819a..00000000 --- a/R/get_paths.R +++ /dev/null @@ -1,227 +0,0 @@ -#' Get Level Paths (DEPRECATED) -#' @description Calculates level paths using the stream-leveling approach of -#' NHD and NHDPlus. In addition to a levelpath identifier, a topological sort and -#' levelpath outlet identifier is provided in output. If arbolate sum is provided in -#' the weight column, this will match the behavior of NHDPlus. Any numeric value can be -#' included in this column and the largest value will be followed when no nameID is available. -#' @param x data.frame with ID, toID, nameID, and weight columns. -#' @param override_factor numeric factor to use to override nameID. -#' If `weight` is `numeric_factor` times larger on a path, it will be followed -#' regardless of the nameID indication. -#' @param status boolean if status updates should be printed. -#' @param cores numeric number of cores to use in initial path ranking calculations. -#' @return data.frame with ID, outletID, topo_sort, and levelpath columns. -#' See details for more info. -#' @importFrom hydroloom add_levelpaths hy -#' @details -#' \enumerate{ -#' \item levelpath provides an identifier for the collection of flowlines -#' that make up the single mainstem flowpath of a total upstream aggregate catchment. -#' \item outletID is the catchment ID (COMID in the case of NHDPlus) for the catchment -#' at the outlet of the levelpath the catchment is part of. -#' \item topo_sort is similar to Hydroseq in NHDPlus in that large topo_sort values -#' are upstream of small topo_sort values. Note that there are many valid topological -#' sort orders of a directed graph. -#' } -#' @export -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' test_flowline <- prepare_nhdplus(walker_flowline, 0, 0, FALSE) -#' -#' test_flowline <- data.frame( -#' ID = test_flowline$COMID, -#' toID = test_flowline$toCOMID, -#' nameID = walker_flowline$GNIS_ID, -#' weight = walker_flowline$ArbolateSu, -#' stringsAsFactors = FALSE) -#' -#' get_levelpaths(test_flowline) -#' -#' -get_levelpaths <- function(x, override_factor = NULL, status = FALSE, cores = NULL) { - - warning("get_levelpaths is deprecated in favor of add_levelpaths in the hydroloom package.") - - if(!is.null(cores)) { - if(inherits(cores, "cluster")) stop("passing a cluster object no longer supported") - message("the future plan is being modified and will be changed back on exit") - oplan <- future::plan(future::multisession, workers = cores) - on.exit(future::plan(oplan), add = TRUE) - } - - get_levelpaths_internal(x, override_factor) - -} - -get_levelpaths_internal <- function(x, override_factor) { - x <- check_names(st_drop_geometry(x), "get_levelpaths") - - x <- hy(select(x, all_of(c(id = "ID", toid = "toID", "nameID", "weight")))) - - x <- add_levelpaths(x, weight_attribute = "weight", name_attribute = "nameID", - override_factor = override_factor) - - return(as.data.frame(select(x, all_of(c("ID" = "id", "outletID" = "levelpath_outlet_id", - "topo_sort" = "topo_sort", "levelpath" = "levelpath"))))) -} - -#' @importFrom hydroloom make_fromids -get_fromids <- function(index_ids, return_list = FALSE) { - - index_ids <- select(index_ids, indid = "id", toindid = "toid") - - make_fromids(index_ids, return_list) - -} - -#' Get Sorted Network -#' @description given a tree with an id and and toid in the -#' first and second columns, returns a sorted and potentially -#' split set of output. -#' -#' Can also be used as a very fast implementation of upstream -#' with tributaries navigation. The full network from each -#' outlet is returned in sorted order. -#' -#' @export -#' @param x data.frame with an identifier and to identifier in the -#' first and second columns. -#' @param split logical if TRUE, the result will be split into -#' independent networks identified by the id of their outlet. The -#' outlet id of each independent network is added as a "terminalID" -#' attribute. -#' @param outlets same as id in x; if specified only the network -#' emanating from these outlets will be considered and returned. -#' @return data.frame containing a topologically sorted version -#' of the requested network and optionally a terminal id. -#' @importFrom hydroloom sort_network -#' @examples -#' source(system.file("extdata/new_hope_data.R", package = "nhdplusTools")) -#' -#' fpath <- get_tocomid( -#' dplyr::select(new_hope_flowline, COMID, FromNode, ToNode, Divergence, FTYPE, -#' AreaSqKM, LENGTHKM, GNIS_ID) -#' ) -#' -#' head(fpath <- get_sorted(fpath, split = TRUE)) -#' -#' fpath['sort_order'] <- 1:nrow(fpath) -#' -#' plot(fpath['sort_order']) -#' -get_sorted <- function(x, split = FALSE, outlets = NULL) { - - orig_names <- names(x)[1:2] - - names(x)[1:2] <- c("id", "toid") - - x <- sort_network(x, split, outlets) - - names(x)[1:2] <- orig_names - - if(split) - names(x)[names(x) == "terminal_id"] <- "terminalID" - - return(x) -} - -#' @noRd -#' @importFrom dplyr all_of -#' @importFrom hydroloom make_index_ids -get_index_ids <- function(x, - innames = c("comid", "tocomid"), - outnames = c("id", "toid")) { - - if(!all(innames %in% names(x))) { - stop(paste(paste(innames, collapse = ", "), "must be in input or provided.")) - } - - out <- select(x, all_of(innames)) - - names(out) <- c("id", "toid") - - out <- make_index_ids(out) - - out <- tidyr::unnest(out$to_list, "toindid") - - out <- select(out, "indid", "toindid") - - names(out) <- outnames - - out - -} - -#' @noRd -#' @param x data.frame with ID and toID -#' @param rev logical if TRUE (default) top down -#' @importFrom tidyr replace_na -topo_sort_network <- function(x, reverse = TRUE) { - - if(any(x$ID == 0)) stop("ID 0 must not be present. It is used as the outlet ID.") - - x[["toID"]] <- replace_na(x[["toID"]], 0) - - x <- get_sorted(x[, c("ID", "toID", names(x)[!names(x) %in% c("ID", "toID")])]) - - if(reverse) { - x <- x[nrow(x):1, ] - } - - x[!is.na(x$ID), ] - -} - -#' Get Terminal ID (DEPRECATED) -#' @description Get the ID of the basin outlet for each flowline. -#' This function has been deprecated in favor of get_sorted. -#' @param x two column data.frame with IDs and toIDs. Names are ignored. -#' @param outlets IDs of outlet flowlines -#' @export -#' @return data.frame containing the terminal ID for each outlet -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' fl <- dplyr::select(prepare_nhdplus(walker_flowline, 0, 0), -#' ID = COMID, toID = toCOMID) -#' -#' outlet <- fl$ID[which(!fl$toID %in% fl$ID)] -#' -#' get_terminal(fl, outlet) -#' -get_terminal <- function(x, outlets) { - - message("Deprecated, use get_sorted.") - - ordered_id <- x$ID - - x <- get_sorted(x, split = TRUE, outlets = outlets) - - left_join(data.frame(ID = ordered_id), - x[c("ID", "terminalID")], by = "ID") -} - -#' Get Path Length -#' @description Generates the main path length to a basin's terminal path. -#' @param x data.frame with ID, toID, length columns. -#' @importFrom hydroloom add_pathlength -#' @export -#' @return data.frame containing pathlength for each ID -#' @examples -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' fl <- dplyr::select(prepare_nhdplus(walker_flowline, 0, 0), -#' ID = COMID, toID = toCOMID, length = LENGTHKM) -#' -#' get_pathlength(fl) -#' -get_pathlength <- function(x) { - - x <- select(st_drop_geometry(x), all_of(c("id" = "ID", "toid" = "toID", - "length_km" = "length"))) - - x <- add_pathlength(x) - - return(data.frame(ID = x$id, pathlength = x$pathlength_km)) -} diff --git a/R/get_vaa.R b/R/get_vaa.R index dfffca22..75ae860a 100644 --- a/R/get_vaa.R +++ b/R/get_vaa.R @@ -1,8 +1,8 @@ #' @title File path to value added attribute (vaa) Cache -#' @description nhdplusTools will download and cache an `fst` file with +#' @description hydrogeofetch will download and cache a parquet file with #' NHDPlusV2 attribute data sans geometry. This function returns the #' file path to the cached file. Will use the user data dir indicated -#' by \link{nhdplusTools_data_dir}. +#' by \link{hydrogeofetch_data_dir}. #' @param updated_network logical default FALSE. If TRUE, returns path to updated #' network parameters. See \link{get_vaa} for more. #' @inherit download_vaa details @@ -16,9 +16,9 @@ get_vaa_path <- function(updated_network = FALSE) { if(updated_network) { - file.path(nhdplusTools_data_dir(), "enhd_nhdplusatts.fst") + file.path(hydrogeofetch_data_dir(), "enhd_nhdplusatts.parquet") } else { - file.path(nhdplusTools_data_dir(), "nhdplusVAA.fst") + file.path(hydrogeofetch_data_dir(), "nhdplusVAA.parquet") } } @@ -27,7 +27,6 @@ get_vaa_path <- function(updated_network = FALSE) { #' @inherit download_vaa details #' @inheritParams get_vaa #' @return character vector -#' @importFrom fst metadata_fst #' @export #' @examples #' \dontrun{ @@ -42,7 +41,7 @@ get_vaa_names <- function(updated_network = FALSE) { check_vaa_path(path, TRUE) - fst::metadata_fst(path)[["columnNames"]] + arrow::open_dataset(path)$schema$names } #' @title NHDPlusV2 Attribute Subset @@ -50,7 +49,7 @@ get_vaa_names <- function(updated_network = FALSE) { #' @inherit download_vaa details #' @param atts character The variable names you would like, always includes comid #' @param path character path where the file should be saved. Default is a -#' persistent system data as retrieved by \link{nhdplusTools_data_dir}. +#' persistent system data as retrieved by \link{hydrogeofetch_data_dir}. #' Also see: \link{get_vaa_path} #' @param download logical if TRUE, the default, will download VAA table if not #' found at path. @@ -58,7 +57,6 @@ get_vaa_names <- function(updated_network = FALSE) { #' from E2NHD and National Water Model retrieved from #' \doi{10.5066/P976XCVT}. #' @return data.frame containing requested VAA data -#' @importFrom fst read.fst #' @export #' @examples #' \dontrun{ @@ -82,11 +80,11 @@ get_vaa <- function(atts = NULL, check_vaa_path(path, download, FALSE) if(updated_network) { - updated_net_path <- file.path(dirname(path), "enhd_nhdplusatts.fst") + updated_net_path <- file.path(dirname(path), "enhd_nhdplusatts.parquet") check_vaa_path(updated_net_path, download, TRUE) - new_names <- fst::metadata_fst(updated_net_path)[["columnNames"]] + new_names <- arrow::open_dataset(updated_net_path)$schema$names } available_names = get_vaa_names(updated_network) @@ -121,17 +119,19 @@ get_vaa <- function(atts = NULL, replace_names <- atts[atts %in% new_names & !atts %in% deprecated_names] - # Grab the original vaas but not the ones we are going to replace. - out <- fst::read.fst(path, include_names) + # as.data.frame strips any data.table class arrow may attach on metadata + # roundtrip failure, keeping the data.frame `[` semantics downstream. + out <- as.data.frame(arrow::read_parquet(path, col_select = include_names)) - # grab all the new attributes. - new_comid <- fst::read.fst(updated_net_path, "comid") + new_comid <- arrow::read_parquet(updated_net_path, col_select = "comid") - # reorder out to match new -- also drop stuff missing from new. out <- out[match(new_comid$comid, out$comid), , drop = FALSE] - out <- cbind(out, fst::read.fst(updated_net_path, - c(replace_names[replace_names != "comid"]))) + cols_to_add <- replace_names[replace_names != "comid"] + if(length(cols_to_add) > 0) { + out <- cbind(out, as.data.frame(arrow::read_parquet(updated_net_path, + col_select = cols_to_add))) + } reorder <- match(get_vaa_names(updated_network), names(out)) @@ -141,7 +141,7 @@ get_vaa <- function(atts = NULL, } else { - return(fst::read_fst(path, c('comid', atts[atts != 'comid']))) + return(as.data.frame(arrow::read_parquet(path, col_select = c('comid', atts[atts != 'comid'])))) } } @@ -171,7 +171,6 @@ check_vaa_path <- function(path, download, updated_network = FALSE) { #' @param force logical. Force data re-download. Default = FALSE #' @return character path to cached data #' @export -#' @importFrom httr GET progress write_disk download_vaa <- function(path = get_vaa_path(updated_network), force = FALSE, updated_network = FALSE) { @@ -186,13 +185,7 @@ download_vaa <- function(path = get_vaa_path(updated_network), force = FALSE, up dir.create(dirname(path), showWarnings = FALSE, recursive = TRUE) - resp <- httr::GET(url, - httr::write_disk(path, overwrite = TRUE), - httr::progress()) - - if (resp$status_code != 200) { - stop("Download unsuccessfull :(") - } + if(is.null(hgf_download(url, path))) return(NULL) } # return file path return(path) @@ -231,7 +224,7 @@ get_characteristics_metadata <- function(search, source = "usgs", cache = TRUE) if(source == "streamcat") { check_pkg("StreamCatTools") - r <- file.path(nhdplusTools_data_dir(), "streamcat_metadata.rds") + r <- file.path(hydrogeofetch_data_dir(), "streamcat_metadata.rds") if(!cache) unlink(r, force = TRUE) @@ -241,8 +234,8 @@ get_characteristics_metadata <- function(search, source = "usgs", cache = TRUE) } else { params <- StreamCatTools::sc_get_params(param = "variable_info") - if(!dir.exists(nhdplusTools_data_dir())) - dir.create(nhdplusTools_data_dir(), recursive = TRUE) + if(!dir.exists(hydrogeofetch_data_dir())) + dir.create(hydrogeofetch_data_dir(), recursive = TRUE) saveRDS(params, r) params @@ -265,8 +258,8 @@ get_characteristics_metadata <- function(search, source = "usgs", cache = TRUE) out <- tryCatch({ u <- "https://prod-is-usgs-sb-prod-publish.s3.amazonaws.com/5669a79ee4b08895842a1d47/metadata_table.tsv" - f <- file.path(nhdplusTools_data_dir(), "metadata_table.tsv") - r <- file.path(nhdplusTools_data_dir(), "metadata_table.rds") + f <- file.path(hydrogeofetch_data_dir(), "metadata_table.tsv") + r <- file.path(hydrogeofetch_data_dir(), "metadata_table.rds") if(!cache) { unlink(r, force = TRUE) @@ -279,7 +272,7 @@ get_characteristics_metadata <- function(search, source = "usgs", cache = TRUE) if(!dir.exists(dirname(f))) dir.create(dirname(f), recursive = TRUE) - if(!file.exists(f)) resp <- httr::RETRY("GET", u, httr::write_disk(f)) + if(!file.exists(f)) hgf_download(u, f, progress = FALSE) out <- read.delim(f, sep = "\t") @@ -332,7 +325,7 @@ get_characteristics_metadata <- function(search, source = "usgs", cache = TRUE) #' (for metrics like BankfullDepth, IWI, etc.). Ignored when #' \code{source = "usgs"} where the area of interest is encoded in the #' variable name prefix (e.g. CAT_, TOT_, ACC_). -#' @importFrom dplyr bind_rows filter select everything collect +#' @importFrom dplyr bind_rows filter select everything collect all_of #' @importFrom arrow s3_bucket open_dataset #' @export #' @examples @@ -388,9 +381,12 @@ get_catchment_characteristics <- function(varname, ids, sub <- filter(select(ds, any_of(cols_to_select)), .data$COMID %in% ids) - att <- collect(sub) + # as.data.frame strips the data.table class arrow may attach when its + # R metadata roundtrip fails (e.g. externalptr columns) so downstream + # `att[, cols, drop = FALSE]` keeps the data.frame semantics it expects. + att <- as.data.frame(collect(sub)) - att <- dplyr::mutate_all(att, ~ifelse(. == -9999, NA, .)) + att <- dplyr::mutate_all(att, \(x) ifelse(x == -9999, NA, x)) has_pct_nodata <- "percent_nodata" %in% names(att) diff --git a/R/index_nhdplus.R b/R/index_nhdplus.R deleted file mode 100644 index 2a57ea3a..00000000 --- a/R/index_nhdplus.R +++ /dev/null @@ -1,205 +0,0 @@ -#' @title Get Flowline Index -#' @description given an sf point geometry column, return COMID, reachcode, -#' and measure for each. -#' @param flines sf data.frame of type LINESTRING or MULTILINESTRING including -#' COMID, REACHCODE, ToMeas, and FromMeas. Can be "download_nhdplusv2" and remote -#' nhdplusv2 data will be downloaded for the bounding box surround the submitted points. -#' NOTE: The download option may not work for large areas, use with caution. -#' @param points sf or sfc of type POINT in analysis projection. NOTE: flines will -#' be projected to the projection of the points layer. -#' @param search_radius units distance for the nearest neighbor search -#' to extend in analysis projection. If missing or NULL, and points are in a lon -#' lat projection, a default of 0.01 degree is used, otherwise 200 m is used. -#' Conversion to the linear unit used by the provided crs of points is attempted. -#' See RANN nn2 documentation for more details. -#' @param precision numeric the resolution of measure precision in the output in meters. -#' @param max_matches numeric the maximum number of matches to return if multiple are -#' found in search_radius -#' @return data.frame with five columns, id, COMID, REACHCODE, REACH_meas, and offset. id is the -#' row or list element in the point input. -#' @details Note 1: Inputs are cast into LINESTRINGS. Because of this, -#' the measure output -#' of inputs that are true multipart lines may be in error. -#' -#' Note 2: This algorithm finds the nearest node in the input flowlines to -#' identify which flowline the point should belong to. As a second pass, -#' it can calculate the measure to greater precision than the nearest flowline -#' geometry node. -#' -#' Note 3: Offset is returned in units consistent with the projection of -#' the input points. -#' -#' Note 4: See `dfMaxLength` input to sf::st_segmentize() for details of -#' handling of precision parameter. -#' -#' Note 5: "from" is downstream -- 0 is the outlet "to" is upstream -- 100 is the inlet -#' -#' @importFrom hydroloom index_points_to_lines -#' @importFrom magrittr `%>%` -#' @export -#' @examples -#' \donttest{ -#' -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) -#' -#' point <- sf::st_sfc(sf::st_point(c(-76.87479, 39.48233)), -#' crs = 4326) -#' -#' get_flowline_index(sample_flines, point) -#' -#' point <- sf::st_transform(point, 5070) -#' -#' get_flowline_index(sample_flines, point, -#' search_radius = units::set_units(200, "m")) -#' -#' get_flowline_index("download_nhdplusv2", point) -#' -#' get_flowline_index(sample_flines, point, precision = 30) -#' -#' get_flowline_index(sample_flines, -#' sf::st_sfc(list(sf::st_point(c(-76.86934, 39.49328)), -#' sf::st_point(c(-76.91711, 39.40884)), -#' sf::st_point(c(-76.88081, 39.36354))), -#' crs = 4326), -#' search_radius = units::set_units(0.2, "degrees"), -#' max_matches = 10) -#' -#' } -#' -get_flowline_index <- function(flines, points, - search_radius = NULL, - precision = NA, - max_matches = 1) { - - if(is.character(flines) && flines == "download_nhdplusv2") { - - if((!is.null(nrow(points)) && nrow(points)) == 1 | length(points) == 1) { - - search_radius <- hydroloom:::check_search_radius(search_radius, points) - - req <- sf::st_buffer(points, search_radius) - - } else { - - req <- points - - } - - flines <- align_nhdplus_names( - get_nhdplus(AOI = sf::st_transform(req, 4326), - realization = "flowline")) |> - sf::st_transform(sf::st_crs(points)) - - } - - flines <- check_names(flines, "get_flowline_index") - - out <- index_points_to_lines(flines, points, search_radius, precision, max_matches) - - rename(out, any_of(c(id = "point_id", REACH_meas = "REACHCODE_measure"))) - -} - -#' @title Disambiguate Flowline Indexes -#' @description Given a set of flowline indexes and numeric or ascii criteria, -#' return closest match. If numeric criteria are used, the minimum difference -#' in the numeric attribute is used for disambiguation. If ascii criteria are used, -#' the \link[utils]{adist} function is used with the following algorithm: -#' `1 - adist_score / max_string_length`. Comparisons ignore case. -#' @param indexes data.frame as output from \link{get_flowline_index} with more than -#' one hydrologic location per indexed point. -#' @param flowpath data.frame with two columns. The first should join to the COMID -#' field of the indexes and the second should be the numeric or ascii metric such as drainage -#' area or GNIS Name. Names of this data.frame are not used. -#' @param hydro_location data.frame with two columns. The first should join to the -#' id field of the indexes and the second should be the numeric or ascii metric such as drainage -#' area or GNIS Name.. Names of this data,frame are not used. -#' @return data.frame indexes deduplicated according to the minimum difference -#' between the values in the metric columns. If two or more result in the same "minimum" -#' value, duplicates will be returned. -#' @importFrom hydroloom disambiguate_indexes -#' @export -#' @examples -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) -#' -#' hydro_location <- sf::st_sf(id = c(1, 2, 3), -#' geom = sf::st_sfc(list(sf::st_point(c(-76.86934, 39.49328)), -#' sf::st_point(c(-76.91711, 39.40884)), -#' sf::st_point(c(-76.88081, 39.36354))), -#' crs = 4326), -#' totda = c(23.6, 7.3, 427.9), -#' nameid = c("Patapsco", "", "Falls Run River")) -#' -#' flowpath <- dplyr::select(sample_flines, -#' comid = COMID, -#' totda = TotDASqKM, -#' nameid = GNIS_NAME, -#' REACHCODE, -#' ToMeas, -#' FromMeas) -#' -#' indexes <- get_flowline_index(flowpath, -#' hydro_location, -#' search_radius = 0.2, -#' max_matches = 10) -#' -#' disambiguate_flowline_indexes(indexes, -#' dplyr::select(flowpath, comid, totda), -#' dplyr::select(hydro_location, id, totda)) -#' -#' result <- disambiguate_flowline_indexes(indexes, -#' dplyr::select(flowpath, comid, nameid), -#' dplyr::select(hydro_location, id, nameid)) -#' -#' result[result$id == 1, ] -#' -#' result[result$id == 2, ] -#' -#' result[result$id == 3, ] -#' -disambiguate_flowline_indexes <- function(indexes, flowpath, hydro_location) { - check_names(indexes, "disambiguate_flowline_indexes") - - indexes <- rename(indexes, all_of(c(point_id = "id"))) - - out <- disambiguate_indexes(indexes, flowpath, hydro_location) - - rename(out, all_of(c(id = "point_id"))) - -} - -#' @title Get Waterbody Index -#' @description given an sf point geometry column, return waterbody id, and -#' COMID of dominant artificial path -#' @param waterbodies sf data.frame of type POLYGON or MULTIPOLYGON including -#' COMID attributes. -#' @param flines sf data.frame of type LINESTRING or MULTILINESTRING including -#' COMID, WBAREACOMI, and Hydroseq attributes -#' @param points sfc of type POINT -#' @param search_radius units class with a numeric value indicating how far to -#' search for a waterbody boundary in units of provided projection. Set units with -#' \link[units]{set_units}. -#' @return data.frame with two columns, COMID, in_wb_COMID, near_wb_COMID, -#' near_wb_dist, and outlet_fline_COMID. Distance is in units of provided projection. -#' @importFrom hydroloom index_points_to_waterbodies -#' @export -#' @examples -#' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) -#' -#' waterbodies <- sf::st_transform( -#' sf::read_sf(sample_data, "NHDWaterbody"), 5070) -#' -#' points <- sf::st_transform( -#' sf::st_sfc(sf::st_point(c(-89.356086, 43.079943)), -#' crs = 4326), 5070) -#' -#' get_waterbody_index(waterbodies, points, -#' search_radius = units::set_units(500, "m")) -#' -get_waterbody_index <- function(waterbodies, points, flines = NULL, - search_radius = NULL) { - - index_points_to_waterbodies(waterbodies, points, flines, search_radius) - -} diff --git a/R/mainstems.R b/R/mainstems.R new file mode 100644 index 00000000..5099a018 --- /dev/null +++ b/R/mainstems.R @@ -0,0 +1,353 @@ +mainstem_lookup_types <- c("nhdpv2", "nhdphr") + +mainstem_lookup_id_col <- function(type) { + if(type == "nhdpv2") "comid" else "nhdplushr_id" +} + +#' @title File path to mainstem lookup table cache +#' @description hydrogeofetch will download and cache a parquet file +#' translating geoconnex mainstem identifiers to NHDPlusV2 comids +#' (\code{type = "nhdpv2"}) or NHDPlusHR NHDPlusIDs (\code{type = "nhdphr"}). +#' Will use the user data dir indicated by \link{hydrogeofetch_data_dir}. +#' @param type character one of "nhdpv2" or "nhdphr" +#' @inherit add_mainstems details +#' @return character file path +#' @keywords internal +get_mainstem_lookup_path <- function(type) { + if(!type %in% mainstem_lookup_types) + stop("type must be one of: ", paste(mainstem_lookup_types, collapse = ", ")) + + release <- get("ref_rivers_release", envir = hydrogeofetch_env) + + file.path(hydrogeofetch_data_dir(), + paste0("ref_rivers_", release, "_", type, "_lookup.parquet")) +} + +#' @title Download mainstem lookup table from ref_rivers +#' @description Downloads and caches a geoconnex mainstem identifier lookup +#' table on your computer. +#' @inherit add_mainstems details +#' @inheritParams get_mainstem_lookup_path +#' @param path character path where the file should be saved. Default is a +#' persistent system data dir as retrieved by \link{hydrogeofetch_data_dir}. +#' Also see: \link{get_mainstem_lookup_path} +#' @param force logical. Force data re-download. Default = FALSE +#' @return character path to cached data +#' @importFrom arrow read_csv_arrow write_parquet +#' @keywords internal + +download_mainstem_lookup <- function(type, path = get_mainstem_lookup_path(type), force = FALSE) { + + if(file.exists(path) & !force) { + message("File already cached") + return(path) + } + + release <- get("ref_rivers_release", envir = hydrogeofetch_env) + base <- get("ref_rivers_base_url", envir = hydrogeofetch_env) + url <- paste0(base, release, "/", type, "_lookup.csv") + + if(nhdplus_debug()) message(url) + + dir.create(dirname(path), showWarnings = FALSE, recursive = TRUE) + + tmp <- tempfile(fileext = ".csv") + + if(is.null(hgf_download(url, tmp))) return(NULL) + + csv <- tryCatch(arrow::read_csv_arrow(tmp), error = function(e) NULL) + + unlink(tmp) + + expect_names <- c("uri", mainstem_lookup_id_col(type)) + + if(is.null(csv) || !all(expect_names %in% names(csv))) { + warning("Mainstem lookup table for ", type, + " did not have the expected columns.", call. = FALSE) + return(NULL) + } + + arrow::write_parquet(csv, path) + + path +} + +#' @noRd +read_mainstem_lookup <- function(type, ids = NULL) { + path <- get_mainstem_lookup_path(type) + + if(!file.exists(path)) { + if(is.null(download_mainstem_lookup(type, path))) return(NULL) + } + + id_col <- mainstem_lookup_id_col(type) + + ds <- open_dataset(path) + + if(!is.null(ids)) ds <- filter(ds, .data[[id_col]] %in% ids) + + as.data.frame(collect(ds)) +} + +#' @noRd +mainstem_id_from_uri <- function(uri) { + as.integer(sub(".*?/mainstems/([0-9]+).*", "\\1", uri)) +} + +#' @noRd +mainstem_uri_from_id <- function(id) { + paste0("https://geoconnex.us/ref/mainstems/", id) +} + +#' @noRd +parse_replacement_uris <- function(x) { + lapply(x, function(s) regmatches(s, gregexpr("https://geoconnex\\.us/[^']+", s))[[1]]) +} + +#' @title Add mainstem identifiers +#' @description Joins geoconnex mainstem identifiers onto a table that +#' contains NHDPlusV2 (comid/featureid) or NHDPlusHR (nhdplusid) +#' identifiers using a lookup table. +#' @details Source data comes from the ref_rivers GitHub release +#' \href{https://github.com/internetofwater/ref_rivers/releases}{here}. The +#' csv source data is downloaded once, converted to parquet, and cached in +#' the user data dir indicated by \link{hydrogeofetch_data_dir}. +#' @param x data.frame or sf containing an identifier column joinable to +#' NHDPlusV2 or NHDPlusHR. +#' @param join_col character name of the identifier column in \code{x}. +#' Detected automatically from comid, featureid, or nhdplusid (any case) +#' if not provided. +#' @param join_col_type character one of "nhdpv2" or "nhdphr". Detected +#' automatically from \code{join_col} if not provided. +#' @return \code{x} with mainstem_uri and mainstemid columns added. +#' @export +#' @examples +#' \donttest{ +#' add_mainstems(data.frame(comid = c(2804607, 2804621))) +#' } +add_mainstems <- function(x, join_col = NULL, join_col_type = NULL) { + + names_x <- names(x) + + if(is.null(join_col)) { + hit <- names_x[tolower(names_x) %in% c("comid", "featureid", "nhdplusid")] + + if(length(hit) == 0) + stop("Could not detect a comid, featureid, or nhdplusid column in x. ", + "Specify join_col.", call. = FALSE) + + join_col <- hit[1] + } + + if(!join_col %in% names_x) + stop("join_col '", join_col, "' not found in x.", call. = FALSE) + + if(is.null(join_col_type)) { + join_col_type <- if(tolower(join_col) == "nhdplusid") "nhdphr" else "nhdpv2" + } + + if(!join_col_type %in% mainstem_lookup_types) + stop("join_col_type must be one of: ", + paste(mainstem_lookup_types, collapse = ", "), call. = FALSE) + + hy_g <- get_hyg(x, inherits(x, "sf"), join_col) + + x <- st_drop_geometry(x) + + id_col <- mainstem_lookup_id_col(join_col_type) + + lk <- read_mainstem_lookup(join_col_type, ids = unique(x[[join_col]])) + + if(is.null(lk) || nrow(lk) == 0) { + + x$mainstem_uri <- NA_character_ + x$mainstemid <- NA_integer_ + + } else { + + lk$mainstemid <- mainstem_id_from_uri(lk$uri) + lk <- rename(lk, mainstem_uri = "uri") + lk <- distinct(select(lk, all_of(c(id_col, "mainstem_uri", "mainstemid")))) + + x <- left_join(x, lk, by = stats::setNames(id_col, join_col)) + + } + + if(!is.null(hy_g)) x <- st_sf(left_join(x, hy_g, by = join_col)) + + x +} + +#' @noRd +fetch_superseded_mainstems <- function() { + base <- get("gocnx_ref_base_url", envir = hydrogeofetch_env) + + base_call <- paste0(base, "collections/mainstems/items?superseded=True") + + out <- get_features_paging(base_call, limit = 500, status = FALSE) + + if(!inherits(out, "sf") || nrow(out) == 0) return(NULL) + + out$new_uris <- parse_replacement_uris(out$new_mainstemid) + + out +} + +#' @noRd +get_mainstem_geometry <- function(uris) { + geoms <- lapply(unique(uris), function(uri) { + feat <- hgf_sf(paste0(uri, "?f=json")) + if(inherits(feat, "sf")) rename(feat["uri"], id = "uri") else feat + }) + + geoms <- geoms[vapply(geoms, inherits, logical(1), what = "sf")] + + if(length(geoms) == 0) return(NULL) + + st_sf(bind_rows(geoms)) +} + +#' @title Check mainstem identifiers for supersession +#' @description Checks whether geoconnex mainstem identifiers have been +#' superseded by a newer reference release. See \link{update_mainstems} to +#' resolve a superseded mainstem id to its replacement. +#' @param x integer or character vector of geoconnex \code{ref/mainstems} ids, +#' or full mainstem uri (any namespace, e.g. \code{ref/mainstems} or +#' \code{usgs/mainstems}) vector of mainstem identifiers to check. Bare +#' integers/character ids are assumed to be \code{ref/mainstems} ids; a full +#' uri is matched literally, so ids minted under a different namespace are +#' never confused with a \code{ref/mainstems} id that happens to share the +#' same trailing number. +#' @return logical vector the same length as \code{x}. \code{TRUE} +#' indicates the mainstem id has been superseded. +#' @export +#' @examples +#' \donttest{ +#' check_mainstems(c(2086165, 2086637)) +#' } +check_mainstems <- function(x) { + + x_chr <- as.character(x) + is_uri <- grepl("^https?://", x_chr) + + uris <- x_chr + uris[!is_uri] <- mainstem_uri_from_id(x_chr[!is_uri]) + + super <- fetch_superseded_mainstems() + + if(is.null(super)) { + warning("Could not retrieve superseded mainstems list.", call. = FALSE) + return(rep(NA, length(x))) + } + + uris %in% super$uri +} + +#' @title Update superseded mainstem identifiers +#' @description Given point features with a known geoconnex mainstem +#' identifier, checks whether the mainstem has been superseded and, if so, +#' re-indexes the point to the replacement mainstem using +#' \link[hydroloom]{index_points_to_lines}. Points whose mainstem has no +#' replacement are left unchanged and flagged "unresolved". See +#' \link{check_mainstems} for a lightweight superseded/current check. +#' @param x sf POINT features with a mainstem identifier column. +#' @param mainstem_col character name of the mainstem identifier column in +#' \code{x}. Detected automatically from a "mainstemid" or "mainstem_id" +#' column if not provided. Values may be bare \code{ref/mainstems} ids or +#' full mainstem uris (any namespace); see \link{check_mainstems}. +#' @param search_radius units distance for the nearest neighbor search +#' passed to \link[hydroloom]{index_points_to_lines}. If \code{NULL}, the +#' default in \link[hydroloom]{index_points_to_lines} is used. +#' @return \code{x} with \code{mainstem_col} updated in place and a +#' \code{mainstem_update_status} column added with values "unchanged", +#' "updated", or "unresolved". +#' @export +#' @examples +#' \donttest{ +#' pt <- sf::st_sf(mainstemid = 2086165, +#' geometry = sf::st_sfc(sf::st_point(c(-75.567, 43.176)), +#' crs = 4326)) +#' update_mainstems(pt) +#' } +update_mainstems <- function(x, mainstem_col = NULL, search_radius = NULL) { + + if(!inherits(x, "sf") || !all(st_geometry_type(x) == "POINT")) + stop("x must be an sf object of POINT geometries.", call. = FALSE) + + if(is.null(mainstem_col)) { + hit <- names(x)[tolower(names(x)) %in% c("mainstemid", "mainstem_id")] + + if(length(hit) == 0) + stop("Could not detect a mainstemid column in x. Specify mainstem_col.", + call. = FALSE) + + mainstem_col <- hit[1] + } + + col_chr <- as.character(x[[mainstem_col]]) + is_uri <- grepl("^https?://", col_chr) + + x_uri <- col_chr + x_uri[!is_uri] <- mainstem_uri_from_id(col_chr[!is_uri]) + + x$mainstem_update_status <- "unchanged" + + super <- fetch_superseded_mainstems() + + if(is.null(super)) { + warning("Could not retrieve superseded mainstems list.", call. = FALSE) + return(x) + } + + is_super <- x_uri %in% super$uri + + if(!any(is_super)) return(x) + + super_uris <- unique(x_uri[is_super]) + + for(suri in super_uris) { + + cands <- super$new_uris[[match(suri, super$uri)]] + + rows <- which(x_uri == suri) + + if(length(cands) == 0) { + x$mainstem_update_status[rows] <- "unresolved" + next + } + + cand_lines <- get_mainstem_geometry(cands) + + if(is.null(cand_lines)) { + x$mainstem_update_status[rows] <- "unresolved" + next + } + + pts_5070 <- st_transform(x[rows, ], 5070) + lines_5070 <- st_transform(cand_lines, 5070) + + idx <- index_points_to_lines(lines_5070, pts_5070, + search_radius = search_radius, + max_matches = 1) + + if(is.null(idx) || nrow(idx) == 0) { + x$mainstem_update_status[rows] <- "unresolved" + next + } + + matched <- rows[idx$point_id] + x[[mainstem_col]][matched] <- + ifelse(is_uri[matched], idx$id, mainstem_id_from_uri(idx$id)) + x$mainstem_update_status[matched] <- "updated" + + unmatched <- setdiff(rows, matched) + if(length(unmatched) > 0) + x$mainstem_update_status[unmatched] <- "unresolved" + } + + if(any(x$mainstem_update_status == "unresolved")) + warning("Some mainstem ids could not be resolved to a replacement. ", + "See mainstem_update_status column.", call. = FALSE) + + x +} diff --git a/R/map_nhdplus.R b/R/map_nhdplus.R index bb3aa42a..d5e2c8c7 100644 --- a/R/map_nhdplus.R +++ b/R/map_nhdplus.R @@ -44,7 +44,7 @@ check_pkg <- function(pkg) { #' #' map_nhdplus(list(13293970, 13293750)) #' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/sample_data.R", package = "hydrogeofetch")) #' #' map_nhdplus(list(13293970, 13293750), streamorder = 3, nhdplus_data = sample_data) #' @@ -90,22 +90,22 @@ map_nhdplus <- function(outlets = NULL, bbox = NULL, streamorder = NULL, ######################## - m <- leaflet::leaflet() %>% - leaflet::addProviderTiles("Esri.NatGeoWorldMap", group = "Terrain") %>% - leaflet::addProviderTiles("CartoDB.Positron", group = "Grayscale") %>% - leaflet::addProviderTiles("Esri.WorldImagery", group = "Imagery") %>% - leaflet::addScaleBar("bottomleft") %>% + m <- leaflet::leaflet() |> + leaflet::addProviderTiles("Esri.NatGeoWorldMap", group = "Terrain") |> + leaflet::addProviderTiles("CartoDB.Positron", group = "Grayscale") |> + leaflet::addProviderTiles("Esri.WorldImagery", group = "Imagery") |> + leaflet::addScaleBar("bottomleft") |> leaflet::addMiniMap( toggleDisplay = TRUE, minimized = TRUE - ) %>% + ) |> leaflet::addMeasure( position = "bottomleft", primaryLengthUnit = "feet", primaryAreaUnit = "sqmiles", activeColor = "red", completedColor = "green" - ) %>% + ) |> leaflet::fitBounds(lng1 = pd$plot_bbox[1,1], lng2 = pd$plot_bbox[1,2], lat1 = pd$plot_bbox[2,1], lat2 = pd$plot_bbox[2,2]) @@ -178,7 +178,7 @@ map_nhdplus <- function(outlets = NULL, bbox = NULL, streamorder = NULL, stroke = FALSE, radius = ~cex*5, group = "outlets" - ) %>% + ) |> leaflet::addLegend(position = 'topright', colors = pal(unique(outs$type)), labels = unique(outs$type), title = "Outlet Type", opacity = .7 diff --git a/R/oafeat_tools.R b/R/oafeat_tools.R index e223dfe2..05438c0c 100644 --- a/R/oafeat_tools.R +++ b/R/oafeat_tools.R @@ -36,7 +36,7 @@ query_usgs_oafeat <- function(AOI = NULL, ids = NULL, properties = NULL, skip_geometry = FALSE) { - base <- get("usgs_water_root", envir = nhdplusTools_env) + base <- get_water_url() source <- data.frame(server = 'usgs_oafeat', user_call = c('huc02_2020', 'huc04_2020', 'huc06_2020', @@ -164,11 +164,21 @@ query_usgs_oafeat <- function(AOI = NULL, ids = NULL, discover_oafeat <- function(landing_url) { landing <- mem_get_json(landing_url) + if(is.null(landing)) { + warning("OGC API landing page unavailable at ", landing_url, + call. = FALSE) + return(NULL) + } collections <- landing$links |> filter_list_kvp("rel", "data", n = 1) |> extract("href") |> mem_get_json() + if(is.null(collections)) { + warning("OGC API collections endpoint unavailable from ", landing_url, + call. = FALSE) + return(NULL) + } collections_meta <- dplyr::bind_rows( lapply(collections$collections, @@ -223,6 +233,8 @@ get_oafeat <- function(base, avail <- discover_oafeat(base) + if(is.null(avail)) return(NULL) + if(is.null(type)) { warning("type is required, returning choices.") return(avail) @@ -243,13 +255,18 @@ get_oafeat <- function(base, if(!is.null(AOI)) { if(is.character(AOI)) { - AOI <- try(sf::read_sf(AOI)) |> - st_transform(4326) + if(grepl("^https?://", AOI)) { + AOI <- hgf_sf(AOI) + } else { + AOI <- try(sf::read_sf(AOI), silent = TRUE) + } if(!inherits(AOI, "sf")) { stop("AOI did not return an sf object when read") } + AOI <- st_transform(AOI, 4326) + } if(!inherits(AOI, "bbox") && @@ -317,50 +334,11 @@ add_sep <- function(bc) { bc } -make_request <- function(req, body = "") { - tryCatch({ - if(nhdplus_debug()) { - message(paste(req, "\n")) - message(body) - } - - if(body != "") { - r <- RETRY("POST", - req, - body = body, - httr::content_type("application/query-cql-json"), - pause_min = 5, pause_base = 5) - - if(r$status != 200) { - return(r) - } - - out <- rawToChar(r$content) - } else { - - out <- rawToChar(RETRY("GET", req)$content) - - } - - - }, error = function(e) { - warning("Something went wrong trying to access a service.") - return(NULL) - }) - - out <- tryCatch({ - st_zm(read_sf(out))}, - error = function(e) return(NULL)) - - out -} - get_features_paging <- function(base_call, ids_list = list(), limit = 1000, status = TRUE) { use_ids <- !identical(ids_list, list()) if(use_ids) { - # we will page through ids ids <- split_equal_size(ids_list$ids, limit) } @@ -377,22 +355,18 @@ get_features_paging <- function(base_call, ids_list = list(), limit = 1000, stat while(keep_going) { if(use_ids) { - post_body <- id_filter_cql(ids[[id_batch]], ids_list$id_attribute) - req <- paste0(base_call, "limit=", limit, "&offset=", offset) - } else { - post_body <- "" - req <- paste0(base_call, "limit=", limit, "&offset=", offset) - } - page <- make_request(req, post_body) + req <- paste0(base_call, "limit=", limit, "&offset=", offset) - if(!is.null(page) & inherits(page, "response")) { - warning("Can't continue, got unexpected response: ", print(page)) - page <- NULL + if(post_body != "") { + page <- hgf_sf(req, body = post_body, + content_type = "application/query-cql-json") + } else { + page <- hgf_sf(req) } if(!inherits(page, "sf")) { @@ -404,7 +378,6 @@ get_features_paging <- function(base_call, ids_list = list(), limit = 1000, stat if(nrow(page) > 0) out <- c(out, list(page)) if(nrow(page) < limit) { - # this batch is exhausted if(use_ids && id_batch < length(ids)) { id_batch <- id_batch + 1 offset <- 0 @@ -481,7 +454,7 @@ get_huc12_by_huc <- function(huc_ids, t_srs = NULL) { if(length(huc_ids) == 0) return(sf::st_sf(geometry = sf::st_sfc(crs = 4326))) - base <- get("usgs_water_root", envir = nhdplusTools_env) + base <- get_water_url() n <- unique(nchar(huc_ids)) if(length(n) != 1 || !n %in% c(8, 10)) diff --git a/R/plot_nhdplus.R b/R/plot_nhdplus.R index 446a6ff9..bfa5cf31 100644 --- a/R/plot_nhdplus.R +++ b/R/plot_nhdplus.R @@ -58,13 +58,14 @@ #' #' @export #' @importFrom graphics par +#' @importFrom hydroloom navigate_hydro_network #' @examples #' \donttest{ #' options("rgdal_show_exportToProj4_warnings"="none") #' # Beware plot_nhdplus caches data to the default location. #' # If you do not want data in "user space" change the default. -#' old_dir <- nhdplusTools::nhdplusTools_data_dir() -#' nhdplusTools_data_dir(tempdir()) +#' old_dir <- hydrogeofetch::hydrogeofetch_data_dir() +#' hydrogeofetch_data_dir(tempdir()) #' #' plot_nhdplus("05428500") #' @@ -72,7 +73,7 @@ #' #' plot_nhdplus(list(13293970, 13293750)) #' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/sample_data.R", package = "hydrogeofetch")) #' #' plot_nhdplus(list(13293970, 13293750), streamorder = 3, nhdplus_data = sample_data) #' @@ -104,7 +105,7 @@ #' crs = "+proj=longlat +datum=WGS84 +no_defs") #' #' fline <- sf::read_sf(sample_data, "NHDFlowline_Network") -#' comids <- nhdplusTools::get_UT(fline, 13293970) +#' comids <- hydroloom::navigate_hydro_network(fline, 13293970, "UT") #' #' plot_nhdplus(comids) #' @@ -120,7 +121,7 @@ #' plot_nhdplus(comids, nhdplus_data = sample_data, streamorder = 3, add = TRUE, #' plot_config = list(flowline = list(col = "darkblue"))) #' -#' nhdplusTools::nhdplusTools_data_dir(old_dir) +#' hydrogeofetch::hydrogeofetch_data_dir(old_dir) #' } plot_nhdplus <- function(outlets = NULL, bbox = NULL, streamorder = NULL, @@ -163,6 +164,9 @@ plot_nhdplus <- function(outlets = NULL, bbox = NULL, streamorder = NULL, suppressWarnings({ if(!add) { tryCatch({ + check_pkg("maptiles") + check_pkg("mapsf") + bb <- if(!is.null(pd$plot_bbox)) { plot_bbox_to_3857(pd$plot_bbox) } else { @@ -231,7 +235,7 @@ plot_nhdplus <- function(outlets = NULL, bbox = NULL, streamorder = NULL, } tile_cache_dir <- function() { - tile_cache_dir <- file.path(nhdplusTools_data_dir(), + tile_cache_dir <- file.path(hydrogeofetch_data_dir(), "tile_cache_dir") # Checks to see if tile_cache_dir is writable @@ -412,7 +416,7 @@ get_plot_data <- function(outlets = NULL, bbox = NULL, return(NULL) } - all_comids <- lapply(nexus, function(x) get_UT(align_nhdplus_names(flowline), x$comid)) + all_comids <- lapply(nexus, function(x) navigate_hydro_network(align_nhdplus_names(flowline), x$comid, "UT")) subsets <- subset_nhdplus(comids = unlist(all_comids), output_file = gpkg, nhdplus_data = nhdplus_data, status = FALSE, @@ -469,7 +473,7 @@ get_plot_data <- function(outlets = NULL, bbox = NULL, flowline <- align_nhdplus_names(nhd_data$flowline) flowline <- do.call(rbind, lapply(nexus$comid, function(x) { - flowline[flowline$COMID %in% get_UT(align_nhdplus_names(flowline), x), ] + flowline[flowline$COMID %in% navigate_hydro_network(align_nhdplus_names(flowline), x, "UT"), ] })) catchment <- nhd_data$catchment @@ -725,7 +729,7 @@ off_network <- function(waterbody, flowline) { #' @examples #' \donttest{ #' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/sample_data.R", package = "hydrogeofetch")) #' #' fline <- sf::read_sf(sample_data, "NHDFlowline_Network") #' wtbdy <- sf::read_sf(sample_data, "NHDWaterbody") @@ -741,9 +745,9 @@ get_wb_outlet <- function(lake_id, network) { network <- check_names(network, "get_wb_outlet_hires", align = FALSE, tolower = FALSE) if (lake_id %in% network$WBArea_Permanent_Identifier){ - outlet <- network %>% - filter(.data$WBArea_Permanent_Identifier == lake_id) %>% - group_by(.data$WBArea_Permanent_Identifier) %>% + outlet <- network |> + filter(.data$WBArea_Permanent_Identifier == lake_id) |> + group_by(.data$WBArea_Permanent_Identifier) |> filter(.data$Hydroseq == min(.data$Hydroseq)) return(outlet) } else { @@ -753,9 +757,9 @@ get_wb_outlet <- function(lake_id, network) { network <- check_names(network, "get_wb_outlet_mres", tolower = TRUE) if (lake_id %in% network$wbareacomi){ - outlet <- network %>% - filter(.data$wbareacomi == lake_id) %>% - group_by(.data$wbareacomi) %>% + outlet <- network |> + filter(.data$wbareacomi == lake_id) |> + group_by(.data$wbareacomi) |> filter(.data$hydroseq == min(.data$hydroseq)) return(outlet) } else { diff --git a/R/prep_nhdplus.R b/R/prep_nhdplus.R index dda9baa5..ceba8758 100644 --- a/R/prep_nhdplus.R +++ b/R/prep_nhdplus.R @@ -1,5 +1,5 @@ #' @title Prep NHDPlus Data -#' @description Function to prep NHDPlus data for use by nhdplusTools functions +#' @description Function to prep NHDPlus data for use by hydrogeofetch functions #' @param flines data.frame NHDPlus flowlines including: #' COMID, LENGTHKM, FTYPE (or FCODE), TerminalFl, FromNode, ToNode, TotDASqKM, #' StartFlag, StreamOrde, StreamCalc, TerminalPa, Pathlength, @@ -18,11 +18,11 @@ #' @param skip_toCOMID logical if TRUE, toCOMID will not be added to output. #' @param align_names logical #' @return data.frame ready to be used with the refactor_flowlines function. -#' @importFrom dplyr select filter left_join group_split group_by bind_rows +#' @importFrom dplyr select filter left_join group_split group_by bind_rows all_of #' @export #' @examples #' -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) +#' source(system.file("extdata", "sample_flines.R", package = "hydrogeofetch")) #' #' prepare_nhdplus(sample_flines, #' min_network_size = 10, @@ -66,7 +66,7 @@ prepare_nhdplus <- function(flines, assign("prepare_nhdplus_attributes", unique(req_names), - envir = nhdplusTools_env) + envir = hydrogeofetch_env) flines <- check_names(flines, "prepare_nhdplus", tolower = TRUE) @@ -201,7 +201,7 @@ filter_coastal <- function(flines) { #' @importFrom hydroloom add_toids #' @export #' @examples -#' source(system.file("extdata", "sample_flines.R", package = "nhdplusTools")) +#' source(system.file("extdata", "sample_flines.R", package = "hydrogeofetch")) #' #' tocomid <- get_tocomid(sample_flines) #' @@ -264,3 +264,12 @@ get_tocomid <- function(x, return_dendritic = TRUE, missing = 0, } } + +#' @noRd +get_hyg <- function(x, add, id = "comid") { + if(add && inherits(x, "sf")) { + select(x, all_of(id)) + } else { + NULL + } +} diff --git a/R/rebuild_topology.R b/R/rebuild_topology.R deleted file mode 100644 index d3e4b7df..00000000 --- a/R/rebuild_topology.R +++ /dev/null @@ -1,65 +0,0 @@ -#' @importFrom dplyr all_of -get_hyg <- function(x, add, id = "comid") { - if(add && inherits(x, "sf")) { - select(x, all_of(id)) - } else { - NULL - } -} - -#' get node topology from edge topology (DEPRECATED) -#' @description creates a node topology table from an edge topology -#' @inheritParams get_sorted -#' @inheritParams get_tocomid -#' @param add_div data.frame containing id and toid diverted paths to add. -#' Should have id and toid fields in the first and second columns. Names -#' are not used. -#' @return data.frame containing id, fromnode, and tonode attributes or all -#' attributes provided with id, fromnode and tonode in the first three columns. -#' @export -#' @examples -#' source(system.file("extdata/new_hope_data.R", package = "nhdplusTools")) -#' -#' x <- dplyr::select(get_tocomid( -#' dplyr::select(new_hope_flowline, COMID, FromNode, ToNode, Divergence, FTYPE, -#' AreaSqKM, LENGTHKM, GNIS_ID) -#' ), -tonode, -fromnode) -#' -#' head(y <- make_node_topology(x)) -#' -#' # just the divergences which have unique fromids in x but don't in new hope. -#' div <- get_tocomid(dplyr::select(new_hope_flowline, COMID, FromNode, ToNode), -#' return_dendritic = FALSE, -#' remove_coastal = FALSE) -#' div <- div[div$tocomid %in% -#' new_hope_flowline$COMID[new_hope_flowline$Divergence == 2],] -#' -#' y <- make_node_topology(x, div) -#' - -make_node_topology <- function(x, add_div = NULL, add = TRUE) { - - warning("nhdplusTools make_node_topology is deprecated. Use hydroloom version.") - - orig_name <- names(x)[1:2] - - names(x)[1:2] <- c("id", "toid") - - x <- hydroloom::make_node_topology(x, add_div, add) - - if(add) { - - names(x)[1:2] <- orig_name[1:2] - - x - - } else { - - x <- as.data.frame(select(x, "id", "fromnode", "tonode")) - - names(x)[1] <- orig_name[1] - - x - - } -} diff --git a/R/rescale_catchments.R b/R/rescale_catchments.R index be0fb555..ed0b862b 100644 --- a/R/rescale_catchments.R +++ b/R/rescale_catchments.R @@ -33,7 +33,7 @@ rescale_characteristics <- function(vars, lookup_table) { across(any_of(cols_max), \(x) max(x, na.rm = TRUE), .names = "{col}_max") ) |> ungroup() |> - rename_with(~gsub("_rescaled", "", .), contains("_rescaled")) + rename_with(\(x) gsub("_rescaled", "", x), contains("_rescaled")) } #' @description @@ -47,7 +47,7 @@ rescale_characteristics <- function(vars, lookup_table) { #' "areasqkm." Used to retrieve adjusted catchment areas in the case of split #' catchments. #' -#' @importFrom dplyr mutate select right_join left_join filter rename bind_rows +#' @importFrom dplyr mutate select right_join left_join filter rename bind_rows all_of #' @importFrom stats weighted.mean #' @noRd #' @@ -58,7 +58,7 @@ get_catchment_areas <- function(comids, refactored_areas = NULL){ mutate(comid = as.integer(.data$member_comid)) # fetch basin area for all comids - catchment_areas <- nhdplusTools::get_vaa(atts = c("comid", "areasqkm", "lengthkm")) |> + catchment_areas <- hydrogeofetch::get_vaa(atts = c("comid", "areasqkm", "lengthkm")) |> select(all_of(c("comid", "areasqkm", "lengthkm"))) |> right_join(comids_fmt, by = "comid", multiple = "all") @@ -170,8 +170,8 @@ get_catchment_areas <- function(comids, refactored_areas = NULL){ #' @param aoi character area of interest, passed through to #' \link{get_catchment_characteristics} when \code{catchment_characteristics} #' is not provided. See \link{get_catchment_characteristics} for details. -#' @importFrom dplyr left_join rename_with mutate across group_by summarize ungroup distinct starts_with any_of -#' @importFrom tidyr pivot_wider contains +#' @importFrom dplyr left_join rename_with mutate across group_by summarize ungroup distinct starts_with any_of contains +#' @importFrom data.table as.data.table dcast #' @export #' rescale_catchment_characteristics <- function(vars, lookup_table, @@ -217,11 +217,12 @@ rescale_catchment_characteristics <- function(vars, lookup_table, var_names <- unique(catchment_characteristics$characteristic_id) # pivot to wide format - catchment_characteristics <- pivot_wider(catchment_characteristics, - names_from = "characteristic_id", - values_from = c("characteristic_value", "percent_nodata")) + catchment_characteristics <- as.data.frame(dcast( + as.data.table(catchment_characteristics), + comid ~ characteristic_id, + value.var = c("characteristic_value", "percent_nodata"))) catchment_characteristics <- rename_with(.data = catchment_characteristics, - .fn = ~gsub("characteristic_value_", "", .x, fixed = TRUE)) + .fn = \(x) gsub("characteristic_value_", "", x, fixed = TRUE)) if(is.null(catchment_areas)) { # get comid catchment areas, adjusting area for catchments that have been "split" @@ -236,7 +237,7 @@ rescale_catchment_characteristics <- function(vars, lookup_table, # rescale the nldi characteristics if needed (i.e., for split catchments) if(!all(lookup_table$comid == lookup_table$member_comid)){ lookup_table <- mutate(lookup_table, - across(any_of(var_names), ~.x*.data$split_area_prop, .names = "{col}_rescaled")) + across(any_of(var_names), \(x) x * .data$split_area_prop, .names = "{col}_rescaled")) } return(rescale_characteristics(vars, lookup_table)) diff --git a/R/run_plus_attributes.R b/R/run_plus_attributes.R deleted file mode 100644 index 290db0b0..00000000 --- a/R/run_plus_attributes.R +++ /dev/null @@ -1,210 +0,0 @@ -#' DEPRECATED: Add NHDPlus Network Attributes to a provided network. -#' @description Given a river network with required base attributes, adds the -#' NHDPlus network attributes: hydrosequence, levelpath, terminalpath, pathlength, -#' down levelpath, down hydroseq, total drainage area, and terminalflag. -#' The function implements two parallelization schemes for small and large basins -#' respectively. If a number of cores is specified, parallel execution will be used. -#' -#' @param net data.frame containing comid, tocomid, nameID, lengthkm, and areasqkm. -#' Additional attributes will be passed through unchanged. -#' tocomid == 0 is the convention used for outlets. -#' If a "weight" column is provided, it will be used in \link{get_levelpaths} -#' otherwise, arbolate sum is calculated for the network and used as the weight. -#' -#' @param override numeric factor to be passed to \link{get_levelpaths} -#' @param cores integer number of processes to spawn if run in parallel. -#' @param split_temp character path to optional temporary copy of the network -#' split into independent sub-networks. If it exists, it will be read from disk -#' rather than recreated. -#' @param status logical should progress be printed? -#' @return data.frame with added attributes -#' @importFrom tidyr replace_na -#' @export -#' @examples -#' -#' source(system.file("extdata", "walker_data.R", package = "nhdplusTools")) -#' -#' test_flowline <- prepare_nhdplus(walker_flowline, 0, 0, FALSE) -#' -#' test_flowline <- data.frame( -#' comid = test_flowline$COMID, -#' tocomid = test_flowline$toCOMID, -#' nameID = walker_flowline$GNIS_ID, -#' lengthkm = test_flowline$LENGTHKM, -#' areasqkm = walker_flowline$AreaSqKM) -#' -#' add_plus_network_attributes(test_flowline) - -add_plus_network_attributes <- function(net, override = 5, - cores = NULL, split_temp = NULL, - status = TRUE) { - - warning("add_plus_network_attributes() is deprecated. Please migrate to hydroloom equivalent(s).") - - add_sf <- FALSE - if(inherits(net, "sf")) { - add_sf <- TRUE - geom <- dplyr::select(net) - net <- st_drop_geometry(net) - } - - check_names(net, "add_plus_network_attributes", align = FALSE) - - if(any((rem <- c("hydroseq", "levelpathi", "terminalpa", - "pathlength", "dnlevelpat", "dnhydroseq", "totdasqkm", - "terminalfl")) %in% names(net))) { - warning("provided flowlines contain attributes that will be over written.") - net <- select(net, -dplyr::any_of(c(rem))) - } - - if(!status) { - old_opt <- pbapply::pboptions(type = "none") - on.exit(pbapply::pboptions(type = old_opt$type)) - } - - net[["tocomid"]] <- replace_na(net[["tocomid"]], 0) - - rename_arb <- FALSE - - if(!"weight" %in% names(net)) { - net$weight <- calculate_arbolate_sum( - select(net, ID = "comid", - toID = "tocomid", length = "lengthkm")) - rename_arb <- TRUE - } - - if(!is.null(split_temp) && file.exists(split_temp)) { - lp <- readRDS(split_temp) - } else { - - lp <- get_sorted( - dplyr::rename(net, - ID = "comid", - toID = "tocomid"), - split = TRUE) - - lp <- split(lp, lp$terminalID) - - if(!is.null(split_temp)) { - saveRDS(lp, split_temp) - } - } - - if(!is.null(cores)) { - - rows <- sapply(lp, nrow) - - small_lp <- lp[rows <= 20000] - lp <- lp[rows > 20000] - - message("running large networks") - - if(inherits(cores, "cluster")) stop("passing a cluster object no longer supported") - message("the future plan is being modified and will be changed back on exit") - oplan <- future::plan(future::multisession, workers = cores) - on.exit(future::plan(oplan), add = TRUE) - - } - - lp <- lapply(X = lp, - FUN = function(x, override) { - get_levelpaths_internal(x, override_factor = override) - }, override = override) - - if(!is.null(cores)) { - - run_small <- function(small_lp, override) { - - message("running small networks") - - small_lp <- pbapply::pblapply(X = small_lp, - FUN = function(x, override, get_levelpaths_internal) { - get_levelpaths_internal(x, override) - }, override = override, - get_levelpaths_internal = get_levelpaths_internal, - cl = "future") - - } - - lp <- c(lp, run_small(small_lp, override)) - - } - - combine_networks <- function(lp) { - # ts stands for toposort here. Given that the networks retrieved above are - # independent, we need to lag them so they don't have overlapping identifiers. - start_ts <- 0 - - for(i in 1:length(lp)) { - - lp[[i]]$levelpath <- lp[[i]]$levelpath + start_ts - lp[[i]]$topo_sort <- lp[[i]]$topo_sort + start_ts - - start_ts <- max(lp[[i]]$topo_sort) - - } - - lp <- lapply(lp, function(x) { - mutate(x, terminalpath = min(.data$topo_sort)) - }) - - bind_rows(lp) - } - - lp <- combine_networks(lp) - - net <- net %>% - left_join(select(lp, - comid = "ID", terminalpa = "terminalpath", - hydroseq = "topo_sort", levelpathi = "levelpath"), - by = "comid") - - in_pathlength <- select(net, ID = "comid", toID = "tocomid", length = "lengthkm") - - pathlength <- get_pathlength(in_pathlength) - - pathlength <- distinct(pathlength) %>% - filter(!is.na(.data$pathlength)) - - net <- left_join(net, - select(pathlength, - comid = "ID", - "pathlength"), - by = "comid") - - dn_lp <- net %>% - left_join(select(net, - "comid", dnlevelpat = "levelpathi"), - by = c("tocomid" = "comid"), ) %>% - filter(!is.na(.data$dnlevelpat)) %>% - select("comid", "dnlevelpat") - - net <- left_join(net, dn_lp, by = "comid") %>% - mutate(dnlevelpat = ifelse(is.na(.data$dnlevelpat), 0, .data$dnlevelpat)) - - dn_hs <- net %>% - left_join(select(net, - "comid", dnhydroseq = "hydroseq"), - by = c("tocomid" = "comid")) %>% - select("comid", "dnhydroseq") - - net <- left_join(net, dn_hs, by = "comid") %>% - mutate(dnhydroseq = ifelse(is.na(.data$dnhydroseq), 0, .data$dnhydroseq)) - - if("areasqkm" %in% names(net)) { - net$totdasqkm <- calculate_total_drainage_area( - select(net, ID = "comid", toID = "tocomid", area = "areasqkm") - ) - } - - net <- net %>% - group_by(.data$terminalpa) %>% - mutate(terminalfl = ifelse(.data$hydroseq == min(.data$hydroseq), 1, 0)) %>% - ungroup() - - if(add_sf) { - net <- sf::st_sf(cbind(net, geom)) - } - - net -} diff --git a/R/subset_nhdplus.R b/R/subset_nhdplus.R index 316b92cf..b8c85c60 100644 --- a/R/subset_nhdplus.R +++ b/R/subset_nhdplus.R @@ -1,6 +1,6 @@ #' @title Subset NHDPlus #' @description Saves a subset of the National Seamless database or other -#' nhdplusTools compatible data based on a specified collection of COMIDs. +#' hydrogeofetch compatible data based on a specified collection of COMIDs. #' This function uses \code{\link{get_nhdplus}} for the "download" data #' source but returns data consistent with local data subsets in a subset #' file. @@ -40,7 +40,7 @@ #' @examples #' \donttest{ #' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/sample_data.R", package = "hydrogeofetch")) #' #' nhdplus_path(sample_data) #' @@ -56,7 +56,7 @@ #' #' start_comid <- discover_nhdplus_id(start_point) #' -#' comids <- get_UT(sample_flines, start_comid) +#' comids <- hydroloom::navigate_hydro_network(sample_flines, start_comid, "UT") #' #' plot(sf::st_geometry(dplyr::filter(sample_flines, COMID %in% comids)), #' add=TRUE, col = "red", lwd = 2) @@ -93,9 +93,9 @@ #' sf::st_layers(output_file) #' #' # NHDPlusHR -#' source(system.file("extdata/nhdplushr_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/nhdplushr_data.R", package = "hydrogeofetch")) #' -#' up_ids <- get_UT(hr_data$NHDFlowline, 15000500028335) +#' up_ids <- hydroloom::navigate_hydro_network(hr_data$NHDFlowline, 15000500028335, "UT") #' #' sub_gpkg <- file.path(work_dir, "sub.gpkg") #' sub_nhdhr <- subset_nhdplus(up_ids, output_file = sub_gpkg, @@ -219,7 +219,7 @@ subset_nhdplus <- function(comids = NULL, output_file = NULL, nhdplus_data = NUL tryCatch({ for (layer_name in intersection_names) { if(is.null(out_list[layer_name][[1]])) { - layer <- sf::st_transform(envelope, 4326) %>% + layer <- sf::st_transform(envelope, 4326) |> get_nhdplus_bybox(layer = tolower(layer_name), streamorder = streamorder) } else { layer <- out_list[layer_name][[1]] @@ -372,15 +372,15 @@ get_nhd_data <- function(nhdplus_data, layer_name, comids, id, status) { comids[start:end] }) - assign("cur_count", 0, envir = nhdplusTools_env) + assign("cur_count", 0, envir = hydrogeofetch_env) out <- lapply(sets, function(x, total) { if(status) { cur_count <- - get("cur_count", envir = nhdplusTools_env) + length(x) + get("cur_count", envir = hydrogeofetch_env) + length(x) - assign("cur_count", cur_count, envir = nhdplusTools_env) + assign("cur_count", cur_count, envir = hydrogeofetch_env) message(paste(cur_count, "comids of", total)) @@ -492,10 +492,11 @@ get_flowline_layer_name <- function(nhdplus_data) { #' should the run_make_standalone function be run on result? #' @export #' @importFrom dplyr filter select +#' @importFrom hydroloom sort_network #' @return data.frame containing subset network #' @examples #' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/sample_data.R", package = "hydrogeofetch")) #' #' sample_flines <- sf::read_sf(sample_data, "NHDFlowline_Network") #' @@ -560,11 +561,11 @@ recase_sf <- function(x, orig_names) { #' have no tributaries in the upstream RPU will be included in the output. #' @export #' @return data.frame containing subset network -#' @importFrom dplyr filter arrange summarize +#' @importFrom dplyr filter arrange summarize desc #' @importFrom sf st_sf st_drop_geometry #' @examples #' -#' source(system.file("extdata/sample_data.R", package = "nhdplusTools")) +#' source(system.file("extdata/sample_data.R", package = "hydrogeofetch")) #' #' sample_flines <- sf::read_sf(sample_data, "NHDFlowline_Network") #' @@ -680,10 +681,11 @@ get_all_navigable <- function(fline, rpu) { outlets <- arrange(outlets, desc(.data$arbolatesu)) - network <- get_sorted(fline[c("comid", "tocomid")], - split = FALSE, outlets = outlets$comid) + network <- sort_network( + stats::setNames(sf::st_drop_geometry(fline[c("comid", "tocomid")]), c("id", "toid")), + split = FALSE, outlets = outlets$comid) - network$comid + network$id } #' @noRd diff --git a/README.Rmd b/README.Rmd index 03c02736..db20ac0f 100644 --- a/README.Rmd +++ b/README.Rmd @@ -15,28 +15,35 @@ knitr::opts_chunk$set( ) ``` -# nhdplusTools +# hydrogeofetch ![R-CMD-check](https://github.com/doi-usgs/nhdplusTools/workflows/R-CMD-check/badge.svg) [![codecov](https://codecov.io/gh/doi-usgs/nhdplusTools/branch/master/graph/badge.svg)](https://app.codecov.io/gh/doi-usgs/nhdplusTools) [![CRAN Downloads](https://cranlogs.r-pkg.org/badges/grand-total/nhdplusTools)](https://cran.r-project.org/package=nhdplusTools) [![CRAN](https://www.r-pkg.org/badges/version/nhdplusTools)](https://cran.r-project.org/package=nhdplusTools) -## Renaming to `hydrogeofetch` +## Renamed from `nhdplusTools` -`nhdplusTools` is being renamed to `hydrogeofetch` (hydrologic geospatial fabric extraction tool chain). The package's scope has grown beyond NHDPlus, and the name is no longer accurate. `hydrogeofetch` will be a pure rename. Functions, function signatures, behavior, will all stay the same. `hydrogeofetch` will be published to CRAN as a new package. +`hydrogeofetch` (hydrologic geospatial fabric extraction tool chain) is the renamed successor to `nhdplusTools`. The package scope has grown beyond NHDPlus, and the new name reflects that broader role. All functions, signatures, and behavior carry over unchanged. -Development on `nhdplusTools` has stopped; only critical bug fixes will be applied. Once `hydrogeofetch` v1.0 is released, `nhdplusTools` will become a deprecation shim that forwards calls to `hydrogeofetch` with a deprecation warnings. The shim will remain on CRAN until **October 2028**, at which point `nhdplusTools` will be archived. Archived packages stay installable from the CRAN archive, so code pinned to `nhdplusTools` will continue to run. +`nhdplusTools` will remain on CRAN as a deprecation shim forwarding calls to `hydrogeofetch` until **October 2028**, at which point it will be archived. Archived packages stay installable from the CRAN archive, so code pinned to `nhdplusTools` will continue to run. -Plan to switch to `hydrogeofetch` once v1.0 is available. +## hydrogeofetch: Tools for Accessing and Working with the NHDPlus and other US hydrographic data. -Follow this repository to keep up to date! +## Major changes in 2.0.0 -## nhdplusTools: Tools for Accessing and Working with the NHDPlus and other US hydrographic data. +Indexing, network navigation, and network attribute functions have moved to [**hydroloom**](https://doi-usgs.github.io/hydroloom/). hydrogeofetch is now focused on data access and NHD-specific preparation; hydroloom handles the general-purpose network analysis that was previously bundled here. + +If you use any of the following, you now need `library(hydroloom)`: + +- **Indexing** (`get_flowline_index`, `get_waterbody_index`, `disambiguate_flowline_indexes`) → [`hydroloom::index_points_to_lines()`](https://doi-usgs.github.io/hydroloom/reference/index_points_to_lines.html), [`index_points_to_waterbodies()`](https://doi-usgs.github.io/hydroloom/reference/index_points_to_waterbodies.html), [`disambiguate_indexes()`](https://doi-usgs.github.io/hydroloom/reference/disambiguate_indexes.html) +- **Navigation** (`get_UT`, `get_DM`, `get_UM`, `get_DD`, `navigate_network`) → [`hydroloom::navigate_hydro_network()`](https://doi-usgs.github.io/hydroloom/reference/navigate_hydro_network.html) +- **Network attributes** (`get_streamorder`, `get_levelpaths`, `get_pathlength`, `calculate_total_drainage_area`, etc.) → hydroloom's [`add_streamorder()`](https://doi-usgs.github.io/hydroloom/reference/add_streamorder.html), [`add_levelpaths()`](https://doi-usgs.github.io/hydroloom/reference/add_levelpaths.html), [`add_pathlength()`](https://doi-usgs.github.io/hydroloom/reference/add_pathlength.html), [`accumulate_downstream()`](https://doi-usgs.github.io/hydroloom/reference/accumulate_downstream.html) + +The typical workflow is now: use hydrogeofetch to fetch and prepare NHD data, then hand off to hydroloom for indexing, navigation, and network analysis. See `vignette("migrating_from_nhdplusTools")` for a complete mapping of old to new function names. ### Recommended Citation: ``` - Blodgett, D., Johnson, J.M., 2022, nhdplusTools: Tools for - Accessing and Working with the NHDPlus, - https://doi.org/10.5066/P97AS8JD + Blodgett, D. 2026, hydrogeofetch: Hydrologic Geospatial + Fabric Data Extraction Tool Chain, TBD-DOI ``` ### Installation: @@ -59,18 +66,19 @@ Detailed documentation of all the package functions can be found at the [**Refer ### Data: -`nhdplusTools`, is designed to provide easy access to data associated with the U.S. National Hydrography Dataset. Many functions provided in `nhdplusTools` are thin wrappers around functions that have been migrated to `hydroloom`. +`hydrogeofetch` provides easy access to data associated with the U.S. National Hydrography Dataset. Network analysis and spatial indexing live in [`hydroloom`](https://doi-usgs.github.io/hydroloom/). ## Package Vision -The `nhdplusTools` package is intended to provide a reusable set of tools to -subset, relate data to, and generate network attributes for U.S. NHDPlus data. +The `hydrogeofetch` package provides tools to discover, download, subset, and +prepare U.S. NHDPlus data. Network analysis (navigation, indexing, attribute +generation) belongs in [`hydroloom`](https://doi-usgs.github.io/hydroloom/). -General, globally applicable functionality has been moved to [`hydroloom`](https://github.com/DOI-USGS/hydroloom) +General, globally applicable functionality has been moved to [`hydroloom`](https://doi-usgs.github.io/hydroloom/). -`nhdplusTools` implements a data model consistent with both the [NHDPlus](https://www.epa.gov/waterdata/nhdplus-national-hydrography-dataset-plus) -dataset and the [HY\_Features](http://opengeospatial.github.io/HY_Features/) data -model. The package aims to provide a set of tools that can be used to build +`hydrogeofetch` implements a data model consistent with both the [NHDPlus](https://www.epa.gov/waterdata/nhdplus-national-hydrography-dataset-plus) +dataset and the [HY\_Features](http://opengeospatial.github.io/HY_Features/) data +model. The package aims to provide a set of tools that can be used to build workflows using NHDPlus data. **This vision is intended as a guide to contributors -- conveying what kinds of @@ -88,16 +96,12 @@ discovery, network navigation, and data export utilities to generate spatial and attribute subsets of the NHDPlus dataset. ##### Indexing -One of the most important roles of the NHDPlus is as a connecting network for -ancillary data and models. The first step in any workflow that uses the -network like this is indexing relevant data to the network. A number of methods -for indexing exist, they can be broken into two main categories: linear -referencing and catchment indexing. Both operate on features represented by -points, lines, and polygons. `nhdplusTools` should eventually support both -linear and catchment indexing. +Indexing data to the hydrographic network (linear referencing and catchment +indexing) is handled by [`hydroloom`](https://doi-usgs.github.io/hydroloom/). +`hydrogeofetch` provides the data access needed to set up indexing workflows. ### Data Model -Given that `nhdplusTools` is focused on working with NHDPlus data, the NHDPlus +Given that `hydrogeofetch` is focused on working with NHDPlus data, the NHDPlus data model will largely govern the data model the package is designed to work with. That said, much of the package functionality also uses concepts from the HY\_Features standard. @@ -118,7 +122,7 @@ together fully describe an NHDPlus catchment. [See the NHDPlus mapping to HY_Features in the HY_Features specification.](https://docs.ogc.org/is/14-111r6/14-111r6.html#annexD_1) Below is a description of the scope of data used by the -`nhdplusTools` package. While other data and attributes may come into scope, +`hydrogeofetch` package. While other data and attributes may come into scope, it should only be done as a naive pass-through, as in data subsetting, or with considerable deliberation. @@ -142,7 +146,7 @@ The NHDPlus is a very large dataset. The architecture of this package as it relates to handling data and what dependencies are used will be very important. ##### Web vs Local Data -`nhdplusTools` offers a mix of web service and local data functionality. +`hydrogeofetch` offers a mix of web service and local data functionality. Web services have generally been avoided for large processes. However, applications that would require loading significant amounts of data to perform something that can be accomplished with a web service very quickly are supported. @@ -162,50 +166,37 @@ https://github.com/ropensci/FedData https://github.com/hyriver/pygeohydro ... others -- please suggest additions? -### Build notes: -This package uses a convention to avoid building vignettes on CRAN. The `BUILD_VIGNETTES` environment variable must be set to `TRUE`. This is done with a .Renviron file in the package directory with the line `BUILD_VIGNETTES=TRUE`. - -Given this, the package should be built locally to include vignettes using: - -```r -devtools::build() -``` +### Build and release: -An additional CRAN build environment variable also must be set to avoid submitting a vignette with very large embedded images. +Development happens on GitHub (doi-usgs/nhdplusTools). Official builds and release candidates are produced on code.usgs.gov (code.usgs.gov/water/nhdplusTools) using GitLab CI. -```r -Sys.setenv(BUILD_VIGNETTES = "TRUE", BUILD_VIGNETTES_CRAN = "TRUE") -devtools::build() +**Vignettes** use the `BUILD_VIGNETTES` environment variable to control code evaluation. Set `BUILD_VIGNETTES=TRUE` in a local `.Renviron` to build vignettes with live code. Without it, vignettes render with static output only. An additional `BUILD_VIGNETTES_CRAN=TRUE` variable controls image size in the drainage area vignette. +**Local development** does not require building the source package. Use `devtools::test()`, `devtools::check()`, and `devtools::document()` directly. -devtools::release() -``` +**Release candidates** are built on code.usgs.gov. Push a branch named `rc/` (e.g. `rc/2.0.0`) to trigger the GitLab CI pipeline, which runs three stages: -### Check notes: -In addition to typical R package checking, a Dockerfile is included in this repository. Once built, it can be run with the following command. +1. **check** -- lightweight structural check (no tests, no examples, no vignettes). Gates the rest of the pipeline. +2. **build** -- `R CMD build .` to produce the source tarball. Uploads the tarball to the GitLab generic package registry. +3. **verify** -- `R CMD check --as-cran` on the built tarball. -``` -docker build -t nhdplustools_test . - -docker run --rm -it -v $PWD:/src nhdplustools_test /bin/bash -c "cp -r /src/* /check/ && cp /src/.Rbuildignore /check/ && cd /check && Rscript -e 'devtools::build()' && R CMD check --as-cran ../nhdplusTools_*" -``` +Once the pipeline passes, download the tarball from the package registry and submit it to CRAN. The tarball that CRAN receives is the exact artifact that passed `--as-cran` in CI. -### Release procedure: -- ensure all checks pass and code coverage is adequate. -- ensure news has been updated -- convert disclaimer to [released form](https://code.usgs.gov/water/sbtools/-/blob/v1.1.14/README.md#L113) -- update version in inst/CITATION file -- update version in code.json file -- Build source package and upload to CRAN -- Once a new version has been accepted by cran, -- ensure pkgdown is up to date -- commit, push, and PR/MR changes -- create release page and tag -- attach cran tar.gz to release page -- update DOI to point to release page -- switch README disclaimer back to ["dev" mode.](https://code.usgs.gov/water/sbtools#disclaimer) -- Update version in Description. -- push an PR/MR changes. +### Release checklist: +- All checks pass and code coverage is adequate +- NEWS.md is up to date +- Disclaimer is in [released form](https://code.usgs.gov/water/sbtools/-/blob/v1.1.14/README.md#L113) +- Version updated in inst/CITATION and code.json +- Push `rc/` branch to code.usgs.gov and confirm pipeline passes +- Download tarball from GitLab package registry and submit to CRAN +- After CRAN acceptance: + - Ensure pkgdown is up to date + - Commit, push, and PR/MR changes + - Create release page and tag + - Attach CRAN tar.gz to release page + - Update DOI to point to release page + - Switch README disclaimer back to ["dev" mode](https://code.usgs.gov/water/sbtools#disclaimer) + - Bump version in DESCRIPTION ### Contributing: diff --git a/README.md b/README.md index 36dc6012..74f09783 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# nhdplusTools +# hydrogeofetch ![R-CMD-check](https://github.com/doi-usgs/nhdplusTools/workflows/R-CMD-check/badge.svg) [![codecov](https://codecov.io/gh/doi-usgs/nhdplusTools/branch/master/graph/badge.svg)](https://app.codecov.io/gh/doi-usgs/nhdplusTools) @@ -9,34 +9,24 @@ Downloads](https://cranlogs.r-pkg.org/badges/grand-total/nhdplusTools)](https://cran.r-project.org/package=nhdplusTools) [![CRAN](https://www.r-pkg.org/badges/version/nhdplusTools)](https://cran.r-project.org/package=nhdplusTools) -## Renaming to `hydrogeofetch` +## Renamed from `nhdplusTools` -`nhdplusTools` is being renamed to `hydrogeofetch` (hydrologic -geospatial fabric extraction tool chain). The package’s scope has grown -beyond NHDPlus, and the name is no longer accurate. `hydrogeofetch` will -be a pure rename. Functions, function signatures, behavior, will all -stay the same. `hydrogeofetch` will be published to CRAN as a new -package. +`hydrogeofetch` (hydrologic geospatial fabric extraction tool chain) is +the renamed successor to `nhdplusTools`. The package scope has grown +beyond NHDPlus, and the new name reflects that broader role. All +functions, signatures, and behavior carry over unchanged. -Development on `nhdplusTools` has stopped; only critical bug fixes will -be applied. Once `hydrogeofetch` v1.0 is released, `nhdplusTools` will -become a deprecation shim that forwards calls to `hydrogeofetch` with a -deprecation warnings. The shim will remain on CRAN until **October -2028**, at which point `nhdplusTools` will be archived. Archived -packages stay installable from the CRAN archive, so code pinned to -`nhdplusTools` will continue to run. +`nhdplusTools` will remain on CRAN as a deprecation shim forwarding +calls to `hydrogeofetch` until **October 2028**, at which point it will +be archived. Archived packages stay installable from the CRAN archive, +so code pinned to `nhdplusTools` will continue to run. -Plan to switch to `hydrogeofetch` once v1.0 is available. - -Follow this repository to keep up to date! - -## nhdplusTools: Tools for Accessing and Working with the NHDPlus and other US hydrographic data. +## hydrogeofetch: Tools for Accessing and Working with the NHDPlus and other US hydrographic data. ### Recommended Citation: - Blodgett, D., Johnson, J.M., 2022, nhdplusTools: Tools for - Accessing and Working with the NHDPlus, - https://doi.org/10.5066/P97AS8JD + Blodgett, D. 2026, hydrogeofetch: Hydrologic Geospatial + Fabric Data Extraction Tool Chain, TBD-DOI ### Installation: @@ -63,21 +53,20 @@ page**](https://doi-usgs.github.io/nhdplusTools/reference/). ### Data: -`nhdplusTools`, is designed to provide easy access to data associated -with the U.S. National Hydrography Dataset. Many functions provided in -`nhdplusTools` are thin wrappers around functions that have been -migrated to `hydroloom`. +`hydrogeofetch` provides easy access to data associated with the U.S. +National Hydrography Dataset. Many functions are thin wrappers around +functions that have been migrated to `hydroloom`. ## Package Vision -The `nhdplusTools` package is intended to provide a reusable set of +The `hydrogeofetch` package is intended to provide a reusable set of tools to subset, relate data to, and generate network attributes for U.S. NHDPlus data. General, globally applicable functionality has been moved to [`hydroloom`](https://github.com/DOI-USGS/hydroloom) -`nhdplusTools` implements a data model consistent with both the +`hydrogeofetch` implements a data model consistent with both the [NHDPlus](https://www.epa.gov/waterdata/nhdplus-national-hydrography-dataset-plus) dataset and the [HY_Features](http://opengeospatial.github.io/HY_Features/) data model. @@ -110,12 +99,11 @@ that uses the network like this is indexing relevant data to the network. A number of methods for indexing exist, they can be broken into two main categories: linear referencing and catchment indexing. Both operate on features represented by points, lines, and polygons. -`nhdplusTools` should eventually support both linear and catchment -indexing. +`hydrogeofetch` supports both linear and catchment indexing. ### Data Model -Given that `nhdplusTools` is focused on working with NHDPlus data, the +Given that `hydrogeofetch` is focused on working with NHDPlus data, the NHDPlus data model will largely govern the data model the package is designed to work with. That said, much of the package functionality also uses concepts from the HY_Features standard. @@ -137,7 +125,7 @@ collection of data that together fully describe an NHDPlus catchment. [See the NHDPlus mapping to HY_Features in the HY_Features specification.](https://docs.ogc.org/is/14-111r6/14-111r6.html#annexD_1) -Below is a description of the scope of data used by the `nhdplusTools` +Below is a description of the scope of data used by the `hydrogeofetch` package. While other data and attributes may come into scope, it should only be done as a naive pass-through, as in data subsetting, or with considerable deliberation. @@ -169,13 +157,13 @@ important. ##### Web vs Local Data -`nhdplusTools` offers a mix of web service and local data functionality. -Web services have generally been avoided for large processes. However, -applications that would require loading significant amounts of data to -perform something that can be accomplished with a web service very -quickly are supported. Systems like the [Network Linked Data -Index](https://waterdata.usgs.gov/blog/nldi-intro/) are used for data -discovery. +`hydrogeofetch` offers a mix of web service and local data +functionality. Web services have generally been avoided for large +processes. However, applications that would require loading significant +amounts of data to perform something that can be accomplished with a web +service very quickly are supported. Systems like the [Network Linked +Data Index](https://waterdata.usgs.gov/blog/nldi-intro/) are used for +data discovery. ##### NHDPlus Version @@ -195,59 +183,55 @@ is also supported. … others – please suggest additions? -### Build notes: - -This package uses a convention to avoid building vignettes on CRAN. The -`BUILD_VIGNETTES` environment variable must be set to `TRUE`. This is -done with a .Renviron file in the package directory with the line -`BUILD_VIGNETTES=TRUE`. +### Build and release: -Given this, the package should be built locally to include vignettes -using: - -``` r -devtools::build() -``` +Development happens on GitHub (doi-usgs/nhdplusTools). Official builds +and release candidates are produced on code.usgs.gov +(code.usgs.gov/water/nhdplusTools) using GitLab CI. -An additional CRAN build environment variable also must be set to avoid -submitting a vignette with very large embedded images. - -``` r -Sys.setenv(BUILD_VIGNETTES = "TRUE", BUILD_VIGNETTES_CRAN = "TRUE") -devtools::build() - - -devtools::release() -``` +**Vignettes** use the `BUILD_VIGNETTES` environment variable to control +code evaluation. Set `BUILD_VIGNETTES=TRUE` in a local `.Renviron` to +build vignettes with live code. Without it, vignettes render with static +output only. An additional `BUILD_VIGNETTES_CRAN=TRUE` variable controls +image size in the drainage area vignette. -### Check notes: +**Local development** does not require building the source package. Use +`devtools::test()`, `devtools::check()`, and `devtools::document()` +directly. -In addition to typical R package checking, a Dockerfile is included in -this repository. Once built, it can be run with the following command. +**Release candidates** are built on code.usgs.gov. Push a branch named +`rc/` (e.g. `rc/2.0.0`) to trigger the GitLab CI pipeline, +which runs three stages: - docker build -t nhdplustools_test . +1. **check** – lightweight structural check (no tests, no examples, no + vignettes). Gates the rest of the pipeline. +2. **build** – `R CMD build .` to produce the source tarball. Uploads + the tarball to the GitLab generic package registry. +3. **verify** – `R CMD check --as-cran` on the built tarball. - docker run --rm -it -v $PWD:/src nhdplustools_test /bin/bash -c "cp -r /src/* /check/ && cp /src/.Rbuildignore /check/ && cd /check && Rscript -e 'devtools::build()' && R CMD check --as-cran ../nhdplusTools_*" +Once the pipeline passes, download the tarball from the package registry +and submit it to CRAN. The tarball that CRAN receives is the exact +artifact that passed `--as-cran` in CI. -### Release procedure: +### Release checklist: -- ensure all checks pass and code coverage is adequate. -- ensure news has been updated -- convert disclaimer to [released +- All checks pass and code coverage is adequate +- NEWS.md is up to date +- Disclaimer is in [released form](https://code.usgs.gov/water/sbtools/-/blob/v1.1.14/README.md#L113) -- update version in inst/CITATION file -- update version in code.json file -- Build source package and upload to CRAN -- Once a new version has been accepted by cran, -- ensure pkgdown is up to date -- commit, push, and PR/MR changes -- create release page and tag -- attach cran tar.gz to release page -- update DOI to point to release page -- switch README disclaimer back to [“dev” - mode.](https://code.usgs.gov/water/sbtools#disclaimer) -- Update version in Description. -- push an PR/MR changes. +- Version updated in inst/CITATION and code.json +- Push `rc/` branch to code.usgs.gov and confirm pipeline + passes +- Download tarball from GitLab package registry and submit to CRAN +- After CRAN acceptance: + - Ensure pkgdown is up to date + - Commit, push, and PR/MR changes + - Create release page and tag + - Attach CRAN tar.gz to release page + - Update DOI to point to release page + - Switch README disclaimer back to [“dev” + mode](https://code.usgs.gov/water/sbtools#disclaimer) + - Bump version in DESCRIPTION ### Contributing: diff --git a/_pkgdown.yml b/_pkgdown.yml index 8d3afa01..868d49dd 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -48,78 +48,43 @@ reference: - '`get_nldi_feature`' - '`get_split_catchment`' - '`get_raindrop_trace`' - - title: Indexing and Network Navigation - desc: > - Functions to index and match data to NHDPlus features. Most of these - work with any data that fits the expected data model. Some are very - specific to the NHDPlus data model but are not specific to a U.S - spatial context. - - contents: - - '`get_flowline_index`' - - '`get_hydro_location`' - - '`get_waterbody_index`' - - '`get_wb_outlet`' - - '`get_nldi_index`' - - '`get_node`' - - '`rescale_measures`' - - '`disambiguate_flowline_indexes`' - - '`get_partial_length`' - - '`get_path_lengths`' - - '`get_path_members`' - title: Network Navigation desc: > - Functions to traverse and return network subsets. All except `navigate_nldi()` - have applicability outside the U.S. context. + Functions to traverse and return network subsets. Navigation and indexing + functions have moved to the hydroloom package. - contents: - - '`get_DD`' - - '`get_DM`' - - '`get_UM`' - - '`get_UT`' - - '`navigate_network`' - '`navigate_nldi`' + - title: NHD-Specific Preparation + desc: > + Functions that prepare NHD data for use with hydroloom and other packages. + - contents: + - '`get_tocomid`' + - '`prepare_nhdplus`' + - '`align_nhdplus_names`' - title: Package Helpers desc: > Functions that make the package easier to work with. - contents: - - '`nhdplusTools_cache_settings`' - - '`st_compatibalize`' - - '`rename_geometry`' - - '`align_nhdplus_names`' + - '`hydrogeofetch_cache_settings`' - '`nhdplus_path`' - - '`nhdplusTools_data_dir`' + - '`hydrogeofetch_data_dir`' - '`plot_nhdplus`' - '`map_nhdplus`' - - '`prepare_nhdplus`' - '`get_vaa_names`' - '`get_vaa_path`' - - '`rescale_catchment_characteristics`' - - title: Network Attributes - desc: > - Functions that generate common river network attributes. These can be - used with any data that adheres to the supported data model. See function - documentation for specifics. - - contents: - - '`fix_flowdir`' - - '`get_tocomid`' - - '`make_node_topology`' - - '`get_sorted`' - - '`calculate_total_drainage_area`' - - '`calculate_arbolate_sum`' - - '`get_streamlevel`' - - '`get_levelpaths`' - - '`get_pathlength`' - - '`get_streamorder`' - - '`get_terminal`' - - '`add_plus_network_attributes`' - - '`get_pfaf`' + - '`get_nldi_index`' + - '`get_wb_outlet`' + - '`add_mainstems`' + - '`check_mainstems`' + - '`update_mainstems`' articles: - title: Vignettes contents: - - nhdplusTools - - get_data_overview - - plot_nhdplus - - indexing - - nhdplushr - - get_3dhp_data - - drainage_area_estimation + - hydrogeofetch + - articles/get_data_overview + - articles/plot_nhdplus + - articles/nhdplushr + - articles/get_3dhp_data + - articles/drainage_area_estimation + - articles/migrating_from_nhdplusTools diff --git a/docs/404.html b/docs/404.html index 01cfce83..5fff8c7c 100644 --- a/docs/404.html +++ b/docs/404.html @@ -5,7 +5,7 @@ -Page not found (404) • nhdplusTools +Page not found (404) • hydrogeofetch @@ -37,15 +37,15 @@ - nhdplusTools - 1.5.0 + hydrogeofetch + 2.0.0 -

+

If you have the whole National Seamless database and want to work at regional to national scales, skip down the the Local Data Subsetting section.

Web Service Data Subsetting

-

nhdplusTools supports discovery and data subsetting +

hydrogeofetch supports discovery and data subsetting using web services made available through the Network Linked Data Index (NLDI) and the Water Mission Area pygeoapir. The code below shows how to use the NLDI functions to @@ -302,13 +298,13 @@

Web Service Data Subsetting
 dataRetrieval::get_nldi_sources()$source
-#>  [1] "ca_gages"          "census2020-nhdpv2" "epa_nrsa"         
-#>  [4] "geoconnex-demo"    "gfv11_pois"        "GRAND"            
-#>  [7] "HILARRI"           "huc12pp"           "huc12pp_102020"   
-#> [10] "nmwdi-st"          "npdes"             "nwisgw"           
-#> [13] "nwissite"          "ref_dams"          "ref_gage"         
-#> [16] "vigil"             "wade"              "wade_rights"      
-#> [19] "wade_timeseries"   "WQP"               "comid"
+#>  [1] "comid"             "huc12pp"           "WQP"              
+#>  [4] "nwissite"          "wade"              "ref_gage"         
+#>  [7] "ca_gages"          "gfv11_pois"        "vigil"            
+#> [10] "nwisgw"            "nmwdi-st"          "geoconnex-demo"   
+#> [13] "epa_nrsa"          "census2020-nhdpv2" "npdes"            
+#> [16] "huc12pp_102020"    "ref_dams"          "wade_rights"      
+#> [19] "wade_timeseries"   "GRAND"             "HILARRI"
 
 nldi_feature <- list(featureSource = "comid", 
                      featureID = as.integer(start_comid$comid)[1])
@@ -320,9 +316,9 @@ 
Web Service Data Subsetting#> Bounding box:  xmin: -89.37037 ymin: 43.08521 xmax: -89.35393 ymax: 43.09491
 #> Geodetic CRS:  WGS 84
 #> # A tibble: 1 × 4
-#>   sourceName    identifier comid                                        geometry
-#>   <chr>         <chr>      <chr>                                <LINESTRING [°]>
-#> 1 NHDPlus comid 13293750   13293750 (-89.37037 43.09491, -89.36997 43.09475, -8…

+#> sourceName identifier comid geometry +#> <chr> <chr> <int> <LINESTRING [°]> +#> 1 NHDPlus comid 13293750 13293750 (-89.37037 43.09491, -89.36997 43.09475, -8…

We can use get_nldi_feature() as a way to make sure the featureID is available for the chosen “featureSource”. Now that we know the NLDI has our comid, we can use the “upstreamTributaries” navigation @@ -336,13 +332,11 @@

Web Service Data Subsettingplot(sf::st_geometry(flowline), lwd = 3, col = "black") plot(sf::st_geometry(flowline_nldi$origin), lwd = 3, col = "red", add = TRUE) plot(sf::st_geometry(flowline_nldi$UT), lwd = 1, col = "red", add = TRUE) -

+

The NLDI only provides geometry and a comid for each of the flowlines. The subset_nhdplus() function has a “download” option that allows us to download four layers and all attributes as -shown below. There is also a navigate_network() function -that will replace navigate_nldi() and -subset_nhdplus() for many use cases.

+shown below.

 output_file_download <- file.path(work_dir, "subset_download.gpkg")
 
@@ -365,7 +359,7 @@ 
Web Service Data Subsetting= 7, col = "darkgrey")
 plot(sf::st_geometry(flowline_nldi$UT), 
      lwd = 3, col = "red", add = TRUE)
-

+

This plot illustrates the kind of thing that’s possible (filtering to specific stream orders) using the attributes that are downloaded.

Before moving on, one more demonstration of what can be done using @@ -406,24 +400,21 @@

Web Service Data Subsetting= 3, col = "blue") plot(sf::st_geometry(upstream_nwis$UT_nwissite), cex = 1, lwd = 2, col = "red", add = TRUE) -

+

Local Data Subsetting

While web service data access is very convenient, some use cases make working with web services impossible or cumbersome such that working -with local data is preferable. nhdplusTools supports such workflows with -hybrid, web-service and local, workflows.

+with local data is preferable. hydrogeofetch supports such workflows +with hybrid, web-service and local, workflows.

With the starting COMID we found with -discover_nhdplus_id() above, we can use one of the network -navigation functions, get_UM(), get_UT(), -get_DM(), or get_DD() to retrieve a collection -of comids along the upstream mainstem, upstream with tributaries, -downstream mainstem, or downstream with diversions network paths. Here -we’ll use upstream with tributaries.

+discover_nhdplus_id() above, we can use hydroloom’s +navigate_hydro_network() to retrieve a collection of COMIDs +along the upstream-with-tributaries network path.

-UT_comids <- get_UT(flowline, start_comid$comid[1])
+UT_comids <- hydroloom::navigate_hydro_network(flowline, start_comid$comid[1], "UT")
 UT_comids
 #>   [1]  13293380  13293384  13293388  13293392  13293398  13293424  13293430
 #>   [8]  13293454  13293504  13293506  13293576  13293750  13294110  13294128
@@ -459,7 +450,7 @@ 
Local Data Subsettingplot(start_point, cex = 1.5, lwd = 2, col = "red", add = TRUE)
 plot(sf::st_geometry(dplyr::filter(flowline, comid %in% UT_comids)),
      add=TRUE, col = "red", lwd = 2)
-

+

Say you want to save the network subset for later use in R or in some other GIS. The subset_nhdplus() function is your friend. If you have the whole national seamless database downloaded, you can pull @@ -499,55 +490,53 @@

Local Data Subsetting add=TRUE, col = "red", lwd = 2) plot(sf::st_geometry(catchment), add = TRUE) plot(sf::st_geometry(waterbody), col = rgb(0, 0, 1, alpha = 0.5), add = TRUE)

-

+

Indexing

-

nhdplusTools supports a number of indexing use cases. See -the function index for specifics.

-

Using the data above, we can use the -get_flowline_index() function to get the comid, reachcode, -and measure of our starting point like this.

+

Indexing functions have moved to the hydroloom package. Use +hydrogeofetch to fetch flowline data and hydroloom to index points to +those flowlines.

-get_flowline_index(flowline, start_point)
-#>   id    COMID      REACHCODE REACH_meas       offset
-#> 1  1 13293750 07090002007373    41.8003 0.0009619034
-

get_flowline_index() will work with a list of points -too. For demonstration purposes, we can use the gages in our subset from +hydroloom::index_points_to_lines(flowline, start_point) +#> point_id comid reachcode reachcode_measure offset +#> 1 1 13293750 07090002007373 41.8003 0.0009619034

+

index_points_to_lines() works with a list of points too. +For demonstration purposes, we can use the gages in our subset from above.

 gage <- sf::read_sf(output_file, "Gage")
 
-get_flowline_index(flowline, sf::st_geometry(gage), precision = 10)
-#>    id    COMID      REACHCODE REACH_meas         offset
-#> 1   2 13294276 07090002008387    14.8355 0.000020891030
-#> 2   3 13294264 07090002007650    56.4439 0.000031104771
-#> 3   4 13293750 07090002007373    42.5089 0.000016222605
-#> 4   5 13294312 07090002008383     1.2211 0.000013118677
-#> 5   6 13294264 07090002007650    41.0519 0.000040136119
-#> 6   7 13294264 07090002007650     2.0872 0.000009600705
-#> 7   9 13294300 07090002008379    85.4441 0.000024421783
-#> 8  10 13293690 07090002007648     0.7642 0.000022907103
-#> 9  11 13294264 07090002007650    71.4042 0.008212854054
-#> 10 13 13294176 07090002007664     6.1174 0.000037883863
-#> 11 15 13294290 07090002008374    88.0213 0.000037222413
-#> 12 16 13294138 07090002007709    73.3717 0.000010914584
-#> 13 18 13293486 07090002007724     8.8530 0.000019633805
-#> 14 19 13293512 07090002007723     5.9868 0.000003962819
-#> 15 20 13294176 07090002007664    16.9958 0.000011728694
-#> 16 22 13293474 07090002007713    72.0814 0.000009896451
-#> 17 23 13293520 07090002007676    18.1942 0.000038997161
-#> 18 26 13293512 07090002007723    68.1716 0.000019188910
-#> 19 27 13294264 07090002007650    97.1673 0.000022821372
-#> 20 29 13293456 07090002007738    90.8364 0.000027257370
-#> 21 31 13293512 07090002007723    67.6223 0.000030945071
-#> 22 33 13293576 07090002008236     2.7112 0.000016468104
-

For more info about get_flowline_index() and other -indexing functions, see the article vignette("indexing") -about it or the reference page that describes it.

+hydroloom::index_points_to_lines(flowline, sf::st_geometry(gage), precision = 10) +#> point_id comid reachcode reachcode_measure offset +#> 1 2 13294276 07090002008387 14.8355 0.000020891030 +#> 2 3 13294264 07090002007650 56.4439 0.000031104771 +#> 3 4 13293750 07090002007373 42.5089 0.000016222605 +#> 4 5 13294312 07090002008383 1.2211 0.000013118677 +#> 5 6 13294264 07090002007650 41.0519 0.000040136119 +#> 6 7 13294264 07090002007650 2.0872 0.000009600705 +#> 7 9 13294300 07090002008379 85.4441 0.000024421783 +#> 8 10 13293690 07090002007648 0.7642 0.000022907103 +#> 9 11 13294264 07090002007650 71.4042 0.008212854054 +#> 10 13 13294176 07090002007664 6.1174 0.000037883863 +#> 11 15 13294290 07090002008374 88.0213 0.000037222413 +#> 12 16 13294138 07090002007709 73.3717 0.000010914584 +#> 13 18 13293486 07090002007724 8.8530 0.000019633805 +#> 14 19 13293512 07090002007723 5.9868 0.000003962819 +#> 15 20 13294176 07090002007664 16.9958 0.000011728694 +#> 16 22 13293474 07090002007713 72.0814 0.000009896451 +#> 17 23 13293520 07090002007676 18.1942 0.000038997161 +#> 18 26 13293512 07090002007723 68.1716 0.000019188910 +#> 19 27 13294264 07090002007650 97.1673 0.000022821372 +#> 20 29 13293456 07090002007738 90.8364 0.000027257370 +#> 21 31 13293512 07090002007723 67.6223 0.000030945071 +#> 22 33 13293576 07090002008236 2.7112 0.000016468104 +

See the hydroloom +indexing documentation and +vignette("migrating_from_nhdplusTools") for more.

@@ -561,7 +550,7 @@

Indexing

-

Developed by David Blodgett, Mike Johnson.

+

Developed by David Blodgett.

diff --git a/docs/articles/nhdplusTools_files/figure-html/discover_nhdplus_id-1.png b/docs/articles/hydrogeofetch_files/figure-html/discover_nhdplus_id-1.png similarity index 100% rename from docs/articles/nhdplusTools_files/figure-html/discover_nhdplus_id-1.png rename to docs/articles/hydrogeofetch_files/figure-html/discover_nhdplus_id-1.png diff --git a/docs/articles/hydrogeofetch_files/figure-html/navigate_nldi-1.png b/docs/articles/hydrogeofetch_files/figure-html/navigate_nldi-1.png new file mode 100644 index 00000000..4ecb124b Binary files /dev/null and b/docs/articles/hydrogeofetch_files/figure-html/navigate_nldi-1.png differ diff --git a/docs/articles/nhdplusTools_files/figure-html/nldi_nwissite-1.png b/docs/articles/hydrogeofetch_files/figure-html/nldi_nwissite-1.png similarity index 100% rename from docs/articles/nhdplusTools_files/figure-html/nldi_nwissite-1.png rename to docs/articles/hydrogeofetch_files/figure-html/nldi_nwissite-1.png diff --git a/docs/articles/nhdplusTools_files/figure-html/plot_fline_subset-1.png b/docs/articles/hydrogeofetch_files/figure-html/plot_fline_subset-1.png similarity index 100% rename from docs/articles/nhdplusTools_files/figure-html/plot_fline_subset-1.png rename to docs/articles/hydrogeofetch_files/figure-html/plot_fline_subset-1.png diff --git a/docs/articles/nhdplusTools_files/figure-html/plot_result-1.png b/docs/articles/hydrogeofetch_files/figure-html/plot_result-1.png similarity index 100% rename from docs/articles/nhdplusTools_files/figure-html/plot_result-1.png rename to docs/articles/hydrogeofetch_files/figure-html/plot_result-1.png diff --git a/docs/articles/nhdplusTools_files/figure-html/point-1.png b/docs/articles/hydrogeofetch_files/figure-html/point-1.png similarity index 100% rename from docs/articles/nhdplusTools_files/figure-html/point-1.png rename to docs/articles/hydrogeofetch_files/figure-html/point-1.png diff --git a/docs/articles/hydrogeofetch_files/figure-html/subset_nhdplus_download-1.png b/docs/articles/hydrogeofetch_files/figure-html/subset_nhdplus_download-1.png new file mode 100644 index 00000000..ade9eb5a Binary files /dev/null and b/docs/articles/hydrogeofetch_files/figure-html/subset_nhdplus_download-1.png differ diff --git a/docs/articles/nhdplusTools_files/figure-html/tldr-1.png b/docs/articles/hydrogeofetch_files/figure-html/tldr-1.png similarity index 100% rename from docs/articles/nhdplusTools_files/figure-html/tldr-1.png rename to docs/articles/hydrogeofetch_files/figure-html/tldr-1.png diff --git a/docs/articles/hydrogeofetch_files/figure-html/tldr2-1.png b/docs/articles/hydrogeofetch_files/figure-html/tldr2-1.png new file mode 100644 index 00000000..21d19fc8 Binary files /dev/null and b/docs/articles/hydrogeofetch_files/figure-html/tldr2-1.png differ diff --git a/docs/articles/index.html b/docs/articles/index.html index 7a3eb3bf..5fdc93f8 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -1,5 +1,5 @@ -Articles • nhdplusToolsArticles • hydrogeofetch @@ -16,14 +16,14 @@ - nhdplusTools - 1.5.0 + hydrogeofetch + 2.0.0