diff --git a/.flake8 b/.flake8 deleted file mode 100644 index f295e07b..00000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length = 88 -extend-ignore = E203, E704 diff --git a/.gitignore b/.gitignore index 17ed0848..182189e4 100644 --- a/.gitignore +++ b/.gitignore @@ -111,4 +111,4 @@ ENV/ .mypy_cache/ # macOS -*.DS_Store \ No newline at end of file +*.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36b65c4b..47750fe7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,44 +6,23 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: + # tests/data/ holds byte-exact API response captures (TSV/RDB with + # significant trailing tabs, plus pinned JSON). Skip whitespace and + # EOL normalization there so we don't silently change fixture bytes. - id: trailing-whitespace + exclude: ^tests/data/ - id: end-of-file-fixer + exclude: ^tests/data/ + - id: mixed-line-ending + exclude: ^tests/data/ - id: check-docstring-first - id: check-json - id: check-yaml - - id: double-quote-string-fixer - id: debug-statements - - id: mixed-line-ending - - - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: - - '--py38-plus' - - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - - id: black-jupyter - - - repo: https://github.com/keewis/blackdoc - rev: v0.3.8 - hooks: - - id: blackdoc - - - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.6 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.15 hooks: - - id: prettier + - id: ruff-check + args: [--fix] + - id: ruff-format diff --git a/.prettierrc.toml b/.prettierrc.toml deleted file mode 100644 index 3346a577..00000000 --- a/.prettierrc.toml +++ /dev/null @@ -1,2 +0,0 @@ -semi = false -singleQuote = true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..ca72daf7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# AGENTS.md + +## Scope +- Python code is in `dataretrieval/`; `dataretrieval/waterdata/` is the modern USGS Water Data API, `dataretrieval/nwis.py` is legacy/deprecated. +- `R/dataRetrieval/` is the R project copy; leave it alone unless the task asks for R work. +- Exclude `.claude/worktrees/` from searches and edits; it contains stale worktrees that pollute results. + +## Example Notebooks +- `demos/*.ipynb` — top-level Water Data tour: `USGS_WaterData_Introduction_Examples.ipynb` is the entry point; `_ContinuousData_`, `_DailyStatistics_`, `_DiscreteSamples_`, `_ReferenceLists_` cover individual collections; `WaterData_demo.ipynb`, `peak_streamflow_trends.ipynb`, and `R Python Vignette equivalents.ipynb` are standalone walkthroughs. +- `demos/hydroshare/*.ipynb` — per-service HydroShare examples (NLDI, NWIS WaterUse, and Water Data DailyValues / GroundwaterLevels / Measurements / ParameterCodes / Peaks / Ratings / Samples / SiteInfo / SiteInventory / Statistics / UnitValues). Mirror these when adding examples for a new collection. +- `demos/nwqn_data_pull/` — non-notebook example: a lithops/Docker batch pipeline (`retrieve_nwqn_samples.py`, `retrieve_nwqn_streamflow.py`) with its own `README.md`. +- Any `Untitled*.ipynb`, `*_test.ipynb`, or notebooks not listed here are untracked local scratch; ignore them. + +## Environment +- Use `pip install .[test,nldi]` (CI uses pip, not uv despite `uv.lock`). Docs: `pip install .[doc,nldi]`. + +## Commands +- Lint: `ruff check .` and `ruff format --check .`. +- Tests: `coverage run -m pytest tests/ && coverage report -m`, or focused like `pytest tests/waterdata_test.py::test_mock_get_samples`. +- Docs: install docs deps, `ipython kernel install --name "python3" --user`, then `make html` from `docs/`. `make docs` adds doctest+linkcheck (network-dependent). + +## Testing Gotchas +- Tests mock HTTP with `pytest-httpx`'s `httpx_mock` fixture and fixtures under `tests/data/`; keep new API tests offline. `tests/conftest.py` relaxes the fixture's strict-mode defaults (unused mocks and unmocked requests are tolerated) so rerun-on-failure works. +- `tests/nwis_test.py::test_nwis_service_live` hits live NWIS. +- `tests/nadp_test.py` is module-skipped (NADP deprecated). +- `tests/waterdata_test.py` and `tests/waterdata_ratings_test.py` skip on Python <3.10, so a 3.9 run does not cover them. + +## Implementation Notes +- HTTP client is `httpx` (migrated from `requests` in #289); new code should use `httpx` and tests should mock with `httpx_mock`. +- Public download helpers return `(DataFrame, metadata)`. +- `dataretrieval/__init__.py` star-imports service modules; `dataretrieval/waterdata/__init__.py` controls Water Data exports via `__all__`. +- `dataretrieval.waterdata.utils._default_headers()` adds `X-Api-Key` from `API_USGS_PAT`; never hard-code tokens in examples or tests. +- Water Data request builders translate Python kwargs to API spellings (`skip_geometry` -> `skipGeometry`, `filter_lang` -> `filter-lang`); tests assert exact URLs/query params. +- Multi-value OGC params are comma-joined GETs, except `monitoring-locations` which POSTs CQL2 JSON. The OGC edge WAF caps total request bytes (URL + body) at ~8200, so `dataretrieval/waterdata/chunking.py` auto-splits oversized queries across sub-requests (both GET and POST paths); preserve this when adding new list-shaped kwargs. +- NLDI requires `geopandas` at import time (`pip install .[nldi]`); other modules fall back to pandas when geopandas is absent. diff --git a/README.md b/README.md index a468c652..e56d31c2 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ from dataretrieval import waterdata # Get daily streamflow data (returns DataFrame and metadata) df, metadata = waterdata.get_daily( - monitoring_location_id='USGS-01646500', + monitoring_location_id='USGS-01646500', parameter_code='00060', # Discharge time='2024-10-01/2025-09-30' ) @@ -98,7 +98,7 @@ windows to avoid timeouts and other issues: ```python # Get continuous data for a single monitoring location and water year df, metadata = waterdata.get_continuous( - monitoring_location_id='USGS-01646500', + monitoring_location_id='USGS-01646500', parameter_code='00065', # Gage height time='2024-10-01/2025-09-30' ) @@ -152,7 +152,7 @@ from dataretrieval import nldi # Get watershed basin for a stream reach basin = nldi.get_basin( feature_source='comid', - feature_id='13293474' # NHD reach identifier + feature_id='13293474' # NHD reach identifier ) print(f"Basin contains {len(basin)} feature(s)") @@ -184,7 +184,7 @@ print(f"Found {len(flowlines)} upstream tributaries within 50km") ### Legacy NWIS Services (Deprecated) - **Daily values (dv)**: Legacy daily statistical data - **Instantaneous values (iv)**: Legacy continuous data -- **Site info (site)**: Basic site information +- **Site info (site)**: Basic site information - **Statistics (stat)**: Statistical summaries - **Discharge peaks (peaks)**: Annual peak discharge events diff --git a/dataretrieval/py.typed b/dataretrieval/py.typed index 8b137891..e69de29b 100644 --- a/dataretrieval/py.typed +++ b/dataretrieval/py.typed @@ -1 +0,0 @@ - diff --git a/demos/nwqn_data_pull/Dockerfile_dataretrieval b/demos/nwqn_data_pull/Dockerfile_dataretrieval index 2d3eaddc..d1793c6a 100644 --- a/demos/nwqn_data_pull/Dockerfile_dataretrieval +++ b/demos/nwqn_data_pull/Dockerfile_dataretrieval @@ -56,4 +56,4 @@ COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] -CMD [ "handler.entry_point.lambda_handler" ] \ No newline at end of file +CMD [ "handler.entry_point.lambda_handler" ] diff --git a/docs/source/examples/USGS_WaterData_DailyValues_Examples.nblink b/docs/source/examples/USGS_WaterData_DailyValues_Examples.nblink index a0cf6297..f0d3e7af 100644 --- a/docs/source/examples/USGS_WaterData_DailyValues_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_DailyValues_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_DailyValues_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_GroundwaterLevels_Examples.nblink b/docs/source/examples/USGS_WaterData_GroundwaterLevels_Examples.nblink index ae9400f4..4e60e860 100644 --- a/docs/source/examples/USGS_WaterData_GroundwaterLevels_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_GroundwaterLevels_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_GroundwaterLevels_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_Measurements_Examples.nblink b/docs/source/examples/USGS_WaterData_Measurements_Examples.nblink index 2bf26789..1c394798 100644 --- a/docs/source/examples/USGS_WaterData_Measurements_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_Measurements_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_Measurements_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_ParameterCodes_Examples.nblink b/docs/source/examples/USGS_WaterData_ParameterCodes_Examples.nblink index 9e69b23b..b1e9ce71 100644 --- a/docs/source/examples/USGS_WaterData_ParameterCodes_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_ParameterCodes_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_ParameterCodes_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_Peaks_Examples.nblink b/docs/source/examples/USGS_WaterData_Peaks_Examples.nblink index 400077bf..f2834c22 100644 --- a/docs/source/examples/USGS_WaterData_Peaks_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_Peaks_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_Peaks_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_Ratings_Examples.nblink b/docs/source/examples/USGS_WaterData_Ratings_Examples.nblink index b145eac3..538de145 100644 --- a/docs/source/examples/USGS_WaterData_Ratings_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_Ratings_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_Ratings_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_Samples_Examples.nblink b/docs/source/examples/USGS_WaterData_Samples_Examples.nblink index f03cb32d..aacd729c 100644 --- a/docs/source/examples/USGS_WaterData_Samples_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_Samples_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_Samples_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_SiteInfo_Examples.nblink b/docs/source/examples/USGS_WaterData_SiteInfo_Examples.nblink index 43c1069a..9de7a10b 100644 --- a/docs/source/examples/USGS_WaterData_SiteInfo_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_SiteInfo_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_SiteInfo_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_SiteInventory_Examples.nblink b/docs/source/examples/USGS_WaterData_SiteInventory_Examples.nblink index e9da45d2..74a2bdb5 100644 --- a/docs/source/examples/USGS_WaterData_SiteInventory_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_SiteInventory_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_SiteInventory_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_Statistics_Examples.nblink b/docs/source/examples/USGS_WaterData_Statistics_Examples.nblink index 1ab51d1a..c1a1c6a3 100644 --- a/docs/source/examples/USGS_WaterData_Statistics_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_Statistics_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_Statistics_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/USGS_WaterData_UnitValues_Examples.nblink b/docs/source/examples/USGS_WaterData_UnitValues_Examples.nblink index 606ced5e..2cbe60c3 100644 --- a/docs/source/examples/USGS_WaterData_UnitValues_Examples.nblink +++ b/docs/source/examples/USGS_WaterData_UnitValues_Examples.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/hydroshare/USGS_WaterData_UnitValues_Examples.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/WaterData_demo.nblink b/docs/source/examples/WaterData_demo.nblink index 7c485f1a..52d9cbd1 100644 --- a/docs/source/examples/WaterData_demo.nblink +++ b/docs/source/examples/WaterData_demo.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/WaterData_demo.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index de6f1b25..91d7bd1f 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -92,4 +92,4 @@ Duplicating the R ``dataRetrieval`` vignettes functionality .. toctree:: :maxdepth: 2 - rvignettes \ No newline at end of file + rvignettes diff --git a/docs/source/examples/rvignettes.nblink b/docs/source/examples/rvignettes.nblink index 39351e6e..15d71447 100644 --- a/docs/source/examples/rvignettes.nblink +++ b/docs/source/examples/rvignettes.nblink @@ -1,3 +1,3 @@ { "path": "../../../demos/R Python Vignette equivalents.ipynb" -} \ No newline at end of file +} diff --git a/docs/source/meta/contributing.rst b/docs/source/meta/contributing.rst index fdc330b5..fb17f6c4 100644 --- a/docs/source/meta/contributing.rst +++ b/docs/source/meta/contributing.rst @@ -156,4 +156,4 @@ Acknowledgements ---------------- This document was adapted from the ``cookiecutter`` project's CONTRIBUTING file, which resides at https://github.com/cookiecutter/cookiecutter/blob/main/CONTRIBUTING.md -Thank you to the ``cookiecutter`` team for helping streamline open-source development for the masses. \ No newline at end of file +Thank you to the ``cookiecutter`` team for helping streamline open-source development for the masses. diff --git a/docs/source/meta/installing.rst b/docs/source/meta/installing.rst index 34caf181..c799c7b6 100644 --- a/docs/source/meta/installing.rst +++ b/docs/source/meta/installing.rst @@ -86,4 +86,4 @@ You can also build the documentation locally by running the following commands: This both tests the documentation (runs code blocks and checks links), and also locally *builds* the documentation, placing the HTML files within the ``docs/build/html`` directory. You can then open the ``index.html`` file in -your browser to view the documentation. \ No newline at end of file +your browser to view the documentation. diff --git a/docs/source/meta/license.rst b/docs/source/meta/license.rst index f42ffe00..a24ec1d9 100644 --- a/docs/source/meta/license.rst +++ b/docs/source/meta/license.rst @@ -9,4 +9,4 @@ Interior. For more information, see the `LICENSE.md`_ file. See the .. _LICENSE.md: https://github.com/DOI-USGS/dataretrieval-python/blob/main/LICENSE.md -.. _Disclaimer.md: https://github.com/DOI-USGS/dataretrieval-python/blob/main/DISCLAIMER.md \ No newline at end of file +.. _Disclaimer.md: https://github.com/DOI-USGS/dataretrieval-python/blob/main/DISCLAIMER.md diff --git a/docs/source/reference/nadp.rst b/docs/source/reference/nadp.rst index e046cb59..023f7cd5 100644 --- a/docs/source/reference/nadp.rst +++ b/docs/source/reference/nadp.rst @@ -5,4 +5,4 @@ dataretrieval.nadp .. automodule:: dataretrieval.nadp :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/docs/source/reference/nldi.rst b/docs/source/reference/nldi.rst index b9f3fe69..d223eb0d 100644 --- a/docs/source/reference/nldi.rst +++ b/docs/source/reference/nldi.rst @@ -5,4 +5,4 @@ dataretrieval.nldi .. automodule:: dataretrieval.nldi :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/docs/source/reference/nwis.rst b/docs/source/reference/nwis.rst index da5f6b69..6d8aef97 100644 --- a/docs/source/reference/nwis.rst +++ b/docs/source/reference/nwis.rst @@ -5,4 +5,4 @@ dataretrieval.nwis .. automodule:: dataretrieval.nwis :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/docs/source/reference/streamstats.rst b/docs/source/reference/streamstats.rst index 2e3baa39..6e4279e3 100644 --- a/docs/source/reference/streamstats.rst +++ b/docs/source/reference/streamstats.rst @@ -5,4 +5,4 @@ dataretrieval.streamstats .. automodule:: dataretrieval.streamstats :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/docs/source/reference/utils.rst b/docs/source/reference/utils.rst index b7ee734f..5c115a9f 100644 --- a/docs/source/reference/utils.rst +++ b/docs/source/reference/utils.rst @@ -5,4 +5,4 @@ dataretrieval.utils .. automodule:: dataretrieval.utils :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/docs/source/reference/waterdata.rst b/docs/source/reference/waterdata.rst index b0e638b8..e6cbc5a1 100644 --- a/docs/source/reference/waterdata.rst +++ b/docs/source/reference/waterdata.rst @@ -5,4 +5,4 @@ dataretrieval.waterdata .. automodule:: dataretrieval.waterdata :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/docs/source/reference/wqp.rst b/docs/source/reference/wqp.rst index bcdfdaf3..5ec7b90e 100644 --- a/docs/source/reference/wqp.rst +++ b/docs/source/reference/wqp.rst @@ -5,4 +5,4 @@ dataretrieval.wqp .. automodule:: dataretrieval.wqp :members: - :special-members: \ No newline at end of file + :special-members: