Skip to content

Commit 64d1a6e

Browse files
committed
docs(waterdata): Add retry-loop example for resume_from
Worked example in get_daily's docstring showing the canonical PartialResult catch / accumulate-partials / sleep-and-retry pattern, capped at a one-hour deadline matched to the API's hourly rate-limit window so structural failures surface rather than spin forever. Module-level multi_value_chunked docstring and the per-getter resume_from parameter doc now point to get_daily for the example.
1 parent baae50a commit 64d1a6e

2 files changed

Lines changed: 70 additions & 12 deletions

File tree

dataretrieval/waterdata/api.py

Lines changed: 67 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,8 @@ def get_daily(
195195
``QuotaExhausted``) exception from a previous call. The chunker
196196
consults its ``chunk_manifest`` to skip already-completed
197197
sub-requests and fetch only the remainder. Pass the same other
198-
kwargs as the original call.
198+
kwargs as the original call. See ``get_daily`` for a worked
199+
retry-loop example.
199200
200201
Returns
201202
-------
@@ -252,6 +253,51 @@ def get_daily(
252253
... parameter_code="00060",
253254
... time="P7D",
254255
... )
256+
257+
>>> # Resume loop: a heavy chunked query may exhaust the hourly
258+
>>> # rate-limit budget partway through or hit a transient upstream
259+
>>> # error. Catch ``PartialResult``, accumulate the partial frames,
260+
>>> # and re-call with ``resume_from=`` to fetch only the
261+
>>> # outstanding chunks. The USGS API rate-limit window is one
262+
>>> # hour, so a total retry window of one hour is a sensible
263+
>>> # ceiling — anything longer means the failure is structural,
264+
>>> # not transient, and the loop should surface the error.
265+
>>> import time
266+
>>> import pandas as pd
267+
>>> from dataretrieval import waterdata
268+
>>> from dataretrieval.waterdata.chunking import PartialResult
269+
>>>
270+
>>> sites = sites_df["monitoring_location_id"].tolist()
271+
>>> deadline = time.monotonic() + 3600 # one hour cap
272+
>>> partials = []
273+
>>> md = None # carries the latest chunk_manifest between attempts
274+
>>> attempt = 0
275+
>>> while True:
276+
... try:
277+
... df, md = waterdata.get_daily(
278+
... monitoring_location_id=sites,
279+
... parameter_code="00060",
280+
... time="P7D",
281+
... resume_from=md, # ``None`` on the first attempt
282+
... )
283+
... break # full result fetched
284+
... except PartialResult as exc:
285+
... partials.append(exc.partial_frame)
286+
... md = exc.partial_metadata
287+
... if time.monotonic() >= deadline:
288+
... raise TimeoutError(
289+
... f"Could not complete chunked query within one hour "
290+
... f"({md.chunk_manifest.completed}/"
291+
... f"{md.chunk_manifest.total} chunks done)."
292+
... ) from exc
293+
... attempt += 1
294+
... # Exponential backoff, capped at 10 minutes. Quota-
295+
... # reset failures benefit from a longer wait; transient
296+
... # transport errors clear quickly. ``min(...)`` ensures
297+
... # a tight cap; the outer deadline ensures we never wait
298+
... # past one hour total.
299+
... time.sleep(min(60 * 2 ** (attempt - 1), 600))
300+
>>> full = pd.concat([*partials, df], ignore_index=True)
255301
"""
256302
service = "daily"
257303
output_id = "daily_id"
@@ -412,7 +458,8 @@ def get_continuous(
412458
``QuotaExhausted``) exception from a previous call. The chunker
413459
consults its ``chunk_manifest`` to skip already-completed
414460
sub-requests and fetch only the remainder. Pass the same other
415-
kwargs as the original call.
461+
kwargs as the original call. See ``get_daily`` for a worked
462+
retry-loop example.
416463
417464
Returns
418465
-------
@@ -727,7 +774,8 @@ def get_monitoring_locations(
727774
``QuotaExhausted``) exception from a previous call. The chunker
728775
consults its ``chunk_manifest`` to skip already-completed
729776
sub-requests and fetch only the remainder. Pass the same other
730-
kwargs as the original call.
777+
kwargs as the original call. See ``get_daily`` for a worked
778+
retry-loop example.
731779
732780
Returns
733781
-------
@@ -957,7 +1005,8 @@ def get_time_series_metadata(
9571005
``QuotaExhausted``) exception from a previous call. The chunker
9581006
consults its ``chunk_manifest`` to skip already-completed
9591007
sub-requests and fetch only the remainder. Pass the same other
960-
kwargs as the original call.
1008+
kwargs as the original call. See ``get_daily`` for a worked
1009+
retry-loop example.
9611010
9621011
Returns
9631012
-------
@@ -1161,7 +1210,8 @@ def get_combined_metadata(
11611210
``QuotaExhausted``) exception from a previous call. The chunker
11621211
consults its ``chunk_manifest`` to skip already-completed
11631212
sub-requests and fetch only the remainder. Pass the same other
1164-
kwargs as the original call.
1213+
kwargs as the original call. See ``get_daily`` for a worked
1214+
retry-loop example.
11651215
11661216
Returns
11671217
-------
@@ -1385,7 +1435,8 @@ def get_latest_continuous(
13851435
``QuotaExhausted``) exception from a previous call. The chunker
13861436
consults its ``chunk_manifest`` to skip already-completed
13871437
sub-requests and fetch only the remainder. Pass the same other
1388-
kwargs as the original call.
1438+
kwargs as the original call. See ``get_daily`` for a worked
1439+
retry-loop example.
13891440
13901441
Returns
13911442
-------
@@ -1589,7 +1640,8 @@ def get_latest_daily(
15891640
``QuotaExhausted``) exception from a previous call. The chunker
15901641
consults its ``chunk_manifest`` to skip already-completed
15911642
sub-requests and fetch only the remainder. Pass the same other
1592-
kwargs as the original call.
1643+
kwargs as the original call. See ``get_daily`` for a worked
1644+
retry-loop example.
15931645
15941646
Returns
15951647
-------
@@ -1784,7 +1836,8 @@ def get_field_measurements(
17841836
``QuotaExhausted``) exception from a previous call. The chunker
17851837
consults its ``chunk_manifest`` to skip already-completed
17861838
sub-requests and fetch only the remainder. Pass the same other
1787-
kwargs as the original call.
1839+
kwargs as the original call. See ``get_daily`` for a worked
1840+
retry-loop example.
17881841
17891842
Returns
17901843
-------
@@ -1909,7 +1962,8 @@ def get_field_measurements_metadata(
19091962
``QuotaExhausted``) exception from a previous call. The chunker
19101963
consults its ``chunk_manifest`` to skip already-completed
19111964
sub-requests and fetch only the remainder. Pass the same other
1912-
kwargs as the original call.
1965+
kwargs as the original call. See ``get_daily`` for a worked
1966+
retry-loop example.
19131967
19141968
Returns
19151969
-------
@@ -2040,7 +2094,8 @@ def get_peaks(
20402094
``QuotaExhausted``) exception from a previous call. The chunker
20412095
consults its ``chunk_manifest`` to skip already-completed
20422096
sub-requests and fetch only the remainder. Pass the same other
2043-
kwargs as the original call.
2097+
kwargs as the original call. See ``get_daily`` for a worked
2098+
retry-loop example.
20442099
20452100
Returns
20462101
-------
@@ -2899,7 +2954,8 @@ def get_channel(
28992954
``QuotaExhausted``) exception from a previous call. The chunker
29002955
consults its ``chunk_manifest`` to skip already-completed
29012956
sub-requests and fetch only the remainder. Pass the same other
2902-
kwargs as the original call.
2957+
kwargs as the original call. See ``get_daily`` for a worked
2958+
retry-loop example.
29032959
29042960
Returns
29052961
-------

dataretrieval/waterdata/chunking.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -617,7 +617,9 @@ def multi_value_chunked(
617617
the caller's kwargs. The wrapper pops it before planning (so it
618618
never reaches the underlying HTTP request), validates the saved
619619
plan matches the fresh plan, and skips the already-completed
620-
cartesian-product combinations.
620+
cartesian-product combinations. See ``get_daily``'s docstring for
621+
a worked retry-loop example using a one-hour deadline matched to
622+
the API's rate-limit window.
621623
"""
622624

623625
def decorator(fetch_once: _FetchOnce) -> _FetchOnce:

0 commit comments

Comments
 (0)