Skip to content

Commit 8605dea

Browse files
committed
convert failures counter to a stop that shows URL that failed
1 parent f0bef3e commit 8605dea

1 file changed

Lines changed: 1 addition & 4 deletions

File tree

dataretrieval/waterdata/utils.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,6 @@ def _walk_pages(
589589
if max_results is None or pd.isna(max_results):
590590
dfs = _get_resp_data(resp, geopd=geopd)
591591
curr_url = _next_req_url(resp)
592-
failures = []
593592
while curr_url:
594593
try:
595594
resp = client.request(
@@ -604,10 +603,8 @@ def _walk_pages(
604603
dfs = pd.concat([dfs, df1], ignore_index=True)
605604
curr_url = _next_req_url(resp)
606605
except Exception:
607-
failures.append(curr_url)
606+
logger.info("Request failed for URL: %s. Stopping pagination and data download.", curr_url)
608607
curr_url = None
609-
if failures:
610-
logger.warning("There were %d failed requests.", len(failures))
611608
return dfs, initial_response
612609
else:
613610
resp.raise_for_status()

0 commit comments

Comments
 (0)