Skip to content

Commit b611e9f

Browse files
thodson-usgsclaude
andcommitted
refactor(waterdata): make max_chunks a positive-integer count (1 = off, reject 0)
Follow-up to the parallel_chunks contract cleanup. max_chunks is a sub-request *count*, so its valid domain is the positive integers. Move the "off" sentinel from 0 to 1 (which already behaves identically — no extra fan-out, per the earlier n=1-passthrough change) and reject anything below 1: - ChunkPlan.max_chunks default 0 -> 1; guard `< 0` -> `< 1`, so 0 and negatives now raise ValueError instead of silently no-oping. - _parallel_chunks ambient default 0 -> 1; read-site + comments updated. - _refine off-guard `<= 0` -> `<= 1` (clearer intent; the loop already no-oped at 1). - Docstrings (class param, Raises, _refine, ambient) updated. Tests: zero-cap passthrough -> default/unit passthrough; negative-raises -> invalid-raises parametrized over {0, -1}; ramp param (0,1) -> (1,1) with the coverage guard keyed on expected_pieces; the two byte-only baselines and the ambient-default assertions move 0 -> 1. The public parallel_chunks(n) already rejected n < 1, so no user-facing behavior changes. Signed-off-by: thodson-usgs <thodson@usgs.gov> Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 8d4c4a1 commit b611e9f

3 files changed

Lines changed: 54 additions & 50 deletions

File tree

dataretrieval/ogc/chunking.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,9 @@ def get_active_client() -> httpx.AsyncClient | None:
184184
# limit alone requires. Scoped to a ``with parallel_chunks(...):`` block (a
185185
# ContextVar), deliberately NOT an env var (see :func:`parallel_chunks` for
186186
# why). The ambient holds ``n`` — the requested cap on the plan's total
187-
# sub-request count; ``0`` (the default, outside any block) means "chunk only as
188-
# much as the byte limit needs".
189-
_parallel_chunks: Ambient[int] = Ambient("ogc_parallel_chunks", 0)
187+
# sub-request count; ``1`` (the default, outside any block) means "off — chunk
188+
# only as much as the byte limit needs, no extra fan-out".
189+
_parallel_chunks: Ambient[int] = Ambient("ogc_parallel_chunks", 1)
190190

191191

192192
@contextmanager
@@ -739,7 +739,7 @@ def wrapper(
739739
) -> tuple[pd.DataFrame, Any]:
740740
limit = _OGC_URL_BYTE_LIMIT if url_limit is None else url_limit
741741
# Read the parallel_chunks dial ``n`` from the ambient set by
742-
# ``parallel_chunks`` (0 = off outside any such block; otherwise the
742+
# ``parallel_chunks`` (1 = off outside any such block; otherwise the
743743
# requested total sub-request cap). It only affects *planning*, done
744744
# here up front, so a later resume — which re-issues the
745745
# already-planned sub-requests — needs no snapshot.

dataretrieval/ogc/planning.py

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -330,18 +330,19 @@ class ChunkPlan:
330330
Byte budget for the request (URL + body) — a hard ceiling every
331331
sub-request must fit.
332332
max_chunks : int, optional
333-
Hard cap on the plan's total sub-request count (default ``0`` = off).
334-
``0`` (off) and ``1`` (no extra fan-out) both chunk only as much as
335-
``url_limit`` requires — the most conservative plan, fewest
336-
sub-requests — so a fitting request is a passthrough under either. A
337-
cap of ``2`` or more fans the plan out to up to ``max_chunks``
338-
sub-requests overall (the cartesian product across axes, never fewer
339-
than the byte budget already forces) — capped as a whole, not per axis,
340-
so several multi-value axes can't multiply past the cap. The plan never
341-
exceeds the cap and may land below it when no whole split lands on it
342-
exactly. A negative cap is a caller error and raises ``ValueError``.
343-
Set from the :func:`~dataretrieval.ogc.chunking.parallel_chunks` ``n``;
344-
see :meth:`_refine`.
333+
Hard cap on the plan's total sub-request count (default ``1`` = off).
334+
``1`` chunks only as much as ``url_limit`` requires — the most
335+
conservative plan, fewest sub-requests — so a fitting request is a
336+
passthrough. A cap of ``2`` or more fans the plan out to up to
337+
``max_chunks`` sub-requests overall (the cartesian product across axes,
338+
never fewer than the byte budget already forces) — capped as a whole,
339+
not per axis, so several multi-value axes can't multiply past the cap.
340+
The plan never exceeds the cap and may land below it when no whole
341+
split lands on it exactly. ``max_chunks`` is a sub-request count, so a
342+
value below ``1`` (``0`` or negative) is a caller error and raises
343+
``ValueError``. Set from the
344+
:func:`~dataretrieval.ogc.chunking.parallel_chunks` ``n``; see
345+
:meth:`_refine`.
345346
346347
Attributes
347348
----------
@@ -367,24 +368,25 @@ class ChunkPlan:
367368
If the request needs chunking but even the singleton plan
368369
doesn't fit ``url_limit``.
369370
ValueError
370-
If ``max_chunks`` is negative.
371+
If ``max_chunks`` is less than 1 (0 or negative).
371372
"""
372373

373374
def __init__(
374375
self,
375376
args: dict[str, Any],
376377
build_request: Callable[..., httpx.Request],
377378
url_limit: int,
378-
max_chunks: int = 0,
379+
max_chunks: int = 1,
379380
) -> None:
380-
if max_chunks < 0:
381-
# ``0`` disables fan-out (the ambient default outside any
382-
# ``parallel_chunks`` block); a negative cap is meaningless and can
383-
# only be a caller bug, so fail loudly rather than silently no-op.
384-
# The public ``parallel_chunks(n)`` already rejects ``n < 1``; this
385-
# guards direct construction.
381+
if max_chunks < 1:
382+
# ``max_chunks`` is a sub-request *count*: the minimum is ``1``
383+
# (the ambient default outside any ``parallel_chunks`` block),
384+
# which means "off — no extra fan-out". ``0`` or negative is a
385+
# meaningless count and can only be a caller bug, so fail loudly
386+
# rather than silently no-op. The public ``parallel_chunks(n)``
387+
# already rejects ``n < 1``; this guards direct construction.
386388
raise ValueError(
387-
f"max_chunks must be >= 0 (0 disables fan-out); got {max_chunks!r}."
389+
f"max_chunks must be >= 1 (1 disables fan-out); got {max_chunks!r}."
388390
)
389391

390392
self.args = args
@@ -438,8 +440,8 @@ def __init__(
438440
# A request that already fits and hasn't opted into finer chunking is
439441
# the common passthrough: leave ``axes``/``chunks`` empty so
440442
# ``total == 1`` and ``iter_sub_args`` yields the original args
441-
# verbatim. ``max_chunks`` of 0 (off) or 1 (no extra fan-out) both mean
442-
# "don't split", so both take this path; only ``max_chunks >= 2`` asks
443+
# verbatim. ``max_chunks == 1`` (off / no extra fan-out) means
444+
# "don't split", so it takes this path; only ``max_chunks >= 2`` asks
443445
# for extra fan-out and sets the axes up to be refined below.
444446
if fits and max_chunks <= 1:
445447
return
@@ -452,7 +454,7 @@ def __init__(
452454
self._plan(build_request, url_limit)
453455
# Soft pass: optionally split further than the byte budget requires.
454456
# Purely additive — never re-raises, and the byte budget stays
455-
# satisfied; a no-op at ``max_chunks <= 0``.
457+
# satisfied; a no-op at ``max_chunks == 1``.
456458
self._refine(max_chunks)
457459

458460
if self.canonical_url is None:
@@ -530,18 +532,18 @@ def _refine(self, max_chunks: int) -> None:
530532
one axis saturating before another is touched. Purely additive — only
531533
ever *splits* existing chunks, so the byte pass's work and the
532534
``url_limit`` invariant are both preserved, and it never raises. A
533-
no-op at ``max_chunks <= 0``.
535+
no-op at ``max_chunks == 1``.
534536
535537
Parameters
536538
----------
537539
max_chunks : int
538-
Hard cap on the plan's total sub-request count (``0`` = off) — the
540+
Hard cap on the plan's total sub-request count (``1`` = off) — the
539541
``parallel_chunks(n)`` value. The cap is on the whole plan (the
540542
cartesian product across axes), not per axis, so several multi-value
541543
axes can't multiply past it; the plan never exceeds it and may fall
542544
short when no whole split lands on it exactly.
543545
"""
544-
if max_chunks <= 0:
546+
if max_chunks <= 1:
545547
return
546548
while True:
547549
total = self.total

tests/waterdata_chunking_test.py

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2122,13 +2122,13 @@ async def fetch(args):
21222122
# ---------------------------------------------------------------------------
21232123

21242124

2125-
def test_zero_cap_preserves_passthrough():
2126-
"""``max_chunks=0`` (the default) must not perturb the existing
2125+
def test_default_preserves_passthrough():
2126+
"""The default ``max_chunks`` (1 = off) must not perturb the existing
21272127
plan: a multi-value request that fits the byte limit is still the trivial
21282128
passthrough (no axes, ``total == 1``), byte-for-byte the pre-feature
21292129
behavior."""
21302130
args = {"monitoring_location_id": ["A", "B", "C", "D"]}
2131-
plan = ChunkPlan(args, _fake_build, url_limit=8000, max_chunks=0)
2131+
plan = ChunkPlan(args, _fake_build, url_limit=8000) # default max_chunks=1
21322132
assert plan.axes == []
21332133
assert plan.total == 1
21342134
assert list(plan.iter_sub_args()) == [args]
@@ -2137,34 +2137,36 @@ def test_zero_cap_preserves_passthrough():
21372137
def test_unit_cap_preserves_passthrough():
21382138
"""``max_chunks=1`` means "no extra fan-out", so a fitting multi-value
21392139
request stays the trivial passthrough (no axes, ``total == 1``,
2140-
``iter_sub_args`` yields the original args verbatim) — identical to the off
2141-
(0) case, not a materialized one-chunk-per-axis plan."""
2140+
``iter_sub_args`` yields the original args verbatim) — identical to the
2141+
default (off), not a materialized one-chunk-per-axis plan."""
21422142
args = {"monitoring_location_id": ["A", "B", "C", "D"]}
21432143
plan = ChunkPlan(args, _fake_build, url_limit=8000, max_chunks=1)
21442144
assert plan.axes == []
21452145
assert plan.total == 1
21462146
assert list(plan.iter_sub_args()) == [args]
21472147

21482148

2149-
def test_negative_cap_raises():
2150-
"""A negative ``max_chunks`` is a caller bug, not a silent no-op: it raises
2151-
``ValueError`` at construction. (The public ``parallel_chunks(n)`` already
2152-
rejects ``n < 1``; this pins the same guard on direct construction.)"""
2149+
@pytest.mark.parametrize("bad", [0, -1])
2150+
def test_invalid_cap_raises(bad):
2151+
"""``max_chunks`` is a sub-request count, so a value below 1 (``0`` or
2152+
negative) is a caller bug, not a silent no-op: it raises ``ValueError`` at
2153+
construction. (The public ``parallel_chunks(n)`` already rejects ``n < 1``;
2154+
this pins the same guard on direct construction.)"""
21532155
args = {"monitoring_location_id": ["A", "B", "C", "D"]}
2154-
with pytest.raises(ValueError, match="max_chunks must be >= 0"):
2155-
ChunkPlan(args, _fake_build, url_limit=8000, max_chunks=-1)
2156+
with pytest.raises(ValueError, match="max_chunks must be >= 1"):
2157+
ChunkPlan(args, _fake_build, url_limit=8000, max_chunks=bad)
21562158

21572159

21582160
@pytest.mark.parametrize(
21592161
("max_chunks", "expected_pieces"),
2160-
[(0, 1), (2, 2), (8, 8), (16, 10), (32, 10)],
2162+
[(1, 1), (2, 2), (8, 8), (16, 10), (32, 10)],
21612163
)
21622164
def test_cap_ramps_then_saturates(max_chunks, expected_pieces):
21632165
"""A single 10-atom axis that fits the byte limit splits into
21642166
``min(10, cap)`` pieces: 1 (off), 2, 8, then saturating at 10 (one atom per
21652167
chunk) once the cap overshoots the atom count. Monotonic and bounded, and
21662168
whenever it splits the partition is a cover — every atom exactly once. (The
2167-
cap-0 passthrough has no axis to cover; see the passthrough test.)"""
2169+
cap-1 passthrough has no axis to cover; see the passthrough test.)"""
21682170
atoms = [f"S{i:02d}" for i in range(10)]
21692171
plan = ChunkPlan(
21702172
{"monitoring_location_id": atoms},
@@ -2173,7 +2175,7 @@ def test_cap_ramps_then_saturates(max_chunks, expected_pieces):
21732175
max_chunks=max_chunks,
21742176
)
21752177
assert plan.total == expected_pieces
2176-
if max_chunks:
2178+
if expected_pieces > 1:
21772179
flattened = [
21782180
a for chunk in plan.chunks["monitoring_location_id"] for a in chunk
21792181
]
@@ -2205,7 +2207,7 @@ def test_cap_below_byte_split_does_not_reduce_fan_out():
22052207
# Heavy axis of four 30-char atoms; a limit tight enough that the byte pass
22062208
# must drive every atom into its own sub-request (4 pieces > the cap of 2).
22072209
args = {"monitoring_location_id": ["X" * 30, "Y" * 30, "Z" * 30, "W" * 30]}
2208-
baseline = ChunkPlan(args, _fake_build, url_limit=250, max_chunks=0)
2210+
baseline = ChunkPlan(args, _fake_build, url_limit=250, max_chunks=1)
22092211
assert baseline.total > 2 # byte pass alone already fanned out past 2
22102212
refined = ChunkPlan(args, _fake_build, url_limit=250, max_chunks=2)
22112213
# cap 2 < baseline pieces → refine is a no-op here.
@@ -2218,7 +2220,7 @@ def test_cap_never_exceeds_the_byte_budget():
22182220
a chunk), and the fan-out is at least what the byte pass required."""
22192221
args = {"monitoring_location_id": ["X" * 30, "Y" * 30, "Z" * 30, "W" * 30]}
22202222
limit = 310
2221-
byte_only = ChunkPlan(args, _fake_build, url_limit=limit, max_chunks=0)
2223+
byte_only = ChunkPlan(args, _fake_build, url_limit=limit, max_chunks=1)
22222224
plan = ChunkPlan(args, _fake_build, url_limit=limit, max_chunks=32)
22232225
assert plan.total >= byte_only.total
22242226
for sub in plan.iter_sub_args():
@@ -2318,13 +2320,13 @@ def test_cap_does_not_mask_unchunkable():
23182320
def test_parallel_chunks_publishes_n_on_the_ambient():
23192321
"""The context manager publishes ``n`` on the ambient for the block and
23202322
restores the previous value on exit — including proper nesting."""
2321-
assert _parallel_chunks.get() == 0
2323+
assert _parallel_chunks.get() == 1 # default (off, = no extra fan-out)
23222324
with parallel_chunks(32):
23232325
assert _parallel_chunks.get() == 32
23242326
with parallel_chunks(2):
23252327
assert _parallel_chunks.get() == 2
23262328
assert _parallel_chunks.get() == 32 # outer restored
2327-
assert _parallel_chunks.get() == 0 # default (off) outside any block
2329+
assert _parallel_chunks.get() == 1 # default (off) outside any block
23282330

23292331

23302332
@pytest.mark.parametrize(
@@ -2348,7 +2350,7 @@ def test_parallel_chunks_rejects_non_positive_int(bad):
23482350
with pytest.raises(ValueError, match="must be a positive integer"):
23492351
with parallel_chunks(bad):
23502352
pass
2351-
assert _parallel_chunks.get() == 0
2353+
assert _parallel_chunks.get() == 1 # default (off) — unchanged by a rejected call
23522354

23532355

23542356
def test_parallel_chunks_drives_end_to_end_fan_out():

0 commit comments

Comments
 (0)