Skip to content

Commit b3eb6d2

Browse files
[API Compatibility] Add decorator for paddle.utils.data.BatchSampler (PaddlePaddle#78382)
* add decorator for BatchSampler * fix error and add first arg type checking * fix condition * fix duplicate args * fix args * remove duplicate code * support Iterable parsing and fix tests
1 parent c042c2c commit b3eb6d2

3 files changed

Lines changed: 92 additions & 4 deletions

File tree

python/paddle/io/dataloader/batch_sampler.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,12 @@
1616

1717
import math
1818
from collections.abc import Iterable, Iterator, Sequence, Sized
19+
from typing import overload
1920

2021
import numpy as np
2122

23+
from paddle.utils.decorator_utils import batch_sampler_decorator
24+
2225
from .dataset import IterableDataset
2326
from .sampler import RandomSampler, Sampler, SequenceSampler
2427

@@ -107,6 +110,25 @@ class BatchSampler(Sampler[Sequence[int]]):
107110
shuffle: bool
108111
drop_last: bool
109112

113+
@overload
114+
def __init__(
115+
self,
116+
dataset: Sized | None = None,
117+
sampler: Sampler | Iterable[int] | None = None,
118+
shuffle: bool = False,
119+
batch_size: int = 1,
120+
drop_last: bool = False,
121+
) -> None: ...
122+
123+
@overload
124+
def __init__(
125+
self,
126+
sampler: Sampler | Iterable[int] | None = None,
127+
batch_size: int = 1,
128+
drop_last: bool = False,
129+
) -> None: ...
130+
131+
@batch_sampler_decorator()
110132
def __init__(
111133
self,
112134
dataset: Sized | None = None,

python/paddle/utils/decorator_utils.py

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,13 @@
1717
import functools
1818
import inspect
1919
import warnings
20-
from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
20+
from collections.abc import Iterable
21+
from typing import Any, Callable, TypeVar, cast
2122

2223
from typing_extensions import ParamSpec, get_overloads
2324

2425
import paddle
2526

26-
if TYPE_CHECKING:
27-
from collections.abc import Iterable
28-
2927
_InputT = ParamSpec("_InputT")
3028
_RetT = TypeVar("_RetT")
3129
_SENTINEL = object()
@@ -994,3 +992,39 @@ def wrapper(*args: _InputT.args, **kwargs: _InputT.kwargs) -> _RetT:
994992
return wrapper
995993

996994
return decorator
995+
996+
997+
def batch_sampler_decorator() -> Callable[
998+
[Callable[_InputT, _RetT]], Callable[_InputT, _RetT]
999+
]:
1000+
"""
1001+
Usage Example:
1002+
PyTorch: torch.utils.data.BatchSampler(sampler, batch_size, drop_last)
1003+
Paddle: paddle.utils.data.BatchSampler(dataset, sampler, shuffle, batch_size, drop_last)
1004+
"""
1005+
1006+
def decorator(func: Callable[_InputT, _RetT]) -> Callable[_InputT, _RetT]:
1007+
@functools.wraps(func)
1008+
def wrapper(*args: _InputT.args, **kwargs: _InputT.kwargs) -> _RetT:
1009+
# args[0] is self
1010+
# args[1] is Sampler / Iterable, use torch signature
1011+
if len(args) >= 2 and isinstance(
1012+
args[1], (paddle.io.Sampler, Iterable)
1013+
):
1014+
kwargs["sampler"] = args[1]
1015+
if len(args) >= 3:
1016+
kwargs["batch_size"] = args[2]
1017+
if len(args) == 4:
1018+
kwargs["drop_last"] = args[3]
1019+
if len(args) > 4:
1020+
raise TypeError(
1021+
"BatchSampler() received too many arguments"
1022+
)
1023+
args = (args[0],)
1024+
1025+
return func(*args, **kwargs)
1026+
1027+
wrapper.__signature__ = inspect.signature(func)
1028+
return wrapper
1029+
1030+
return decorator

test/legacy_test/test_batch_sampler.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,38 @@ def init_batch_sampler(self):
225225
return bs
226226

227227

228+
class TestBatchSamplerTorchPositionalArg(TestBatchSampler):
229+
def init_batch_sampler(self):
230+
dataset = RandomDataset(1000, 10)
231+
sampler = SequenceSampler(dataset)
232+
bs = BatchSampler(sampler, self.batch_size, self.drop_last)
233+
return bs
234+
235+
236+
class TestBatchSamplerTorchPositionalArgWithIterableSampler(TestBatchSampler):
237+
def init_batch_sampler(self):
238+
sampler = range(1000)
239+
bs = BatchSampler(sampler, self.batch_size, self.drop_last)
240+
return bs
241+
242+
243+
class TestBatchSamplerPositionalArgError(TestBatchSampler):
244+
def init_batch_sampler(self):
245+
dataset = RandomDataset(1000, 10)
246+
sampler = SequenceSampler(dataset)
247+
bs = BatchSampler(
248+
sampler, self.batch_size, self.drop_last, self.shuffle
249+
)
250+
return bs
251+
252+
def test_main(self):
253+
try:
254+
bs = self.init_batch_sampler()
255+
self.assertTrue(False)
256+
except TypeError:
257+
pass
258+
259+
228260
class TestBatchSamplerWithSamplerDropLast(unittest.TestCase):
229261
def setUp(self):
230262
self.num_samples = 1000

0 commit comments

Comments
 (0)