fix: retry and timeout values do not propagate in requests during pagination (#2065)

Author: parthea

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2

@@ -509,6 +509,8 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             method=rpc,
             request=request,
             response=response,
+            retry=retry,
+            timeout=timeout,
             metadata=metadata,
         )
         {% endif %}

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/pagers.py.j2

@@ -7,7 +7,13 @@
 {# This lives within the loop in order to ensure that this template
    is empty if there are no paged methods.
  -#}
-from typing import Any, Callable, Iterable, Sequence, Tuple, Optional, Iterator
+from google.api_core import gapic_v1
+from google.api_core import retry as retries
+from typing import Any, Callable, Iterable, Sequence, Tuple, Optional, Iterator, Union
+try:
+    OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None]
+except AttributeError:  # pragma: NO COVER
+    OptionalRetry = Union[retries.Retry, object, None]  # type: ignore
 
 {% filter sort_lines %}
 {% for method in service.methods.values() | selectattr('paged_result_field') %}
@@ -41,6 +47,8 @@ class {{ method.name }}Pager:
             method: Callable[..., {{ method.output.ident }}],
             request: {{ method.input.ident }},
             response: {{ method.output.ident }},
+            retry: OptionalRetry = gapic_v1.method.DEFAULT,
+            timeout: Union[float, object] = gapic_v1.method.DEFAULT,
             metadata: Sequence[Tuple[str, str]] = ()):
         """Instantiate the pager.
 
@@ -51,12 +59,17 @@ class {{ method.name }}Pager:
                 The initial request object.
             response (:class:`{{ method.output.ident.sphinx }}`):
                 The initial response object.
+            retry (google.api_core.retry.Retry): Designation of what errors,
+                if any, should be retried.
+            timeout (float): The timeout for this request.
             metadata (Sequence[Tuple[str, str]]): Strings which should be
                 sent along with the request as metadata.
         """
         self._method = method
         self._request = {{ method.input.ident }}(request)
         self._response = response
+        self._retry = retry
+        self._timeout = timeout
         self._metadata = metadata
 
     def __getattr__(self, name: str) -> Any:
@@ -67,7 +80,7 @@ class {{ method.name }}Pager:
         yield self._response
         while self._response.next_page_token:
             self._request.page_token = self._response.next_page_token
-            self._response = self._method(self._request, metadata=self._metadata)
+            self._response = self._method(self._request, retry=self._retry, timeout=self._timeout, metadata=self._metadata)
             yield self._response
 
     {% if method.paged_result_field.map %}

gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -45,6 +45,7 @@ from google.api_core import client_options
 from google.api_core import exceptions as core_exceptions
 from google.api_core import grpc_helpers
 from google.api_core import path_template
+from google.api_core import retry as retries
 {% if service.has_lro %}
 from google.api_core import future
 from google.api_core import operation
@@ -938,6 +939,8 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
         )
 
         expected_metadata = ()
+        retry = retries.Retry()
+        timeout = 5
         {% if method.field_headers %}
         expected_metadata = tuple(expected_metadata) + (
             gapic_v1.routing_header.to_grpc_metadata((
@@ -949,7 +952,7 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
             )),
         )
         {% endif %}
-        pager = client.{{ method_name }}(request={})
+        pager = client.{{ method_name }}(request={}, retry=retry, timeout=timeout)
 
         {% if service.version %}
         if HAS_GOOGLE_API_CORE_VERSION_HEADER:
@@ -958,6 +961,8 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
             )
         {% endif %}
         assert pager._metadata == expected_metadata
+        assert pager._retry == retry
+        assert pager._timeout == timeout
 
         results = list(pager)
         assert len(results) == 6

gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2

@@ -217,6 +217,8 @@
             method=rpc,
             request=request,
             response=response,
+            retry=retry,
+            timeout=timeout,
             metadata=metadata,
         )
         {% elif method.extended_lro and full_extended_lro %}

gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2

@@ -414,6 +414,8 @@ class {{ service.async_client_name }}:
             method=rpc,
             request=request,
             response=response,
+            retry=retry,
+            timeout=timeout,
             metadata=metadata,
         )
         {% endif %}

gapic/templates/%namespace/%name_%version/%sub/services/%service/pagers.py.j2

@@ -7,7 +7,15 @@
 {# This lives within the loop in order to ensure that this template
    is empty if there are no paged methods.
  #}
-from typing import Any, AsyncIterator, Awaitable, Callable, Sequence, Tuple, Optional, Iterator
+from google.api_core import gapic_v1
+from google.api_core import retry as retries
+from typing import Any, AsyncIterator, Awaitable, Callable, Sequence, Tuple, Optional, Iterator, Union
+try:
+    OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None]
+    OptionalAsyncRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None]
+except AttributeError:  # pragma: NO COVER
+    OptionalRetry = Union[retries.Retry, object, None]  # type: ignore
+    OptionalAsyncRetry = Union[retries.AsyncRetry, object, None]  # type: ignore
 
 {% filter sort_lines %}
 {% for method in service.methods.values() | selectattr('paged_result_field') %}
@@ -43,6 +51,8 @@ class {{ method.name }}Pager:
             request: {{ method.input.ident }},
             response: {{ method.output.ident }},
             *,
+            retry: OptionalRetry = gapic_v1.method.DEFAULT,
+            timeout: Union[float, object] = gapic_v1.method.DEFAULT,
             metadata: Sequence[Tuple[str, str]] = ()):
         """Instantiate the pager.
 
@@ -53,12 +63,17 @@ class {{ method.name }}Pager:
                 The initial request object.
             response ({{ method.output.ident.sphinx }}):
                 The initial response object.
+            retry (google.api_core.retry.Retry): Designation of what errors,
+                if any, should be retried.
+            timeout (float): The timeout for this request.
             metadata (Sequence[Tuple[str, str]]): Strings which should be
                 sent along with the request as metadata.
         """
         self._method = method
         self._request = {{ method.input.ident }}(request)
         self._response = response
+        self._retry = retry
+        self._timeout = timeout
         self._metadata = metadata
 
     def __getattr__(self, name: str) -> Any:
@@ -69,7 +84,7 @@ class {{ method.name }}Pager:
         yield self._response
         while self._response.next_page_token:
             self._request.page_token = self._response.next_page_token
-            self._response = self._method(self._request, metadata=self._metadata)
+            self._response = self._method(self._request, retry=self._retry, timeout=self._timeout, metadata=self._metadata)
             yield self._response
 
     {% if method.paged_result_field.map %}
@@ -113,6 +128,8 @@ class {{ method.name }}AsyncPager:
             request: {{ method.input.ident }},
             response: {{ method.output.ident }},
             *,
+            retry: OptionalAsyncRetry = gapic_v1.method.DEFAULT,
+            timeout: Union[float, object] = gapic_v1.method.DEFAULT,
             metadata: Sequence[Tuple[str, str]] = ()):
         """Instantiates the pager.
 
@@ -123,12 +140,17 @@ class {{ method.name }}AsyncPager:
                 The initial request object.
             response ({{ method.output.ident.sphinx }}):
                 The initial response object.
+            retry (google.api_core.retry.AsyncRetry): Designation of what errors,
+                if any, should be retried.
+            timeout (float): The timeout for this request.
             metadata (Sequence[Tuple[str, str]]): Strings which should be
                 sent along with the request as metadata.
         """
         self._method = method
         self._request = {{ method.input.ident }}(request)
         self._response = response
+        self._retry = retry
+        self._timeout = timeout
         self._metadata = metadata
 
     def __getattr__(self, name: str) -> Any:
@@ -139,7 +161,7 @@ class {{ method.name }}AsyncPager:
         yield self._response
         while self._response.next_page_token:
             self._request.page_token = self._response.next_page_token
-            self._response = await self._method(self._request, metadata=self._metadata)
+            self._response = await self._method(self._request, retry=self._retry, timeout=self._timeout, metadata=self._metadata)
             yield self._response
     {% if method.paged_result_field.map %}
     def __aiter__(self) -> Iterator[Tuple[str, {{ method.paged_result_field.type.fields.get('value').ident }}]]:

gapic/templates/noxfile.py.j2

@@ -176,7 +176,9 @@ def cover(session):
 def mypy(session):
     """Run the type checker."""
     session.install(
-        'mypy',
+        # TODO(https://github.com/googleapis/gapic-generator-python/issues/2066):
+        # Ignore release of mypy 1.11.0 which may have a regression
+        'mypy!=1.11.0',
         'types-requests',
         'types-protobuf'
     )

gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -51,6 +51,7 @@ from google.api_core import exceptions as core_exceptions
 from google.api_core import grpc_helpers
 from google.api_core import grpc_helpers_async
 from google.api_core import path_template
+from google.api_core import retry as retries
 {% if service.has_lro or service.has_extended_lro %}
 from google.api_core import future
 {% endif %}

gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2

@@ -850,6 +850,8 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
         )
 
         expected_metadata = ()
+        retry = retries.Retry()
+        timeout = 5
         {% if not method.explicit_routing and method.field_headers %}
         expected_metadata = tuple(expected_metadata) + (
             gapic_v1.routing_header.to_grpc_metadata((
@@ -867,9 +869,11 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
                 version_header.to_api_version_header("{{ service.version }}"),
             )
         {% endif %}
-        pager = client.{{ method_name }}(request={})
+        pager = client.{{ method_name }}(request={}, retry=retry, timeout=timeout)
 
         assert pager._metadata == expected_metadata
+        assert pager._retry == retry
+        assert pager._timeout == timeout
 
         results = list(pager)
         assert len(results) == 6

noxfile.py

@@ -434,7 +434,9 @@ def showcase_mypy(
     """Perform typecheck analysis on the generated Showcase library."""
 
     # Install pytest and gapic-generator-python
-    session.install("mypy", "types-pkg-resources", "types-protobuf", "types-requests", "types-dataclasses")
+    # TODO(https://github.com/googleapis/gapic-generator-python/issues/2066):
+    # Ignore release of mypy 1.11.0 which may have a regression
+    session.install("mypy!=1.11.0", "types-pkg-resources", "types-protobuf", "types-requests", "types-dataclasses")
 
     with showcase_library(session, templates=templates, other_opts=other_opts) as lib:
         session.chdir(lib)