Merge branch 'allowablepixels' of github.com:Open-EO/openeo-python-client into allowablepixels

Author: manugv

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `openeo.testing.io.TestDataLoader`: unit test utility to compactly load (and optionally preprocess) tests data (text/JSON/...)
+- `openeo.Connection`: automatically retry API requests on `429 Too Many Requests` HTTP errors, with appropriate delay if possible ([#441](https://github.com/Open-EO/openeo-python-client/issues/441))
+
 ### Changed
 
 - `DataCube.apply_dimension()`: not explicitly specifying the `dimension` argument is deprecated and will trigger warnings ([#774](https://github.com/Open-EO/openeo-python-client/issues/774))

Jenkinsfile

@@ -23,4 +23,5 @@ pythonPipeline {
     "PYTEST_DEBUG_TEMPROOT=pytest-tmp",
   ]
   pre_test_script = 'pre_test.sh'
+  enable_uv = true
 }

docs/basics.rst

@@ -317,7 +317,7 @@ and observations are filtered out:
 
 .. image:: _static/images/basics/evi-masked-composite.png
 
-.. _aggregate-spatial-evi
+.. _aggregate-spatial-evi:
 
 Aggregated EVI timeseries
 ===========================

docs/conf.py

@@ -41,6 +41,7 @@
     'sphinx.ext.viewcode',
     'sphinx.ext.doctest',
     'myst_parser',
+    "sphinx.ext.intersphinx",
 ]
 
 import sphinx_autodoc_typehints
@@ -194,3 +195,13 @@
      author, 'openeo', 'One line description of project.',
      'Miscellaneous'),
 ]
+
+
+# Mapping for external documentation
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "numpy": ("https://numpy.org/doc/stable/", None),
+    "xarray": ("https://docs.xarray.dev/en/stable/", None),
+    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
+    "urllib3": ("https://urllib3.readthedocs.io/en/stable/", None),
+}

docs/cookbook/job_manager.rst

@@ -1,4 +1,4 @@
-.. _job-manager
+.. _job-manager:
 
 ====================================
 Multi Backend Job Manager

docs/datacube_construction.rst

@@ -200,6 +200,7 @@ TODO
 
 
 .. _multi-result-process-graphs:
+
 Building process graphs with multiple result nodes
 ===================================================
 

docs/udf.rst

@@ -374,11 +374,10 @@ the datacube.
 
 
 .. warning::
-
-The ``apply_neighborhood`` is the most versatile, but also most complex process. Make sure to keep an eye on the dimensions
-and the shape of the DataArray returned by your UDF. For instance, a very common error is to somehow 'flip' the spatial dimensions.
-Debugging the UDF locally can help, but then you will want to try and reproduce the input that you get also on the backend.
-This can typically be achieved by using logging to inspect the DataArrays passed into your UDF backend side.
+    The ``apply_neighborhood`` is the most versatile, but also most complex process. Make sure to keep an eye on the dimensions
+    and the shape of the DataArray returned by your UDF. For instance, a very common error is to somehow 'flip' the spatial dimensions.
+    Debugging the UDF locally can help, but then you will want to try and reproduce the input that you get also on the backend.
+    This can typically be achieved by using logging to inspect the DataArrays passed into your UDF backend side.
 
 
 

openeo/_version.py

@@ -1 +1 @@
-__version__ = "0.42.0a1"
+__version__ = "0.42.0a2"

openeo/extra/job_management/__init__.py

@@ -29,7 +29,8 @@
 import shapely.errors
 import shapely.geometry.base
 import shapely.wkt
-from requests.adapters import HTTPAdapter, Retry
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
 from openeo import BatchJob, Connection
 from openeo.internal.processes.parse import (
@@ -284,7 +285,7 @@ def _make_resilient(connection):
         503 Service Unavailable
         504 Gateway Timeout
         """
-        # TODO: refactor this helper out of this class and unify with `openeo_driver.util.http.requests_with_retry`
+        # TODO: migrate this to now built-in retry configuration of `Connection` or `openeo.util.http.retry_adapter`?
         status_forcelist = [500, 502, 503, 504]
         retries = Retry(
             total=MAX_RETRIES,

openeo/rest/_connection.py

@@ -5,13 +5,15 @@
 from typing import Iterable, Optional, Union
 
 import requests
+import urllib3.util
 from requests import Response
 from requests.auth import AuthBase
 
 import openeo
 from openeo.rest import OpenEoApiError, OpenEoApiPlainError, OpenEoRestError
 from openeo.rest.auth.auth import NullAuth
 from openeo.util import ContextTimer, ensure_list, str_truncate, url_join
+from openeo.utils.http import HTTP_502_BAD_GATEWAY, session_with_retries
 
 _log = logging.getLogger(__name__)
 
@@ -26,15 +28,22 @@ class RestApiConnection:
     def __init__(
         self,
         root_url: str,
+        *,
         auth: Optional[AuthBase] = None,
         session: Optional[requests.Session] = None,
         default_timeout: Optional[int] = None,
         slow_response_threshold: Optional[float] = None,
+        retry: Union[urllib3.util.Retry, dict, bool, None] = None,
     ):
         self._root_url = root_url
         self._auth = None
         self.auth = auth or NullAuth()
-        self.session = session or requests.Session()
+        if session:
+            self.session = session
+        elif retry is not False:
+            self.session = session_with_retries(retry=retry)
+        else:
+            self.session = requests.Session()
         self.default_timeout = default_timeout or DEFAULT_TIMEOUT
         self.default_headers = {
             "User-Agent": "openeo-python-client/{cv} {py}/{pv} {pl}".format(
@@ -165,7 +174,7 @@ def _raise_api_error(self, response: requests.Response):
         _log.warning(f"Failed to parse API error response: [{status_code}] {text!r} (headers: {response.headers})")
 
         # TODO: eliminate this VITO-backend specific error massaging?
-        if status_code == 502 and "Proxy Error" in text:
+        if status_code == HTTP_502_BAD_GATEWAY and "Proxy Error" in text:
             error_message = (
                 "Received 502 Proxy Error."
                 " This typically happens when a synchronous openEO processing request takes too long and is aborted."