App Configuration - Startup Retry (#44873)

* Adding StartupOptions

* Adding Tests

* Update CHANGELOG.md

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fixing checks

* Update README.md

* review items

* review updates

* remove is failoverable change

* Update test_startup_retry.py

* Fixing formatting

* remove 404 logic

* Update _azureappconfigurationproviderasync.py

* remove broad exception

* Updated startup_timeout check

* rename of backoff items

* Update test_startup_retry.py

* Update _azureappconfigurationproviderbase.py

* Update test_startup_retry.py

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: mrm9084

sdk/appconfiguration/azure-appconfiguration-provider/CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features Added
 
+- Added startup retry, if the initial load fails, the provider will retry until the startup timeout is reached. By default the retry period is 100s, and can be configured via the `startup_timeout` kwarg on the `load` method.
 - Adds support for adding `audience` to the kwargs for `load` allowing it to specify the audience for the request.
 
 ### Breaking Changes

sdk/appconfiguration/azure-appconfiguration-provider/azure/appconfiguration/provider/_azureappconfigurationprovider.py

@@ -4,6 +4,7 @@
 # license information.
 # -------------------------------------------------------------------------
 import datetime
+import time
 import logging
 from typing import (
     Any,
@@ -27,13 +28,15 @@
 from ._constants import (
     FEATURE_MANAGEMENT_KEY,
     FEATURE_FLAG_KEY,
+    DEFAULT_STARTUP_TIMEOUT,
     SNAPSHOT_REF_CONTENT_TYPE,
 )
 from ._azureappconfigurationproviderbase import (
     AzureAppConfigurationProviderBase,
     delay_failure,
     process_load_parameters,
     sdk_allowed_kwargs,
+    _get_startup_backoff,
 )
 from ._client_manager import ConfigurationClientManager, _ConfigurationClientWrapper as ConfigurationClient
 from ._user_agent import USER_AGENT
@@ -60,6 +63,7 @@ def load(  # pylint: disable=docstring-keyword-should-match-keyword-only
     feature_flag_enabled: bool = False,
     feature_flag_selectors: Optional[List[SettingSelector]] = None,
     feature_flag_refresh_enabled: bool = False,
+    startup_timeout: int = DEFAULT_STARTUP_TIMEOUT,
     **kwargs,
 ) -> "AzureAppConfigurationProvider":
     """
@@ -104,6 +108,9 @@ def load(  # pylint: disable=docstring-keyword-should-match-keyword-only
     :keyword configuration_mapper: Optional function to map configuration settings. Enables transformation of
     configurations before they are added to the provider.
     :paramtype configuration_mapper: Optional[Callable[[ConfigurationSetting], None]]
+    :keyword startup_timeout: The amount of time in seconds allowed to load data from Azure App Configuration on
+     startup. The default value is 100 seconds.
+    :paramtype startup_timeout: int
     """
 
 
@@ -124,6 +131,7 @@ def load(  # pylint: disable=docstring-keyword-should-match-keyword-only
     feature_flag_enabled: bool = False,
     feature_flag_selectors: Optional[List[SettingSelector]] = None,
     feature_flag_refresh_enabled: bool = False,
+    startup_timeout: int = DEFAULT_STARTUP_TIMEOUT,
     **kwargs,
 ) -> "AzureAppConfigurationProvider":
     """
@@ -170,6 +178,9 @@ def load(  # pylint: disable=docstring-keyword-should-match-keyword-only
     :keyword configuration_mapper: Optional function to map configuration settings. Enables transformation of
     configurations before they are added to the provider.
     :paramtype configuration_mapper: Optional[Callable[[ConfigurationSetting], None]]
+    :keyword startup_timeout: The amount of time in seconds allowed to load data from Azure App Configuration on
+     startup. The default value is 100 seconds.
+    :paramtype startup_timeout: int
     """
 
 
@@ -184,6 +195,7 @@ def load(*args, **kwargs) -> "AzureAppConfigurationProvider":
         params["endpoint"],
         params["credential"],
         uses_key_vault=params["uses_key_vault"],
+        startup_timeout=params["startup_timeout"],
         **params["kwargs"],
     )
     kwargs = sdk_allowed_kwargs(params["kwargs"])
@@ -234,6 +246,8 @@ def __init__(self, **kwargs: Any) -> None:
         min_backoff: int = min(kwargs.pop("min_backoff", 30), interval)
         max_backoff: int = min(kwargs.pop("max_backoff", 600), interval)
 
+        self._startup_timeout: int = kwargs.pop("startup_timeout", DEFAULT_STARTUP_TIMEOUT)
+
         self._replica_client_manager = ConfigurationClientManager(
             connection_string=kwargs.pop("connection_string"),
             endpoint=kwargs.pop("endpoint"),
@@ -363,16 +377,47 @@ def refresh(self, **kwargs) -> None:
             self._refresh_lock.release()
 
     def _load_all(self, **kwargs: Any) -> None:
+        startup_start_time = datetime.datetime.now()
+        exponential_backoff_attempts = 0
+        startup_exceptions: List[Exception] = []
+
+        while True:
+            # Try to initialize from all available clients
+            if self._try_initialize(startup_exceptions, **kwargs):
+                return  # Successfully loaded
+
+            # Calculate delay before next retry attempt
+            elapsed_seconds = (datetime.datetime.now() - startup_start_time).total_seconds()
+            delay, is_exponential_backoff = _get_startup_backoff(elapsed_seconds, exponential_backoff_attempts)
+
+            if is_exponential_backoff:
+                exponential_backoff_attempts += 1
+
+            # Check if delay would exceed remaining timeout
+            remaining_timeout = self._startup_timeout - elapsed_seconds
+            if delay > remaining_timeout:
+                raise TimeoutError(
+                    "The provider timed out while attempting to load.",
+                    startup_exceptions,
+                )
+
+            if delay > 0:
+                time.sleep(delay)
+
+    def _try_initialize(self, startup_exceptions: List[Exception], **kwargs: Any) -> bool:
+        """
+        Try to initialize the provider from all available clients.
+
+        :param startup_exceptions: List to collect exceptions from failed attempts.
+        :type startup_exceptions: List[Exception]
+        :return: True if initialization succeeded, False otherwise.
+        :rtype: bool
+        """
         self._replica_client_manager.refresh_clients()
         self._replica_client_manager.find_active_clients()
         is_failover_request = False
         replica_count = self._replica_client_manager.get_client_count() - 1
 
-        error_message = """
-        Failed to load configuration settings. No Azure App Configuration stores successfully loaded from.
-        """
-        exception: Exception = RuntimeError(error_message)
-
         while client := self._replica_client_manager.get_next_active_client():
             headers = self._update_correlation_context_header(
                 kwargs.pop("headers", {}),
@@ -415,13 +460,15 @@ def _load_all(self, **kwargs: Any) -> None:
                 with self._update_lock:
                     self._watched_settings = watched_settings
                     self._dict = processed_settings
-                return
+                return True
             except AzureError as e:
-                exception = e
                 logger.warning("Failed to load configurations from endpoint %s.\n %s", client.endpoint, e.message)
                 self._replica_client_manager.backoff(client)
                 is_failover_request = True
-        raise exception
+
+                startup_exceptions.append(e)
+
+        return False
 
     def _expand_snapshot_references(
         self, configuration_settings: List[ConfigurationSetting], client: ConfigurationClient

sdk/appconfiguration/azure-appconfiguration-provider/azure/appconfiguration/provider/_azureappconfigurationproviderbase.py

@@ -8,6 +8,7 @@
 import json
 import time
 import datetime
+import random
 from threading import Lock
 import logging
 from typing import (
@@ -41,6 +42,11 @@
     APP_CONFIG_AICC_MIME_PROFILE,
     FEATURE_MANAGEMENT_KEY,
     FEATURE_FLAG_KEY,
+    DEFAULT_STARTUP_TIMEOUT,
+    MAX_STARTUP_BACKOFF_DURATION,
+    MIN_STARTUP_EXPONENTIAL_BACKOFF_DURATION,
+    JITTER_RATIO,
+    STARTUP_BACKOFF_INTERVALS,
 )
 from ._refresh_timer import _RefreshTimer
 from ._request_tracing_context import _RequestTracingContext
@@ -122,12 +128,18 @@ def process_load_parameters(*args, **kwargs: Any) -> Dict[str, Any]:
         or kwargs.get("uses_key_vault", False)
     )
 
+    # Get startup timeout
+    startup_timeout = kwargs.pop("startup_timeout", DEFAULT_STARTUP_TIMEOUT)
+    if startup_timeout < 0:
+        raise ValueError("Startup timeout must be greater than or equal to 0 seconds.")
+
     return {
         "endpoint": endpoint,
         "credential": credential,
         "connection_string": connection_string,
         "uses_key_vault": uses_key_vault,
         "start_time": start_time,
+        "startup_timeout": startup_timeout,
         "kwargs": kwargs,
     }
 
@@ -199,6 +211,71 @@ def sdk_allowed_kwargs(kwargs):
     return {k: v for k, v in kwargs.items() if k in allowed_kwargs}
 
 
+def _jitter(duration: float, ratio: float = JITTER_RATIO) -> float:
+    """
+    Apply jitter to a duration value.
+
+    :param duration: The base duration in seconds.
+    :type duration: float
+    :param ratio: The jitter ratio (0 to 1). Default is 0.25 (25% jitter means +/- 25% variation).
+    :type ratio: float
+    :return: The jittered duration in seconds.
+    :rtype: float
+    """
+    if ratio < 0 or ratio > 1:
+        raise ValueError("Jitter ratio must be between 0 and 1.")
+    if ratio == 0:
+        return duration
+    jitter = ratio * (random.random() * 2 - 1)
+    return duration * (1 + jitter)
+
+
+def _get_startup_backoff(elapsed_seconds: float, attempts: int) -> Tuple[float, bool]:
+    """
+    Get a backoff duration based on elapsed startup time.
+
+    :param elapsed_seconds: The time elapsed since startup began, in seconds.
+    :type elapsed_seconds: float
+    :param attempts: The number of retry attempts made (1-based).
+    :type attempts: int
+    :return: A tuple where the first element is the backoff duration in seconds,
+             and the second element indicates if the fixed backoff window has been exceeded.
+    :rtype: Tuple[float, bool]
+    """
+    for threshold, backoff in STARTUP_BACKOFF_INTERVALS:
+        if elapsed_seconds < threshold:
+            return backoff, False
+    return _calculate_backoff_duration(attempts), True
+
+
+def _calculate_backoff_duration(attempts: int) -> float:
+    """
+    Calculate the jittered exponential backoff duration.
+
+    :param attempts: The number of retry attempts made (1-based).
+    :type attempts: int
+    :return: The calculated backoff duration with jitter applied.
+    :rtype: float
+    """
+    attempts += 1
+    if attempts < 1:
+        raise ValueError("Number of attempts must be at least 1.")
+
+    if attempts == 1:
+        return MIN_STARTUP_EXPONENTIAL_BACKOFF_DURATION
+
+    # Calculate exponential backoff: min * 2^(attempts-1)
+    # Cap the shift amount to prevent overflow
+    safe_shift = min(attempts - 1, 63)
+    calculated = MIN_STARTUP_EXPONENTIAL_BACKOFF_DURATION * (1 << safe_shift)
+
+    # Cap at max duration
+    if calculated > MAX_STARTUP_BACKOFF_DURATION or calculated <= 0:  # Check for overflow
+        calculated = MAX_STARTUP_BACKOFF_DURATION
+
+    return _jitter(calculated, JITTER_RATIO)
+
+
 class AzureAppConfigurationProviderBase(Mapping[str, Union[str, JSON]]):  # pylint: disable=too-many-instance-attributes
     """
     Provides a dictionary-like interface to Azure App Configuration settings. Enables loading of sets of configuration

sdk/appconfiguration/azure-appconfiguration-provider/azure/appconfiguration/provider/_constants.py

@@ -37,11 +37,32 @@
 # ------------------------------------------------------------------------
 APP_CONFIG_AI_MIME_PROFILE = "https://azconfig.io/mime-profiles/ai/"
 APP_CONFIG_AICC_MIME_PROFILE = "https://azconfig.io/mime-profiles/ai/chat-completion"
-SNAPSHOT_REF_CONTENT_TYPE = 'application/json; profile="https://azconfig.io/mime-profiles/snapshot-ref"; charset=utf-8'
+
+# =============================================================================
+# Startup Retry Constants
+# =============================================================================
+# Timeout
+DEFAULT_STARTUP_TIMEOUT = 100  # seconds
+
+# Backoff durations
+MIN_STARTUP_EXPONENTIAL_BACKOFF_DURATION = 30  # seconds - minimum backoff after fixed window
+MAX_STARTUP_BACKOFF_DURATION = 600  # seconds (10 minutes) - caps exponential backoff
+
+# Jitter ratio for randomizing backoff durations (+/- 25% variation)
+JITTER_RATIO = 0.25
+
+# Fixed backoff intervals: (elapsed_time_threshold, backoff_duration)
+# Defines fixed backoff durations based on how long startup has been attempting
+STARTUP_BACKOFF_INTERVALS = [
+    (100, 5),  # 0-100 seconds elapsed: 5 second backoff
+    (200, 10),  # 100-200 seconds elapsed: 10 second backoff
+    (600, MIN_STARTUP_EXPONENTIAL_BACKOFF_DURATION),  # 200-600 seconds elapsed: 30 second backoff
+]
 
 # ------------------------------------------------------------------------
 # Snapshot Reference Constants
 # ------------------------------------------------------------------------
+SNAPSHOT_REF_CONTENT_TYPE = 'application/json; profile="https://azconfig.io/mime-profiles/snapshot-ref"; charset=utf-8'
 SNAPSHOT_NAME_FIELD = "snapshot_name"
 
 # ------------------------------------------------------------------------