feat: add retry decorator configuration and update deployment templates

Author: a-klos

infrastructure/rag/templates/_helpers.tpl

@@ -10,6 +10,10 @@
 {{- printf "%s-usecase-configmap" .Release.Name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{- define "configmap.retryDecoratorName" -}}
+{{- printf "%s-retry-decorator-configmap" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{- define "secret.usecaseName" -}}
 {{- printf "%s-usecase-secret" .Release.Name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

infrastructure/rag/templates/admin-backend/deployment.yaml

@@ -106,6 +106,8 @@ spec:
               name: {{ template "configmap.keyValueStoreName" . }}
           - configMapRef:
               name: {{ template "configmap.sourceUploaderName" . }}
+          - configMapRef:
+              name: {{ template "configmap.retryDecoratorName" . }}
           - secretRef:
               name: {{ template "secret.langfuseName" . }}
           - secretRef:

infrastructure/rag/templates/backend/deployment.yaml

@@ -131,6 +131,8 @@ spec:
               name: {{ template "configmap.fakeEmbedderName" . }}
           - configMapRef:
               name: {{ template "configmap.chatHistoryName" . }}
+          - configMapRef:
+              name: {{ template "configmap.retryDecoratorName" . }}
           - secretRef:
               name: {{ template "secret.langfuseName" . }}
           - secretRef:

infrastructure/rag/templates/configmap.yaml

@@ -24,3 +24,12 @@ data:
   {{- range $key, $value := .Values.shared.envs.usecase }}
   {{ $key }}: {{ $value | quote }}
   {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "configmap.retryDecoratorName" . }}
+data:
+  {{- range $key, $value := .Values.shared.envs.retryDecorator }}
+  {{ $key }}: {{ $value | quote }}
+  {{- end }}

infrastructure/rag/values.yaml

@@ -441,6 +441,14 @@ shared:
     s3:
       S3_ENDPOINT: http://rag-minio:9000
       S3_BUCKET: documents
+    retryDecorator:
+      RETRY_DECORATOR_MAX_RETRIES: "5"
+      RETRY_DECORATOR_RETRY_BASE_DELAY: "0.5"
+      RETRY_DECORATOR_RETRY_MAX_DELAY: "600"
+      RETRY_DECORATOR_BACKOFF_FACTOR: "2"
+      RETRY_DECORATOR_ATTEMPT_CAP: "6"
+      RETRY_DECORATOR_JITTER_MIN: "0.05"
+      RETRY_DECORATOR_JITTER_MAX: "0.25"
     usecase:
 
 

libs/rag-core-lib/src/rag_core_lib/impl/settings/retry_decorator_settings.py

@@ -26,10 +26,8 @@ class RetryDecoratorSettings(BaseSettings):
         Maximum jitter to add to wait times.
     """
 
-    # Pydantic v2 settings configuration
     model_config = SettingsConfigDict(env_prefix="RETRY_DECORATOR_", case_sensitive=False)
 
-    # Constrained fields
     max_retries: PositiveInt = Field(
         default=5,
         title="Max Retries",

libs/rag-core-lib/tests/retry_decorator_test.py

@@ -1,5 +1,4 @@
 import asyncio
-import logging
 import time
 from typing import Optional
 
@@ -19,160 +18,138 @@ def __init__(self, headers: Optional[dict[str, str]] = None, status_code: Option
         super().__init__("rate limit")
 
 
-@pytest.mark.asyncio
-async def test_async_success_first_try():
-    calls = {"n": 0}
+@pytest.fixture
+def counter():
+    class C:
+        def __init__(self) -> None:
+            self.n = 0
 
-    @retry_with_backoff(settings=RetryDecoratorSettings(max_retries=2))
-    async def fn():
-        calls["n"] += 1
-        return 42
+        def inc(self) -> int:
+            self.n += 1
+            return self.n
 
-    assert await fn() == 42
-    assert calls["n"] == 1
+    return C()
 
 
-@pytest.mark.asyncio
-async def test_async_retries_then_success(monkeypatch):
-    calls = {"n": 0}
-    slept = []
+@pytest.fixture
+def async_sleeps(monkeypatch):
+    sleeps: list[float] = []
 
     async def fake_sleep(x):
-        slept.append(x)
+        sleeps.append(x)
 
     monkeypatch.setattr(asyncio, "sleep", fake_sleep)
+    return sleeps
+
+
+@pytest.fixture
+def sync_sleeps(monkeypatch):
+    sleeps: list[float] = []
+
+    def fake_sleep(x):
+        sleeps.append(x)
+
+    monkeypatch.setattr(time, "sleep", fake_sleep)
+    return sleeps
+
 
+@pytest.mark.asyncio
+async def test_async_success_first_try(counter):
+    @retry_with_backoff(settings=RetryDecoratorSettings(max_retries=2))
+    async def fn():
+        counter.inc()
+        return 42
+
+    assert await fn() == 42
+    assert counter.n == 1
+
+
+@pytest.mark.asyncio
+async def test_async_retries_then_success(counter, async_sleeps):
     @retry_with_backoff(settings=RetryDecoratorSettings(max_retries=3, retry_base_delay=0.01))
     async def fn():
-        calls["n"] += 1
-        if calls["n"] < 3:
+        if counter.inc() < 3:
             raise DummyError("boom")
         return "ok"
 
     assert await fn() == "ok"
-    assert calls["n"] == 3
-    # Expect at least two sleeps due to two failures
-    assert len(slept) >= 2
+    assert counter.n == 3
+    assert len(async_sleeps) >= 2  # two failures -> two sleeps
 
 
-def test_sync_success_first_try():
-    calls = {"n": 0}
-
+def test_sync_success_first_try(counter):
     @retry_with_backoff(settings=RetryDecoratorSettings(max_retries=2))
     def fn():
-        calls["n"] += 1
+        counter.inc()
         return 7
 
     assert fn() == 7
-    assert calls["n"] == 1
-
-
-def test_sync_retries_then_fail(monkeypatch):
-    calls = {"n": 0}
-    slept = []
+    assert counter.n == 1
 
-    def fake_sleep(x):
-        slept.append(x)
-
-    monkeypatch.setattr(time, "sleep", fake_sleep)
 
+def test_sync_retries_then_fail(counter, sync_sleeps):
     @retry_with_backoff(settings=RetryDecoratorSettings(max_retries=2, retry_base_delay=0.01))
     def fn():
-        calls["n"] += 1
+        counter.inc()
         raise DummyError("always")
 
     with pytest.raises(DummyError):
         fn()
-    # 1 initial + 2 retries = 3 calls total
-    assert calls["n"] == 3
-    # Two sleeps (after two failures)
-    assert len(slept) == 2
+    assert counter.n == 3  # 1 initial + 2 retries
+    assert len(sync_sleeps) == 2
 
 
 @pytest.mark.asyncio
-async def test_async_rate_limit_uses_header_wait(monkeypatch):
-    calls = {"n": 0}
-    slept = []
-
-    async def fake_sleep(x):
-        slept.append(x)
-
-    monkeypatch.setattr(asyncio, "sleep", fake_sleep)
-
-    headers = {
-        "x-ratelimit-reset-requests": "1.5s",
-        "x-ratelimit-remaining-requests": "0",
-    }
+async def test_async_rate_limit_uses_header_wait(counter, async_sleeps):
+    headers = {"x-ratelimit-reset-requests": "1.5s", "x-ratelimit-remaining-requests": "0"}
 
     @retry_with_backoff(
         settings=RetryDecoratorSettings(max_retries=1, retry_base_delay=0.01),
         rate_limit_exceptions=(RateLimitError,),
     )
     async def fn():
-        calls["n"] += 1
-        if calls["n"] == 1:
+        if counter.inc() == 1:
             raise RateLimitError(headers=headers, status_code=429)
         return "ok"
 
     out = await fn()
     assert out == "ok"
-    assert calls["n"] == 2
-    # Should sleep roughly ~1.5s (+jitter). Verify >= 1.5
-    assert any(x >= 1.5 for x in slept)
+    assert counter.n == 2
+    assert any(x >= 1.5 for x in async_sleeps)
 
 
 @pytest.mark.asyncio
-async def test_is_rate_limited_callback(monkeypatch):
-    calls = {"n": 0}
-    slept = []
-
-    async def fake_sleep(x):
-        slept.append(x)
-
-    monkeypatch.setattr(asyncio, "sleep", fake_sleep)
-
-    def mark_rate_limited(exc: BaseException) -> bool:
+async def test_is_rate_limited_callback(counter, async_sleeps):
+    def mark_rate_limited(exc: BaseException) -> bool:  # noqa: ANN001 - explicit for clarity
         return isinstance(exc, DummyError)
 
     @retry_with_backoff(
         settings=RetryDecoratorSettings(max_retries=1, retry_base_delay=0.01),
         is_rate_limited=mark_rate_limited,
     )
     async def fn():
-        calls["n"] += 1
-        if calls["n"] == 1:
+        if counter.inc() == 1:
             raise DummyError("treated as rate limited")
         return "ok"
 
     out = await fn()
     assert out == "ok"
-    assert calls["n"] == 2
-    assert len(slept) == 1
+    assert counter.n == 2
+    assert len(async_sleeps) == 1
 
 
-def test_sync_rate_limit_headers(monkeypatch):
-    calls = {"n": 0}
-    slept = []
-
-    def fake_sleep(x):
-        slept.append(x)
-
-    monkeypatch.setattr(time, "sleep", fake_sleep)
-
-    headers = {
-        "x-ratelimit-reset-requests": "2s",
-    }
+def test_sync_rate_limit_headers(counter, sync_sleeps):
+    headers = {"x-ratelimit-reset-requests": "2s"}
 
     @retry_with_backoff(
         settings=RetryDecoratorSettings(max_retries=1, retry_base_delay=0.01),
         rate_limit_exceptions=(RateLimitError,),
     )
     def fn():
-        calls["n"] += 1
-        if calls["n"] == 1:
+        if counter.inc() == 1:
             raise RateLimitError(headers=headers, status_code=429)
         return "ok"
 
     assert fn() == "ok"
-    assert calls["n"] == 2
-    assert any(x >= 2.0 for x in slept)
+    assert counter.n == 2
+    assert any(x >= 2.0 for x in sync_sleeps)