Merge pull request #865 from ninan-nn/hotfix/hotfix_pool

fix(sdks): fix backoff

Author: Pangjiping

sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/ReconcileState.kt

@@ -28,7 +28,7 @@ import java.time.Instant
 internal class ReconcileState(
     private val degradedThreshold: Int,
     private val backoffBase: Duration = Duration.ofSeconds(1),
-    private val backoffMax: Duration = Duration.ofSeconds(60),
+    private val backoffMax: Duration = Duration.ofDays(1),
 ) {
     @Volatile
     var failureCount: Int = 0
@@ -63,7 +63,7 @@ internal class ReconcileState(
         if (failureCount >= degradedThreshold) {
             state = PoolState.DEGRADED
             backoffAttempts++
-            val exponent = backoffAttempts.coerceAtMost(10)
+            val exponent = backoffAttempts.coerceAtMost(30)
             val delaySeconds = backoffBase.seconds * (1L shl exponent)
             val delayMs =
                 minOf(

sdks/sandbox/kotlin/sandbox/src/test/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/pool/PoolReconcilerStateTest.kt

@@ -44,6 +44,18 @@ class PoolReconcilerStateTest {
         assertEquals(3, state.failureCount)
     }
 
+    @Test
+    fun `default degraded backoff caps at one day`() {
+        val state = ReconcileState(degradedThreshold = 1)
+
+        repeat(20) { state.recordFailure("boom") }
+
+        assertEquals(PoolState.DEGRADED, state.state)
+        assertEquals(20, state.failureCount)
+        assertEquals(true, state.isBackoffActive(Instant.now().plus(Duration.ofHours(23))))
+        assertFalse(state.isBackoffActive(Instant.now().plus(Duration.ofHours(25))))
+    }
+
     @Test
     fun `reconcile create exception increments failure count once per task`() {
         val stateStore = InMemoryPoolStateStore()

sdks/sandbox/python/src/opensandbox/_pool_reconciler.py

@@ -32,7 +32,7 @@
 class ReconcileState:
     degraded_threshold: int
     backoff_base: timedelta = timedelta(seconds=1)
-    backoff_max: timedelta = timedelta(seconds=60)
+    backoff_max: timedelta = timedelta(days=1)
     failure_count: int = 0
     state: PoolState = PoolState.HEALTHY
     last_error: str | None = None
@@ -53,7 +53,7 @@ def record_failure(self, error_message: str | None) -> None:
         if self.failure_count >= self.degraded_threshold:
             self.state = PoolState.DEGRADED
             self.backoff_attempts += 1
-            exponent = min(self.backoff_attempts, 10)
+            exponent = min(self.backoff_attempts, 30)
             delay = min(
                 self.backoff_base.total_seconds() * (1 << exponent),
                 self.backoff_max.total_seconds(),

sdks/sandbox/python/tests/test_pool_sync.py

@@ -2,12 +2,13 @@
 
 import threading
 import time
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Any, cast
 
 import httpx
 import pytest
 
+from opensandbox._pool_reconciler import ReconcileState
 from opensandbox.config.connection_sync import ConnectionConfigSync
 from opensandbox.exceptions import (
     PoolAcquireFailedException,
@@ -19,6 +20,17 @@
 from opensandbox.sync.pool import SandboxPoolSync
 
 
+def test_degraded_backoff_caps_at_one_day() -> None:
+    state = ReconcileState(degraded_threshold=1)
+
+    for _ in range(20):
+        state.record_failure("boom")
+
+    assert state.failure_count == 20
+    assert state.is_backoff_active(datetime.now(timezone.utc) + timedelta(hours=23))
+    assert not state.is_backoff_active(datetime.now(timezone.utc) + timedelta(hours=25))
+
+
 def test_acquire_fail_fast_empty_raises_pool_empty() -> None:
     pool = _create_pool(max_idle=0)
     pool.start()