Retry Connect with Exponential Backoff (#12397)

masaori335 · web-flow · commit dd8f0d6eb36b · 2025-07-30T08:18:13.000+09:00
diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst
@@ -1705,6 +1705,14 @@ Origin Server Connect Attempts
    `proxy.config.http.connect_attempts_max_retries`_ so an error is returned to the client faster and also to reduce the load on the down origin.
    The timeout interval `proxy.config.http.connect_attempts_timeout`_ in seconds is used with this setting.
 
+.. ts:cv:: CONFIG proxy.config.http.connect_attempts_retry_backoff_base INT 0
+   :reloadable:
+   :overridable:
+   :units: milliseconds
+
+   The base delay, in milliseconds, used for exponential backoff between retry attempts to connect to the origin server. After each
+   failure the delay doubles - e.g. 64m, 128m, 256m. When this is set to `0` (the default value), |TS| retries immediately without exponential backoff.
+
 .. ts:cv:: CONFIG proxy.config.http.connect.down.policy INT 2
    :overridable:
 
diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst
@@ -118,6 +118,7 @@ TSOverridableConfigKey Value                                              Config
 :enumerator:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES`               :ts:cv:`proxy.config.http.connect_attempts_max_retries`
 :enumerator:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES`                :ts:cv:`proxy.config.http.connect_attempts_rr_retries`
 :enumerator:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT`                   :ts:cv:`proxy.config.http.connect_attempts_timeout`
+:enumerator:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE`        :ts:cv:`proxy.config.http.connect_attempts_retry_backoff_base`
 :enumerator:`TS_CONFIG_HTTP_DEFAULT_BUFFER_SIZE`                        :ts:cv:`proxy.config.http.default_buffer_size`
 :enumerator:`TS_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK`                  :ts:cv:`proxy.config.http.default_buffer_water_mark`
 :enumerator:`TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS`                      :ts:cv:`proxy.config.http.doc_in_cache_skip_dns`
diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst
@@ -81,6 +81,7 @@ Enumeration Members
 .. enumerator:: TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DOWN_SERVER
 .. enumerator:: TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES
 .. enumerator:: TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT
+.. enumerator:: TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE
 .. enumerator:: TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT
 .. enumerator:: TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME
 .. enumerator:: TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS
diff --git a/include/proxy/http/HttpConfig.h b/include/proxy/http/HttpConfig.h
@@ -660,6 +660,7 @@ struct OverridableHttpConfigParams {
   MgmtInt connect_attempts_max_retries_down_server = 3;
   MgmtInt connect_attempts_rr_retries              = 3;
   MgmtInt connect_attempts_timeout                 = 30;
+  MgmtInt connect_attempts_retry_backoff_base      = 0;
 
   MgmtInt connect_down_policy = 2;
 
diff --git a/include/proxy/http/HttpSM.h b/include/proxy/http/HttpSM.h
@@ -58,6 +58,7 @@
 
 #define CONNECT_EVENT_TXN    (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 0
 #define CONNECT_EVENT_DIRECT (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 1
+#define CONNECT_EVENT_RETRY  (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 2
 
 // The default size for http header buffers when we don't
 //   need to include extra space for the document
diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in
@@ -821,6 +821,7 @@ enum TSOverridableConfigKey {
   TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DOWN_SERVER,
   TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES,
   TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT,
+  TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE,
   TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME,
   TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS,
   TS_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT,
diff --git a/plugins/lua/ts_lua_http_config.cc b/plugins/lua/ts_lua_http_config.cc
@@ -74,6 +74,7 @@ typedef enum {
   TS_LUA_CONFIG_HTTP_CONNECT_DOWN_POLICY                      = TS_CONFIG_HTTP_CONNECT_DOWN_POLICY,
   TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES              = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES,
   TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT                 = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT,
+  TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE      = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE,
   TS_LUA_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME                   = TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME,
   TS_LUA_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS                    = TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS,
   TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT           = TS_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT,
@@ -215,6 +216,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_DOWN_POLICY),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT),
+  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT),
diff --git a/src/api/InkAPI.cc b/src/api/InkAPI.cc
@@ -7259,6 +7259,9 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr
   case TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT:
     ret = _memberp_to_generic(&overridableHttpConfig->connect_attempts_timeout, conv);
     break;
+  case TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE:
+    ret = _memberp_to_generic(&overridableHttpConfig->connect_attempts_retry_backoff_base, conv);
+    break;
   case TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME:
     conv = &HttpDownServerCacheTimeConv;
     ret  = &overridableHttpConfig->down_server_timeout;
diff --git a/src/api/InkAPITest.cc b/src/api/InkAPITest.cc
@@ -8704,6 +8704,7 @@ std::array<std::string_view, TS_CONFIG_LAST_ENTRY> SDK_Overridable_Configs = {
    "proxy.config.http.connect_attempts_max_retries_down_server",
    "proxy.config.http.connect_attempts_rr_retries",
    "proxy.config.http.connect_attempts_timeout",
+   "proxy.config.http.connect_attempts_retry_backoff_base",
    "proxy.config.http.down_server.cache_time",
    "proxy.config.http.doc_in_cache_skip_dns",
    "proxy.config.http.background_fill_active_timeout",
diff --git a/src/proxy/http/HttpConfig.cc b/src/proxy/http/HttpConfig.cc
@@ -878,6 +878,8 @@ HttpConfig::startup()
   HttpEstablishStaticConfigLongLong(c.oride.connect_attempts_max_retries, "proxy.config.http.connect_attempts_max_retries");
   HttpEstablishStaticConfigLongLong(c.oride.connect_attempts_max_retries_down_server,
                                     "proxy.config.http.connect_attempts_max_retries_down_server");
+  HttpEstablishStaticConfigLongLong(c.oride.connect_attempts_retry_backoff_base,
+                                    "proxy.config.http.connect_attempts_retry_backoff_base");
 
   HttpEstablishStaticConfigLongLong(c.oride.connect_down_policy, "proxy.config.http.connect.down.policy");
 
@@ -1173,6 +1175,8 @@ HttpConfig::reconfigure()
             "will never redispatch to another server",
             m_master.oride.connect_attempts_rr_retries, params->oride.connect_attempts_max_retries);
   }
+  params->oride.connect_attempts_retry_backoff_base = m_master.oride.connect_attempts_retry_backoff_base;
+
   params->oride.connect_attempts_rr_retries     = m_master.oride.connect_attempts_rr_retries;
   params->oride.connect_attempts_timeout        = m_master.oride.connect_attempts_timeout;
   params->oride.connect_down_policy             = m_master.oride.connect_down_policy;
diff --git a/src/proxy/http/HttpDebugNames.cc b/src/proxy/http/HttpDebugNames.cc
@@ -152,6 +152,16 @@ HttpDebugNames::get_event_name(int event)
     static_assert(static_cast<int>(NET_EVENT_OPEN_FAILED) == static_cast<int>(TS_EVENT_NET_CONNECT_FAILED));
     return "NET_EVENT_OPEN_FAILED/TS_EVENT_NET_CONNECT_FAILED";
 
+  ////////////////////
+  // CONNECT_EVENTS //
+  ////////////////////
+  case CONNECT_EVENT_TXN:
+    return "CONNECT_EVENT_TXN";
+  case CONNECT_EVENT_DIRECT:
+    return "CONNECT_EVENT_DIRECT";
+  case CONNECT_EVENT_RETRY:
+    return "CONNECT_EVENT_RETRY";
+
   ////////////////////
   // HOSTDB  EVENTS //
   ////////////////////
diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc
@@ -23,6 +23,7 @@
  */
 
 #include "proxy/http/HttpConfig.h"
+#include "tscore/ink_hrtime.h"
 #include "tsutil/Metrics.h"
 #include "tsutil/ts_bw_format.h"
 #include "proxy/ProxyTransaction.h"
@@ -1781,6 +1782,11 @@ HttpSM::state_http_server_open(int event, void *data)
 {
   SMDbg(dbg_ctl_http_track, "entered inside state_http_server_open: %s", HttpDebugNames::get_event_name(event));
   STATE_ENTER(&HttpSM::state_http_server_open, event);
+
+  if (event == CONNECT_EVENT_RETRY) {
+    pending_action.clear_if_action_is(reinterpret_cast<Action *>(data));
+  }
+
   ink_release_assert(event == EVENT_INTERVAL || event == NET_EVENT_OPEN || event == NET_EVENT_OPEN_FAILED ||
                      pending_action.empty());
   if (event != NET_EVENT_OPEN) {
@@ -1820,6 +1826,9 @@ HttpSM::state_http_server_open(int event, void *data)
     // Try it again, but direct this time
     do_http_server_open(false, true);
     break;
+  case CONNECT_EVENT_RETRY:
+    do_http_server_open();
+    break;
   case CONNECT_EVENT_TXN:
     SMDbg(dbg_ctl_http, "Connection handshake complete via CONNECT_EVENT_TXN");
     if (this->create_server_txn(static_cast<PoolableSession *>(data))) {
@@ -8080,7 +8089,17 @@ HttpSM::set_next_state()
       }
     }
 
-    do_http_server_open();
+    ink_hrtime backoff_base = t_state.txn_conf->connect_attempts_retry_backoff_base;
+    if (backoff_base > 0) {
+      // Retry with exponential backoff
+      ink_hrtime backoff = backoff_base << t_state.current.retry_attempts.get();
+      SMDbg(dbg_ctl_http_connect, "Retry connect with %" PRId64 " msec backoff", backoff);
+      this->pending_action = nullptr;
+      this->pending_action = this_ethread()->schedule_in(this, HRTIME_MSECONDS(backoff), CONNECT_EVENT_RETRY);
+    } else {
+      do_http_server_open();
+    }
+
     break;
   }
 
diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc
@@ -486,6 +486,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http.connect_attempts_timeout", RECD_INT, "30", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http.connect_attempts_retry_backoff_base", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.http.connect.down.policy", RECD_INT, "2", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.http.down_server.cache_time", RECD_INT, "60", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
diff --git a/src/shared/overridable_txn_vars.cc b/src/shared/overridable_txn_vars.cc
@@ -86,6 +86,8 @@ const std::unordered_map<std::string_view, std::tuple<const TSOverridableConfigK
     {"proxy.config.net.sock_recv_buffer_size_out",                     {TS_CONFIG_NET_SOCK_RECV_BUFFER_SIZE_OUT, TS_RECORDDATATYPE_INT}         },
     {"proxy.config.net.sock_send_buffer_size_out",                     {TS_CONFIG_NET_SOCK_SEND_BUFFER_SIZE_OUT, TS_RECORDDATATYPE_INT}         },
     {"proxy.config.http.connect_attempts_timeout",                     {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT, TS_RECORDDATATYPE_INT}         },
+    {"proxy.config.http.connect_attempts_retry_backoff_base",
+     {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE, TS_RECORDDATATYPE_INT}                                                                },
     {"proxy.config.websocket.no_activity_timeout",                     {TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT, TS_RECORDDATATYPE_INT}         },
     {"proxy.config.http.negative_caching_lifetime",                    {TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME, TS_RECORDDATATYPE_INT}        },
     {"proxy.config.http.default_buffer_water_mark",                    {TS_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK, TS_RECORDDATATYPE_INT}        },
