CDRIVER-6092 CDRIVER-6262 CDRIVER-6268 implement exponential backoff and jitter in retry loops (#2240)

* Implement backpressure prose tests 1-4

* Add backpressure and other related unified tests

* Add mongoc_retry_backoff_generator_t

* Refactor with_transaction to use mongoc_retry_backoff_generator_t

* Add mongoc_token_bucket_t

* Implement overload retry loops

* Add mongoc_retryable_cmd_t

* Refactor retry loops to use mongoc_retryable_cmd_t

* Fix unified test runner

---------

Co-authored-by: Kevin Albertson <kevin.albertson@mongodb.com>

Author: connorsmacd

NEWS

@@ -1,3 +1,15 @@
+libmongoc 2.3.0 [UNRELEASED]
+============================
+
+## New Features
+
+- Support retry for server overload errors.
+    - Supported on all commands.
+    - Custom application retry logic may need to be adjusted to avoid retrying too long.
+    - Upgrade is recommended to avoid impacts of server changes related to overload errors.
+        - If not upgrading, custom application retry logic may need to be adjusted to handle higher rates of overload errors.
+    - Set the URI option `adaptiveRetry=true` limit retries with a token bucket system.
+
 libmongoc 2.2.3
 ===============
 

src/libmongoc/CMakeLists.txt

@@ -567,7 +567,6 @@ set (MONGOC_SOURCES
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-cursor.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-cursor-cmd.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-cursor-change-stream.c
-   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-cursor-cmd-deprecated.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-cursor-find.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-cursor-array.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-database.c
@@ -607,6 +606,8 @@ set (MONGOC_SOURCES
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-read-concern.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-read-prefs.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-rpc.c
+   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-retryable-cmd.c
+   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-retry-backoff-generator.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-server-api.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-server-description.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-server-stream.c
@@ -624,6 +625,7 @@ set (MONGOC_SOURCES
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-timeout.c
+   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-token-bucket.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-background-monitoring.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-description.c
@@ -1091,7 +1093,6 @@ set (test-libmongoc-sources
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-hedged-reads.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-http.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-interrupt.c
-   ${PROJECT_SOURCE_DIR}/tests/test-mongoc-jitter-source.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-linux-distro-scanner.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-list.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-loadbalanced.c
@@ -1108,6 +1109,7 @@ set (test-libmongoc-sources
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-read-concern.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-read-prefs.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-read-write-concern.c
+   ${PROJECT_SOURCE_DIR}/tests/test-mongoc-retry-backoff-generator.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-retryability-helpers.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-retryable-reads.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-retryable-writes.c
@@ -1130,6 +1132,7 @@ set (test-libmongoc-sources
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-structured-log.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-thread.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-timeout.c
+   ${PROJECT_SOURCE_DIR}/tests/test-mongoc-token-bucket.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-topology-description.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-topology-reconcile.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-topology-scanner.c

src/libmongoc/doc/mongoc_uri_t.rst

@@ -97,6 +97,7 @@ Constant                                   Key                               Def
 MONGOC_URI_RETRYREADS                      retryreads                        true                              If "true" and the server is a MongoDB 3.6+ standalone, replica set, or sharded cluster, the driver safely retries a read that failed due to a network error or replica set failover.
 MONGOC_URI_RETRYWRITES                     retrywrites                       true if driver built w/ TLS       If "true" and the server is a MongoDB 3.6+ replica set or sharded cluster, the driver safely retries a write that failed due to a network error or replica set failover. Only inserts, updates of single documents, or deletes of single
                                                                                                                documents are retried.
+MONGOC_URI_ADAPTIVERETRIES                 adaptiveretries                   false                             If "true", the driver will limit retry attempts during periods of prolonged server overload.
 MONGOC_URI_APPNAME                         appname                           Empty (no appname)                The client application name. This value is used by MongoDB when it logs connection information and profile information, such as slow queries.
 MONGOC_URI_TLS                             tls                               Empty (not set, same as false)    {true|false}, indicating if TLS must be used. (See also :symbol:`mongoc_client_set_ssl_opts` and :symbol:`mongoc_client_pool_set_ssl_opts`.)
 MONGOC_URI_COMPRESSORS                     compressors                       Empty (no compressors)            Comma separated list of compressors, if any, to use to compress the wire protocol messages. Snappy, zlib, and zstd are optional build time dependencies, and enable the "snappy", "zlib", and "zstd" values respectively.

src/libmongoc/src/mongoc/mongoc-bulkwrite.c

@@ -1853,8 +1853,14 @@ mongoc_bulkwrite_execute(mongoc_bulkwrite_t *self, const mongoc_bulkwriteopts_t
          // Send command.
          {
             mongoc_server_stream_t *new_ss = NULL;
-            bool ok = mongoc_cluster_run_retryable_write(
-               &self->client->cluster, &parts.assembled, parts.is_retryable_write, &new_ss, &cmd_reply, &error);
+            bool ok = mongoc_cluster_run_retryable_write(&self->client->cluster,
+                                                         &parts.assembled,
+                                                         parts.is_retryable_write,
+                                                         self->client->jitter_source,
+                                                         self->client->topology->token_bucket,
+                                                         &new_ss,
+                                                         &cmd_reply,
+                                                         &error);
             if (new_ss) {
                // A retry occurred. Save the newly created stream to use for subsequent commands.
                mongoc_server_stream_cleanup(ss);

src/libmongoc/src/mongoc/mongoc-client-private.h

@@ -26,7 +26,9 @@
 #include <mongoc/mongoc-apm-private.h>
 #include <mongoc/mongoc-buffer-private.h>
 #include <mongoc/mongoc-cluster-private.h>
+#include <mongoc/mongoc-jitter-source-private.h>
 #include <mongoc/mongoc-rpc-private.h>
+#include <mongoc/mongoc-token-bucket-private.h>
 
 #include <mongoc/mongoc-config.h>
 #include <mongoc/mongoc-host-list.h>
@@ -90,6 +92,8 @@ BSON_BEGIN_DECLS
 #define WIRE_VERSION_MIN WIRE_VERSION_4_2 /* a.k.a. minWireVersion */
 #define WIRE_VERSION_MAX WIRE_VERSION_8_0 /* a.k.a. maxWireVersion */
 
+#define MONGOC_DEFAULT_RETRY_TOKEN_CAPACITY 1000.0
+
 struct _mongoc_collection_t;
 
 struct _mongoc_client_t {
@@ -121,6 +125,8 @@ struct _mongoc_client_t {
    unsigned int csid_rand_seed;
 
    uint32_t generation;
+
+   mongoc_jitter_source_t *jitter_source;
 };
 
 /* Defines whether _mongoc_client_command_with_opts() is acting as a read
@@ -233,6 +239,9 @@ _mongoc_client_set_stream_initiator_single_or_pooled(mongoc_client_t *client,
                                                      mongoc_stream_initiator_t initiator,
                                                      void *user_data);
 
+void
+_mongoc_client_set_jitter_source(mongoc_client_t *client, mongoc_jitter_source_t *source);
+
 BSON_END_DECLS
 
 #endif /* MONGOC_CLIENT_PRIVATE_H */

src/libmongoc/src/mongoc/mongoc-client-session.c

@@ -23,6 +23,7 @@
 #include <mongoc/mongoc-rand-private.h>
 #include <mongoc/mongoc-read-concern-private.h>
 #include <mongoc/mongoc-read-prefs-private.h>
+#include <mongoc/mongoc-retry-backoff-generator-private.h>
 #include <mongoc/mongoc-trace-private.h>
 #include <mongoc/mongoc-util-private.h>
 
@@ -891,6 +892,10 @@ _max_time_ms_failure(bson_t *reply)
    return false;
 }
 
+#define MONGOC_WITH_TRANSACTION_RETRY_BACKOFF_GROWTH_FACTOR 1.5
+#define MONGOC_WITH_TRANSACTION_RETRY_BACKOFF_INITIAL mlib_duration(5, ms)
+#define MONGOC_WITH_TRANSACTION_RETRY_BACKOFF_MAX mlib_duration(500, ms)
+
 bool
 mongoc_client_session_with_transaction(mongoc_client_session_t *session,
                                        mongoc_client_session_with_transaction_cb_t cb,
@@ -911,7 +916,16 @@ mongoc_client_session_with_transaction(mongoc_client_session_t *session,
 
    const mlib_timer timer = mlib_expires_after(timeout, ms);
 
-   int transaction_attempt = 0;
+   const mongoc_retry_backoff_params_t retry_backoff_params = {
+      .growth_factor = MONGOC_WITH_TRANSACTION_RETRY_BACKOFF_GROWTH_FACTOR,
+      .backoff_initial = MONGOC_WITH_TRANSACTION_RETRY_BACKOFF_INITIAL,
+      .backoff_max = MONGOC_WITH_TRANSACTION_RETRY_BACKOFF_MAX,
+   };
+
+   mongoc_retry_backoff_generator_t *const retry_backoff_generator =
+      _mongoc_retry_backoff_generator_new(retry_backoff_params, session->jitter_source);
+
+   bool is_first_attempt = true;
 
    /* Attempt to wrap a user callback in start- and end- transaction semantics.
       If this fails for transient reasons, restart, either from the very
@@ -921,10 +935,10 @@ mongoc_client_session_with_transaction(mongoc_client_session_t *session,
       At the top of this loop, active_reply should always be NULL, and
       local_reply should always be uninitialized. */
    while (true) {
-      if (transaction_attempt > 0) {
-         const double jitter = _mongoc_jitter_source_generate(session->jitter_source);
-
-         const mlib_duration backoff_duration = _mongoc_compute_backoff_duration(jitter, transaction_attempt);
+      if (is_first_attempt) {
+         is_first_attempt = false;
+      } else {
+         const mlib_duration backoff_duration = _mongoc_retry_backoff_generator_next(retry_backoff_generator);
 
          const mlib_timer backoff_timer = mlib_expires_after(backoff_duration);
 
@@ -944,8 +958,6 @@ mongoc_client_session_with_transaction(mongoc_client_session_t *session,
          GOTO(done);
       }
 
-      transaction_attempt = BSON_MIN(transaction_attempt + 1, MONGOC_BACKOFF_ATTEMPT_LIMIT);
-
       res = cb(session, ctx, &active_reply, error);
       state = session->txn.state;
 
@@ -1030,6 +1042,8 @@ mongoc_client_session_with_transaction(mongoc_client_session_t *session,
    }
 
 done:
+   _mongoc_retry_backoff_generator_destroy(retry_backoff_generator);
+
    /* At this point, active_reply is either pointing to the user's reply
       object, or our local one on the stack, or is NULL. */
    if (reply && active_reply) {