Add S3 timeout config to bottomless replicator and fix integration tests

- Add LIBSQL_BOTTOMLESS_S3_READ_TIMEOUT_SECS (default 5s)
- Add LIBSQL_BOTTOMLESS_S3_CONNECT_TIMEOUT_SECS (default 5s)
- Add LIBSQL_BOTTOMLESS_S3_OPERATION_ATTEMPT_TIMEOUT_SECS (default 10s)
- Configure TimeoutConfig on aws_sdk_s3::Config in bottomless::replicator::Options::client_config()
- Update meta_store.rs Options construction to include new timeout fields
- Remove #[ignore] from network_partition test
- Fix test fixtures: endpoint timing, image caching, mut minio

Author: aredridel

.gitignore

@@ -12,4 +12,4 @@ libsql-sqlite3/**.o.tmp
 
 /bindings/c/generated
 /bindings/c/**.xcframework
-/bindings/**/.DS_Store
+/bindings/**/.DS_Store

Cargo.lock

@@ -13,6 +13,7 @@ anyhow = "1.0.66"
 async-compression = { version = "0.4.4", features = ["tokio", "gzip", "zstd"] }
 aws-config = { version = "1" }
 aws-sdk-s3 = { version = "1" }
+aws-smithy-types = { version = "1" }
 bytes = "1"
 libsql-sys = { path = "../libsql-sys" }
 libsql_replication = { path = "../libsql-replication" }

bottomless/Cargo.toml

@@ -15,6 +15,7 @@ use aws_sdk_s3::operation::list_objects::builders::ListObjectsFluentBuilder;
 use aws_sdk_s3::operation::list_objects::ListObjectsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::{Client, Config};
+use aws_smithy_types::timeout::TimeoutConfig;
 use bytes::{Buf, Bytes};
 use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
 use libsql_replication::injector::Injector as _;
@@ -121,6 +122,12 @@ pub struct Options {
     pub s3_max_parallelism: usize,
     /// Max number of retries for S3 operations
     pub s3_max_retries: u32,
+    /// Timeout for reading the first byte of an S3 response (seconds)
+    pub s3_read_timeout_secs: u64,
+    /// Timeout for establishing a TCP connection to S3 (seconds)
+    pub s3_connect_timeout_secs: u64,
+    /// Timeout for a single S3 operation attempt, including retries (seconds)
+    pub s3_operation_attempt_timeout_secs: u64,
     /// Skip snapshot upload per checkpoint.
     pub skip_snapshot: bool,
     /// Skip uploading snapshots on shutdown
@@ -145,6 +152,11 @@ impl Options {
             "LIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY was not set"
         ))?;
         let session_token: Option<String> = self.session_token.clone();
+        let timeout_config = TimeoutConfig::builder()
+            .read_timeout(Duration::from_secs(self.s3_read_timeout_secs))
+            .connect_timeout(Duration::from_secs(self.s3_connect_timeout_secs))
+            .operation_attempt_timeout(Duration::from_secs(self.s3_operation_attempt_timeout_secs))
+            .build();
         let conf = loader
             .behavior_version(BehaviorVersion::latest())
             .region(Region::new(region))
@@ -159,6 +171,7 @@ impl Options {
                 aws_sdk_s3::config::retry::RetryConfig::standard()
                     .with_max_attempts(self.s3_max_retries),
             )
+            .timeout_config(timeout_config)
             .build();
 
         let s3_config = aws_sdk_s3::config::Builder::from(&conf)
@@ -233,6 +246,12 @@ impl Options {
             ),
         };
         let s3_max_retries = env_var_or("LIBSQL_BOTTOMLESS_S3_MAX_RETRIES", 10).parse::<u32>()?;
+        let s3_read_timeout_secs =
+            env_var_or("LIBSQL_BOTTOMLESS_S3_READ_TIMEOUT_SECS", 5).parse::<u64>()?;
+        let s3_connect_timeout_secs =
+            env_var_or("LIBSQL_BOTTOMLESS_S3_CONNECT_TIMEOUT_SECS", 5).parse::<u64>()?;
+        let s3_operation_attempt_timeout_secs =
+            env_var_or("LIBSQL_BOTTOMLESS_S3_OPERATION_ATTEMPT_TIMEOUT_SECS", 10).parse::<u64>()?;
         let cipher = match encryption_cipher {
             Some(cipher) => Cipher::from_str(&cipher)?,
             None => Cipher::default(),
@@ -261,6 +280,9 @@ impl Options {
             region,
             bucket_name,
             s3_max_retries,
+            s3_read_timeout_secs,
+            s3_connect_timeout_secs,
+            s3_operation_attempt_timeout_secs,
             skip_snapshot,
             skip_shutdown_upload,
         })

bottomless/src/replicator.rs

@@ -129,6 +129,9 @@ pub async fn metastore_connection_maker(
                 max_batch_interval: config.backup_interval,
                 s3_max_parallelism: 32,
                 s3_max_retries: 10,
+                s3_read_timeout_secs: 5,
+                s3_connect_timeout_secs: 5,
+                s3_operation_attempt_timeout_secs: 10,
                 skip_snapshot: false,
                 skip_shutdown_upload: false,
             };

libsql-server/src/namespace/meta_store.rs

@@ -36,10 +36,10 @@ async fn test_basic_restore() {
         .expect("Failed to cleanup dbs dir");
 
     // Phase 3: Start sqld - should restore from minio
-    let endpoint2 = sqld.http_endpoint();
     sqld.start(data_dir.path())
         .await
         .expect("Failed to restart sqld");
+    let endpoint2 = sqld.http_endpoint();
     sqld.wait_for_ready(Duration::from_secs(60))
         .await
         .expect("sqld did not become ready after restore");