Skip to content

Commit d1548b0

Browse files
authored
Reduce the AWS artifact upload concurrency and increase retries (#1041)
Attempting to resolve release failures ``` Running `target/release/pythonbuild upload-mirror-distributions --dist dist --datetime 20260324T0736 --tag 20260324 --bucket *** --prefix github/python-build-standalone/releases/download/20260324/` found all 852 release artifacts uploading cpython-3.10.20-aarch64-apple-darwin-debug-20260324T0736.tar.zst -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-apple-darwin-debug-full.tar.zst uploading cpython-3.10.20-aarch64-apple-darwin-install_only-20260324T0736.tar.gz -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-apple-darwin-install_only.tar.gz uploading cpython-3.10.20-aarch64-apple-darwin-install_only_stripped-20260324T0736.tar.gz -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-apple-darwin-install_only_stripped.tar.gz uploading cpython-3.10.20-aarch64-apple-darwin-pgo+lto-20260324T0736.tar.zst -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-apple-darwin-pgo+lto-full.tar.zst uploading cpython-3.10.20-aarch64-unknown-linux-gnu-debug-20260324T0736.tar.zst -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-unknown-linux-gnu-debug-full.tar.zst uploading cpython-3.10.20-aarch64-unknown-linux-gnu-install_only-20260324T0736.tar.gz -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-unknown-linux-gnu-install_only.tar.gz uploading cpython-3.10.20-aarch64-unknown-linux-gnu-install_only_stripped-20260324T0736.tar.gz -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz uploading cpython-3.10.20-aarch64-unknown-linux-gnu-pgo+lto-20260324T0736.tar.zst -> s3://***/github/python-build-standalone/releases/download/20260324/cpython-3.10.20+20260324-aarch64-unknown-linux-gnu-pgo+lto-full.tar.zst Error: dispatch failure Caused by: 0: io error 1: client error (SendRequest) 2: connection error 3: Connection reset by peer (os error 104) error: Recipe `release-upload-mirror` failed with exit code 1 ```
1 parent d0b47bd commit d1548b0

1 file changed

Lines changed: 19 additions & 7 deletions

File tree

src/s3.rs

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ use {
1414
},
1515
};
1616

17+
/// Maximum number of concurrent S3 uploads.
18+
const UPLOAD_CONCURRENCY: usize = 4;
19+
20+
/// Maximum number of attempts per S3 request (includes the initial attempt).
21+
/// The AWS SDK uses exponential backoff with jitter between attempts.
22+
const S3_MAX_ATTEMPTS: u32 = 5;
23+
1724
/// Upload a single file to S3 under `key`, setting an immutable cache-control header.
1825
async fn upload_s3_file(
1926
s3: &aws_sdk_s3::Client,
@@ -32,9 +39,9 @@ async fn upload_s3_file(
3239
return Ok(());
3340
}
3441
// A single PUT is sufficient here: individual artifacts are well under the 5 GB
35-
// single-request limit, and we already upload up to 8 files concurrently, so
36-
// splitting each file into multipart chunks would add complexity without
37-
// meaningfully improving throughput.
42+
// single-request limit, and we already upload up to UPLOAD_CONCURRENCY files
43+
// concurrently, so splitting each file into multipart chunks would add complexity
44+
// without meaningfully improving throughput.
3845
let body = ByteStream::from_path(path).await?;
3946
s3.put_object()
4047
.bucket(bucket)
@@ -101,11 +108,16 @@ pub async fn command_upload_mirror_distributions(args: &ArgMatches) -> Result<()
101108

102109
// Initialise the AWS S3 client. Credentials and endpoint are read from the standard
103110
// AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
104-
// AWS_ENDPOINT_URL, AWS_DEFAULT_REGION)
111+
// AWS_ENDPOINT_URL, AWS_DEFAULT_REGION).
112+
let retry_config =
113+
aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(S3_MAX_ATTEMPTS);
105114
let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
106-
let s3 = aws_sdk_s3::Client::new(&config);
115+
let s3_config = aws_sdk_s3::config::Builder::from(&config)
116+
.retry_config(retry_config)
117+
.build();
118+
let s3 = aws_sdk_s3::Client::from_conf(s3_config);
107119

108-
// Upload all files concurrently (up to 8 in-flight at a time).
120+
// Upload all files concurrently (up to UPLOAD_CONCURRENCY in-flight at a time).
109121
let upload_futs = wanted_filenames
110122
.iter()
111123
.filter(|(source, _)| filenames.contains(*source))
@@ -118,7 +130,7 @@ pub async fn command_upload_mirror_distributions(args: &ArgMatches) -> Result<()
118130
});
119131

120132
futures::stream::iter(upload_futs)
121-
.buffer_unordered(8)
133+
.buffer_unordered(UPLOAD_CONCURRENCY)
122134
.try_collect::<Vec<_>>()
123135
.await?;
124136

0 commit comments

Comments
 (0)