main sync develop

Author: vikramdevtron

ci-runner/Dockerfile

@@ -18,7 +18,7 @@ FROM docker:20.10.24-dind
 RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && \
     ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
     apk -Uuv add groff less python3 py3-pip && \
-    pip3 install awscli && \
+    pip3 install awscli==1.38.11 && \
     apk --purge -v del py-pip && \
     rm /var/cache/apk/*
 

ci-runner/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go

@@ -203,31 +203,55 @@ func (r *Resolver) ResolveEndpoint(_ context.Context, params s3v2.EndpointParame
 	return transport.Endpoint{URI: u}, nil
 }
 
-func GetS3BucketBasicsClient(ctx context.Context, region string, accessKey, secretKey string, endpointUrl string) (BucketBasics, error) {
-	cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(accessKey, secretKey, "")))
-	if err != nil {
-		return BucketBasics{}, err
+func getS3DefaultSDKConfig(ctx context.Context, region, accessKey, secretKey, endpointUrl string) (s3Cfg awsv2.Config, err error) {
+	if len(endpointUrl) != 0 && len(region) == 0 {
+		// case handled for minio
+		region = "us-east-1"
+		s3Cfg = awsv2.Config{Region: region}
+		return s3Cfg, nil
 	}
-	sdkConfig := awsv2.Config{Region: region}
-	sdkConfig.Credentials = cfg.Credentials
-	var s3Client *s3v2.Client
-	if len(endpointUrl) > 0 {
-		if len(region) == 0 {
-			region = "us-east-1" //for minio
-			sdkConfig = awsv2.Config{Region: region}
+	var cfg awsv2.Config
+	if len(accessKey) == 0 || len(secretKey) == 0 {
+		// case handled for S3 IAM role
+		cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region))
+		if err != nil {
+			return awsv2.Config{}, err
 		}
-		endpointURL, err := url.Parse(endpointUrl)
+	} else {
+		// case handled for S3 with access key and secret key
+		cfg, err = config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(accessKey, secretKey, "")))
+		if err != nil {
+			return awsv2.Config{}, err
+		}
+	}
+	s3Cfg = awsv2.Config{Region: region, Credentials: cfg.Credentials}
+	return s3Cfg, nil
+}
+
+func getS3Client(s3Cfg awsv2.Config, endpointUrl string) (s3Client *s3v2.Client, err error) {
+	if len(endpointUrl) > 0 {
+		parsedEndpointUrl, err := url.Parse(endpointUrl)
 		if err != nil {
-			return BucketBasics{}, err
+			return s3Client, err
 		}
-		s3Client = s3v2.NewFromConfig(sdkConfig, func(o *s3v2.Options) {
+		return s3v2.NewFromConfig(s3Cfg, func(o *s3v2.Options) {
 			o.UsePathStyle = true
-			o.EndpointResolverV2 = &Resolver{URL: endpointURL}
-		})
+			o.EndpointResolverV2 = &Resolver{URL: parsedEndpointUrl}
+		}), nil
 	} else {
-		s3Client = s3v2.NewFromConfig(sdkConfig)
+		return s3v2.NewFromConfig(s3Cfg), nil
 	}
+}
 
+func GetS3BucketBasicsClient(ctx context.Context, region, accessKey, secretKey, endpointUrl string) (BucketBasics, error) {
+	s3Cfg, err := getS3DefaultSDKConfig(ctx, region, accessKey, secretKey, endpointUrl)
+	if err != nil {
+		return BucketBasics{}, err
+	}
+	s3Client, err := getS3Client(s3Cfg, endpointUrl)
+	if err != nil {
+		return BucketBasics{}, err
+	}
 	bucketBasics := BucketBasics{S3Client: s3Client}
 	return bucketBasics, nil
 }

ci-runner/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go

@@ -22,11 +22,25 @@ import (
 	"os/exec"
 )
 
+const (
+	WhenSupported = "when_supported"
+	WhenRequired  = "when_required"
+)
+
 func setAWSEnvironmentVariables(s3Config *AwsS3BaseConfig, command *exec.Cmd) {
 	if s3Config.AccessKey != "" && s3Config.Passkey != "" {
-		command.Env = append(os.Environ(),
+		command.Env = os.Environ()
+		command.Env = append(command.Env,
 			fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", s3Config.AccessKey),
 			fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", s3Config.Passkey),
 		)
 	}
+	if s3Config.EndpointUrl != "" {
+		command.Env = append(command.Env,
+			// The below is required for https://github.com/aws/aws-cli/issues/9214
+			// This is only required for secure endpoints only https://github.com/boto/boto3/issues/4398#issuecomment-2712259341
+			fmt.Sprintf("AWS_REQUEST_CHECKSUM_CALCULATION=%s", WhenRequired),
+			fmt.Sprintf("AWS_RESPONSE_CHECKSUM_VALIDATION=%s", WhenRequired),
+		)
+	}
 }