Skip to content

Commit c49b4aa

Browse files
authored
Merge pull request #227 from devtron-labs/fix-ci-blob-upload-issue-main
fix: S3 default integrity change
2 parents 6d3037d + 971c1aa commit c49b4aa

26 files changed

Lines changed: 148 additions & 72 deletions

File tree

chart-sync/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.22.4
55
toolchain go1.22.6
66

77
replace (
8-
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea
8+
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f
99
helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42
1010
)
1111

chart-sync/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
5454
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5555
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5656
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU=
58-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA=
57+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8=
58+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA=
5959
github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY=
6060
github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
6161
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=

chart-sync/vendor/modules.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ github.com/containerd/platforms
9393
# github.com/davecgh/go-spew v1.1.1
9494
## explicit
9595
github.com/davecgh/go-spew/spew
96-
# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea
96+
# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f
9797
## explicit; go 1.21
9898
github.com/devtron-labs/common-lib/fetchAllEnv
9999
github.com/devtron-labs/common-lib/git-manager/util
@@ -790,4 +790,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
790790
# sigs.k8s.io/yaml v1.3.0
791791
## explicit; go 1.12
792792
sigs.k8s.io/yaml
793-
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea
793+
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f

ci-runner/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ FROM docker:20.10.24-dind
1818
RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && \
1919
ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
2020
apk -Uuv add groff less python3 py3-pip && \
21-
pip3 install awscli && \
21+
pip3 install awscli==1.38.11 && \
2222
apk --purge -v del py-pip && \
2323
rm /var/cache/apk/*
2424

ci-runner/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
toolchain go1.21.8
66

7-
replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea
7+
replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f
88

99
require (
1010
github.com/Knetic/govaluate v3.0.0+incompatible

ci-runner/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
9595
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9696
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
9797
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
98-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU=
99-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA=
98+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8=
99+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA=
100100
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
101101
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
102102
github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=

ci-runner/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go

Lines changed: 41 additions & 17 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

ci-runner/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go

Lines changed: 15 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

ci-runner/vendor/modules.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ github.com/cespare/xxhash/v2
248248
# github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
249249
## explicit
250250
github.com/davecgh/go-spew/spew
251-
# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea
251+
# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f
252252
## explicit; go 1.21
253253
github.com/devtron-labs/common-lib/blob-storage
254254
github.com/devtron-labs/common-lib/constants
@@ -991,4 +991,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
991991
# sigs.k8s.io/yaml v1.3.0
992992
## explicit; go 1.12
993993
sigs.k8s.io/yaml
994-
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea
994+
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f

common-lib/blob-storage/AwsS3Blob.go

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -203,31 +203,55 @@ func (r *Resolver) ResolveEndpoint(_ context.Context, params s3v2.EndpointParame
203203
return transport.Endpoint{URI: u}, nil
204204
}
205205

206-
func GetS3BucketBasicsClient(ctx context.Context, region string, accessKey, secretKey string, endpointUrl string) (BucketBasics, error) {
207-
cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(accessKey, secretKey, "")))
208-
if err != nil {
209-
return BucketBasics{}, err
206+
func getS3DefaultSDKConfig(ctx context.Context, region, accessKey, secretKey, endpointUrl string) (s3Cfg awsv2.Config, err error) {
207+
if len(endpointUrl) != 0 && len(region) == 0 {
208+
// case handled for minio
209+
region = "us-east-1"
210+
s3Cfg = awsv2.Config{Region: region}
211+
return s3Cfg, nil
210212
}
211-
sdkConfig := awsv2.Config{Region: region}
212-
sdkConfig.Credentials = cfg.Credentials
213-
var s3Client *s3v2.Client
214-
if len(endpointUrl) > 0 {
215-
if len(region) == 0 {
216-
region = "us-east-1" //for minio
217-
sdkConfig = awsv2.Config{Region: region}
213+
var cfg awsv2.Config
214+
if len(accessKey) == 0 || len(secretKey) == 0 {
215+
// case handled for S3 IAM role
216+
cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region))
217+
if err != nil {
218+
return awsv2.Config{}, err
218219
}
219-
endpointURL, err := url.Parse(endpointUrl)
220+
} else {
221+
// case handled for S3 with access key and secret key
222+
cfg, err = config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(accessKey, secretKey, "")))
223+
if err != nil {
224+
return awsv2.Config{}, err
225+
}
226+
}
227+
s3Cfg = awsv2.Config{Region: region, Credentials: cfg.Credentials}
228+
return s3Cfg, nil
229+
}
230+
231+
func getS3Client(s3Cfg awsv2.Config, endpointUrl string) (s3Client *s3v2.Client, err error) {
232+
if len(endpointUrl) > 0 {
233+
parsedEndpointUrl, err := url.Parse(endpointUrl)
220234
if err != nil {
221-
return BucketBasics{}, err
235+
return s3Client, err
222236
}
223-
s3Client = s3v2.NewFromConfig(sdkConfig, func(o *s3v2.Options) {
237+
return s3v2.NewFromConfig(s3Cfg, func(o *s3v2.Options) {
224238
o.UsePathStyle = true
225-
o.EndpointResolverV2 = &Resolver{URL: endpointURL}
226-
})
239+
o.EndpointResolverV2 = &Resolver{URL: parsedEndpointUrl}
240+
}), nil
227241
} else {
228-
s3Client = s3v2.NewFromConfig(sdkConfig)
242+
return s3v2.NewFromConfig(s3Cfg), nil
229243
}
244+
}
230245

246+
func GetS3BucketBasicsClient(ctx context.Context, region, accessKey, secretKey, endpointUrl string) (BucketBasics, error) {
247+
s3Cfg, err := getS3DefaultSDKConfig(ctx, region, accessKey, secretKey, endpointUrl)
248+
if err != nil {
249+
return BucketBasics{}, err
250+
}
251+
s3Client, err := getS3Client(s3Cfg, endpointUrl)
252+
if err != nil {
253+
return BucketBasics{}, err
254+
}
231255
bucketBasics := BucketBasics{S3Client: s3Client}
232256
return bucketBasics, nil
233257
}

0 commit comments

Comments
 (0)