Skip to content

Commit a3b1d59

Browse files
OmpragashOmpragash
authored
Merge pull request #73 from chhawchharia/CI-21342
CI-21342: migrate AWS SDK Go v1 to v2 for S3 and CloudFront services
2 parents 55a53b2 + d6065ee commit a3b1d59

3 files changed

Lines changed: 167 additions & 95 deletions

File tree

‎aws.go‎

Lines changed: 106 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,75 @@
11
package main
22

33
import (
4+
"context"
45
"crypto/md5"
6+
"errors"
57
"fmt"
68
"io"
79
"mime"
810
"os"
911
"path/filepath"
10-
"strings"
1112
"time"
1213

13-
"github.com/aws/aws-sdk-go/aws"
14-
"github.com/aws/aws-sdk-go/aws/awserr"
15-
"github.com/aws/aws-sdk-go/aws/credentials"
16-
"github.com/aws/aws-sdk-go/aws/session"
17-
"github.com/aws/aws-sdk-go/service/cloudfront"
18-
"github.com/aws/aws-sdk-go/service/s3"
14+
"github.com/aws/aws-sdk-go-v2/aws"
15+
"github.com/aws/aws-sdk-go-v2/config"
16+
"github.com/aws/aws-sdk-go-v2/credentials"
17+
"github.com/aws/aws-sdk-go-v2/service/cloudfront"
18+
cftypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types"
19+
"github.com/aws/aws-sdk-go-v2/service/s3"
20+
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
21+
"github.com/aws/smithy-go"
1922
"github.com/ryanuber/go-glob"
2023
)
2124

2225
type AWS struct {
23-
client *s3.S3
24-
cfClient *cloudfront.CloudFront
26+
client *s3.Client
27+
cfClient *cloudfront.Client
2528
remote []string
2629
local []string
2730
plugin *Plugin
2831
}
2932

3033
func NewAWS(p *Plugin) AWS {
34+
ctx := context.Background()
3135

32-
sessCfg := &aws.Config{
33-
S3ForcePathStyle: aws.Bool(p.PathStyle),
34-
Region: aws.String(p.Region),
36+
optFns := []func(*config.LoadOptions) error{
37+
config.WithRegion(p.Region),
3538
}
3639

37-
if p.Endpoint != "" {
38-
sessCfg.Endpoint = &p.Endpoint
39-
sessCfg.DisableSSL = aws.Bool(strings.HasPrefix(p.Endpoint, "http://"))
40+
if p.Key != "" && p.Secret != "" {
41+
optFns = append(optFns, config.WithCredentialsProvider(
42+
credentials.NewStaticCredentialsProvider(p.Key, p.Secret, ""),
43+
))
4044
}
4145

42-
// allowing to use the instance role or provide a key and secret
43-
if p.Key != "" && p.Secret != "" {
44-
sessCfg.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "")
46+
cfg, err := config.LoadDefaultConfig(ctx, optFns...)
47+
if err != nil {
48+
panic(fmt.Sprintf("unable to load AWS config: %v", err))
4549
}
4650

47-
sess, _ := session.NewSession(sessCfg)
51+
s3Opts := []func(*s3.Options){}
52+
if p.Endpoint != "" {
53+
s3Opts = append(s3Opts, func(o *s3.Options) {
54+
o.BaseEndpoint = aws.String(p.Endpoint)
55+
o.UsePathStyle = p.PathStyle
56+
})
57+
} else if p.PathStyle {
58+
s3Opts = append(s3Opts, func(o *s3.Options) {
59+
o.UsePathStyle = true
60+
})
61+
}
4862

49-
c := s3.New(sess)
50-
cf := cloudfront.New(sess)
63+
c := s3.NewFromConfig(cfg, s3Opts...)
64+
cf := cloudfront.NewFromConfig(cfg)
5165
r := make([]string, 1)
5266
l := make([]string, 1)
5367

5468
return AWS{c, cf, r, l, p}
5569
}
5670

5771
func (a *AWS) Upload(local, remote string) error {
72+
ctx := context.Background()
5873
p := a.plugin
5974
if local == "" {
6075
return nil
@@ -109,29 +124,67 @@ func (a *AWS) Upload(local, remote string) error {
109124
}
110125
}
111126

112-
metadata := map[string]*string{}
127+
metadata := map[string]string{}
113128
for pattern := range p.Metadata {
114129
if match := glob.Glob(pattern, local); match {
115130
for k, v := range p.Metadata[pattern] {
116-
metadata[k] = aws.String(v)
131+
metadata[k] = v
117132
}
118133
break
119134
}
120135
}
121136

122-
head, err := a.client.HeadObject(&s3.HeadObjectInput{
137+
head, err := a.client.HeadObject(ctx, &s3.HeadObjectInput{
123138
Bucket: aws.String(p.Bucket),
124139
Key: aws.String(remote),
125140
})
126-
if err != nil && err.(awserr.Error).Code() != "404" {
141+
if err != nil {
142+
var apiErr smithy.APIError
143+
isNotFound := false
144+
if ok := errors.As(err, &apiErr); ok {
145+
if apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NotFound" {
146+
isNotFound = true
147+
}
148+
}
149+
var nsb *s3types.NoSuchKey
150+
if errors.As(err, &nsb) {
151+
isNotFound = true
152+
}
153+
154+
if !isNotFound {
155+
debug("\"%s\" not found in bucket, uploading with Content-Type \"%s\" and permissions \"%s\"", local, contentType, access)
156+
var putObject = &s3.PutObjectInput{
157+
Bucket: aws.String(p.Bucket),
158+
Key: aws.String(remote),
159+
Body: file,
160+
ContentType: aws.String(contentType),
161+
ACL: s3types.ObjectCannedACL(access),
162+
Metadata: metadata,
163+
}
164+
165+
if len(cacheControl) > 0 {
166+
putObject.CacheControl = aws.String(cacheControl)
167+
}
168+
169+
if len(contentEncoding) > 0 {
170+
putObject.ContentEncoding = aws.String(contentEncoding)
171+
}
172+
173+
if a.plugin.DryRun {
174+
return nil
175+
}
176+
177+
_, err = a.client.PutObject(ctx, putObject)
178+
return err
179+
}
127180

128181
debug("\"%s\" not found in bucket, uploading with Content-Type \"%s\" and permissions \"%s\"", local, contentType, access)
129182
var putObject = &s3.PutObjectInput{
130183
Bucket: aws.String(p.Bucket),
131184
Key: aws.String(remote),
132185
Body: file,
133186
ContentType: aws.String(contentType),
134-
ACL: aws.String(access),
187+
ACL: s3types.ObjectCannedACL(access),
135188
Metadata: metadata,
136189
}
137190

@@ -143,12 +196,11 @@ func (a *AWS) Upload(local, remote string) error {
143196
putObject.ContentEncoding = aws.String(contentEncoding)
144197
}
145198

146-
// skip upload during dry run
147199
if a.plugin.DryRun {
148200
return nil
149201
}
150202

151-
_, err = a.client.PutObject(putObject)
203+
_, err = a.client.PutObject(ctx, putObject)
152204
return err
153205
}
154206

@@ -197,7 +249,7 @@ func (a *AWS) Upload(local, remote string) error {
197249
if !shouldCopy && len(metadata) > 0 {
198250
for k, v := range metadata {
199251
if hv, ok := head.Metadata[k]; ok {
200-
if *v != *hv {
252+
if v != hv {
201253
debug("Metadata values have changed for %s", local)
202254
shouldCopy = true
203255
break
@@ -208,7 +260,7 @@ func (a *AWS) Upload(local, remote string) error {
208260

209261
if !shouldCopy {
210262
debug("Retrieving ACL for \"%s\"", local)
211-
grant, err := a.client.GetObjectAcl(&s3.GetObjectAclInput{
263+
grant, err := a.client.GetObjectAcl(ctx, &s3.GetObjectAclInput{
212264
Bucket: aws.String(p.Bucket),
213265
Key: aws.String(remote),
214266
})
@@ -218,12 +270,12 @@ func (a *AWS) Upload(local, remote string) error {
218270

219271
previousAccess := "private"
220272
for _, g := range grant.Grants {
221-
gt := *g.Grantee
273+
gt := g.Grantee
222274
if gt.URI != nil {
223275
if *gt.URI == "http://acs.amazonaws.com/groups/global/AllUsers" {
224-
if *g.Permission == "READ" {
276+
if g.Permission == s3types.PermissionRead {
225277
previousAccess = "public-read"
226-
} else if *g.Permission == "WRITE" {
278+
} else if g.Permission == s3types.PermissionWrite {
227279
previousAccess = "public-read-write"
228280
}
229281
}
@@ -246,10 +298,10 @@ func (a *AWS) Upload(local, remote string) error {
246298
Bucket: aws.String(p.Bucket),
247299
Key: aws.String(remote),
248300
CopySource: aws.String(fmt.Sprintf("%s/%s", p.Bucket, remote)),
249-
ACL: aws.String(access),
301+
ACL: s3types.ObjectCannedACL(access),
250302
ContentType: aws.String(contentType),
251303
Metadata: metadata,
252-
MetadataDirective: aws.String("REPLACE"),
304+
MetadataDirective: s3types.MetadataDirectiveReplace,
253305
}
254306

255307
if len(cacheControl) > 0 {
@@ -260,12 +312,11 @@ func (a *AWS) Upload(local, remote string) error {
260312
copyObject.ContentEncoding = aws.String(contentEncoding)
261313
}
262314

263-
// skip update if dry run
264315
if a.plugin.DryRun {
265316
return nil
266317
}
267318

268-
_, err = a.client.CopyObject(copyObject)
319+
_, err = a.client.CopyObject(ctx, copyObject)
269320
return err
270321
} else {
271322
_, err = file.Seek(0, 0)
@@ -279,7 +330,7 @@ func (a *AWS) Upload(local, remote string) error {
279330
Key: aws.String(remote),
280331
Body: file,
281332
ContentType: aws.String(contentType),
282-
ACL: aws.String(access),
333+
ACL: s3types.ObjectCannedACL(access),
283334
Metadata: metadata,
284335
}
285336

@@ -291,52 +342,54 @@ func (a *AWS) Upload(local, remote string) error {
291342
putObject.ContentEncoding = aws.String(contentEncoding)
292343
}
293344

294-
// skip upload if dry run
295345
if a.plugin.DryRun {
296346
return nil
297347
}
298348

299-
_, err = a.client.PutObject(putObject)
349+
_, err = a.client.PutObject(ctx, putObject)
300350
return err
301351
}
302352
}
303353

304354
func (a *AWS) Redirect(path, location string) error {
355+
ctx := context.Background()
305356
p := a.plugin
306357
debug("Adding redirect from \"%s\" to \"%s\"", path, location)
307358

308359
if a.plugin.DryRun {
309360
return nil
310361
}
311362

312-
_, err := a.client.PutObject(&s3.PutObjectInput{
363+
_, err := a.client.PutObject(ctx, &s3.PutObjectInput{
313364
Bucket: aws.String(p.Bucket),
314365
Key: aws.String(path),
315-
ACL: aws.String("public-read"),
366+
ACL: s3types.ObjectCannedACLPublicRead,
316367
WebsiteRedirectLocation: aws.String(location),
317368
})
318369
return err
319370
}
320371

321372
func (a *AWS) Delete(remote string) error {
373+
ctx := context.Background()
322374
p := a.plugin
323375
debug("Removing remote file \"%s\"", remote)
324376

325377
if a.plugin.DryRun {
326378
return nil
327379
}
328380

329-
_, err := a.client.DeleteObject(&s3.DeleteObjectInput{
381+
_, err := a.client.DeleteObject(ctx, &s3.DeleteObjectInput{
330382
Bucket: aws.String(p.Bucket),
331383
Key: aws.String(remote),
332384
})
333385
return err
334386
}
335387

336388
func (a *AWS) List(path string) ([]string, error) {
389+
ctx := context.Background()
337390
p := a.plugin
338391
remote := make([]string, 1)
339-
resp, err := a.client.ListObjects(&s3.ListObjectsInput{
392+
resp, err := a.client.ListObjects(ctx, &s3.ListObjectsInput{
340393
Bucket: aws.String(p.Bucket),
341394
Prefix: aws.String(path),
342395
})
@@ -348,8 +401,8 @@ func (a *AWS) List(path string) ([]string, error) {
348401
remote = append(remote, *item.Key)
349402
}
350403

351-
for *resp.IsTruncated {
352-
resp, err = a.client.ListObjects(&s3.ListObjectsInput{
404+
for aws.ToBool(resp.IsTruncated) {
405+
resp, err = a.client.ListObjects(ctx, &s3.ListObjectsInput{
353406
Bucket: aws.String(p.Bucket),
354407
Prefix: aws.String(path),
355408
Marker: aws.String(remote[len(remote)-1]),
@@ -368,16 +421,17 @@ func (a *AWS) List(path string) ([]string, error) {
368421
}
369422

370423
func (a *AWS) Invalidate(invalidatePath string) error {
424+
ctx := context.Background()
371425
p := a.plugin
372426
debug("Invalidating \"%s\"", invalidatePath)
373-
_, err := a.cfClient.CreateInvalidation(&cloudfront.CreateInvalidationInput{
427+
_, err := a.cfClient.CreateInvalidation(ctx, &cloudfront.CreateInvalidationInput{
374428
DistributionId: aws.String(p.CloudFrontDistribution),
375-
InvalidationBatch: &cloudfront.InvalidationBatch{
429+
InvalidationBatch: &cftypes.InvalidationBatch{
376430
CallerReference: aws.String(time.Now().Format(time.RFC3339Nano)),
377-
Paths: &cloudfront.Paths{
378-
Quantity: aws.Int64(1),
379-
Items: []*string{
380-
aws.String(invalidatePath),
431+
Paths: &cftypes.Paths{
432+
Quantity: aws.Int32(1),
433+
Items: []string{
434+
invalidatePath,
381435
},
382436
},
383437
},

‎go.mod‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,38 @@
11
module github.com/drone-plugins/drone-s3-sync
22

3-
go 1.19
3+
go 1.24
44

55
require (
6-
github.com/aws/aws-sdk-go v1.44.155
6+
github.com/aws/aws-sdk-go-v2 v1.41.2
7+
github.com/aws/aws-sdk-go-v2/config v1.32.10
8+
github.com/aws/aws-sdk-go-v2/credentials v1.19.10
9+
github.com/aws/aws-sdk-go-v2/service/cloudfront v1.60.1
10+
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2
11+
github.com/aws/smithy-go v1.24.2
712
github.com/joho/godotenv v1.4.0
813
github.com/ryanuber/go-glob v1.0.0
914
github.com/sirupsen/logrus v1.9.0
1015
github.com/urfave/cli v1.22.10
1116
)
1217

1318
require (
19+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
20+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect
21+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
22+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
23+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
24+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect
25+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect
26+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect
27+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect
28+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect
29+
github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect
30+
github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect
31+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect
32+
github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect
1433
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
15-
github.com/jmespath/go-jmespath v0.4.0 // indirect
1634
github.com/russross/blackfriday/v2 v2.0.1 // indirect
1735
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
1836
github.com/stretchr/testify v1.8.1 // indirect
19-
golang.org/x/net v0.4.0 // indirect
2037
golang.org/x/sys v0.3.0 // indirect
2138
)

0 commit comments

Comments
 (0)