Merge pull request #169 from concourse/aws-sdk-v2

Upgrade to v2 of the aws-sdk

Author: taylorsilva

README.md

@@ -65,7 +65,7 @@ the bucket.
 
 * `region_name`: *Optional. Default `us-east-1`.* The region the bucket is in.
 
-* `endpoint`: *Optional.* Custom endpoint for using S3 compatible provider.
+* `endpoint`: *Optional.* Custom endpoint for using S3 compatible provider. Can be a hostname or URL.
 
 * `disable_ssl`: *Optional.* Disable SSL for the endpoint, useful for S3 compatible providers without SSL.
 
@@ -74,7 +74,7 @@ the bucket.
 * `server_side_encryption`: *Optional.* The server-side encryption algorithm
 used when storing the version object (e.g. `AES256`, `aws:kms`).
 
-* `use_v2_signing`: *Optional.* Use v2 signing, default false.
+* `use_v2_signing`: *Deprecated.* No longer used after upgrading to v2 of the AWS Go SDK.
 
 ### `swift` Driver
 

check/check_test.go

@@ -2,16 +2,17 @@ package main_test
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"os"
 	"os/exec"
 	"path"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/concourse/semver-resource/models"
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
@@ -45,24 +46,24 @@ var _ = Describe("Check", func() {
 	Context("when executed", func() {
 		var request models.CheckRequest
 		var response models.CheckResponse
-		var svc *s3.S3
+		var svc *s3.Client
 
 		BeforeEach(func() {
 			guid, err := uuid.NewRandom()
 			Expect(err).NotTo(HaveOccurred())
 
 			key = guid.String()
 
-			creds := credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
-			awsConfig := &aws.Config{
-				Region:           aws.String(regionName),
-				Credentials:      creds,
-				S3ForcePathStyle: aws.Bool(true),
-				MaxRetries:       aws.Int(12),
-			}
-			sess, err := session.NewSession(awsConfig)
+			creds := credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")
+			cfg, err := config.LoadDefaultConfig(context.TODO(),
+				config.WithRegion(regionName),
+				config.WithRetryMaxAttempts(12),
+				config.WithCredentialsProvider(creds),
+			)
 			Expect(err).NotTo(HaveOccurred())
-			svc = s3.New(sess)
+			svc = s3.NewFromConfig(cfg, func(o *s3.Options) {
+				o.UsePathStyle = true
+			})
 
 			request = models.CheckRequest{
 				Version: models.Version{},
@@ -80,7 +81,7 @@ var _ = Describe("Check", func() {
 		})
 
 		AfterEach(func() {
-			_, err := svc.DeleteObject(&s3.DeleteObjectInput{
+			_, err := svc.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
 				Bucket: aws.String(bucketName),
 				Key:    aws.String(key),
 			})
@@ -105,12 +106,11 @@ var _ = Describe("Check", func() {
 		})
 
 		putVersion := func(version string) {
-			_, err := svc.PutObject(&s3.PutObjectInput{
+			_, err := svc.PutObject(context.TODO(), &s3.PutObjectInput{
 				Bucket:      aws.String(bucketName),
 				Key:         aws.String(key),
 				ContentType: aws.String("text/plain"),
 				Body:        bytes.NewReader([]byte(version)),
-				ACL:         aws.String(s3.ObjectCannedACLPrivate),
 			})
 			Expect(err).NotTo(HaveOccurred())
 		}

driver/driver.go

@@ -1,15 +1,18 @@
 package driver
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net/http"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"net/url"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/blang/semver"
 	"github.com/concourse/semver-resource/models"
 	"github.com/concourse/semver-resource/version"
@@ -38,16 +41,14 @@ func FromSource(source models.Source) (Driver, error) {
 
 	switch source.Driver {
 	case models.DriverUnspecified, models.DriverS3:
-		var creds *credentials.Credentials
+		var credsProvider aws.CredentialsProvider
 
-		if source.AccessKeyID == "" && source.SecretAccessKey == "" {
-			creds = credentials.AnonymousCredentials
-		} else {
-			creds = credentials.NewStaticCredentials(source.AccessKeyID, source.SecretAccessKey, source.SessionToken)
+		if source.AccessKeyID != "" && source.SecretAccessKey != "" {
+			credsProvider = credentials.NewStaticCredentialsProvider(source.AccessKeyID, source.SecretAccessKey, source.SessionToken)
 		}
 
 		regionName := source.RegionName
-		if len(regionName) == 0 {
+		if regionName == "" {
 			regionName = "us-east-1"
 		}
 
@@ -60,39 +61,67 @@ func FromSource(source models.Source) (Driver, error) {
 			httpClient = http.DefaultClient
 		}
 
-		awsConfig := &aws.Config{
-			Region:           aws.String(regionName),
-			Credentials:      creds,
-			S3ForcePathStyle: aws.Bool(true),
-			MaxRetries:       aws.Int(maxRetries),
-			DisableSSL:       aws.Bool(source.DisableSSL),
-			HTTPClient:       httpClient,
+		cfg, err := config.LoadDefaultConfig(context.TODO(),
+			config.WithRegion(regionName),
+			config.WithHTTPClient(httpClient),
+			config.WithRetryMaxAttempts(maxRetries),
+			config.WithCredentialsProvider(credsProvider),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error loading default aws config: %w", err)
 		}
 
-		if len(source.Endpoint) != 0 {
-			awsConfig.Endpoint = aws.String(source.Endpoint)
+		if source.AssumeRoleArn != "" {
+			stsClient := sts.NewFromConfig(cfg)
+			roleCreds := stscreds.NewAssumeRoleProvider(stsClient, source.AssumeRoleArn)
+			creds, err := roleCreds.Retrieve(context.TODO())
+			if err != nil {
+				return nil, fmt.Errorf("error assuming role: %w", err)
+			}
+
+			cfg.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(
+				creds.AccessKeyID,
+				creds.SecretAccessKey,
+				creds.SessionToken,
+			))
 		}
 
-		s3Session := session.New(awsConfig)
+		s3Opts := []func(*s3.Options){
+			func(o *s3.Options) {
+				o.UsePathStyle = true
+			},
+		}
 
-		var s3Client *s3.S3
-		if source.AssumeRoleArn != "" {
-			creds := stscreds.NewCredentials(s3Session, source.AssumeRoleArn)
-			s3Client = s3.New(s3Session, &aws.Config{Credentials: creds})
-		} else {
-			s3Client = s3.New(s3Session)
+		if source.Endpoint != "" {
+			endpoint := source.Endpoint
+			u, err := url.Parse(source.Endpoint)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing given endpoint: %w", err)
+			}
+			if u.Scheme == "" {
+				// source.Endpoint is a hostname
+				scheme := "https://"
+				if source.DisableSSL {
+					scheme = "http://"
+				}
+				endpoint = scheme + source.Endpoint
+			}
+
+			s3Opts = append(s3Opts, func(o *s3.Options) {
+				o.BaseEndpoint = &endpoint
+			})
 		}
 
-		svc := s3Client
+		s3Client := s3.NewFromConfig(cfg, s3Opts...)
 
 		if source.UseV2Signing {
-			setv2Handlers(svc)
+			//TODO: warn this setting is deprecated. The SDK only has v4 signing
 		}
 
 		return &S3Driver{
 			InitialVersion: initialVersion,
 
-			Svc:                  svc,
+			Svc:                  s3Client,
 			BucketName:           source.Bucket,
 			Key:                  source.Key,
 			ServerSideEncryption: source.ServerSideEncryption,

driver/driver_test.go

@@ -3,7 +3,7 @@ package driver_test
 import (
 	"net/http"
 
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/concourse/semver-resource/driver"
 	"github.com/concourse/semver-resource/models"
 	. "github.com/onsi/ginkgo/v2"
@@ -25,9 +25,9 @@ var _ = Describe("Driver", func() {
 			s3Driver, ok := aDriver.(*driver.S3Driver)
 			Expect(ok).To(BeTrue())
 			Expect(s3Driver.Svc).To(Not(BeNil()))
-			svc, ok := s3Driver.Svc.(*s3.S3)
+			svc, ok := s3Driver.Svc.(*s3.Client)
 			Expect(ok).To(BeTrue())
-			Expect(svc.Client.Config.HTTPClient).Should(BeEquivalentTo(http.DefaultClient))
+			Expect(svc.Options().HTTPClient).Should(BeEquivalentTo(http.DefaultClient))
 		})
 		It("returns a s3 driver with a transport that ignores ssl verification", func() {
 			src.SkipSSLVerification = true
@@ -37,10 +37,11 @@ var _ = Describe("Driver", func() {
 			s3Driver, ok := aDriver.(*driver.S3Driver)
 			Expect(ok).To(BeTrue())
 			Expect(s3Driver.Svc).To(Not(BeNil()))
-			svc, ok := s3Driver.Svc.(*s3.S3)
+			svc, ok := s3Driver.Svc.(*s3.Client)
 			Expect(ok).To(BeTrue())
-			Expect(svc.Client.Config.HTTPClient.Transport).ToNot(BeNil())
-			transport, ok := svc.Client.Config.HTTPClient.Transport.(*http.Transport)
+			httpClient, ok := svc.Options().HTTPClient.(*http.Client)
+			Expect(httpClient.Transport).ToNot(BeNil())
+			transport, ok := httpClient.Transport.(*http.Transport)
 			Expect(ok).To(BeTrue())
 			Expect(transport.TLSClientConfig.InsecureSkipVerify).Should(BeTrue())
 		})

driver/s3.go

@@ -2,20 +2,22 @@ package driver
 
 import (
 	"bytes"
+	"context"
+	"errors"
 	"fmt"
 	"io"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/blang/semver"
 	"github.com/concourse/semver-resource/version"
 )
 
 type Servicer interface {
-	GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error)
-	PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error)
+	GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
+	PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)
 }
 
 type S3Driver struct {
@@ -30,10 +32,11 @@ type S3Driver struct {
 func (driver *S3Driver) Bump(bump version.Bump) (semver.Version, error) {
 	var currentVersion semver.Version
 
-	resp, err := driver.Svc.GetObject(&s3.GetObjectInput{
-		Bucket: aws.String(driver.BucketName),
-		Key:    aws.String(driver.Key),
-	})
+	resp, err := driver.Svc.GetObject(context.TODO(),
+		&s3.GetObjectInput{
+			Bucket: aws.String(driver.BucketName),
+			Key:    aws.String(driver.Key),
+		})
 	if err == nil {
 		bucketNumberPayload, err := io.ReadAll(resp.Body)
 		if err != nil {
@@ -46,10 +49,13 @@ func (driver *S3Driver) Bump(bump version.Bump) (semver.Version, error) {
 		if err != nil {
 			return semver.Version{}, err
 		}
-	} else if s3err, ok := err.(awserr.RequestFailure); ok && s3err.StatusCode() == 404 {
-		currentVersion = driver.InitialVersion
 	} else {
-		return semver.Version{}, err
+		var noSuchKey *types.NoSuchKey
+		if errors.As(err, &noSuchKey) {
+			currentVersion = driver.InitialVersion
+		} else {
+			return semver.Version{}, err
+		}
 	}
 
 	newVersion := bump.Apply(currentVersion)
@@ -68,21 +74,20 @@ func (driver *S3Driver) Set(newVersion semver.Version) error {
 		Key:         aws.String(driver.Key),
 		ContentType: aws.String("text/plain"),
 		Body:        bytes.NewReader([]byte(newVersion.String())),
-		ACL:         aws.String(s3.ObjectCannedACLPrivate),
 	}
 
 	if len(driver.ServerSideEncryption) > 0 {
-		params.ServerSideEncryption = aws.String(driver.ServerSideEncryption)
+		params.ServerSideEncryption = types.ServerSideEncryption(driver.ServerSideEncryption)
 	}
 
-	_, err := driver.Svc.PutObject(params)
+	_, err := driver.Svc.PutObject(context.TODO(), params)
 	return err
 }
 
 func (driver *S3Driver) Check(cursor *semver.Version) ([]semver.Version, error) {
 	var bucketNumber string
 
-	resp, err := driver.Svc.GetObject(&s3.GetObjectInput{
+	resp, err := driver.Svc.GetObject(context.TODO(), &s3.GetObjectInput{
 		Bucket: aws.String(driver.BucketName),
 		Key:    aws.String(driver.Key),
 	})
@@ -94,14 +99,17 @@ func (driver *S3Driver) Check(cursor *semver.Version) ([]semver.Version, error)
 		defer resp.Body.Close()
 
 		bucketNumber = string(bucketNumberPayload)
-	} else if s3err, ok := err.(awserr.RequestFailure); ok && s3err.StatusCode() == 404 {
-		if cursor == nil {
-			return []semver.Version{driver.InitialVersion}, nil
+	} else {
+		var noSuchKey *types.NoSuchKey
+		if errors.As(err, &noSuchKey) {
+			if cursor == nil {
+				return []semver.Version{driver.InitialVersion}, nil
+			} else {
+				return []semver.Version{}, nil
+			}
 		} else {
-			return []semver.Version{}, nil
+			return nil, err
 		}
-	} else {
-		return nil, err
 	}
 
 	bucketVersion, err := semver.Parse(bucketNumber)