Merge pull request #13 from aws-samples/develop

feat: support payer request for none public bucket

Author: YikaiHu

Dockerfile

@@ -28,6 +28,7 @@ ENV SRC_ENDPOINT ''
 ENV SRC_CREDENTIALS ''
 ENV SRC_IN_CURRENT_ACCOUNT false
 ENV SKIP_COMPARE false
+ENV PAYER_REQUEST false
 
 ENV DEST_BUCKET ''
 ENV DEST_PREFIX ''

cmd/root.go

@@ -96,6 +96,7 @@ func initConfig() {
 	viper.BindEnv("srcCredential", "SRC_CREDENTIALS")
 	viper.BindEnv("srcInCurrentAccount", "SRC_IN_CURRENT_ACCOUNT")
 	viper.BindEnv("skipCompare", "SKIP_COMPARE")
+	viper.BindEnv("payerRequest", "PAYER_REQUEST")
 
 	viper.BindEnv("destBucket", "DEST_BUCKET")
 	viper.BindEnv("destPrefix", "DEST_PREFIX")
@@ -155,6 +156,7 @@ func initConfig() {
 		SrcCredential:        viper.GetString("srcCredential"),
 		SrcInCurrentAccount:  viper.GetBool("srcInCurrentAccount"),
 		SkipCompare:          viper.GetBool("skipCompare"),
+		PayerRequest:         viper.GetBool("payerRequest"),
 		DestBucket:           viper.GetString("destBucket"),
 		DestPrefix:           viper.GetString("destPrefix"),
 		DestRegion:           viper.GetString("destRegion"),

config-example.yaml

@@ -6,6 +6,7 @@ srcEndpoint:
 srcCredential: src
 srcInCurrentAccount: false
 skipCompare: false
+payerRequest: false
 
 destBucket: dest-bucket
 destPrefix:

dth/client.go

@@ -60,6 +60,7 @@ type Client interface {
 type S3Client struct {
 	bucket, prefix, prefixList, region, sourceType string
 	isSrcClient                                    bool
+	isPayerRequest                                 bool
 	client                                         *s3.Client
 }
 
@@ -171,10 +172,22 @@ func (c *S3Client) GetObject(ctx context.Context, key *string, size, start, chun
 		return nil, nil
 	}
 	bodyRange := fmt.Sprintf("bytes=%d-%d", start, start+chunkSize-1)
-	input := &s3.GetObjectInput{
-		Bucket: &c.bucket,
-		Key:    key,
-		Range:  &bodyRange,
+
+	var input *s3.GetObjectInput
+
+	if c.isPayerRequest {
+		input = &s3.GetObjectInput{
+			Bucket:       &c.bucket,
+			Key:          key,
+			Range:        &bodyRange,
+			RequestPayer: types.RequestPayerRequester,
+		}
+	} else {
+		input = &s3.GetObjectInput{
+			Bucket: &c.bucket,
+			Key:    key,
+			Range:  &bodyRange,
+		}
 	}
 
 	output, err := c.client.GetObject(ctx, input, getClientCredentialsModifyFn(c.isSrcClient, SRC_CRED, DST_CRED))
@@ -199,12 +212,25 @@ func (c *S3Client) GetObject(ctx context.Context, key *string, size, start, chun
 // Internal func to call API to list objects.
 func (c *S3Client) listObjectFn(ctx context.Context, continuationToken, prefix, delimiter *string, maxKeys int32) (*s3.ListObjectsV2Output, error) {
 
-	input := &s3.ListObjectsV2Input{
-		Bucket:       &c.bucket,
-		Prefix:       prefix,
-		MaxKeys:      maxKeys,
-		Delimiter:    delimiter,
-		EncodingType: "url",
+	var input *s3.ListObjectsV2Input
+
+	if c.isPayerRequest {
+		input = &s3.ListObjectsV2Input{
+			Bucket:       &c.bucket,
+			Prefix:       prefix,
+			MaxKeys:      maxKeys,
+			Delimiter:    delimiter,
+			EncodingType: "url",
+			RequestPayer: types.RequestPayerRequester,
+		}
+	} else {
+		input = &s3.ListObjectsV2Input{
+			Bucket:       &c.bucket,
+			Prefix:       prefix,
+			MaxKeys:      maxKeys,
+			Delimiter:    delimiter,
+			EncodingType: "url",
+		}
 	}
 
 	if *continuationToken != "" {

dth/config.go

@@ -69,7 +69,7 @@ type JobConfig struct {
 	SrcType, SrcBucket, SrcPrefix, SrcPrefixList, SrcRegion, SrcEndpoint, SrcCredential string
 	DestBucket, DestPrefix, DestRegion, DestCredential, DestStorageClass, DestAcl       string
 	JobTableName, JobQueueName                                                          string
-	SrcInCurrentAccount, DestInCurrentAccount, SkipCompare                              bool
+	SrcInCurrentAccount, DestInCurrentAccount, SkipCompare, PayerRequest                bool
 	*JobOptions
 }
 

dth/job.go

@@ -121,6 +121,10 @@ func NewFinder(ctx context.Context, cfg *JobConfig) (f *Finder) {
 		srcClient.isSrcClient = true
 	}
 
+	if cfg.PayerRequest {
+		srcClient.isPayerRequest = true
+	}
+
 	SRC_CRED = srcCred
 	DST_CRED = desCred
 
@@ -183,6 +187,10 @@ func (f *Finder) Run(ctx context.Context) {
 	var prefixes []*string
 	log.Printf("Prefix List File: %s", f.cfg.SrcPrefixList)
 
+	if f.cfg.PayerRequest {
+		log.Printf("Enable Payer Request Mode")
+	}
+
 	if len(f.cfg.SrcPrefixList) > 0 {
 		prefixes = f.srcClient.ListSelectedPrefixes(ctx, &f.cfg.SrcPrefixList)
 	} else {
@@ -458,6 +466,10 @@ func NewWorker(ctx context.Context, cfg *JobConfig) (w *Worker) {
 		srcClient.isSrcClient = true
 	}
 
+	if cfg.PayerRequest {
+		srcClient.isPayerRequest = true
+	}
+
 	SRC_CRED = srcCred
 	DST_CRED = desCred
 

go.mod

@@ -3,16 +3,16 @@ module golang.a2z.com/dthcli
 go 1.16
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.8.1
+	github.com/aws/aws-sdk-go-v2 v1.17.7
 	github.com/aws/aws-sdk-go-v2/config v1.6.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.3.3
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.1.1
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.1
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.3.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.3.1
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.4.1
-	github.com/aws/smithy-go v1.7.0
+	github.com/aws/smithy-go v1.13.5
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/viper v1.7.1
 )

go.sum

@@ -21,8 +21,11 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go-v2 v1.6.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w=
-github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag=
 github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0=
+github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg=
+github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
 github.com/aws/aws-sdk-go-v2/config v1.6.1 h1:qrZINaORyr78syO1zfD4l7r4tZjy0Z1l0sy4jiysyOM=
 github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA=
 github.com/aws/aws-sdk-go-v2/credentials v1.3.3 h1:A13QPatmUl41SqUfnuT3V0E3XiNGL6qNTOINbE8cZL4=
@@ -33,21 +36,33 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1 h1:rc+fRGvlKbeSd9IFhFS1KWBs
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.1 h1:iJMRP1dYMC0FNzPmNKJOEv7ZqyIfQl1H0+1Dpv+z3Bs=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.1/go.mod h1:Ir1Mb3X4bIw0dxJQVW55CdXGlVy+t+I9zZpqPJKjHto=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1 h1:IkqRRUZTKaS16P2vpX+FNc2jq3JWa3c478gykQp4ow4=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 h1:DWYZIsyqagnWL00f8M/SOr9fN063OEQWn9LLTbdYXsk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23/go.mod h1:uIiFgURZbACBEQJfqTZPb/jxO7R+9LeoHUFudtIdeQI=
 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.3.1 h1:uAn7miAtNHewvTnGbXOA0dW1yE/3spgq9lB2D3sfpeQ=
 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.3.1/go.mod h1:zL1ZppTfOSWcDLRURlZn9w4Wl0jceSBF5PoUXa1amYc=
 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.2.1 h1:7qlg0PmBJJoP6LDTNWfAxVuENYscckFSbnT7GrsNRRk=
 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.2.1/go.mod h1:iLSTe7bH0WakfRQ4ga7mhEvrTDwoB1UfXMztiVFBkzQ=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.1.0/go.mod h1:zdjOOy0ojUn3iNELo6ycIHSMCp4xUbycSHfb8PnbbyM=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3 h1:VxFCgxsqWe7OThOwJ5IpFX3xrObtuIH9Hg/NW7oot1Y=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 h1:CeuSeq/8FnYpPtnuIeLQEEvDv9zUjneuYi8EghMBdwQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26/go.mod h1:2UqAAwMUXKeRkAHIlDJqvMVgOWkUi/AUXPk/YIe+Dg4=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3 h1:7tPSbUWzuoMJ2woUKgOfIPuZS88hMdFHJBBB2vR0bHI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3/go.mod h1:/ugW3qFkJe/h7sNtI6/zJnwRbvavs6GyOid69uI9eek=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 h1:2oMLrNpOSpkDTocIVv3Fut1XrmlbKPlgnnYMGYqFp0Y=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 h1:e2ooMhpYGhDnBfSvIyusvAwX7KexuZaHbQY2Dyei7VU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0/go.mod h1:bh2E0CXKZsQN+faiKVqC40vfNMAWheoULBCnEgO9K+8=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0/go.mod h1:Tzxhu3GnCpj45WJqXyxcLF2gUHzTcmY7CzpQ9x9KVls=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1 h1:PJH4I+qYjPXclKRbVCW47iYUvtXEh1u6YmDhn5J8VQE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1/go.mod h1:ncltU6n4Nof5uJttDtcNQ537uNuwYqsZZQcpkd2/GUQ=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.3.1 h1:atHdsCczZyM/y9QIoCQnxudoKk8+ya2EPIplDOofkjw=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.3.1/go.mod h1:ayQUSrG5QyIl2jRSB0YnoJ1e9swNxsBWaCK3hNI2caI=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.4.1 h1:BBW0EAZrJh3pesbs9m7ghgC8kylLjTPmWN3yG7POjnA=
@@ -57,8 +72,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4Pg
 github.com/aws/aws-sdk-go-v2/service/sts v1.6.2 h1:l504GWCoQi1Pk68vSUFGLmDIEMzRfVGNgLakDK+Uj58=
 github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM=
 github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
-github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg=
 github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
+github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
+github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -100,8 +116,9 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -317,7 +334,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=