chore: Add stability

Author: daixba

dth/client.go

@@ -490,7 +490,7 @@ func (c *S3Client) ListParts(ctx context.Context, key, uploadID *string) (parts
 		output, err := c.client.ListParts(ctx, input)
 		if err != nil {
 			log.Printf("Failed to list parts for %s - %s\n", *key, err.Error())
-			// return nil
+			break
 		}
 
 		for _, part := range output.Parts {

dth/job.go

@@ -172,11 +172,13 @@ func (f *Finder) Run(ctx context.Context) {
 func (f *Finder) getTargetObjects(ctx context.Context, prefix *string) (objects map[string]*int64) {
 
 	destPrefix := appendPrefix(prefix, &f.cfg.DestPrefix)
-	// log.Printf("Getting target list in destination prefix /%s\n", *destPrefix)
+	log.Printf("Scanning in destination prefix /%s\n", *destPrefix)
 
 	token := ""
 	objects = make(map[string]*int64, 1<<17)
 
+	i := 0
+	batch := 10
 	for token != "End" {
 		tar, err := f.desClient.ListObjects(ctx, &token, destPrefix, f.cfg.MaxKeys)
 		if err != nil {
@@ -190,6 +192,10 @@ func (f *Finder) getTargetObjects(ctx context.Context, prefix *string) (objects
 			srcKey := removePrefix(&obj.Key, &f.cfg.DestPrefix)
 			objects[*srcKey] = &obj.Size
 		}
+		i++
+		if (i % batch) == 0 {
+			log.Printf("Scanned %d objects...", i*1000)
+		}
 	}
 	log.Printf("Totally %d objects in destination prefix /%s\n", len(objects), *destPrefix)
 	return
@@ -200,7 +206,7 @@ func (f *Finder) getTargetObjects(ctx context.Context, prefix *string) (objects
 func (f *Finder) compareAndSend(ctx context.Context, prefix *string, batchCh chan struct{}, msgCh chan *string, compareCh chan struct{}, wg *sync.WaitGroup) {
 	defer wg.Done()
 
-	log.Printf("Comparing in source prefix /%s\n", *prefix)
+	log.Printf("Comparing within prefix /%s\n", *prefix)
 	target := f.getTargetObjects(ctx, prefix)
 
 	token := ""

dth/service.go

@@ -18,6 +18,7 @@ package dth
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"log"
 	"strconv"
@@ -268,7 +269,7 @@ func (ss *SqsService) IsQueueEmpty(ctx context.Context) (isEmpty bool) {
 }
 
 // GetSecret is a function to read the value of a secret in Secrets Manager
-func (s *SecretService) GetSecret(ctx context.Context, secretName *string) *string {
+func (s *SecretService) GetSecret(ctx context.Context, secretName *string) (value *string) {
 	log.Printf("Get secret Value of %s from Secrets Manager\n", *secretName)
 
 	input := &sm.GetSecretValueInput{
@@ -279,7 +280,20 @@ func (s *SecretService) GetSecret(ctx context.Context, secretName *string) *stri
 		log.Printf("Error getting secret Value of %s from Secrets Manager - %s", *secretName, err.Error())
 		return nil
 	}
-	return output.SecretString
+
+	if output.SecretString != nil {
+		value = output.SecretString
+	} else {
+		decodedBinarySecretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(output.SecretBinary)))
+		len, err := base64.StdEncoding.Decode(decodedBinarySecretBytes, output.SecretBinary)
+		if err != nil {
+			log.Println("Error decoding Binary Secret - ", err)
+			return
+		}
+		decodedBinarySecret := string(decodedBinarySecretBytes[:len])
+		value = &decodedBinarySecret
+	}
+	return
 }
 
 // PutItem is a function to creates a new item, or replaces an old item with a new item in DynamoDB