Adding endpoint config to enable minio backend (#6)

adds endpoint and disable ssl config to enable support for minio

Co-authored-by: Aleksandar Nikolikj <aleksandar.nikolikj@ericsson.com>
Co-authored-by: Qingping Hou <qph@scribd.com>

Author: 3 people

main.go

@@ -26,6 +26,8 @@ var (
 	FlagExclude         []string
 	FlagScratch         bool
 	FlagDefaultFileMode = "0664"
+	FlagS3Endpoint      = ""
+	FlagDisableSSL      = false
 
 	metricsSyncTime = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "objinsync",
@@ -102,6 +104,8 @@ func main() {
 			if err != nil {
 				log.Fatal(err)
 			}
+			puller.DisableSSL = FlagDisableSSL
+			puller.S3Endpoint = FlagS3Endpoint
 			if FlagExclude != nil {
 				puller.AddExcludePatterns(FlagExclude)
 			}
@@ -156,6 +160,8 @@ func main() {
 
 	pullCmd.PersistentFlags().BoolVarP(
 		&FlagRunOnce, "once", "o", false, "run action once and then exit")
+	pullCmd.PersistentFlags().BoolVarP(
+		&FlagDisableSSL, "disable-ssl", "", false, "disable SSL for object storage connection")
 	pullCmd.PersistentFlags().StringVarP(
 		&FlagStatusAddr, "status-addr", "s", ":8087", "binding address for status endpoint")
 	pullCmd.PersistentFlags().StringSliceVarP(
@@ -169,6 +175,8 @@ func main() {
 	)
 	pullCmd.PersistentFlags().StringVarP(
 		&FlagDefaultFileMode, "default-file-mode", "m", "0664", "default mode to use for creating local file")
+	pullCmd.PersistentFlags().StringVarP(
+		&FlagS3Endpoint, "s3-endpoint", "", "", "override endpoint to use for remote object store (e.g. minio)")
 
 	rootCmd.AddCommand(pullCmd)
 	rootCmd.Execute()

pkg/sync/pull.go

@@ -99,8 +99,10 @@ func uidFromLocalPath(localPath string) (string, error) {
 }
 
 type Puller struct {
-	RemoteUri string
-	LocalDir  string
+	RemoteUri  string
+	LocalDir   string
+	DisableSSL bool
+	S3Endpoint string
 
 	workingDir  string
 	defaultMode os.FileMode
@@ -308,7 +310,16 @@ func (self *Puller) Pull() string {
 		}
 	}
 
-	svc := s3.New(sess, aws.NewConfig().WithRegion(region))
+	s3Config := &aws.Config{Region: aws.String(region)}
+	if self.DisableSSL {
+		s3Config.DisableSSL = aws.Bool(true)
+	}
+	if self.S3Endpoint != "" {
+		s3Config.Endpoint = aws.String(self.S3Endpoint)
+		s3Config.S3ForcePathStyle = aws.Bool(true)
+	}
+	svc := s3.New(sess, s3Config)
+
 	downloader := s3manager.NewDownloaderWithClient(svc)
 
 	if err := self.SetupWorkingDir(); err != nil {
@@ -460,6 +471,7 @@ func NewPuller(remoteUri string, localDir string) (*Puller, error) {
 	return &Puller{
 		RemoteUri:   remoteUri,
 		LocalDir:    localDir,
+		DisableSSL:  false,
 		workingDir:  filepath.Join(localDir, ".objinsync"),
 		defaultMode: 0664,
 		workerCnt:   5,