feat(rst): add xtreemstore provider

Author: swartzn

common/rst/rst.go

@@ -152,6 +152,8 @@ func New(ctx context.Context, config *flex.RemoteStorageTarget, mountPoint files
 	switch config.Type.(type) {
 	case *flex.RemoteStorageTarget_S3_:
 		return newS3(ctx, config, mountPoint)
+	case *flex.RemoteStorageTarget_Xtreemstore:
+		return newXtreemstore(ctx, config, mountPoint)
 	case *flex.RemoteStorageTarget_Mock:
 		// This handles setting up a Mock RST for testing from external packages like WorkerMgr. See
 		// the documentation ion `MockClient` in mock.go for how to setup expectations.

common/rst/xtreemstore.go

@@ -0,0 +1,61 @@
+package rst
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/thinkparq/beegfs-go/common/filesystem"
+	"github.com/thinkparq/protobuf/go/beeremote"
+	"github.com/thinkparq/protobuf/go/flex"
+)
+
+type xtreemstoreS3Provider struct {
+	s3Provider
+	s3ApiClient
+}
+
+var _ Provider = &xtreemstoreS3Provider{}
+
+// newXtreemstore initializes an xtreemstore provider by reusing the S3 client implementation.
+func newXtreemstore(ctx context.Context, rstConfig *flex.RemoteStorageTarget, mountPoint filesystem.Provider) (Provider, error) {
+	xtreemstore := rstConfig.GetXtreemstore()
+	if xtreemstore == nil || xtreemstore.GetS3() == nil {
+		return nil, fmt.Errorf("xtreemstore configuration must include s3 settings")
+	}
+
+	wrapper := &xtreemstoreS3Provider{}
+	provider, err := newS3WithOptions(ctx, rstConfig, xtreemstore.GetS3(), mountPoint,
+		withS3ApiClient(func(base s3ApiClient) s3ApiClient {
+			wrapper.s3ApiClient = base
+			return wrapper
+		}),
+		withS3Provider(func(base s3Provider) s3Provider {
+			wrapper.s3Provider = base
+			return wrapper
+		}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to initialize xtreemstore provider: %w", err)
+	}
+
+	return provider, nil
+}
+
+func (x *xtreemstoreS3Provider) HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+	// Update xtreemstore head-object api request headers to include storage details.
+	optFns = append(optFns, func(options *s3.Options) {
+		options.APIOptions = append(options.APIOptions, smithyhttp.AddHeaderValue("x-amz-meta-xts-request-storage-details", "true"))
+	})
+
+	return x.s3ApiClient.HeadObject(ctx, in, optFns...)
+}
+
+func (x *xtreemstoreS3Provider) IncludeInBulkRequest(ctx context.Context, request *beeremote.JobRequest) (includeInBulk bool) {
+	return false
+}
+
+func (x *xtreemstoreS3Provider) BuildBulkRequest(ctx context.Context) (submitBulkRequest SubmitBulkRequestFn, appendBulkRequestCfg AppendBulkRequestCfgFn, err error) {
+	return nil, nil, nil
+}

common/rst/xtreemstore_test.go

@@ -0,0 +1 @@
+package rst

ctl/internal/cmd/rst/list.go

@@ -52,18 +52,16 @@ func runListCmd(cmd *cobra.Command, cfg rst.GetRSTCfg) error {
 
 		switch rst.WhichType() {
 		case flex.RemoteStorageTarget_S3_case:
-			stringBuilder := strings.Builder{}
 			rstType = "s3"
-			rst.GetS3().ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
-				if string(fd.Name()) == "secret_key" && !cfg.ShowSecrets {
-					stringBuilder.WriteString(fmt.Sprintf("%s: *****, ", fd.Name()))
-				} else {
-					stringBuilder.WriteString(fmt.Sprintf("%s: %s, ", fd.Name(), v))
-				}
-				return true
-			})
-			// Get rid of the last comma+space in the printed configuration.
-			rstConfiguration = stringBuilder.String()[:stringBuilder.Len()-2]
+			rstConfiguration = formatS3RSTConfiguration(rst.GetS3(), cfg.ShowSecrets)
+		case flex.RemoteStorageTarget_Xtreemstore_case:
+			rstType = "xtreemstore"
+			xtreemstoreConfig := rst.GetXtreemstore()
+			if xtreemstoreConfig == nil {
+				rstConfiguration = "xtreemstore configuration is not set"
+				break
+			}
+			rstConfiguration = formatS3RSTConfiguration(xtreemstoreConfig.GetS3(), cfg.ShowSecrets)
 		default:
 			if !cfg.ShowSecrets {
 				rstType = "unknown"
@@ -85,3 +83,25 @@ func runListCmd(cmd *cobra.Command, cfg rst.GetRSTCfg) error {
 
 	return nil
 }
+
+func formatS3RSTConfiguration(s3Config *flex.RemoteStorageTarget_S3, showSecrets bool) string {
+	if s3Config == nil {
+		return "s3 configuration is not set"
+	}
+
+	stringBuilder := strings.Builder{}
+	s3Config.ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+		if string(fd.Name()) == "secret_key" && !showSecrets {
+			fmt.Fprintf(&stringBuilder, "%s: *****, ", fd.Name())
+		} else {
+			fmt.Fprintf(&stringBuilder, "%s: %s, ", fd.Name(), v)
+		}
+		return true
+	})
+
+	if stringBuilder.Len() == 0 {
+		return ""
+	}
+	// Get rid of the last comma+space in the printed configuration.
+	return stringBuilder.String()[:stringBuilder.Len()-2]
+}

rst/remote/build/beegfs-remote.toml

@@ -107,6 +107,18 @@ path-db = "/var/lib/beegfs/remote/path.badger"
 #   access-key = "<ACCESS_KEY>"
 #   secret-key = "<SECRET_KEY>"
 
+# [[remote-storage-target]]
+# id = "4"
+# name = "xtreemstore"
+# policies = { FastStartMaxSize = 104857600 }
+#
+#   [remote-storage-target.xtreemstore]
+#   endpoint-url = "https://<XTREEMSTORE_ENDPOINT>:<PORT>"
+#   region = "<REGION>"
+#   bucket = "<BUCKET>"
+#   access-key = "<ACCESS_KEY>"
+#   secret-key = "<SECRET_KEY>"
+
 #
 # --- Section 2: [Command Line Arguments] ---
 #