feat(artifact): add support for artifact storage with inline and external backends

Introduces a pluggable artifact storage system supporting multiple backends (S3, GCS, SMB, SFTP, local filesystem) and inline database storage. Includes compression support (gzip), filesystem abstraction interfaces, and database schema for storing artifacts either inline or in external cloud/object storage systems. External connections can be configured via connection URL properties.

Author: moshloop

artifact/clients/aws/doc.go

@@ -0,0 +1 @@
+package aws

artifact/clients/aws/fileinfo.go

@@ -0,0 +1,45 @@
+//go:build !fast
+
+package aws
+
+import (
+	"io/fs"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/flanksource/commons/utils"
+	"github.com/samber/lo"
+)
+
+type S3FileInfo struct {
+	Object types.Object
+}
+
+func (obj S3FileInfo) Name() string {
+	return *obj.Object.Key
+}
+
+func (obj S3FileInfo) Size() int64 {
+	return utils.Deref(obj.Object.Size)
+}
+
+func (obj S3FileInfo) Mode() fs.FileMode {
+	return fs.FileMode(0644)
+}
+
+func (obj S3FileInfo) ModTime() time.Time {
+	return lo.FromPtr(obj.Object.LastModified)
+}
+
+func (obj S3FileInfo) FullPath() string {
+	return *obj.Object.Key
+}
+
+func (obj S3FileInfo) IsDir() bool {
+	return strings.HasSuffix(obj.Name(), "/")
+}
+
+func (obj S3FileInfo) Sys() interface{} {
+	return obj.Object
+}

artifact/clients/gcp/fileinfo.go

@@ -0,0 +1,40 @@
+package gcp
+
+import (
+	"io/fs"
+	"time"
+
+	gcs "cloud.google.com/go/storage"
+)
+
+type GCSFileInfo struct {
+	Object *gcs.ObjectAttrs
+}
+
+func (GCSFileInfo) IsDir() bool {
+	return false
+}
+
+func (obj GCSFileInfo) ModTime() time.Time {
+	return obj.Object.Updated
+}
+
+func (obj GCSFileInfo) Mode() fs.FileMode {
+	return fs.FileMode(0644)
+}
+
+func (obj GCSFileInfo) Name() string {
+	return obj.Object.Name
+}
+
+func (obj GCSFileInfo) Size() int64 {
+	return obj.Object.Size
+}
+
+func (obj GCSFileInfo) Sys() interface{} {
+	return obj.Object
+}
+
+func (obj GCSFileInfo) FullPath() string {
+	return obj.Object.Name
+}

artifact/clients/sftp/sftp.go

@@ -0,0 +1,28 @@
+package sftp
+
+import (
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+func SSHConnect(host, user, password string) (*sftp.Client, error) {
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+
+	conn, err := ssh.Dial("tcp", host, config)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := sftp.NewClient(conn)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}

artifact/clients/smb/smb.go

@@ -0,0 +1,63 @@
+package smb
+
+import (
+	"net"
+
+	"github.com/flanksource/duty/types"
+	"github.com/hirochachacha/go-smb2"
+)
+
+type SMBSession struct {
+	net.Conn
+	*smb2.Session
+	*smb2.Share
+}
+
+func SMBConnect(server string, port, share string, auth types.Authentication) (*SMBSession, error) {
+	var err error
+	var smb *SMBSession
+	server = server + ":" + port
+	conn, err := net.Dial("tcp", server)
+	if err != nil {
+		return nil, err
+	}
+	smb = &SMBSession{
+		Conn: conn,
+	}
+
+	d := &smb2.Dialer{
+		Initiator: &smb2.NTLMInitiator{
+			User:     auth.GetUsername(),
+			Password: auth.GetPassword(),
+			Domain:   auth.GetDomain(),
+		},
+	}
+
+	s, err := d.Dial(conn)
+	if err != nil {
+		return nil, err
+	}
+	smb.Session = s
+	fs, err := s.Mount(share)
+	if err != nil {
+		return nil, err
+	}
+
+	smb.Share = fs
+
+	return smb, err
+}
+
+func (s *SMBSession) Close() error {
+	if s.Conn != nil {
+		_ = s.Conn.Close()
+	}
+	if s.Session != nil {
+		_ = s.Logoff()
+	}
+	if s.Share != nil {
+		_ = s.Umount()
+	}
+
+	return nil
+}

artifact/fs.go

@@ -0,0 +1,20 @@
+package artifact
+
+import (
+	gocontext "context"
+	"io"
+	"os"
+)
+
+type FileInfo interface {
+	os.FileInfo
+	FullPath() string
+}
+
+type FilesystemRW interface {
+	io.Closer
+	Read(ctx gocontext.Context, path string) (io.ReadCloser, error)
+	Write(ctx gocontext.Context, path string, data io.Reader) (os.FileInfo, error)
+	ReadDir(name string) ([]FileInfo, error)
+	Stat(name string) (os.FileInfo, error)
+}

artifact/fs/gcs.go

@@ -0,0 +1,86 @@
+package fs
+
+import (
+	gocontext "context"
+	"errors"
+	"io"
+	"os"
+	"strings"
+
+	gcs "cloud.google.com/go/storage"
+	"github.com/flanksource/duty/artifact"
+	gcpUtil "github.com/flanksource/duty/artifact/clients/gcp"
+	"google.golang.org/api/iterator"
+)
+
+type gcsFS struct {
+	*gcs.Client
+	Bucket string
+}
+
+func NewGCSFS(client *gcs.Client, bucket string) *gcsFS {
+	return &gcsFS{
+		Bucket: strings.TrimPrefix(bucket, "gcs://"),
+		Client: client,
+	}
+}
+
+func (t *gcsFS) Close() error {
+	return t.Client.Close()
+}
+
+func (t *gcsFS) ReadDir(name string) ([]artifact.FileInfo, error) {
+	bucket := t.Client.Bucket(t.Bucket)
+	objs := bucket.Objects(gocontext.TODO(), &gcs.Query{Prefix: name})
+
+	var output []artifact.FileInfo
+	for {
+		obj, err := objs.Next()
+		if err != nil {
+			if errors.Is(err, iterator.Done) {
+				break
+			}
+			return nil, err
+		}
+		if obj == nil {
+			break
+		}
+
+		output = append(output, gcpUtil.GCSFileInfo{Object: obj})
+	}
+
+	return output, nil
+}
+
+func (t *gcsFS) Stat(path string) (os.FileInfo, error) {
+	obj := t.Client.Bucket(t.Bucket).Object(path)
+	attrs, err := obj.Attrs(gocontext.TODO())
+	if err != nil {
+		return nil, err
+	}
+
+	return &gcpUtil.GCSFileInfo{Object: attrs}, nil
+}
+
+func (t *gcsFS) Read(ctx gocontext.Context, path string) (io.ReadCloser, error) {
+	return t.Client.Bucket(t.Bucket).Object(path).NewReader(ctx)
+}
+
+func (t *gcsFS) Write(ctx gocontext.Context, path string, data io.Reader) (os.FileInfo, error) {
+	obj := t.Client.Bucket(t.Bucket).Object(path)
+
+	content, err := io.ReadAll(data)
+	if err != nil {
+		return nil, err
+	}
+
+	writer := obj.NewWriter(ctx)
+	if _, err := writer.Write(content); err != nil {
+		return nil, err
+	}
+	if err := writer.Close(); err != nil {
+		return nil, err
+	}
+
+	return t.Stat(path)
+}

artifact/fs/local.go

@@ -0,0 +1,109 @@
+package fs
+
+import (
+	gocontext "context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/bmatcuk/doublestar/v4"
+	"github.com/flanksource/duty/artifact"
+)
+
+type localFS struct {
+	base string
+}
+
+type localFileInfo struct {
+	os.FileInfo
+	fullpath string
+}
+
+func (t localFileInfo) FullPath() string {
+	return t.fullpath
+}
+
+func NewLocalFS(base string) *localFS {
+	return &localFS{base: base}
+}
+
+func (t *localFS) Close() error {
+	return nil
+}
+
+func (t *localFS) ReadDir(name string) ([]artifact.FileInfo, error) {
+	if strings.Contains(name, "*") {
+		return t.ReadDirGlob(name)
+	}
+
+	path := filepath.Join(t.base, name)
+	files, err := os.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+
+	output := make([]artifact.FileInfo, 0, len(files))
+	for _, match := range files {
+		fullPath := filepath.Join(path, match.Name())
+		info, err := os.Stat(fullPath)
+		if err != nil {
+			return nil, err
+		}
+
+		output = append(output, localFileInfo{FileInfo: info, fullpath: fullPath})
+	}
+
+	return output, nil
+}
+
+func (t *localFS) ReadDirGlob(name string) ([]artifact.FileInfo, error) {
+	base, pattern := doublestar.SplitPattern(filepath.Join(t.base, name))
+	matches, err := doublestar.Glob(os.DirFS(base), pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	output := make([]artifact.FileInfo, 0, len(matches))
+	for _, match := range matches {
+		fullPath := filepath.Join(base, match)
+		info, err := os.Stat(fullPath)
+		if err != nil {
+			return nil, err
+		}
+
+		output = append(output, localFileInfo{FileInfo: info, fullpath: fullPath})
+	}
+
+	return output, nil
+}
+
+func (t *localFS) Stat(name string) (os.FileInfo, error) {
+	return os.Stat(filepath.Join(t.base, name))
+}
+
+func (t *localFS) Read(_ gocontext.Context, path string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(t.base, path))
+}
+
+func (t *localFS) Write(_ gocontext.Context, path string, data io.Reader) (os.FileInfo, error) {
+	fullpath := filepath.Join(t.base, path)
+
+	err := os.MkdirAll(filepath.Dir(fullpath), os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("error creating base directory: %w", err)
+	}
+
+	f, err := os.Create(fullpath)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(f, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return t.Stat(path)
+}