feat(artifact): add support for artifact storage with inline and external backends

Introduces a pluggable artifact storage system supporting multiple backends (S3, GCS, SMB, SFTP, local filesystem) and inline database storage. Includes compression support (gzip), filesystem abstraction interfaces, and database schema for storing artifacts either inline or in external cloud/object storage systems. External connections can be configured via connection URL properties.

Author: moshloop

.github/workflows/test.yaml

@@ -40,13 +40,13 @@ jobs:
             postgrest
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Install Go
-        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: 1.25.x
           cache: false # Using custom cache action below for .bin directory
       - name: Checkout code
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - uses: buildjet/cache@3e70d19e31d6a8030aeddf6ed8dbe601f94d09f4 # v4.0.2
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/go/pkg/mod
@@ -76,13 +76,13 @@ jobs:
           --health-retries 5
     steps:
       - name: Install Go
-        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: 1.25.x
           cache: false # Using custom cache action below for .bin directory
       - name: Checkout code
         uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
-      - uses: buildjet/cache@3e70d19e31d6a8030aeddf6ed8dbe601f94d09f4 # v4.0.2
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/go/pkg/mod
@@ -99,6 +99,30 @@ jobs:
           DUTY_DB_DISABLE_RLS: "true"
           LOKI_URL: http://localhost:3100
 
+  e2e-blobs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version: 1.25.x
+          cache: false
+      - name: Checkout code
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            .bin
+          key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
+          restore-keys: |
+            cache-
+      - name: E2E Blob Store Tests
+        run: make test-e2e-blobs
+        env:
+          DUTY_DB_DISABLE_RLS: "true"
+
   migrate:
     runs-on: ubuntu-latest
     strategy:
@@ -115,11 +139,11 @@ jobs:
         go build main.go && ./main --db-url 'postgres://postgres:postgres@localhost:5432/test?sslmode=disable'
     steps:
       - name: Install Go
-        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: 1.25.x
           cache: false # Using custom cache action below for .bin directory
-      - uses: buildjet/cache@3e70d19e31d6a8030aeddf6ed8dbe601f94d09f4 # v4.0.2
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/go/pkg/mod

Makefile

@@ -12,7 +12,7 @@ ginkgo:
 	go install github.com/onsi/ginkgo/v2/ginkgo
 
 test: ginkgo
-	ginkgo -r   --succinct --skip-package=tests/e2e,bench --label-filter "!e2e"
+	ginkgo -r   --succinct --skip-package=tests/e2e,tests/e2e-blobs,bench --label-filter "!e2e"
 
 test-concurrent: ginkgo
 	ginkgo -r -v --nodes=4 --skip-package=bench --label-filter "!e2e"
@@ -30,6 +30,10 @@ e2e-services: ## Run e2e test services in foreground with automatic cleanup on e
 	trap 'docker-compose down -v && docker-compose rm -f' EXIT INT TERM && \
 	docker-compose up --remove-orphans
 
+.PHONY: test-e2e-blobs
+test-e2e-blobs: ginkgo
+	ginkgo -v --label-filter="e2e" ./tests/e2e-blobs/
+
 .PHONY: bench
 bench:
 	go test -run ^$$ -bench=. -benchtime=10s -timeout 30m github.com/flanksource/duty/bench

artifact/blob_store.go

@@ -0,0 +1,107 @@
+package artifact
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+
+	"github.com/flanksource/duty/models"
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+)
+
+type BlobStore interface {
+	Write(data Data, artifact *models.Artifact) (*models.Artifact, error)
+	Read(artifactID uuid.UUID) (*Data, error)
+	io.Closer
+}
+
+type blobStore struct {
+	fs      FilesystemRW
+	db      *gorm.DB
+	backend string
+}
+
+func NewBlobStore(fs FilesystemRW, db *gorm.DB, backend string) BlobStore {
+	return &blobStore{fs: fs, db: db, backend: backend}
+}
+
+func (s *blobStore) Write(data Data, a *models.Artifact) (*models.Artifact, error) {
+	if data.Content == nil {
+		return nil, fmt.Errorf("artifact data content is nil")
+	}
+	defer func() { _ = data.Content.Close() }()
+
+	checksum := sha256.New()
+	mimeReader := io.TeeReader(data.Content, checksum)
+
+	mw := &mimeWriter{Max: maxBytesForMimeDetection}
+	fileReader := io.TeeReader(mimeReader, mw)
+
+	info, err := s.fs.Write(s.db.Statement.Context, data.Filename, fileReader)
+	if err != nil {
+		return nil, fmt.Errorf("writing artifact %s: %w", data.Filename, err)
+	}
+
+	if data.ContentType == "" {
+		data.ContentType = mw.Detect().String()
+	}
+
+	// For inline store, the artifact already has content set
+	if inlineArt := InlineArtifact(info); inlineArt != nil {
+		a.Content = inlineArt.Content
+		a.CompressionType = inlineArt.CompressionType
+	}
+
+	a.Path = data.Filename
+	a.Filename = info.Name()
+	a.Size = info.Size()
+	a.ContentType = data.ContentType
+	a.Checksum = hex.EncodeToString(checksum.Sum(nil))
+
+	if err := s.db.Create(a).Error; err != nil {
+		return nil, fmt.Errorf("saving artifact to db: %w", err)
+	}
+
+	return a, nil
+}
+
+func (s *blobStore) Read(artifactID uuid.UUID) (*Data, error) {
+	var a models.Artifact
+	if err := s.db.Where("id = ?", artifactID).First(&a).Error; err != nil {
+		return nil, fmt.Errorf("finding artifact %s: %w", artifactID, err)
+	}
+
+	if a.IsInline() {
+		content, err := a.GetContent()
+		if err != nil {
+			return nil, fmt.Errorf("decompressing inline artifact %s: %w", artifactID, err)
+		}
+		return &Data{
+			Content:       io.NopCloser(bytes.NewReader(content)),
+			ContentLength: a.Size,
+			Checksum:      a.Checksum,
+			ContentType:   a.ContentType,
+			Filename:      a.Filename,
+		}, nil
+	}
+
+	r, err := s.fs.Read(s.db.Statement.Context, a.Path)
+	if err != nil {
+		return nil, fmt.Errorf("reading artifact %s from %s: %w", artifactID, a.Path, err)
+	}
+
+	return &Data{
+		Content:       r,
+		ContentLength: a.Size,
+		Checksum:      a.Checksum,
+		ContentType:   a.ContentType,
+		Filename:      a.Filename,
+	}, nil
+}
+
+func (s *blobStore) Close() error {
+	return s.fs.Close()
+}

artifact/clients/aws/doc.go

@@ -0,0 +1 @@
+package aws

artifact/clients/aws/fileinfo.go

@@ -0,0 +1,48 @@
+//go:build !fast
+
+package aws
+
+import (
+	"io/fs"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/flanksource/commons/utils"
+	"github.com/samber/lo"
+)
+
+type S3FileInfo struct {
+	Object types.Object
+}
+
+func (obj S3FileInfo) Name() string {
+	if obj.Object.Key == nil {
+		return ""
+	}
+	return *obj.Object.Key
+}
+
+func (obj S3FileInfo) Size() int64 {
+	return utils.Deref(obj.Object.Size)
+}
+
+func (obj S3FileInfo) Mode() fs.FileMode {
+	return fs.FileMode(0644)
+}
+
+func (obj S3FileInfo) ModTime() time.Time {
+	return lo.FromPtr(obj.Object.LastModified)
+}
+
+func (obj S3FileInfo) FullPath() string {
+	return *obj.Object.Key
+}
+
+func (obj S3FileInfo) IsDir() bool {
+	return strings.HasSuffix(obj.Name(), "/")
+}
+
+func (obj S3FileInfo) Sys() interface{} {
+	return obj.Object
+}

artifact/clients/azure/fileinfo.go

@@ -0,0 +1,21 @@
+package azure
+
+import (
+	"io/fs"
+	"time"
+)
+
+type BlobFileInfo struct {
+	BlobName    string
+	BlobSize    int64
+	LastMod     time.Time
+	ContentType string
+}
+
+func (f BlobFileInfo) Name() string      { return f.BlobName }
+func (f BlobFileInfo) Size() int64       { return f.BlobSize }
+func (f BlobFileInfo) Mode() fs.FileMode { return 0644 }
+func (f BlobFileInfo) ModTime() time.Time { return f.LastMod }
+func (f BlobFileInfo) IsDir() bool       { return false }
+func (f BlobFileInfo) Sys() any          { return nil }
+func (f BlobFileInfo) FullPath() string  { return f.BlobName }

artifact/clients/gcp/fileinfo.go

@@ -0,0 +1,40 @@
+package gcp
+
+import (
+	"io/fs"
+	"time"
+
+	gcs "cloud.google.com/go/storage"
+)
+
+type GCSFileInfo struct {
+	Object *gcs.ObjectAttrs
+}
+
+func (GCSFileInfo) IsDir() bool {
+	return false
+}
+
+func (obj GCSFileInfo) ModTime() time.Time {
+	return obj.Object.Updated
+}
+
+func (obj GCSFileInfo) Mode() fs.FileMode {
+	return fs.FileMode(0644)
+}
+
+func (obj GCSFileInfo) Name() string {
+	return obj.Object.Name
+}
+
+func (obj GCSFileInfo) Size() int64 {
+	return obj.Object.Size
+}
+
+func (obj GCSFileInfo) Sys() interface{} {
+	return obj.Object
+}
+
+func (obj GCSFileInfo) FullPath() string {
+	return obj.Object.Name
+}

artifact/clients/sftp/sftp.go

@@ -0,0 +1,32 @@
+package sftp
+
+import (
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// SSHConnect creates an SFTP client connection.
+// NOTE: Uses InsecureIgnoreHostKey because artifact storage targets are
+// configured by admins via trusted connection objects, not user input.
+func SSHConnect(host, user, password string) (*sftp.Client, error) {
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec
+	}
+
+	conn, err := ssh.Dial("tcp", host, config)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := sftp.NewClient(conn)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return client, nil
+}