Pr/inline artifacts

.github/workflows/test.yaml

@@ -40,13 +40,13 @@ jobs:
             postgrest
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Install Go
-        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: 1.25.x
           cache: false # Using custom cache action below for .bin directory
       - name: Checkout code
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - uses: buildjet/cache@3e70d19e31d6a8030aeddf6ed8dbe601f94d09f4 # v4.0.2
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/go/pkg/mod
@@ -76,13 +76,13 @@ jobs:
           --health-retries 5
     steps:
       - name: Install Go
-        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: 1.25.x
           cache: false # Using custom cache action below for .bin directory
       - name: Checkout code
         uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
-      - uses: buildjet/cache@3e70d19e31d6a8030aeddf6ed8dbe601f94d09f4 # v4.0.2
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/go/pkg/mod
@@ -99,6 +99,30 @@ jobs:
           DUTY_DB_DISABLE_RLS: "true"
           LOKI_URL: http://localhost:3100
 
+  e2e-blobs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version: 1.25.x
+          cache: false
+      - name: Checkout code
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            .bin
+          key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
+          restore-keys: |
+            cache-
+      - name: E2E Blob Store Tests
+        run: make test-e2e-blobs
+        env:
+          DUTY_DB_DISABLE_RLS: "true"
+
   migrate:
     runs-on: ubuntu-latest
     strategy:
@@ -115,11 +139,11 @@ jobs:
         go build main.go && ./main --db-url 'postgres://postgres:postgres@localhost:5432/test?sslmode=disable'
     steps:
       - name: Install Go
-        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: 1.25.x
           cache: false # Using custom cache action below for .bin directory
-      - uses: buildjet/cache@3e70d19e31d6a8030aeddf6ed8dbe601f94d09f4 # v4.0.2
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/go/pkg/mod

Makefile

@@ -12,7 +12,7 @@ ginkgo:
 	go install github.com/onsi/ginkgo/v2/ginkgo
 
 test: ginkgo
-	ginkgo -r   --succinct --skip-package=tests/e2e,bench --label-filter "!e2e"
+	ginkgo -r   --succinct --skip-package=tests/e2e,tests/e2e-blobs,bench --label-filter "!e2e"
 
 test-concurrent: ginkgo
 	ginkgo -r -v --nodes=4 --skip-package=bench --label-filter "!e2e"
@@ -30,6 +30,10 @@ e2e-services: ## Run e2e test services in foreground with automatic cleanup on e
 	trap 'docker-compose down -v && docker-compose rm -f' EXIT INT TERM && \
 	docker-compose up --remove-orphans
 
+.PHONY: test-e2e-blobs
+test-e2e-blobs: ginkgo
+	ginkgo -v --label-filter="e2e" ./tests/e2e-blobs/
+
 .PHONY: bench
 bench:
 	go test -run ^$$ -bench=. -benchtime=10s -timeout 30m github.com/flanksource/duty/bench

artifact/blob_store.go

@@ -0,0 +1,110 @@
+package artifact
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+
+	"github.com/flanksource/duty/models"
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+)
+
+type BlobStore interface {
+	Write(data Data, artifact *models.Artifact) (*models.Artifact, error)
+	Read(artifactID uuid.UUID) (*Data, error)
+	io.Closer
+}
+
+type blobStore struct {
+	fs      FilesystemRW
+	db      *gorm.DB
+	backend string
+}
+
+func NewBlobStore(fs FilesystemRW, db *gorm.DB, backend string) BlobStore {
+	return &blobStore{fs: fs, db: db, backend: backend}
+}
+
+func (s *blobStore) Write(data Data, a *models.Artifact) (*models.Artifact, error) {
+	if a == nil {
+		a = &models.Artifact{}
+	}
+	if data.Content == nil {
+		return nil, fmt.Errorf("artifact data content is nil")
+	}
+	defer func() { _ = data.Content.Close() }()
+
+	checksum := sha256.New()
+	mimeReader := io.TeeReader(data.Content, checksum)
+
+	mw := &mimeWriter{Max: maxBytesForMimeDetection}
+	fileReader := io.TeeReader(mimeReader, mw)
+
+	info, err := s.fs.Write(s.db.Statement.Context, data.Filename, fileReader)
+	if err != nil {
+		return nil, fmt.Errorf("writing artifact %s: %w", data.Filename, err)
+	}
+
+	if data.ContentType == "" {
+		data.ContentType = mw.Detect().String()
+	}
+
+	// For inline store, the artifact already has content set
+	if inlineArt := InlineArtifact(info); inlineArt != nil {
+		a.Content = inlineArt.Content
+		a.CompressionType = inlineArt.CompressionType
+	}
+
+	a.Path = data.Filename
+	a.Filename = info.Name()
+	a.Size = info.Size()
+	a.ContentType = data.ContentType
+	a.Checksum = hex.EncodeToString(checksum.Sum(nil))
+
+	if err := s.db.Create(a).Error; err != nil {
+		return nil, fmt.Errorf("saving artifact to db: %w", err)
+	}
+
+	return a, nil
+}
+
+func (s *blobStore) Read(artifactID uuid.UUID) (*Data, error) {
+	var a models.Artifact
+	if err := s.db.Where("id = ?", artifactID).First(&a).Error; err != nil {
+		return nil, fmt.Errorf("finding artifact %s: %w", artifactID, err)
+	}
+
+	if a.IsInline() {
+		content, err := a.GetContent()
+		if err != nil {
+			return nil, fmt.Errorf("decompressing inline artifact %s: %w", artifactID, err)
+		}
+		return &Data{
+			Content:       io.NopCloser(bytes.NewReader(content)),
+			ContentLength: a.Size,
+			Checksum:      a.Checksum,
+			ContentType:   a.ContentType,
+			Filename:      a.Filename,
+		}, nil
+	}
+
+	r, err := s.fs.Read(s.db.Statement.Context, a.Path)
+	if err != nil {
+		return nil, fmt.Errorf("reading artifact %s from %s: %w", artifactID, a.Path, err)
+	}
+
+	return &Data{
+		Content:       r,
+		ContentLength: a.Size,
+		Checksum:      a.Checksum,
+		ContentType:   a.ContentType,
+		Filename:      a.Filename,
+	}, nil
+}
+
+func (s *blobStore) Close() error {
+	return s.fs.Close()
+}

artifact/clients/aws/doc.go

@@ -0,0 +1 @@
+package aws

artifact/clients/aws/fileinfo.go

@@ -0,0 +1,48 @@
+//go:build !fast
+
+package aws
+
+import (
+	"io/fs"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/flanksource/commons/utils"
+	"github.com/samber/lo"
+)
+
+type S3FileInfo struct {
+	Object types.Object
+}
+
+func (obj S3FileInfo) Name() string {
+	if obj.Object.Key == nil {
+		return ""
+	}
+	return *obj.Object.Key
+}
+
+func (obj S3FileInfo) Size() int64 {
+	return utils.Deref(obj.Object.Size)
+}
+
+func (obj S3FileInfo) Mode() fs.FileMode {
+	return fs.FileMode(0644)
+}
+
+func (obj S3FileInfo) ModTime() time.Time {
+	return lo.FromPtr(obj.Object.LastModified)
+}
+
+func (obj S3FileInfo) FullPath() string {
+	return *obj.Object.Key
+}
+
+func (obj S3FileInfo) IsDir() bool {
+	return strings.HasSuffix(obj.Name(), "/")
+}
+
+func (obj S3FileInfo) Sys() interface{} {
+	return obj.Object
+}

artifact/clients/azure/fileinfo.go

@@ -0,0 +1,21 @@
+package azure
+
+import (
+	"io/fs"
+	"time"
+)
+
+type BlobFileInfo struct {
+	BlobName    string
+	BlobSize    int64
+	LastMod     time.Time
+	ContentType string
+}
+
+func (f BlobFileInfo) Name() string      { return f.BlobName }
+func (f BlobFileInfo) Size() int64       { return f.BlobSize }
+func (f BlobFileInfo) Mode() fs.FileMode { return 0644 }
+func (f BlobFileInfo) ModTime() time.Time { return f.LastMod }
+func (f BlobFileInfo) IsDir() bool       { return false }
+func (f BlobFileInfo) Sys() any          { return nil }
+func (f BlobFileInfo) FullPath() string  { return f.BlobName }

artifact/clients/gcp/fileinfo.go

@@ -0,0 +1,40 @@
+package gcp
+
+import (
+	"io/fs"
+	"time"
+
+	gcs "cloud.google.com/go/storage"
+)
+
+type GCSFileInfo struct {
+	Object *gcs.ObjectAttrs
+}
+
+func (GCSFileInfo) IsDir() bool {
+	return false
+}
+
+func (obj GCSFileInfo) ModTime() time.Time {
+	return obj.Object.Updated
+}
+
+func (obj GCSFileInfo) Mode() fs.FileMode {
+	return fs.FileMode(0644)
+}
+
+func (obj GCSFileInfo) Name() string {
+	return obj.Object.Name
+}
+
+func (obj GCSFileInfo) Size() int64 {
+	return obj.Object.Size
+}
+
+func (obj GCSFileInfo) Sys() interface{} {
+	return obj.Object
+}
+
+func (obj GCSFileInfo) FullPath() string {
+	return obj.Object.Name
+}

artifact/clients/sftp/sftp.go

@@ -0,0 +1,35 @@
+package sftp
+
+import (
+	"time"
+
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// SSHConnect creates an SFTP client connection.
+// NOTE: Uses InsecureIgnoreHostKey because artifact storage targets are
+// configured by admins via trusted connection objects, not user input.
+func SSHConnect(host, user, password string) (*sftp.Client, error) {
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec
+		Timeout:         30 * time.Second,
+	}
+
+	conn, err := ssh.Dial("tcp", host, config)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := sftp.NewClient(conn)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return client, nil
+}