chore: move to storage package

mykeelium · mykeelium · commit f1acc3a2f1d2 · 2026-06-08T12:53:25.000-04:00
diff --git a/cmd/api/src/api/tools/ingest.go b/cmd/api/src/api/tools/ingest.go
@@ -31,9 +31,9 @@ import (
 	"github.com/specterops/bloodhound/cmd/api/src/api"
 	"github.com/specterops/bloodhound/cmd/api/src/database/types"
 	"github.com/specterops/bloodhound/cmd/api/src/model/appcfg"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 	"github.com/specterops/bloodhound/packages/go/headers"
+	"github.com/specterops/bloodhound/packages/go/storage"
 )
 
 type IngestControl struct {
diff --git a/cmd/api/src/api/tools/ingest_test.go b/cmd/api/src/api/tools/ingest_test.go
@@ -30,9 +30,9 @@ import (
 
 	"github.com/specterops/bloodhound/cmd/api/src/database/types"
 	"github.com/specterops/bloodhound/cmd/api/src/model/appcfg"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
-	storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
 	"github.com/specterops/bloodhound/packages/go/headers"
+	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 )
diff --git a/cmd/api/src/api/v2/collectors.go b/cmd/api/src/api/v2/collectors.go
@@ -26,7 +26,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/specterops/bloodhound/cmd/api/src/api"
 	"github.com/specterops/bloodhound/cmd/api/src/config"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 	"github.com/specterops/bloodhound/packages/go/storage"
 )
diff --git a/cmd/api/src/api/v2/fileingest.go b/cmd/api/src/api/v2/fileingest.go
@@ -35,9 +35,9 @@ import (
 	ingestModel "github.com/specterops/bloodhound/cmd/api/src/model/ingest"
 	"github.com/specterops/bloodhound/packages/go/bhlog/measure"
 	"github.com/specterops/bloodhound/packages/go/headers"
+	"github.com/specterops/bloodhound/packages/go/storage"
 
 	"github.com/specterops/bloodhound/cmd/api/src/services/job"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/cmd/api/src/services/upload"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 )
diff --git a/cmd/api/src/api/v2/fileingest_test.go b/cmd/api/src/api/v2/fileingest_test.go
@@ -43,9 +43,10 @@ import (
 	"github.com/specterops/bloodhound/cmd/api/src/database/types/null"
 	"github.com/specterops/bloodhound/cmd/api/src/model"
 	"github.com/specterops/bloodhound/cmd/api/src/model/ingest"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
-	storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
+	storageServiceMocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
 	"github.com/specterops/bloodhound/packages/go/headers"
+	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 
 	"github.com/specterops/bloodhound/cmd/api/src/utils/test"
 	"github.com/stretchr/testify/assert"
@@ -493,7 +494,7 @@ func TestResources_ProcessIngestTask(t *testing.T) {
 		mockDatabase            *dbmocks.MockDatabase
 		mockFileService         *storagemocks.MockFileService
 		trueFileService         storage.FileService
-		mockFileServiceResolver *storagemocks.MockFileServiceResolver
+		mockFileServiceResolver *storageServiceMocks.MockFileServiceResolver
 	}
 	type expected struct {
 		responseBody   string
@@ -840,7 +841,7 @@ func TestResources_ProcessIngestTask(t *testing.T) {
 			mocks := &mock{
 				mockDatabase:            dbmocks.NewMockDatabase(ctrl),
 				mockFileService:         storagemocks.NewMockFileService(ctrl),
-				mockFileServiceResolver: storagemocks.NewMockFileServiceResolver(ctrl),
+				mockFileServiceResolver: storageServiceMocks.NewMockFileServiceResolver(ctrl),
 			}
 
 			if testCase.fileServiceOvrd != nil {
diff --git a/cmd/api/src/config/config.go b/cmd/api/src/config/config.go
@@ -259,10 +259,6 @@ func (s Configuration) ScratchDirectory() string {
 	return filepath.Join(s.WorkDir, "ingest_scratch")
 }
 
-func (s Configuration) ScratchDirectory() string {
-	return filepath.Join(s.WorkDir, "ingest_scratch")
-}
-
 func (s Configuration) TempDirectory() string {
 	return filepath.Join(s.WorkDir, "tmp")
 }
diff --git a/cmd/api/src/daemons/api/toolapi/api.go b/cmd/api/src/daemons/api/toolapi/api.go
@@ -33,8 +33,8 @@ import (
 	"github.com/specterops/bloodhound/cmd/api/src/bootstrap"
 	"github.com/specterops/bloodhound/cmd/api/src/config"
 	"github.com/specterops/bloodhound/cmd/api/src/database"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
+	"github.com/specterops/bloodhound/packages/go/storage"
 	"github.com/specterops/dawgs/graph"
 )
 
diff --git a/cmd/api/src/daemons/datapipe/cleanup.go b/cmd/api/src/daemons/datapipe/cleanup.go
@@ -29,8 +29,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
+	"github.com/specterops/bloodhound/packages/go/storage"
 )
 
 const (
diff --git a/cmd/api/src/daemons/datapipe/cleanup_internal_test.go b/cmd/api/src/daemons/datapipe/cleanup_internal_test.go
@@ -25,8 +25,8 @@ import (
 	"time"
 
 	"github.com/specterops/bloodhound/cmd/api/src/daemons/datapipe/mocks"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
-	storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
+	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
diff --git a/cmd/api/src/daemons/datapipe/cleanup_test.go b/cmd/api/src/daemons/datapipe/cleanup_test.go
@@ -27,8 +27,8 @@ import (
 
 	"github.com/specterops/bloodhound/cmd/api/src/daemons/datapipe"
 	"github.com/specterops/bloodhound/cmd/api/src/daemons/datapipe/mocks"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
-	storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
+	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 	"go.uber.org/mock/gomock"
 )
 
diff --git a/cmd/api/src/daemons/datapipe/pipeline.go b/cmd/api/src/daemons/datapipe/pipeline.go
@@ -29,14 +29,15 @@ import (
 	"github.com/specterops/bloodhound/cmd/api/src/model/appcfg"
 	"github.com/specterops/bloodhound/cmd/api/src/services/graphify"
 	"github.com/specterops/bloodhound/cmd/api/src/services/job"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
+	storageService "github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/cmd/api/src/services/upload"
 	"github.com/specterops/bloodhound/packages/go/analysis"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 	"github.com/specterops/bloodhound/packages/go/bhlog/measure"
 	"github.com/specterops/bloodhound/packages/go/cache"
 	"github.com/specterops/bloodhound/packages/go/graphschema/ad"
 	"github.com/specterops/bloodhound/packages/go/graphschema/azure"
+	"github.com/specterops/bloodhound/packages/go/storage"
 	"github.com/specterops/dawgs/graph"
 )
 
@@ -49,13 +50,13 @@ type BHCEPipeline struct {
 	cfg                 config.Configuration
 	orphanedFileSweeper *OrphanFileSweeper
 	ingestSchema        upload.IngestSchema
-	fileServiceResolver storage.FileServiceResolver
+	fileServiceResolver storageService.FileServiceResolver
 	jobService          job.JobService
 	graphifyService     graphify.GraphifyService
 	changelog           *changelog.Changelog
 }
 
-func NewPipeline(ctx context.Context, cfg config.Configuration, db database.Database, graphDB graph.Database, cache cache.Cache, ingestSchema upload.IngestSchema, fileServiceResolver storage.FileServiceResolver, cl *changelog.Changelog) *BHCEPipeline {
+func NewPipeline(ctx context.Context, cfg config.Configuration, db database.Database, graphDB graph.Database, cache cache.Cache, ingestSchema upload.IngestSchema, fileServiceResolver storageService.FileServiceResolver, cl *changelog.Changelog) *BHCEPipeline {
 	return &BHCEPipeline{
 		db:                  db,
 		graphdb:             graphDB,
diff --git a/cmd/api/src/services/entrypoint_test.go b/cmd/api/src/services/entrypoint_test.go
@@ -25,7 +25,9 @@ import (
 	"github.com/specterops/bloodhound/cmd/api/src/config"
 	"github.com/specterops/bloodhound/cmd/api/src/database"
 	"github.com/specterops/bloodhound/cmd/api/src/services"
+	storageService "github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 	"github.com/specterops/dawgs/graph"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
@@ -81,7 +83,7 @@ func TestPreMigrationDaemonsResolvesRetainedFileService(t *testing.T) {
 			connections     = bootstrap.DatabaseConnections[*database.BloodhoundDB, *graph.DatabaseSwitch]{}
 		)
 
-		fileServiceResolver, err := storage.NewFileServiceResolver(map[storage.FileServiceName]storage.FileService{
+		fileServiceResolver, err := storageService.NewFileServiceResolver(map[storage.FileServiceName]storage.FileService{
 			storage.FileServiceIngest: mockFileService,
 		})
 		require.NoError(t, err)
@@ -102,7 +104,7 @@ func TestPreMigrationDaemonsResolvesRetainedFileService(t *testing.T) {
 			connections         = bootstrap.DatabaseConnections[*database.BloodhoundDB, *graph.DatabaseSwitch]{}
 		)
 
-		fileServiceResolver, err := storage.NewFileServiceResolver(map[storage.FileServiceName]storage.FileService{
+		fileServiceResolver, err := storageService.NewFileServiceResolver(map[storage.FileServiceName]storage.FileService{
 			storage.FileServiceRetained: mockRetainedService,
 		})
 		require.NoError(t, err)
diff --git a/cmd/api/src/services/graphify/ingest_storage.go b/cmd/api/src/services/graphify/ingest_storage.go
@@ -24,9 +24,9 @@ import (
 	"os"
 
 	"github.com/specterops/bloodhound/cmd/api/src/model"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 	"github.com/specterops/bloodhound/packages/go/bomenc"
+	"github.com/specterops/bloodhound/packages/go/storage"
 	"github.com/specterops/dawgs/util"
 )
 
diff --git a/cmd/api/src/services/graphify/ingest_storage_test.go b/cmd/api/src/services/graphify/ingest_storage_test.go
@@ -28,8 +28,8 @@ import (
 
 	"github.com/specterops/bloodhound/cmd/api/src/model"
 	"github.com/specterops/bloodhound/cmd/api/src/services/graphify"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
-	storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
+	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 )
diff --git a/cmd/api/src/services/graphify/tasks.go b/cmd/api/src/services/graphify/tasks.go
@@ -28,10 +28,10 @@ import (
 	"github.com/specterops/bloodhound/cmd/api/src/model"
 	"github.com/specterops/bloodhound/cmd/api/src/model/appcfg"
 	"github.com/specterops/bloodhound/cmd/api/src/services/graphify/endpoint"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 	"github.com/specterops/bloodhound/packages/go/bhlog/measure"
 	"github.com/specterops/bloodhound/packages/go/errorlist"
+	"github.com/specterops/bloodhound/packages/go/storage"
 	"github.com/specterops/dawgs/graph"
 )
 
diff --git a/cmd/api/src/services/upload/upload.go b/cmd/api/src/services/upload/upload.go
@@ -27,11 +27,11 @@ import (
 
 	"github.com/specterops/bloodhound/cmd/api/src/model"
 	"github.com/specterops/bloodhound/cmd/api/src/model/ingest"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
 	"github.com/specterops/bloodhound/cmd/api/src/utils"
 	"github.com/specterops/bloodhound/packages/go/bhlog/attr"
 	"github.com/specterops/bloodhound/packages/go/headers"
 	"github.com/specterops/bloodhound/packages/go/mediatypes"
+	"github.com/specterops/bloodhound/packages/go/storage"
 )
 
 var ErrInvalidJSON = errors.New("file is not valid json")
@@ -79,6 +79,10 @@ func cleanupTempFile(ctx context.Context, fileService storage.FileService, tempF
 }
 
 func WriteAndValidateFile(ctx context.Context, fileService storage.FileService, fileData io.Reader, prefix string, validationFunc FileValidator) (string, error) {
+	if validationFunc == nil {
+		return "", fmt.Errorf("validation function is required")
+	}
+
 	// Create a pipe: pr (read end) and pw (write end).
 	// Data written to pw can be read from pr.
 	pr, pw := io.Pipe()
@@ -88,17 +92,6 @@ func WriteAndValidateFile(ctx context.Context, fileService storage.FileService,
 	// and gives the main goroutine a synchronization point to wait for the result.
 	validationErrCh := make(chan error, 1)
 
-func WriteAndValidateFileWithPrefix(fileData io.Reader, location string, tempFilePrefix string, validationFunc FileValidator) (string, error) {
-	if validationFunc == nil {
-		return "", fmt.Errorf("validation function is required")
-	}
-
-	var (
-		tempFile      *os.File
-		tempFileName  string
-		err           error
-		validationErr error
-	)
 	// Start validation in a separate goroutine.
 	// validationFunc reads from pr, writes to io.Discard.
 	// This validates the stream without storing the data twice.
diff --git a/cmd/api/src/services/upload/upload_test.go b/cmd/api/src/services/upload/upload_test.go
@@ -27,8 +27,8 @@ import (
 	"testing"
 
 	"github.com/specterops/bloodhound/cmd/api/src/model/ingest"
-	"github.com/specterops/bloodhound/cmd/api/src/services/storage"
-	storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"
+	"github.com/specterops/bloodhound/packages/go/storage"
+	storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"

Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,6 @@ import (`
`26`	`26`	`"github.com/gorilla/mux"`
`27`	`27`	`"github.com/specterops/bloodhound/cmd/api/src/api"`
`28`	`28`	`"github.com/specterops/bloodhound/cmd/api/src/config"`
`29`		`- "github.com/specterops/bloodhound/cmd/api/src/services/storage"`
`30`	`29`	`"github.com/specterops/bloodhound/packages/go/bhlog/attr"`
`31`	`30`	`"github.com/specterops/bloodhound/packages/go/storage"`
`32`	`31`	`)`
Original file line number	Diff line number	Diff line change
`@@ -259,10 +259,6 @@ func (s Configuration) ScratchDirectory() string {`
`259`	`259`	`return filepath.Join(s.WorkDir, "ingest_scratch")`
`260`	`260`	`}`
`261`	`261`
`262`		`-func (s Configuration) ScratchDirectory() string {`
`263`		`- return filepath.Join(s.WorkDir, "ingest_scratch")`
`264`		`-}`
`265`		`-`
`266`	`262`	`func (s Configuration) TempDirectory() string {`
`267`	`263`	`return filepath.Join(s.WorkDir, "tmp")`
`268`	`264`	`}`
Original file line number	Diff line number	Diff line change
`@@ -33,8 +33,8 @@ import (`
`33`	`33`	`"github.com/specterops/bloodhound/cmd/api/src/bootstrap"`
`34`	`34`	`"github.com/specterops/bloodhound/cmd/api/src/config"`
`35`	`35`	`"github.com/specterops/bloodhound/cmd/api/src/database"`
`36`		`- "github.com/specterops/bloodhound/cmd/api/src/services/storage"`
`37`	`36`	`"github.com/specterops/bloodhound/packages/go/bhlog/attr"`
	`37`	`+ "github.com/specterops/bloodhound/packages/go/storage"`
`38`	`38`	`"github.com/specterops/dawgs/graph"`
`39`	`39`	`)`
`40`	`40`
Original file line number	Diff line number	Diff line change
`@@ -29,8 +29,8 @@ import (`
`29`	`29`	`"strings"`
`30`	`30`	`"sync"`
`31`	`31`
`32`		`- "github.com/specterops/bloodhound/cmd/api/src/services/storage"`
`33`	`32`	`"github.com/specterops/bloodhound/packages/go/bhlog/attr"`
	`33`	`+ "github.com/specterops/bloodhound/packages/go/storage"`
`34`	`34`	`)`
`35`	`35`
`36`	`36`	`const (`
Original file line number	Diff line number	Diff line change
`@@ -27,8 +27,8 @@ import (`
`27`	`27`
`28`	`28`	`"github.com/specterops/bloodhound/cmd/api/src/daemons/datapipe"`
`29`	`29`	`"github.com/specterops/bloodhound/cmd/api/src/daemons/datapipe/mocks"`
`30`		`- "github.com/specterops/bloodhound/cmd/api/src/services/storage"`
`31`		`- storagemocks "github.com/specterops/bloodhound/cmd/api/src/services/storage/mocks"`
	`30`	`+ "github.com/specterops/bloodhound/packages/go/storage"`
	`31`	`+ storagemocks "github.com/specterops/bloodhound/packages/go/storage/mocks"`
`32`	`32`	`"go.uber.org/mock/gomock"`
`33`	`33`	`)`
`34`	`34`