Add periodic garbage collector for key-value store

Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>

Author: ajfloeder

pkg/manager-nnf/manager.go

@@ -500,6 +500,7 @@ func (s *StorageService) Initialize(log ec.Logger, ctrl NnfControllerInterface)
 	// Create the key-value storage database
 	{
 		path := "nnf.db"
+		persistent.SetLogger(log)
 		s.store, err = persistent.Open(path, false)
 		if err != nil {
 			log.Error(err, "Unable to open database", "path", path)

pkg/persistent/storage_api.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2022-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -17,15 +17,37 @@
  * limitations under the License.
  */
 
+// Package persistent provides persistent storage functionality for key-value operations.
+// This package implements durable storage mechanisms with support for various backends.
 package persistent
 
+import (
+	"github.com/NearNodeFlash/nnf-ec/pkg/ec"
+	"github.com/go-logr/logr"
+)
+
+// Package-level logger that can be configured
+var packageLogger ec.Logger = logr.Discard()
+
+// SetLogger configures the package-level logger for all persistent storage operations
+func SetLogger(log ec.Logger) {
+	packageLogger = log.WithName("persistent")
+}
+
+// GetLogger returns the current package logger
+func GetLogger() ec.Logger {
+	return packageLogger
+}
+
+// StorageProvider is the default storage provider instance for the package
 var StorageProvider = NewLocalPersistentStorageProvider()
 
+// PersistentStorageProvider provides methods for creating persistent storage interfaces
 type PersistentStorageProvider interface {
 	NewPersistentStorageInterface(path string, readOnly bool) (PersistentStorageApi, error)
 }
 
-// Persistent Storage API provides an interface for interacting with persistent storage
+// PersistentStorageApi provides an interface for interacting with persistent storage.
 type PersistentStorageApi interface {
 	View(func(txn PersistentStorageTransactionApi) error) error
 	Update(func(txn PersistentStorageTransactionApi) error) error
@@ -34,7 +56,7 @@ type PersistentStorageApi interface {
 	Close() error
 }
 
-// Persistent Storage Transaction API provides an interface for interacting with persistent storage transactions
+// PersistentStorageTransactionApi provides an interface for interacting with persistent storage transactions
 type PersistentStorageTransactionApi interface {
 	NewIterator(prefix string) PersistentStorageIteratorApi
 	Set(key string, value []byte) error

pkg/persistent/storage_json.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2022-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -22,7 +22,6 @@ package persistent
 import (
 	"encoding/json"
 	"io/ioutil"
-
 )
 
 func NewJsonFilePersistentStorageProvider(filename string) PersistentStorageProvider {
@@ -34,16 +33,22 @@ type jsonFilePersisentStorageProvider struct {
 }
 
 func (p *jsonFilePersisentStorageProvider) NewPersistentStorageInterface(name string, readOnly bool) (PersistentStorageApi, error) {
+	log := GetLogger()
+	log.Info("Opening JSON file storage", "file", p.filename, "name", name, "readOnly", readOnly)
+
 	content, err := ioutil.ReadFile(p.filename)
 	if err != nil {
+		log.Error(err, "Failed to read JSON file", "file", p.filename)
 		return nil, err
 	}
 
 	var payload map[string]map[string]string
 	if err := json.Unmarshal(content, &payload); err != nil {
+		log.Error(err, "Failed to unmarshal JSON content", "file", p.filename)
 		return nil, err
 	}
 
+	log.Info("Successfully opened JSON file storage", "name", name)
 	return &jsonPersistentStorageInterface{data: payload[name]}, nil
 }
 
@@ -66,7 +71,3 @@ func (*jsonPersistentStorageInterface) Delete(key string) error {
 func (*jsonPersistentStorageInterface) Close() error {
 	panic("unimplemented")
 }
-
-
-
-

pkg/persistent/storage_local.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2022-2025 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -20,9 +20,13 @@
 package persistent
 
 import (
+	"time"
+
 	"github.com/dgraph-io/badger/v3"
 )
 
+const garbageCollectPeriod = 24 * time.Hour
+
 func NewLocalPersistentStorageProvider() PersistentStorageProvider {
 	return &localPersistentStorageProvider{}
 }
@@ -40,27 +44,51 @@ type localPersistentStorage struct {
 }
 
 func (s *localPersistentStorage) open(path string, readOnly bool) (err error) {
+	log := GetLogger().WithValues("path", path, "readOnly", readOnly)
+	log.Info("BadgerDB: Opening database")
+
 	opts := badger.DefaultOptions(path)
 	opts.SyncWrites = true
-	//opts.ReadOnly = readOnly // Causes ErrLogTruncate
 	opts.BypassLockGuard = readOnly
+	opts.VerifyValueChecksum = true
 
-	// Shrink the in-memory and on-disk size to a more manageable 8 MiB and 16 MiB, respectively;
+	// Shrink the in-memory and on-disk size to a more manageable 8 MiB and 32 MiB, respectively;
 	// We use very little data and the 64 MiB and 256 MiB defaults will cause OOM issues in kubernetes.
-	// 8MiB seems to be the lower limit within badger, anything smaller and badger will complain with
+	// 8MiB seems to be the lower limit within badger, anything smaller and badger complains with
 	//   """
 	//   Valuethreshold 1048576 greater than max batch size of 629145. Either reduce opt.ValueThreshold
 	//   or increase opt.MaxTableSize.
 	//   """
 	opts.MemTableSize = 8 << 20
-	opts.BlockCacheSize = 16 << 20
+	opts.BlockCacheSize = 32 << 20 // Increased to 32 MiB for better cache hit ratio
 
 	s.DB, err = badger.Open(opts)
-	return err
+	if err != nil {
+		log.Error(err, "BadgerDB: Failed to open database")
+		return err
+	}
+
+	log.WithValues("mem_table_size", opts.MemTableSize, "block_cache_size", opts.BlockCacheSize).Info("BadgerDB: Database opened successfully")
+
+	// Run garbage collection on existing database during initialization
+	// Skip GC for read-only databases to avoid potential issues
+	if !readOnly {
+		s.RunPeriodicGC(garbageCollectPeriod)
+	}
+
+	return nil
 }
 
 func (s *localPersistentStorage) Close() error {
-	return s.DB.Close()
+	log := GetLogger()
+	log.Info("BadgerDB: Closing database")
+	err := s.DB.Close()
+	if err != nil {
+		log.Error(err, "BadgerDB: Failed to close database")
+	} else {
+		log.Info("BadgerDB: Database closed successfully")
+	}
+	return err
 }
 
 func (s *localPersistentStorage) View(fn func(PersistentStorageTransactionApi) error) error {
@@ -84,6 +112,38 @@ func (s *localPersistentStorage) Delete(key string) error {
 	return txn.Commit()
 }
 
+func (s *localPersistentStorage) RunGC() error {
+	log := GetLogger().WithName("gc")
+	log.Info("BadgerDB: Starting garbage collection")
+
+	err := s.DB.RunValueLogGC(0.5)
+	if err != nil {
+		if err == badger.ErrNoRewrite {
+			log.Info("BadgerDB: GC completed - no rewrite needed")
+			return nil
+		}
+		log.Error(err, "BadgerDB: GC failed")
+		return err
+	}
+	log.Info("BadgerDB: GC completed successfully")
+	return nil
+}
+
+func (s *localPersistentStorage) RunPeriodicGC(interval time.Duration) {
+	log := GetLogger().WithName("periodic-gc").WithValues("interval", interval)
+	log.Info("BadgerDB: Starting periodic GC")
+
+	ticker := time.NewTicker(interval)
+	go func() {
+		defer ticker.Stop()
+		for range ticker.C {
+			if err := s.RunGC(); err != nil {
+				log.Error(err, "BadgerDB: Periodic GC encountered error")
+			}
+		}
+	}()
+}
+
 type localPersistentStorageTransaction struct {
 	*badger.Txn
 }