Use badger opts

Author: AlCutter

cmd/experimental/migrate/posix/main.go

@@ -29,6 +29,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dgraph-io/badger/v4"
+	"github.com/dustin/go-humanize"
 	"github.com/transparency-dev/tessera"
 	"github.com/transparency-dev/tessera/api/layout"
 	"github.com/transparency-dev/tessera/client"
@@ -38,12 +40,15 @@ import (
 )
 
 var (
-	storageDir         = flag.String("storage_dir", "", "Path to directory in which to store the migrated data.")
-	sourceURL          = flag.String("source_url", "", "Base monitoring URL for the source log.")
-	numWorkers         = flag.Uint("num_workers", 30, "Number of migration worker goroutines.")
-	persistentAntispam = flag.Bool("antispam", true, "Set to true to populate antispam storage.")
-	antispamBatchSize  = flag.Uint("antispam_batch_size", 50000, "Maximum number of antispam rows to insert per batch update.")
-	clientHTTPTimeout  = flag.Duration("client_http_timeout", 30*time.Second, "Timeout for outgoing HTTP requests")
+	storageDir                 = flag.String("storage_dir", "", "Path to directory in which to store the migrated data.")
+	sourceURL                  = flag.String("source_url", "", "Base monitoring URL for the source log.")
+	numWorkers                 = flag.Uint("num_workers", 30, "Number of migration worker goroutines.")
+	persistentAntispam         = flag.Bool("antispam", true, "Set to true to populate antispam storage.")
+	antispamBatchSize          = flag.Uint("antispam_batch_size", 10000, "Maximum number of antispam rows to insert per batch update.")
+	antispamBlockCacheSize     = flag.String("antispam_block_cache_size", "768MB", "Amount of RAM to allocate for antispam block cache, set to zero to disable.")
+	antispamIndexCacheSize     = flag.String("antispam_index_cache_size", "768MB", "Amount of RAM to allocate for antispam index cache, set to zero for unlimited.")
+	antispamCompactionInterval = flag.Duration("antispam_compaction_interval", tposix_as.DefaultCompactionInterval, "Interval between GC/compaction runs on the antispam index.")
+	clientHTTPTimeout          = flag.Duration("client_http_timeout", 30*time.Second, "Timeout for outgoing HTTP requests")
 )
 
 func main() {
@@ -103,10 +108,24 @@ func main() {
 	// Configure antispam storage, if necessary
 	var antispam tessera.Antispam
 	if *persistentAntispam {
-		as_opts := tposix_as.AntispamOpts{
-			MaxBatchSize: *antispamBatchSize,
+		antispamIndexCacheBytes, error := humanize.ParseBytes(*antispamIndexCacheSize)
+		if error != nil {
+			klog.Exitf("Invalid antispam index cache size: %v", error)
 		}
-		antispam, err = tposix_as.NewAntispam(ctx, filepath.Join(*storageDir, ".state", "antispam"), as_opts)
+		antispamBlockCacheBytes, error := humanize.ParseBytes(*antispamBlockCacheSize)
+		if error != nil {
+			klog.Exitf("Invalid antispam block cache size: %v", error)
+		}
+		asOpts := tposix_as.AntispamOpts{
+			MaxBatchSize:       *antispamBatchSize,
+			CompactionInterval: *antispamCompactionInterval,
+			BadgerOptions: func(o badger.Options) badger.Options {
+				return o.
+					WithIndexCacheSize(int64(antispamIndexCacheBytes)).
+					WithBlockCacheSize(int64(antispamBlockCacheBytes))
+			},
+		}
+		antispam, err = tposix_as.NewAntispam(ctx, filepath.Join(*storageDir, ".state", "antispam"), asOpts)
 		if err != nil {
 			klog.Exitf("Failed to create new POSIX antispam storage: %v", err)
 		}

cmd/tesseract/posix/main.go

@@ -34,6 +34,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/dgraph-io/badger/v4"
 	"github.com/dustin/go-humanize"
 	"github.com/transparency-dev/tessera"
 	tposix "github.com/transparency-dev/tessera/storage/posix"
@@ -98,6 +99,10 @@ var (
 	clientHTTPMaxIdle           = flag.Int("client_http_max_idle", 20, "Maximum number of idle HTTP connections for outgoing requests.")
 	clientHTTPMaxIdlePerHost    = flag.Int("client_http_max_idle_per_host", 10, "Maximum number of idle HTTP connections per host for outgoing requests.")
 	garbageCollectionInterval   = flag.Duration("garbage_collection_interval", 10*time.Second, "Interval between scans to remove obsolete partial tiles and entry bundles. Set to 0 to disable.")
+	antispamBatchSize           = flag.Uint("antispam_batch_size", 20000, "Maximum number of antispam rows to insert per batch update.")
+	antispamBlockCacheSize      = flag.String("antispam_block_cache_size", "512MB", "Amount of RAM to allocate for antispam block cache, set to zero to disable.")
+	antispamIndexCacheSize      = flag.String("antispam_index_cache_size", "512MB", "Amount of RAM to allocate for antispam index cache, set to zero for unlimited.")
+	antispamCompactionInterval  = flag.Duration("antispam_compaction_interval", tposix_as.DefaultCompactionInterval, "Interval between GC/compaction runs on the antispam index.")
 
 	// Infrastructure setup flags
 	storageDir    = flag.String("storage_dir", "", "Path to root of log storage.")
@@ -225,8 +230,23 @@ func newStorage(ctx context.Context, signer note.Signer) (st *storage.CTStorage,
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize POSIX Tessera storage driver: %v", err)
 	}
+	antispamIndexCacheBytes, error := humanize.ParseBytes(*antispamIndexCacheSize)
+	if error != nil {
+		return nil, fmt.Errorf("invalid antispam index cache size: %v", error)
+	}
+	antispamBlockCacheBytes, error := humanize.ParseBytes(*antispamBlockCacheSize)
+	if error != nil {
+		return nil, fmt.Errorf("invalid antispam block cache size: %v", error)
+	}
 	asOpts := tposix_as.AntispamOpts{
-		PushbackThreshold: *pushbackMaxAntispamLag,
+		PushbackThreshold:  *pushbackMaxAntispamLag,
+		MaxBatchSize:       *antispamBatchSize,
+		CompactionInterval: *antispamCompactionInterval,
+		BadgerOptions: func(o badger.Options) badger.Options {
+			return o.
+				WithIndexCacheSize(int64(antispamIndexCacheBytes)).
+				WithBlockCacheSize(int64(antispamBlockCacheBytes))
+		},
 	}
 	antispam, err := tposix_as.NewAntispam(ctx, filepath.Join(*storageDir, ".state", "antispam"), asOpts)
 	if err != nil {

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/cenkalti/backoff/v5 v5.0.3
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/x/ansi v0.11.6
+	github.com/dgraph-io/badger/v4 v4.9.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fsouza/fake-gcs-server v1.54.0
 	github.com/gdamore/tcell/v2 v2.13.8
@@ -90,7 +91,6 @@ require (
 	github.com/clipperhouse/stringish v0.1.1 // indirect
 	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
 	github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
-	github.com/dgraph-io/badger/v4 v4.9.1 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect