Merge pull request #2427 from CortexFoundation/dev

change the mechanism to schedule freezer sync

Author: ucwong

core/rawdb/freezer_batch.go

@@ -26,9 +26,16 @@ import (
 	"github.com/CortexFoundation/CortexTheseus/rlp"
 )
 
-// This is the maximum amount of data that will be buffered in memory
-// for a single freezer table batch.
-const freezerBatchBufferLimit = 2 * 1024 * 1024
+const (
+	// This is the maximum amount of data that will be buffered in memory
+	// for a single freezer table batch.
+	freezerBatchBufferLimit = 2 * 1024 * 1024
+
+	// freezerTableFlushThreshold defines the threshold for triggering a freezer
+	// table sync operation. If the number of accumulated uncommitted items exceeds
+	// this value, a sync will be scheduled.
+	freezerTableFlushThreshold = 512
+)
 
 // freezerBatch is a write operation of multiple items on a freezer.
 type freezerBatch struct {
@@ -202,14 +209,17 @@ func (batch *freezerTableBatch) commit() error {
 
 	// Update headBytes of table.
 	batch.t.headBytes += dataSize
+	items := batch.curItem - batch.t.items.Load()
 	batch.t.items.Store(batch.curItem)
 
 	// Update metrics.
 	batch.t.sizeGauge.Inc(dataSize + indexSize)
 	batch.t.writeMeter.Mark(dataSize + indexSize)
 
 	// Periodically sync the table, todo (rjl493456442) make it configurable?
-	if time.Since(batch.t.lastSync) > 30*time.Second {
+	batch.t.uncommitted += items
+	if batch.t.uncommitted > freezerTableFlushThreshold && time.Since(batch.t.lastSync) > 30*time.Second {
+		batch.t.uncommitted = 0
 		batch.t.lastSync = time.Now()
 		return batch.t.Sync()
 	}

core/rawdb/freezer_table.go

@@ -113,8 +113,9 @@ type freezerTable struct {
 	headId uint32              // number of the currently active head file
 	tailId uint32              // number of the earliest file
 
-	metadata *freezerTableMeta // metadata of the table
-	lastSync time.Time         // Timestamp when the last sync was performed
+	metadata    *freezerTableMeta // metadata of the table
+	uncommitted uint64            // Count of items written without flushing to file
+	lastSync    time.Time         // Timestamp when the last sync was performed
 
 	headBytes  int64         // Number of bytes written to the head file
 	readMeter  metrics.Meter // Meter for measuring the effective amount of data read