temp (pruning): Logs and timers for debugging (#29)

* Added logs to indicate pruning op started and timers to count time

* Added pruner max batch size to config

* Add UTs for block and state pruning in batches

* Added UTs for tx indexer batch removals

* Removed println

* Expanded UT to confirm subsequent batch is pruned

* Add batch limit to ABCI response pruning

* Revert "Add batch limit to ABCI response pruning"

This reverts commit 886333f.

* Revert "Expanded UT to confirm subsequent batch is pruned"

This reverts commit e98850a.

* Revert "Removed println"

This reverts commit 38767be.

* Revert "Added UTs for tx indexer batch removals"

This reverts commit 18eae71.

* Revert "Add UTs for block and state pruning in batches"

This reverts commit 0a01939.

* Revert "Added pruner max batch size to config"

This reverts commit bec5c52.

* Unified pruning count prefix

* Attemp to fix linter error

Author: jmalicevic

state/pruner.go

@@ -384,23 +384,32 @@ func (p *Pruner) pruneIndexesRoutine() {
 	}
 }
 
+func (p *Pruner) parseError(err error, indexerType string, lastRetainHeight int64) int64 {
+	// Indexer retain height has not yet been set - do not log any
+	// errors at this time.
+	if errors.Is(err, ErrKeyNotFound) {
+		return 0
+	}
+	p.logger.Error("Failed to get "+indexerType+" retain height", "err", err)
+	return lastRetainHeight
+}
+
 func (p *Pruner) pruneTxIndexerToRetainHeight(lastRetainHeight int64) int64 {
+
 	targetRetainHeight, err := p.GetTxIndexerRetainHeight()
+	p.logger.Info("txIndex pruning started", "currentHeight", lastRetainHeight, "targetRetainHeight", targetRetainHeight)
 	if err != nil {
-		// Indexer retain height has not yet been set - do not log any
-		// errors at this time.
-		if errors.Is(err, ErrKeyNotFound) {
-			return 0
-		}
-		p.logger.Error("Failed to get Indexer retain height", "err", err)
-		return lastRetainHeight
+		return p.parseError(err, "txindexer", lastRetainHeight)
 	}
 
 	if lastRetainHeight >= targetRetainHeight {
 		return lastRetainHeight
 	}
 
+	tStart := time.Now()
 	numPrunedTxIndexer, newTxIndexerRetainHeight, err := p.txIndexer.Prune(targetRetainHeight)
+	tElapsed := time.Since(tStart)
+	p.logger.Info("txIndex pruning time", "elapsed", tElapsed.String(), "pruned", numPrunedTxIndexer, "newTxIndexerRetainHeight", newTxIndexerRetainHeight)
 	if err != nil {
 		p.logger.Error("Failed to prune tx indexer", "err", err, "targetRetainHeight", targetRetainHeight, "newTxIndexerRetainHeight", newTxIndexerRetainHeight)
 	} else if numPrunedTxIndexer > 0 {
@@ -411,22 +420,21 @@ func (p *Pruner) pruneTxIndexerToRetainHeight(lastRetainHeight int64) int64 {
 }
 
 func (p *Pruner) pruneBlockIndexerToRetainHeight(lastRetainHeight int64) int64 {
+
 	targetRetainHeight, err := p.GetBlockIndexerRetainHeight()
+	p.logger.Info("block pruning started", "currentHeight", lastRetainHeight, "targetRetainHeight", targetRetainHeight)
 	if err != nil {
-		// Indexer retain height has not yet been set - do not log any
-		// errors at this time.
-		if errors.Is(err, ErrKeyNotFound) {
-			return 0
-		}
-		p.logger.Error("Failed to get Indexer retain height", "err", err)
-		return lastRetainHeight
+		return p.parseError(err, "blockIndexer", lastRetainHeight)
 	}
 
 	if lastRetainHeight >= targetRetainHeight {
 		return lastRetainHeight
 	}
 
+	tStart := time.Now()
 	numPrunedBlockIndexer, newBlockIndexerRetainHeight, err := p.blockIndexer.Prune(targetRetainHeight)
+	tElapsed := time.Since(tStart)
+	p.logger.Info("block pruning time", "elapsed", tElapsed.String(), "pruned", numPrunedBlockIndexer, "newBlockIndexerRetainHeight", newBlockIndexerRetainHeight)
 	if err != nil {
 		p.logger.Error("Failed to prune block indexer", "err", err, "targetRetainHeight", targetRetainHeight, "newBlockIndexerRetainHeight", newBlockIndexerRetainHeight)
 	} else if numPrunedBlockIndexer > 0 {
@@ -437,11 +445,16 @@ func (p *Pruner) pruneBlockIndexerToRetainHeight(lastRetainHeight int64) int64 {
 }
 
 func (p *Pruner) pruneBlocksToRetainHeight(lastRetainHeight int64) int64 {
+
 	targetRetainHeight := p.findMinBlockRetainHeight()
+	p.logger.Info("block pruning started", "currentHeight", lastRetainHeight, "targetRetainHeight", targetRetainHeight)
 	if targetRetainHeight == lastRetainHeight {
 		return lastRetainHeight
 	}
+	tStart := time.Now()
 	pruned, evRetainHeight, err := p.pruneBlocksToHeight(targetRetainHeight)
+	elapsed := time.Since(tStart)
+	p.logger.Info("block pruning time", "elapsed", elapsed.String(), "pruned", pruned, "evidenceRetainHeight", evRetainHeight)
 	// The new retain height is the current lowest point of the block store
 	// indicated by Base()
 	newRetainHeight := p.bs.Base()
@@ -455,7 +468,9 @@ func (p *Pruner) pruneBlocksToRetainHeight(lastRetainHeight int64) int64 {
 }
 
 func (p *Pruner) pruneABCIResToRetainHeight(lastRetainHeight int64) int64 {
+
 	targetRetainHeight, err := p.stateStore.GetABCIResRetainHeight()
+	p.logger.Info("abcires pruning started", "currentHeight", lastRetainHeight, "targetRetainHeight", targetRetainHeight)
 	if err != nil {
 		p.logger.Error("Failed to get ABCI response retain height", "err", err)
 		if errors.Is(err, ErrKeyNotFound) {
@@ -477,7 +492,10 @@ func (p *Pruner) pruneABCIResToRetainHeight(lastRetainHeight int64) int64 {
 	// newRetainHeight is the height just after that which we have successfully
 	// pruned. In case of an error it will be 0, but then it will also be
 	// ignored.
+	tStart := time.Now()
 	numPruned, newRetainHeight, err := p.stateStore.PruneABCIResponses(targetRetainHeight, forceCompact)
+	tElapsed := time.Since(tStart)
+	p.logger.Info("abcires pruning time", "elapsed", tElapsed.String(), "pruned", numPruned, "newRetainHeight", newRetainHeight)
 	if err != nil {
 		p.logger.Error("Failed to prune ABCI responses", "err", err, "targetRetainHeight", targetRetainHeight)
 		return lastRetainHeight