RHINENG-19880: move deleted system cleanup to background tasks

refactor deleted system pruning from immediate event processing to
asynchronous background task

Author: MichaelMraka

listener/events.go

@@ -102,11 +102,5 @@ func HandleDelete(event mqueue.PlatformEvent) error {
 	utils.LogInfo("inventoryID", event.ID, "count", query.RowsAffected, "Systems deleted")
 	eventMsgsReceivedCnt.WithLabelValues(EventDelete, ReceivedSuccess).Inc()
 
-	err = database.DB.
-		Delete(&models.DeletedSystem{}, "when_deleted < ?", time.Now().Add(-deletionThreshold)).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
-		utils.LogWarn("inventoryID", event.ID, WarnNoRowsModified)
-		return nil
-	}
 	return nil
 }

tasks/config.go

@@ -2,6 +2,7 @@ package tasks
 
 import (
 	"app/base/utils"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -39,4 +40,6 @@ var (
 	// How ofter run full vmaas sync, 7 days by default
 	FullSyncCadence    = utils.PodConfig.GetInt("full_sync_cadence", 24*7)
 	MaxChangedPackages = utils.PodConfig.GetInt("max_changed_packages", 30000)
+	// prune deleted_system table records older than threshold
+	DeletedSystemsThreshold = time.Hour * time.Duration(utils.PodConfig.GetInt("system_delete_hrs", 4))
 )

tasks/system_culling/system_culling.go

@@ -1,8 +1,10 @@
 package system_culling
 
 import (
+	"app/base/models"
 	"app/base/utils"
 	"app/tasks"
+	"time"
 
 	"github.com/pkg/errors"
 	"gorm.io/gorm"
@@ -27,6 +29,13 @@ func runSystemCulling() {
 		utils.LogInfo("nMarked", nMarked, "Stale systems marked")
 		staleSystemsMarkedCnt.Add(float64(nMarked))
 
+		// pruning deleted_system
+		nPruned, err := pruneDeletedSystems(tx, tasks.DeleteCulledSystemsLimit)
+		if err != nil {
+			return errors.Wrap(err, "Prune deleted_systems")
+		}
+		utils.LogInfo("nPruned", nPruned, "Deleted_systems items pruned")
+
 		return nil
 	})
 
@@ -59,3 +68,13 @@ func markSystemsStale(tx *gorm.DB, markedLimit int) (nMarked int, err error) {
 
 	return nMarked, err
 }
+
+func pruneDeletedSystems(tx *gorm.DB, limitDeleted int) (int64, error) {
+	// postgres delete does not support limit
+	subQ := tx.Model(&models.DeletedSystem{}).
+		Where("when_deleted < ?", time.Now().Add(-tasks.DeletedSystemsThreshold)).
+		Limit(limitDeleted).
+		Select("inventory_id")
+	query := tx.Delete(&models.DeletedSystem{}, "inventory_id in (?)", subQ)
+	return query.RowsAffected, query.Error
+}