RHINENG-25740: mark systems stale one by one

to aviod locking system_inventory for long time

Author: MichaelMraka

tasks/system_culling/system_culling.go

@@ -78,15 +78,44 @@ func deleteCulledSystems(tx *gorm.DB, limitDeleted int) (nDeleted int64, err err
 	return nDeleted, nil
 }
 
-func markSystemsStale(tx *gorm.DB, markedLimit int) (nMarked int, err error) {
-	var nMarkedArr []int
-	err = tx.Raw("select mark_stale_systems(?)", markedLimit).
-		Find(&nMarkedArr).Error
-	if len(nMarkedArr) > 0 {
-		nMarked = nMarkedArr[0]
+// each update runs in its own transaction to avoid holding locks across many rows
+func markSystemsStale(tx *gorm.DB, markedLimit int) (nMarked int64, err error) {
+	var candidates []struct {
+		RhAccountID int   `gorm:"column:rh_account_id"`
+		ID          int64 `gorm:"column:id"`
+		Expired     bool  `gorm:"column:expired"`
+	}
+	now := time.Now()
+	err = tx.Model(&models.SystemInventory{}).
+		Select("rh_account_id, id, (stale_warning_timestamp < ?) as expired", now).
+		Where("stale != (stale_warning_timestamp < ?)", now).
+		Order("rh_account_id").Order("id").
+		Limit(markedLimit).
+		Find(&candidates).Error
+	if err != nil {
+		return 0, err
+	}
+
+	for _, c := range candidates {
+		var rowsAffected int64
+		markErr := tasks.CancelableDB().Transaction(func(tx2 *gorm.DB) error {
+			res := tx2.Model(&models.SystemInventory{}).
+				Where("rh_account_id = ? AND id = ?", c.RhAccountID, c.ID).
+				Update("stale", c.Expired)
+			if res.Error != nil {
+				return res.Error
+			}
+			rowsAffected = res.RowsAffected
+			return nil
+		})
+		if markErr != nil {
+			utils.LogWarn("rhAccountID", c.RhAccountID, "systemID", c.ID, "err", markErr.Error(), "Mark stale system")
+			continue
+		}
+		nMarked += rowsAffected
 	}
 
-	return nMarked, err
+	return nMarked, nil
 }
 
 func pruneDeletedSystems(tx *gorm.DB, limitDeleted int) (int64, error) {