RHINENG-21214: use new tables for insert/update/delete directly

Views do not support these operations.

Author: Dugowitch

base/models/models.go

@@ -92,6 +92,69 @@ func (s *SystemPlatform) GetInventoryID() string {
 	return s.InventoryID
 }
 
+type SystemInventory struct {
+	ID                               int64  `gorm:"primaryKey"`
+	InventoryID                      string `gorm:"unique"`
+	RhAccountID                      int    `gorm:"primaryKey"`
+	VmaasJSON                        *string
+	JSONChecksum                     *string
+	LastUpdated                      *time.Time `gorm:"default:null"`
+	UnchangedSince                   *time.Time `gorm:"default:null"`
+	LastUpload                       *time.Time `gorm:"default:null"`
+	Stale                            bool
+	DisplayName                      string
+	ReporterID                       *int
+	YumUpdates                       []byte  `gorm:"column:yum_updates"`
+	YumChecksum                      *string `gorm:"column:yum_checksum"`
+	SatelliteManaged                 bool    `gorm:"column:satellite_managed"`
+	BuiltPkgcache                    bool    `gorm:"column:built_pkgcache"`
+	Arch                             *string
+	Bootc                            bool
+	Tags                             []byte `gorm:"column:tags"`
+	Created                          time.Time
+	Workspaces                       []string
+	StaleTimestamp                   time.Time
+	StaleWarningTimestamp            time.Time
+	CulledTimestamp                  time.Time
+	OSName                           *string
+	OSMajor                          *int16
+	OSMinor                          *int16
+	RhsmVersion                      *string
+	SubscriptionManagerID            *string
+	SapWorkload                      bool
+	SapWorkloadSIDs                  []string
+	AnsibleWorkload                  bool
+	AnsibleWorkloadControllerVersion *string
+	MssqlWorkload                    bool
+	MssqlWorkloadVersion             *string
+}
+
+func (s *SystemInventory) GetInventoryID() string {
+	if s == nil {
+		return ""
+	}
+	return s.InventoryID
+}
+
+type SystemPatch struct {
+	SystemID                         int64      `gorm:"primaryKey"`
+	RhAccountID                      int        `gorm:"primaryKey"`
+	LastEvaluation                   *time.Time `gorm:"default:null"` // TODO: trigger sets it to current time?
+	InstallableAdvisoryCountCache    int
+	InstallableAdvisoryEnhCountCache int
+	InstallableAdvisoryBugCountCache int
+	InstallableAdvisorySecCountCache int
+	PackagesInstalled                int
+	PackagesInstallable              int
+	PackagesApplicable               int
+	ThirdParty                       bool
+	ApplicableAdvisoryCountCache     int
+	ApplicableAdvisoryEnhCountCache  int
+	ApplicableAdvisoryBugCountCache  int
+	ApplicableAdvisorySecCountCache  int
+	TemplateID                       *int64 `gorm:"column:template_id"`
+}
+
 type String struct {
 	ID    []byte `gorm:"primaryKey"`
 	Value string

base/notification/notification.go

@@ -81,7 +81,7 @@ type SystemTag struct {
 	Value     string `json:"value,omitempty"`
 }
 
-func MakeNotification(system *models.SystemPlatform, systemTags []SystemTag, orgID string,
+func MakeNotification(system *models.SystemInventory, systemTags []SystemTag, orgID string,
 	eventType string, events []Event) (*Notification, error) {
 	if orgID == "" || orgID == "null" {
 		return nil, errors.New("invalid orgID")

base/utils/vmaas.go

@@ -122,7 +122,7 @@ func RemoveNonLatestPackages(updates *vmaas.UpdatesV3Response) {
 	updates.UpdateList = &updateList
 }
 
-func ParseVmaasJSON(system *models.SystemPlatform) (vmaas.UpdatesV3Request, error) {
+func ParseVmaasJSON(system *models.SystemInventory) (vmaas.UpdatesV3Request, error) {
 	var updatesReq vmaas.UpdatesV3Request
 	err := sonic.Unmarshal([]byte(*system.VmaasJSON), &updatesReq)
 	return updatesReq, err

evaluator/evaluate.go

@@ -200,8 +200,8 @@ func runEvaluate(
 }
 
 func evaluateInDatabase(ctx context.Context, event *mqueue.PlatformEvent, inventoryID string) (
-	*models.SystemPlatform, *vmaas.UpdatesV3Response, error) {
-	system, err := tryGetSystem(event.AccountID, inventoryID, event.Timestamp)
+	*models.SystemInventory, *vmaas.UpdatesV3Response, error) {
+	system, systemPatch, err := tryGetSystem(event.AccountID, inventoryID, event.Timestamp)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to get system")
 	}
@@ -214,9 +214,9 @@ func evaluateInDatabase(ctx context.Context, event *mqueue.PlatformEvent, invent
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "repo analysis failed")
 	}
-	system.ThirdParty = thirdParty // to set "system_platform.third_party" column
+	systemPatch.ThirdParty = thirdParty // to set "system_platform.third_party" column
 
-	updatesData, err := getUpdatesData(ctx, system)
+	updatesData, err := getUpdatesData(ctx, system, systemPatch)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to get updates data")
 	}
@@ -225,15 +225,15 @@ func evaluateInDatabase(ctx context.Context, event *mqueue.PlatformEvent, invent
 		return nil, nil, nil
 	}
 
-	vmaasData, err := evaluateWithVmaas(updatesData, system, event)
+	vmaasData, err := evaluateWithVmaas(updatesData, system, systemPatch, event)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "evaluation with vmaas failed")
 	}
 
 	return system, vmaasData, nil
 }
 
-func tryGetYumUpdates(system *models.SystemPlatform) (*vmaas.UpdatesV3Response, error) {
+func tryGetYumUpdates(system *models.SystemInventory) (*vmaas.UpdatesV3Response, error) {
 	if system.YumUpdates == nil {
 		return nil, nil
 	}
@@ -246,7 +246,7 @@ func tryGetYumUpdates(system *models.SystemPlatform) (*vmaas.UpdatesV3Response,
 	updatesMap := resp.GetUpdateList()
 	if len(updatesMap) == 0 {
 		// TODO: do we need evaluationCnt.WithLabelValues("error-no-yum-packages").Inc()?
-		utils.LogWarn("inventoryID", system.GetInventoryID(), "No yum_updates")
+		utils.LogWarn("inventoryID", system.InventoryID, "No yum_updates")
 		return nil, nil
 	}
 
@@ -276,19 +276,20 @@ func tryGetYumUpdates(system *models.SystemPlatform) (*vmaas.UpdatesV3Response,
 	return &resp, nil
 }
 
-func evaluateWithVmaas(updatesData *vmaas.UpdatesV3Response,
-	system *models.SystemPlatform, event *mqueue.PlatformEvent) (*vmaas.UpdatesV3Response, error) {
+func evaluateWithVmaas(updatesData *vmaas.UpdatesV3Response, system *models.SystemInventory,
+	systemPatch *models.SystemPatch, event *mqueue.PlatformEvent) (*vmaas.UpdatesV3Response, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, evaluationPartDuration.WithLabelValues("evaluate-with-vmaas-full"))
 
-	err := evaluateAndStore(system, updatesData, event)
+	err := evaluateAndStore(system, systemPatch, updatesData, event)
 	if err != nil {
 		return nil, errors.Wrap(err, "Unable to evaluate and store results")
 	}
 	return updatesData, nil
 }
 
-func getUpdatesData(ctx context.Context, system *models.SystemPlatform) (*vmaas.UpdatesV3Response, error) {
+func getUpdatesData(ctx context.Context, system *models.SystemInventory, systemPatch *models.SystemPatch,
+) (*vmaas.UpdatesV3Response, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, evaluationPartDuration.WithLabelValues("get-updates-data"))
 
@@ -302,7 +303,7 @@ func getUpdatesData(ctx context.Context, system *models.SystemPlatform) (*vmaas.
 		}
 	}
 
-	vmaasData, vmaasErr := getVmaasUpdates(ctx, system)
+	vmaasData, vmaasErr := getVmaasUpdates(ctx, system, systemPatch)
 	if vmaasErr != nil {
 		if errors.Is(vmaasErr, errVmaasBadRequest) {
 			// vmaas bad request means we either created wrong vmaas request
@@ -318,7 +319,7 @@ func getUpdatesData(ctx context.Context, system *models.SystemPlatform) (*vmaas.
 		utils.LogWarn("Vmaas response error, continuing with yum updates only", vmaasErr.Error())
 	}
 
-	if system.SatelliteManaged || system.TemplateID != nil {
+	if system.SatelliteManaged || systemPatch.TemplateID != nil {
 		// satellite managed systems and systems using template has vmaas updates APPLICABLE instead of INSTALLABLE
 		mergedUpdateList := vmaasData.GetUpdateList()
 		for nevra := range mergedUpdateList {
@@ -330,7 +331,8 @@ func getUpdatesData(ctx context.Context, system *models.SystemPlatform) (*vmaas.
 	return merged, nil
 }
 
-func getVmaasUpdates(ctx context.Context, system *models.SystemPlatform) (*vmaas.UpdatesV3Response, error) {
+func getVmaasUpdates(ctx context.Context, system *models.SystemInventory, systemPatch *models.SystemPatch,
+) (*vmaas.UpdatesV3Response, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, evaluationPartDuration.WithLabelValues("vmaas-updates-prepare"))
 
@@ -355,8 +357,8 @@ func getVmaasUpdates(ctx context.Context, system *models.SystemPlatform) (*vmaas
 		return nil, nil
 	}
 
-	updatesReq.ThirdParty = utils.PtrBool(system.ThirdParty) // enable "third_party" updates in VMaaS if needed
-	useOptimisticUpdates := system.ThirdParty || vmaasCallUseOptimisticUpdates
+	updatesReq.ThirdParty = utils.PtrBool(systemPatch.ThirdParty) // enable "third_party" updates in VMaaS if needed
+	useOptimisticUpdates := systemPatch.ThirdParty || vmaasCallUseOptimisticUpdates
 	updatesReq.OptimisticUpdates = utils.PtrBool(useOptimisticUpdates)
 	updatesReq.EpochRequired = utils.PtrBool(true)
 
@@ -377,10 +379,10 @@ func getVmaasUpdates(ctx context.Context, system *models.SystemPlatform) (*vmaas
 	return vmaasData, nil
 }
 
-func tryGetVmaasRequest(system *models.SystemPlatform) (*vmaas.UpdatesV3Request, error) {
+func tryGetVmaasRequest(system *models.SystemInventory) (*vmaas.UpdatesV3Request, error) {
 	if system == nil || system.VmaasJSON == nil {
 		evaluationCnt.WithLabelValues("error-parse-vmaas-json").Inc()
-		utils.LogWarn("inventoryID", system.GetInventoryID(), "system with empty vmaas json")
+		utils.LogWarn("inventoryID", system.InventoryID, "system with empty vmaas json")
 		// skip the system
 		// don't return error as it will cause panic of evaluator pod
 		return nil, nil
@@ -394,44 +396,44 @@ func tryGetVmaasRequest(system *models.SystemPlatform) (*vmaas.UpdatesV3Request,
 
 	if len(updatesReq.PackageList) == 0 {
 		evaluationCnt.WithLabelValues("error-no-packages").Inc()
-		utils.LogWarn("inventoryID", system.GetInventoryID(), "Empty package list")
+		utils.LogWarn("inventoryID", system.InventoryID, "Empty package list")
 		return nil, nil
 	}
 
 	if len(updatesReq.RepositoryList) == 0 {
 		// system without any repositories won't have any advisories evaluated by vmaas
 		evaluationCnt.WithLabelValues("error-no-repositories").Inc()
-		utils.LogWarn("inventoryID", system.GetInventoryID(), "Empty repository list")
+		utils.LogWarn("inventoryID", system.InventoryID, "Empty repository list")
 		return nil, nil
 	}
 	return &updatesReq, nil
 }
 
 func tryGetSystem(accountID int, inventoryID string,
-	requested *types.Rfc3339Timestamp) (*models.SystemPlatform, error) {
-	system, err := loadSystemData(accountID, inventoryID)
+	requested *types.Rfc3339Timestamp) (*models.SystemInventory, *models.SystemPatch, error) {
+	system, systemPatch, err := loadSystemData(accountID, inventoryID)
 	if err != nil {
 		evaluationCnt.WithLabelValues("error-db-read-inventory-data").Inc()
-		return nil, base.WrapFatalDBError(err, "error loading system from DB")
+		return nil, nil, base.WrapFatalDBError(err, "error loading system from DB")
 	}
 	if system.ID == 0 {
 		evaluationCnt.WithLabelValues("error-db-read-inventory-data").Inc()
 		utils.LogWarn("inventoryID", inventoryID, "System not found in DB")
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	if system.Stale && !enableStaleSysEval {
 		evaluationCnt.WithLabelValues("skipping-stale").Inc()
 		utils.LogWarn("inventoryID", inventoryID, "Skipping stale system")
-		return nil, nil
+		return nil, nil, nil
 	}
 
-	if requested != nil && system.LastEvaluation != nil && requested.Time().Before(*system.LastEvaluation) {
+	if requested != nil && systemPatch.LastEvaluation != nil && requested.Time().Before(*systemPatch.LastEvaluation) {
 		evaluationCnt.WithLabelValues("skip-old-msg").Inc()
 		utils.LogWarn("inventoryID", inventoryID, "Skipping old message")
-		return nil, nil
+		return nil, nil, nil
 	}
-	return system, nil
+	return system, systemPatch, nil
 }
 
 func commitWithObserve(tx *gorm.DB) error {
@@ -447,7 +449,7 @@ func commitWithObserve(tx *gorm.DB) error {
 
 // EvaluateAndStore first loads advisories and packages (including change evaluation)
 // and then executes all deletions, updates, and insertions in a single transaction.
-func evaluateAndStore(system *models.SystemPlatform,
+func evaluateAndStore(system *models.SystemInventory, systemPatch *models.SystemPatch,
 	vmaasData *vmaas.UpdatesV3Response, event *mqueue.PlatformEvent) error {
 	advisoriesByName, err := lazySaveAndLoadAdvisories(system, vmaasData)
 	if err != nil {
@@ -475,7 +477,7 @@ func evaluateAndStore(system *models.SystemPlatform,
 		return errors.Wrap(err, "Unable to update system packages")
 	}
 
-	err = updateSystemPlatform(tx, system, systemAdvisoriesNew, installed, installable, applicable)
+	err = updateSystemPatch(tx, system, systemPatch, systemAdvisoriesNew, installed, installable, applicable)
 	if err != nil {
 		evaluationCnt.WithLabelValues("error-update-system").Inc()
 		return errors.Wrap(err, "Unable to update system")
@@ -500,7 +502,7 @@ func evaluateAndStore(system *models.SystemPlatform,
 	return nil
 }
 
-func analyzeRepos(system *models.SystemPlatform) (thirdParty bool, err error) {
+func analyzeRepos(system *models.SystemInventory) (thirdParty bool, err error) {
 	if !enableRepoAnalysis {
 		utils.LogInfo("repo analysis disabled, skipping")
 		return false, nil
@@ -539,15 +541,15 @@ func incrementAdvisoryTypeCounts(advisory models.AdvisoryMetadata, enhCount, bug
 }
 
 // nolint: funlen
-func updateSystemPlatform(tx *gorm.DB, system *models.SystemPlatform,
+func updateSystemPatch(tx *gorm.DB, system *models.SystemInventory, systemPatch *models.SystemPatch,
 	advisories SystemAdvisoryMap, installed, installable, applicable int) error {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, evaluationPartDuration.WithLabelValues("system-update"))
 	if system.LastUpload != nil {
 		defer utils.ObserveSecondsSince(*system.LastUpload, uploadEvaluationDelay)
 	}
-	if system.LastEvaluation != nil {
-		defer utils.ObserveHoursSince(*system.LastEvaluation, twoEvaluationsInterval)
+	if systemPatch.LastEvaluation != nil {
+		defer utils.ObserveHoursSince(*systemPatch.LastEvaluation, twoEvaluationsInterval)
 	}
 
 	data := make(map[string]interface{}, 8)
@@ -592,10 +594,10 @@ func updateSystemPlatform(tx *gorm.DB, system *models.SystemPlatform,
 	}
 
 	if enableRepoAnalysis {
-		data["third_party"] = system.ThirdParty
+		data["third_party"] = systemPatch.ThirdParty
 	}
 
-	err := tx.Model(system).Updates(data).Error
+	err := tx.Model(systemPatch).Updates(data).Error
 
 	now := time.Now()
 	if system.LastUpload != nil && system.LastUpload.Sub(now) > time.Hour {
@@ -630,20 +632,29 @@ func callVMaas(ctx context.Context, request *vmaas.UpdatesV3Request) (*vmaas.Upd
 	return vmaasDataPtr.(*vmaas.UpdatesV3Response), nil
 }
 
-func loadSystemData(accountID int, inventoryID string) (*models.SystemPlatform, error) {
+func loadSystemData(accountID int, inventoryID string) (*models.SystemInventory, *models.SystemPatch, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, evaluationPartDuration.WithLabelValues("data-loading"))
 
-	var system models.SystemPlatform
+	var system models.SystemInventory
 	err := database.DB.Where("rh_account_id = ?", accountID).
-		Where("inventory_id = ?::uuid", inventoryID).
+		Where("system_id = ?", system.ID).
 		Find(&system).Error
-	return &system, err
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var systemPatch models.SystemPatch
+	err = database.DB.Where("rh_account_id = ?", accountID).
+		Where("rh_account_id = ?", system.RhAccountID).
+		Find(&systemPatch).Error
+
+	return &system, &systemPatch, err
 }
 
 func validSystem(accountID int, systemID int64) bool {
 	var foundID int64
-	err := database.DB.Model(models.SystemPlatform{}).
+	err := database.DB.Model(models.SystemInventory{}).
 		Where("rh_account_id = ? and id = ?", accountID, systemID).
 		Select("id").
 		Clauses(clause.Locking{Strength: "SHARE", Table: clause.Table{Name: clause.CurrentTable}}).
@@ -654,7 +665,7 @@ func validSystem(accountID int, systemID int64) bool {
 	return systemID == foundID
 }
 
-func parseVmaasJSON(system *models.SystemPlatform) (vmaas.UpdatesV3Request, error) {
+func parseVmaasJSON(system *models.SystemInventory) (vmaas.UpdatesV3Request, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, evaluationPartDuration.WithLabelValues("parse-vmaas-json"))
 	return utils.ParseVmaasJSON(system)