Move release metadata from BigQuery to PostgreSQL

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: mstaeble

cmd/sippy/load.go

@@ -37,6 +37,7 @@ import (
 	"github.com/openshift/sippy/pkg/dataloader/prowloader"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/gcs"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/github"
+	releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader"
 	"github.com/openshift/sippy/pkg/dataloader/releaseloader"
 	"github.com/openshift/sippy/pkg/dataloader/testownershiploader"
 	"github.com/openshift/sippy/pkg/db"
@@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) {
 	f.JiraFlags.BindFlags(fs)
 
 	fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading")
-	fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading")
+	fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading")
 	fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)")
 	fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)")
 	fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader")
@@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command {
 				cacheClient = nil // error hygiene, since we pass this down to quite a few functions
 			}
 
-			releaseConfigs := []sippyv1.Release{}
-
 			// initializing a bigquery client different from the normal one
 			opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load")
 			bqc, bigqueryErr := bqcachedclient.New(
@@ -164,10 +163,14 @@ func NewLoadCommand() *cobra.Command {
 				if f.CacheFlags.EnablePersistentCaching {
 					bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc)
 				}
-				releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc)
-				if err != nil {
-					return errors.Wrapf(err, "error querying releases from bq")
-				}
+			}
+
+			// Read release definitions from PG for downstream loader construction.
+			// On the first run the table may be empty; the release-definitions
+			// loader will populate it for subsequent runs.
+			releaseConfigs := []sippyv1.Release{}
+			if dbErr == nil {
+				releaseConfigs, _ = api.GetReleasesFromDB(context.Background(), dbc)
 			}
 
 			// Ensure partitions exist for all releases (only when InitDatabase is true)
@@ -201,6 +204,17 @@ func NewLoadCommand() *cobra.Command {
 
 			var regressionCacheAdded bool
 			for _, l := range f.Loaders {
+				if l == "release-definitions" {
+					if bigqueryErr != nil {
+						return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading")
+					}
+					if dbErr != nil {
+						return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading")
+					}
+					rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc)
+					loaders = append(loaders, rdl)
+				}
+
 				// TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob
 				// manifests are updated to use "regression-cache".
 				if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" {

cmd/sippy/seed_data.go

@@ -411,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error {
 		return nil
 	}
 
+	if err := seedReleaseDefinitions(dbc); err != nil {
+		return errors.WithMessage(err, "failed to seed release definitions")
+	}
+	log.Info("Seeded release definitions")
+
 	if err := createTestSuite(dbc, "synthetic"); err != nil {
 		return errors.WithMessage(err, "failed to create test suite")
 	}
@@ -447,6 +452,53 @@ func seedSyntheticData(dbc *db.DB) error {
 	return nil
 }
 
+func seedReleaseDefinitions(dbc *db.DB) error {
+	now := time.Now().UTC()
+	allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic}
+
+	type relMeta struct {
+		previous string
+		gaDays   int // negative = days before now; 0 = no GA (in development)
+	}
+	meta := map[string]relMeta{
+		"4.19": {previous: "4.18", gaDays: -289},
+		"4.20": {previous: "4.19", gaDays: -163},
+		"4.21": {previous: "4.20", gaDays: -58},
+		"4.22": {previous: "4.21"},
+	}
+
+	for _, release := range syntheticReleases {
+		m := meta[release]
+		parts := strings.Split(release, ".")
+		major, minor := 0, 0
+		if len(parts) >= 2 {
+			fmt.Sscanf(parts[0], "%d", &major)
+			fmt.Sscanf(parts[1], "%d", &minor)
+		}
+
+		develStart := now.AddDate(0, 0, m.gaDays-180)
+		def := models.ReleaseDefinition{
+			Release:              release,
+			Major:                major,
+			Minor:                minor,
+			PreviousRelease:      m.previous,
+			DevelopmentStartDate: &develStart,
+			Product:              "OCP",
+			Status:               "Full Support",
+			Capabilities:         allCaps,
+		}
+		if m.gaDays != 0 {
+			ga := now.AddDate(0, 0, m.gaDays)
+			def.GADate = &ga
+		}
+
+		if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil {
+			return fmt.Errorf("failed to create release definition %s: %w", release, err)
+		}
+	}
+	return nil
+}
+
 func seedProwJobs(dbc *db.DB) error {
 	for _, release := range syntheticReleases {
 		for _, job := range syntheticJobs {

pkg/api/componentreadiness/dataprovider/postgres/provider.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/lib/pq"
 
+	"github.com/openshift/sippy/pkg/api"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus"
@@ -132,60 +133,8 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
 	return variants, nil
 }
 
-// releaseMetadata holds hardcoded release info for known releases.
-// This avoids needing a releases table — we derive release names from prow_jobs
-// and fill in metadata from this map.
-var releaseMetadata = map[string]struct {
-	previousRelease string
-	gaOffsetDays    int    // 0 = no GA date (in development)
-	product         string // empty = defaults to "OCP"
-}{
-	"4.17": {previousRelease: "4.16", gaOffsetDays: -540},
-	"4.18": {previousRelease: "4.17", gaOffsetDays: -395},
-	"4.19": {previousRelease: "4.18", gaOffsetDays: -289},
-	"4.20": {previousRelease: "4.19", gaOffsetDays: -163},
-	"4.21": {previousRelease: "4.20", gaOffsetDays: -58},
-	"4.22": {previousRelease: "4.21"},
-	"5.0":  {previousRelease: "4.22"},
-}
-
 func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
-	var releaseNames []string
-	err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`).
-		Pluck("release", &releaseNames).Error
-	if err != nil {
-		return nil, fmt.Errorf("querying releases: %w", err)
-	}
-
-	caps := map[v1.ReleaseCapability]bool{
-		v1.ComponentReadinessCap: true,
-		v1.FeatureGatesCap:       true,
-		v1.MetricsCap:            true,
-		v1.PayloadTagsCap:        true,
-		v1.SippyClassicCap:       true,
-	}
-
-	now := time.Now().UTC()
-	var releases []v1.Release
-	for _, name := range releaseNames {
-		rel := v1.Release{
-			Release:      name,
-			Capabilities: caps,
-			Product:      "OCP",
-		}
-		if meta, ok := releaseMetadata[name]; ok {
-			rel.PreviousRelease = meta.previousRelease
-			if meta.gaOffsetDays != 0 {
-				ga := now.AddDate(0, 0, meta.gaOffsetDays)
-				rel.GADate = &ga
-			}
-			if meta.product != "" {
-				rel.Product = meta.product
-			}
-		}
-		releases = append(releases, rel)
-	}
-	return releases, nil
+	return api.GetReleasesFromDB(ctx, p.dbc)
 }
 
 func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {

pkg/api/releases.go

@@ -498,6 +498,66 @@ func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release {
 	return release
 }
 
+// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table.
+func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) {
+	var rows []sippyv1.ReleaseRow
+
+	queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable)
+
+	q := client.Query(ctx, bqlabel.ReleaseAllReleases, queryString)
+	it, err := q.Read(ctx)
+	if err != nil {
+		log.WithError(err).Error("error querying releases data from bigquery")
+		return rows, err
+	}
+
+	for {
+		r := sippyv1.ReleaseRow{}
+		err := it.Next(&r)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			log.WithError(err).Error("error parsing release row from bigquery")
+			return rows, err
+		}
+		rows = append(rows, r)
+	}
+	return rows, nil
+}
+
+// GetReleasesFromDB queries release metadata from the release_definitions table
+// and converts to []sippyv1.Release for use by existing callers.
+func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]sippyv1.Release, error) {
+	var defs []models.ReleaseDefinition
+	err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error
+	if err != nil {
+		return nil, fmt.Errorf("querying release definitions: %w", err)
+	}
+	releases := make([]sippyv1.Release, 0, len(defs))
+	for _, def := range defs {
+		releases = append(releases, DefinitionToRelease(def))
+	}
+	return releases, nil
+}
+
+// DefinitionToRelease converts a models.ReleaseDefinition to a sippyv1.Release.
+func DefinitionToRelease(def models.ReleaseDefinition) sippyv1.Release {
+	caps := make(map[sippyv1.ReleaseCapability]bool, len(def.Capabilities))
+	for _, cap := range def.Capabilities {
+		caps[sippyv1.ReleaseCapability(cap)] = true
+	}
+	return sippyv1.Release{
+		Release:              def.Release,
+		Status:               def.Status,
+		GADate:               def.GADate,
+		DevelopmentStartDate: def.DevelopmentStartDate,
+		PreviousRelease:      def.PreviousRelease,
+		Capabilities:         caps,
+		Product:              def.Product,
+	}
+}
+
 // BuildReleasesResponse creates the API response structure for releases
 func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases {
 	gaDateMap := make(map[string]time.Time)

pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go

@@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics {
 	return loader
 }
 
-var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"}
+var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"}
 
 // sortLoaders guarantees that the loaders run in a predictable and proper order
 func (l *LoaderWithMetrics) sortLoaders() {

pkg/dataloader/releasedefloader/releasedefloader.go

@@ -0,0 +1,103 @@
+package releasedefloader
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+	"gorm.io/gorm/clause"
+
+	"github.com/openshift/sippy/pkg/api"
+	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
+	bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
+	"github.com/openshift/sippy/pkg/db"
+	"github.com/openshift/sippy/pkg/db/models"
+)
+
+// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL.
+type ReleaseDefinitionLoader struct {
+	ctx      context.Context
+	dbc      *db.DB
+	bqClient *bqcachedclient.Client
+	errs     []error
+}
+
+func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader {
+	return &ReleaseDefinitionLoader{
+		ctx:      ctx,
+		dbc:      dbc,
+		bqClient: bqClient,
+	}
+}
+
+func (l *ReleaseDefinitionLoader) Name() string {
+	return "release-definitions"
+}
+
+func (l *ReleaseDefinitionLoader) Load() {
+	releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient)
+	if err != nil {
+		l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err))
+		return
+	}
+	defs := make([]models.ReleaseDefinition, 0, len(releaseRows))
+	for _, row := range releaseRows {
+		defs = append(defs, ReleaseRowToDefinition(row))
+	}
+	if err := syncReleaseDefinitions(l.dbc, defs); err != nil {
+		l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err))
+	}
+}
+
+func (l *ReleaseDefinitionLoader) Errors() []error {
+	return l.errs
+}
+
+func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error {
+	for _, def := range defs {
+		err := dbc.DB.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "release"}},
+			DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}),
+		}).Create(&def).Error
+		if err != nil {
+			return fmt.Errorf("upserting release definition %s: %w", def.Release, err)
+		}
+	}
+	log.WithField("count", len(defs)).Info("synced release definitions to postgres")
+	return nil
+}
+
+// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model.
+func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition {
+	caps := make(pq.StringArray, 0, len(r.Capabilities))
+	for _, cap := range r.Capabilities {
+		caps = append(caps, string(cap))
+	}
+	sort.Strings(caps)
+
+	def := models.ReleaseDefinition{
+		Release:         r.Release,
+		Major:           r.Major,
+		Minor:           r.Minor,
+		PreviousRelease: r.PreviousRelease.StringVal,
+		Product:         r.Product.StringVal,
+		Status:          r.ReleaseStatus.StringVal,
+		Capabilities:    caps,
+	}
+	if r.Patch.Valid {
+		p := int(r.Patch.Int64)
+		def.Patch = &p
+	}
+	if r.GADate.Valid {
+		ga := r.GADate.Date.In(time.UTC)
+		def.GADate = &ga
+	}
+	if r.DevelStartDate.IsValid() {
+		ds := r.DevelStartDate.In(time.UTC)
+		def.DevelopmentStartDate = &ds
+	}
+	return def
+}

pkg/db/db.go

@@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error {
 
 	// List of all models to migrate
 	modelsToMigrate := []any{
+		&models.ReleaseDefinition{},
 		&models.ReleaseTag{},
 		&models.ReleasePullRequest{},
 		&models.ReleaseRepository{},