Move release metadata from BigQuery to PostgreSQL

Create a release_definitions table to store release metadata (GA dates,
development start dates, previous release, capabilities, etc.) that was
previously only available in BigQuery. During the data load cycle,
releases fetched from BQ are converted directly from ReleaseRow to
ReleaseDefinition and synced to PostgreSQL.

All release data consumers now read from PostgreSQL via GetReleasesFromDB
and GetReleaseDatesFromDB. The BigQuery release functions
(GetReleasesFromBigQuery, GetReleases, QueryReleaseDates) are removed,
along with the hardcoded releaseMetadata map from the PostgreSQL data
provider and the QueryReleases/QueryReleaseDates methods from the
DataProvider interface.

Internally, all consumers use models.ReleaseDefinition directly instead
of converting through the v1.Release intermediary. Capability constants
are defined in pkg/db/models alongside the ReleaseDefinition type.

Ref: TRT-2734

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: mstaeble

cmd/sippy/automatejira.go

@@ -152,7 +152,12 @@ func NewAutomateJiraCommand() *cobra.Command {
 			if err != nil {
 				log.WithError(err).Fatal("unable to load views")
 			}
-			releases, err := api.GetReleases(context.Background(), bigQueryClient, false)
+
+			dbc, err := f.PostgresFlags.GetDBClient()
+			if err != nil {
+				log.WithError(err).Fatal("unable to connect to postgres")
+			}
+			releases, err := api.GetReleasesFromDB(ctx, dbc)
 			if err != nil {
 				log.WithError(err).Fatal("error querying releases")
 			}
@@ -178,10 +183,6 @@ func NewAutomateJiraCommand() *cobra.Command {
 				return errors.WithMessage(err, "error validating options")
 			}
 
-			dbc, err := f.PostgresFlags.GetDBClient()
-			if err != nil {
-				log.WithError(err).Fatal("unable to connect to postgres")
-			}
 			j, err := jiraautomator.NewJiraAutomator(
 				jiraClient, bigQueryClient, provider, dbc, cacheOpts,
 				views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount,

cmd/sippy/load.go

@@ -11,7 +11,6 @@ import (
 	"cloud.google.com/go/bigquery"
 	"github.com/openshift/sippy/pkg/api"
 	"github.com/openshift/sippy/pkg/api/componentreadiness"
-	sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	"github.com/openshift/sippy/pkg/dataloader/regressioncacheloader"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -37,9 +36,11 @@ import (
 	"github.com/openshift/sippy/pkg/dataloader/prowloader"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/gcs"
 	"github.com/openshift/sippy/pkg/dataloader/prowloader/github"
+	releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader"
 	"github.com/openshift/sippy/pkg/dataloader/releaseloader"
 	"github.com/openshift/sippy/pkg/dataloader/testownershiploader"
 	"github.com/openshift/sippy/pkg/db"
+	"github.com/openshift/sippy/pkg/db/models"
 	"github.com/openshift/sippy/pkg/flags"
 	"github.com/openshift/sippy/pkg/github/commenter"
 )
@@ -99,7 +100,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) {
 	f.JiraFlags.BindFlags(fs)
 
 	fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading")
-	fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading")
+	fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading")
 	fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)")
 	fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)")
 	fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader")
@@ -151,8 +152,6 @@ func NewLoadCommand() *cobra.Command {
 				cacheClient = nil // error hygiene, since we pass this down to quite a few functions
 			}
 
-			releaseConfigs := []sippyv1.Release{}
-
 			// initializing a bigquery client different from the normal one
 			opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load")
 			bqc, bigqueryErr := bqcachedclient.New(
@@ -163,26 +162,14 @@ func NewLoadCommand() *cobra.Command {
 				if f.CacheFlags.EnablePersistentCaching {
 					bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc)
 				}
-				releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc)
-				if err != nil {
-					return errors.Wrapf(err, "error querying releases from bq")
-				}
 			}
 
-			// Ensure partitions exist for all releases (only when InitDatabase is true)
-			if f.InitDatabase && dbErr == nil {
-				err = ensurePartitionsForReleases(dbc, releaseConfigs)
-				if err != nil {
-					return errors.Wrapf(err, "error ensuring partitions")
-				}
-
-				// Clean up old partitions
-				detached, dropped, err := dbc.CleanupPartitions(false)
-				if err != nil {
-					log.WithError(err).Warning("failed to cleanup old partitions, continuing with load")
-				} else {
-					log.Infof("Partition cleanup complete: detached %d, dropped %d", detached, dropped)
-				}
+			// Read release definitions from PG for downstream loader construction.
+			// On the first run the table may be empty; the release-definitions
+			// loader will populate it for subsequent runs.
+			var releaseDefs []models.ReleaseDefinition
+			if dbErr == nil {
+				releaseDefs, _ = api.GetReleasesFromDB(context.Background(), dbc)
 			}
 
 			// Sippy Config
@@ -200,6 +187,17 @@ func NewLoadCommand() *cobra.Command {
 
 			var regressionCacheAdded bool
 			for _, l := range f.Loaders {
+				if l == "release-definitions" {
+					if bigqueryErr != nil {
+						return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading")
+					}
+					if dbErr != nil {
+						return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading")
+					}
+					rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc)
+					loaders = append(loaders, rdl)
+				}
+
 				// TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob
 				// manifests are updated to use "regression-cache".
 				if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" {
@@ -236,7 +234,7 @@ func NewLoadCommand() *cobra.Command {
 					regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, jiraClient)
 
 					rcl, err := regressioncacheloader.New(
-						dbc, bqc, config, views.ComponentReadiness, releaseConfigs,
+						dbc, bqc, config, views.ComponentReadiness, releaseDefs,
 						f.ComponentReadinessFlags.CRTimeRoundingFactor,
 						f.ComponentReadinessFlags.CRTimeRoundingOffset,
 						regressionStore,
@@ -251,7 +249,7 @@ func NewLoadCommand() *cobra.Command {
 					if dbErr != nil {
 						return dbErr
 					}
-					loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseConfigs))
+					loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseDefs))
 				}
 
 				// Prow Loader
@@ -260,7 +258,7 @@ func NewLoadCommand() *cobra.Command {
 					if dbErr != nil {
 						return dbErr
 					}
-					prowLoader, err := f.prowLoader(ctx, dbc, config, releaseConfigs, promPusher)
+					prowLoader, err := f.prowLoader(ctx, dbc, config, releaseDefs, promPusher)
 					if err != nil {
 						return err
 					}
@@ -330,7 +328,7 @@ func NewLoadCommand() *cobra.Command {
 				// Feature gates
 				if l == "feature-gates" {
 					refreshMatviews = true
-					fgLoader := featuregateloader.New(dbc, releaseConfigs)
+					fgLoader := featuregateloader.New(dbc, releaseDefs)
 					loaders = append(loaders, fgLoader)
 				}
 
@@ -343,6 +341,23 @@ func NewLoadCommand() *cobra.Command {
 				allErrs = append(allErrs, l.Errors()...)
 			}
 
+			// Ensure partitions exist for all releases (only when InitDatabase is true).
+			// Done after loaders run so the release-definitions loader has populated the table.
+			if f.InitDatabase && dbErr == nil {
+				releaseDefs, _ := api.GetReleasesFromDB(context.Background(), dbc)
+				err = ensurePartitionsForReleases(dbc, releaseDefs)
+				if err != nil {
+					return errors.Wrapf(err, "error ensuring partitions")
+				}
+
+				detached, dropped, err := dbc.CleanupPartitions(false)
+				if err != nil {
+					log.WithError(err).Warning("failed to cleanup old partitions, continuing with load")
+				} else {
+					log.Infof("Partition cleanup complete: detached %d, dropped %d", detached, dropped)
+				}
+			}
+
 			elapsed := time.Since(start)
 			log.WithField("elapsed", elapsed).Info("database load complete")
 
@@ -417,7 +432,7 @@ func (f *LoadFlags) jobVariantsLoader(ctx context.Context) (dataloader.DataLoade
 
 }
 
-func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseConfigs []sippyv1.Release, promPusher *push.Pusher) (dataloader.DataLoader, error) {
+func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseDefs []models.ReleaseDefinition, promPusher *push.Pusher) (dataloader.DataLoader, error) {
 	gcsClient, err := gcs.NewGCSClient(ctx,
 		f.GoogleCloudFlags.ServiceAccountCredentialFile,
 		f.GoogleCloudFlags.OAuthClientCredentialFile,
@@ -452,8 +467,8 @@ func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.
 
 	releases := f.Releases
 	if len(releases) == 0 { // if not specified, use those defined in the Releases table
-		for _, config := range releaseConfigs {
-			releases = append(releases, config.Release) // could filter by capability if needed
+		for _, def := range releaseDefs {
+			releases = append(releases, def.Release) // could filter by capability if needed
 		}
 	}
 
@@ -503,10 +518,9 @@ func parseProwLoadSince(val string) (time.Time, error) {
 // ensurePartitionsForReleases creates partitions for all configured releases.
 // It uses a 7 day lookback window plus 2 days forward from today.
 // Errors are logged but ignored to prevent blocking the load process.
-func ensurePartitionsForReleases(dbc *db.DB, releaseConfigs []sippyv1.Release) error {
-	// Extract release names from release configs
-	releases := make([]string, 0, len(releaseConfigs))
-	for _, r := range releaseConfigs {
+func ensurePartitionsForReleases(dbc *db.DB, releaseDefs []models.ReleaseDefinition) error {
+	releases := make([]string, 0, len(releaseDefs))
+	for _, r := range releaseDefs {
 		releases = append(releases, r.Release)
 	}
 

cmd/sippy/seed_data.go

@@ -16,6 +16,7 @@ import (
 	"github.com/spf13/pflag"
 	"gopkg.in/yaml.v3"
 
+	"github.com/openshift/sippy/pkg/api"
 	componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness"
 	pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
@@ -410,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error {
 		return nil
 	}
 
+	if err := seedReleaseDefinitions(dbc); err != nil {
+		return errors.WithMessage(err, "failed to seed release definitions")
+	}
+	log.Info("Seeded release definitions")
+
 	if err := createTestSuite(dbc, "synthetic"); err != nil {
 		return errors.WithMessage(err, "failed to create test suite")
 	}
@@ -445,6 +451,53 @@ func seedSyntheticData(dbc *db.DB) error {
 	return nil
 }
 
+func seedReleaseDefinitions(dbc *db.DB) error {
+	now := time.Now().UTC()
+	allCaps := pq.StringArray{"componentReadiness", "featureGates", "metrics", "payloadTags", "sippyClassic"}
+
+	type relMeta struct {
+		previous string
+		gaDays   int // negative = days before now; 0 = no GA (in development)
+	}
+	meta := map[string]relMeta{
+		"4.19": {previous: "4.18", gaDays: -289},
+		"4.20": {previous: "4.19", gaDays: -163},
+		"4.21": {previous: "4.20", gaDays: -58},
+		"4.22": {previous: "4.21"},
+	}
+
+	for _, release := range syntheticReleases {
+		m := meta[release]
+		parts := strings.Split(release, ".")
+		major, minor := 0, 0
+		if len(parts) >= 2 {
+			fmt.Sscanf(parts[0], "%d", &major)
+			fmt.Sscanf(parts[1], "%d", &minor)
+		}
+
+		develStart := now.AddDate(0, 0, m.gaDays-180)
+		def := models.ReleaseDefinition{
+			Release:              release,
+			Major:                major,
+			Minor:                minor,
+			PreviousRelease:      m.previous,
+			DevelopmentStartDate: &develStart,
+			Product:              "OCP",
+			Status:               "Full Support",
+			Capabilities:         allCaps,
+		}
+		if m.gaDays != 0 {
+			ga := now.AddDate(0, 0, m.gaDays)
+			def.GADate = &ga
+		}
+
+		if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil {
+			return fmt.Errorf("failed to create release definition %s: %w", release, err)
+		}
+	}
+	return nil
+}
+
 func seedProwJobs(dbc *db.DB) error {
 	for _, release := range syntheticReleases {
 		for _, job := range syntheticJobs {
@@ -692,7 +745,7 @@ func syncRegressions(dbc *db.DB) error {
 	provider := pgprovider.NewPostgresProvider(dbc, nil)
 	ctx := context.Background()
 
-	releases, err := provider.QueryReleases(ctx)
+	releases, err := api.GetReleasesFromDB(ctx, dbc)
 	if err != nil {
 		return fmt.Errorf("querying releases: %w", err)
 	}

pkg/api/componentreadiness/component_report.go

@@ -31,8 +31,8 @@ import (
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
 	crtype "github.com/openshift/sippy/pkg/apis/api/componentreport"
 	"github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	"github.com/openshift/sippy/pkg/db"
+	"github.com/openshift/sippy/pkg/db/models"
 	"github.com/openshift/sippy/pkg/util/sets"
 )
 
@@ -82,7 +82,7 @@ func GetComponentReport(
 	reqOptions reqopts.RequestOptions,
 	baseURL string,
 ) (report crtype.ComponentReport, errs []error) {
-	releaseConfigs, err := provider.QueryReleases(ctx)
+	releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc)
 	if err != nil {
 		return report, []error{err}
 	}
@@ -151,7 +151,7 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport)
 	return nil
 }
 
-func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator {
+func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []models.ReleaseDefinition, baseURL string) ComponentReportGenerator {
 	slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match
 	generator := ComponentReportGenerator{
 		dataProvider:   provider,
@@ -175,7 +175,7 @@ type ComponentReportGenerator struct {
 	dbc            *db.DB
 	ReqOptions     reqopts.RequestOptions
 	middlewares    middleware.List
-	releaseConfigs []v1.Release
+	releaseConfigs []models.ReleaseDefinition
 	baseURL        string
 }
 
@@ -278,7 +278,7 @@ func (c *ComponentReportGenerator) initializeMiddleware() {
 	c.middlewares = middleware.List{}
 	// Initialize all our middleware applicable to this request.
 	if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil {
-		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs))
+		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.dbc, c.ReqOptions, c.releaseConfigs))
 	}
 	if c.dbc != nil {
 		c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions))

pkg/api/componentreadiness/dataprovider/bigquery/provider.go

@@ -18,7 +18,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	apiCache "github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
 	"github.com/openshift/sippy/pkg/bigquery/bqlabel"
 	"github.com/openshift/sippy/pkg/util/param"
@@ -170,14 +169,6 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
 	return variants, nil
 }
 
-func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
-	return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions)
-}
-
-func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
-	return apiPkg.GetReleasesFromBigQuery(ctx, p.client)
-}
-
 func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {
 	unnest := ""
 	if nested {