Move release metadata from BigQuery to PostgreSQL

Create a release_definitions table to store release metadata (GA dates,
development start dates, previous release, capabilities, etc.) that was
previously only available in BigQuery. During the data load cycle,
releases fetched from BQ are converted directly from ReleaseRow to
ReleaseDefinition and synced to PostgreSQL.

All release data consumers now read from PostgreSQL via GetReleasesFromDB
and GetReleaseDatesFromDB. The BigQuery release functions
(GetReleasesFromBigQuery, GetReleases, QueryReleaseDates) are removed,
along with the hardcoded releaseMetadata map from the PostgreSQL data
provider and the QueryReleases/QueryReleaseDates methods from the
DataProvider interface.

Internally, all consumers use models.ReleaseDefinition directly instead
of converting through the v1.Release intermediary. Capability constants
are defined in pkg/db/models alongside the ReleaseDefinition type.

Ref: TRT-2734

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: mstaeble

cmd/sippy/automatejira.go

@@ -152,7 +152,12 @@ func NewAutomateJiraCommand() *cobra.Command {
 			if err != nil {
 				log.WithError(err).Fatal("unable to load views")
 			}
-			releases, err := api.GetReleases(context.Background(), bigQueryClient, false)
+
+			dbc, err := f.PostgresFlags.GetDBClient()
+			if err != nil {
+				log.WithError(err).Fatal("unable to connect to postgres")
+			}
+			releases, err := api.GetReleasesFromDB(ctx, dbc)
 			if err != nil {
 				log.WithError(err).Fatal("error querying releases")
 			}
@@ -178,10 +183,6 @@ func NewAutomateJiraCommand() *cobra.Command {
 				return errors.WithMessage(err, "error validating options")
 			}
 
-			dbc, err := f.PostgresFlags.GetDBClient()
-			if err != nil {
-				log.WithError(err).Fatal("unable to connect to postgres")
-			}
 			j, err := jiraautomator.NewJiraAutomator(
 				jiraClient, bigQueryClient, provider, dbc, cacheOpts,
 				views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount,

cmd/sippy/load.go

@@ -11,7 +11,6 @@ import (
 	"cloud.google.com/go/bigquery"
 	"github.com/openshift/sippy/pkg/api"
 	"github.com/openshift/sippy/pkg/api/componentreadiness"
-	sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	"github.com/openshift/sippy/pkg/dataloader/regressioncacheloader"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -40,6 +39,7 @@ import (
 	"github.com/openshift/sippy/pkg/dataloader/releaseloader"
 	"github.com/openshift/sippy/pkg/dataloader/testownershiploader"
 	"github.com/openshift/sippy/pkg/db"
+	"github.com/openshift/sippy/pkg/db/models"
 	"github.com/openshift/sippy/pkg/flags"
 	"github.com/openshift/sippy/pkg/github/commenter"
 )
@@ -151,7 +151,7 @@ func NewLoadCommand() *cobra.Command {
 				cacheClient = nil // error hygiene, since we pass this down to quite a few functions
 			}
 
-			releaseConfigs := []sippyv1.Release{}
+			var releaseDefs []models.ReleaseDefinition
 
 			// initializing a bigquery client different from the normal one
 			opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load")
@@ -163,15 +163,27 @@ func NewLoadCommand() *cobra.Command {
 				if f.CacheFlags.EnablePersistentCaching {
 					bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc)
 				}
-				releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc)
+				releaseRows, err := api.GetReleaseRowsFromBigQuery(context.Background(), bqc)
 				if err != nil {
 					return errors.Wrapf(err, "error querying releases from bq")
 				}
+				releaseDefs = make([]models.ReleaseDefinition, 0, len(releaseRows))
+				for _, row := range releaseRows {
+					releaseDefs = append(releaseDefs, releaseloader.ReleaseRowToDefinition(row))
+				}
+
+				// Sync release definitions from BQ to PostgreSQL so the serving
+				// path can read them without a BigQuery dependency.
+				if dbErr == nil && len(releaseDefs) > 0 {
+					if syncErr := releaseloader.SyncReleaseDefinitions(dbc, releaseDefs); syncErr != nil {
+						log.WithError(syncErr).Warning("failed to sync release definitions to postgres")
+					}
+				}
 			}
 
 			// Ensure partitions exist for all releases (only when InitDatabase is true)
 			if f.InitDatabase && dbErr == nil {
-				err = ensurePartitionsForReleases(dbc, releaseConfigs)
+				err = ensurePartitionsForReleases(dbc, releaseDefs)
 				if err != nil {
 					return errors.Wrapf(err, "error ensuring partitions")
 				}
@@ -236,7 +248,7 @@ func NewLoadCommand() *cobra.Command {
 					regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, jiraClient)
 
 					rcl, err := regressioncacheloader.New(
-						dbc, bqc, config, views.ComponentReadiness, releaseConfigs,
+						dbc, bqc, config, views.ComponentReadiness, releaseDefs,
 						f.ComponentReadinessFlags.CRTimeRoundingFactor,
 						f.ComponentReadinessFlags.CRTimeRoundingOffset,
 						regressionStore,
@@ -251,7 +263,7 @@ func NewLoadCommand() *cobra.Command {
 					if dbErr != nil {
 						return dbErr
 					}
-					loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseConfigs))
+					loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseDefs))
 				}
 
 				// Prow Loader
@@ -260,7 +272,7 @@ func NewLoadCommand() *cobra.Command {
 					if dbErr != nil {
 						return dbErr
 					}
-					prowLoader, err := f.prowLoader(ctx, dbc, config, releaseConfigs, promPusher)
+					prowLoader, err := f.prowLoader(ctx, dbc, config, releaseDefs, promPusher)
 					if err != nil {
 						return err
 					}
@@ -330,7 +342,7 @@ func NewLoadCommand() *cobra.Command {
 				// Feature gates
 				if l == "feature-gates" {
 					refreshMatviews = true
-					fgLoader := featuregateloader.New(dbc, releaseConfigs)
+					fgLoader := featuregateloader.New(dbc, releaseDefs)
 					loaders = append(loaders, fgLoader)
 				}
 
@@ -417,7 +429,7 @@ func (f *LoadFlags) jobVariantsLoader(ctx context.Context) (dataloader.DataLoade
 
 }
 
-func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseConfigs []sippyv1.Release, promPusher *push.Pusher) (dataloader.DataLoader, error) {
+func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseDefs []models.ReleaseDefinition, promPusher *push.Pusher) (dataloader.DataLoader, error) {
 	gcsClient, err := gcs.NewGCSClient(ctx,
 		f.GoogleCloudFlags.ServiceAccountCredentialFile,
 		f.GoogleCloudFlags.OAuthClientCredentialFile,
@@ -452,8 +464,8 @@ func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.
 
 	releases := f.Releases
 	if len(releases) == 0 { // if not specified, use those defined in the Releases table
-		for _, config := range releaseConfigs {
-			releases = append(releases, config.Release) // could filter by capability if needed
+		for _, def := range releaseDefs {
+			releases = append(releases, def.Release) // could filter by capability if needed
 		}
 	}
 
@@ -503,10 +515,9 @@ func parseProwLoadSince(val string) (time.Time, error) {
 // ensurePartitionsForReleases creates partitions for all configured releases.
 // It uses a 7 day lookback window plus 2 days forward from today.
 // Errors are logged but ignored to prevent blocking the load process.
-func ensurePartitionsForReleases(dbc *db.DB, releaseConfigs []sippyv1.Release) error {
-	// Extract release names from release configs
-	releases := make([]string, 0, len(releaseConfigs))
-	for _, r := range releaseConfigs {
+func ensurePartitionsForReleases(dbc *db.DB, releaseDefs []models.ReleaseDefinition) error {
+	releases := make([]string, 0, len(releaseDefs))
+	for _, r := range releaseDefs {
 		releases = append(releases, r.Release)
 	}
 

cmd/sippy/seed_data.go

@@ -16,6 +16,7 @@ import (
 	"github.com/spf13/pflag"
 	"gopkg.in/yaml.v3"
 
+	"github.com/openshift/sippy/pkg/api"
 	componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness"
 	pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
@@ -410,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error {
 		return nil
 	}
 
+	if err := seedReleaseDefinitions(dbc); err != nil {
+		return errors.WithMessage(err, "failed to seed release definitions")
+	}
+	log.Info("Seeded release definitions")
+
 	if err := createTestSuite(dbc, "synthetic"); err != nil {
 		return errors.WithMessage(err, "failed to create test suite")
 	}
@@ -445,6 +451,53 @@ func seedSyntheticData(dbc *db.DB) error {
 	return nil
 }
 
+func seedReleaseDefinitions(dbc *db.DB) error {
+	now := time.Now().UTC()
+	allCaps := pq.StringArray{"componentReadiness", "featureGates", "metrics", "payloadTags", "sippyClassic"}
+
+	type relMeta struct {
+		previous string
+		gaDays   int // negative = days before now; 0 = no GA (in development)
+	}
+	meta := map[string]relMeta{
+		"4.19": {previous: "4.18", gaDays: -289},
+		"4.20": {previous: "4.19", gaDays: -163},
+		"4.21": {previous: "4.20", gaDays: -58},
+		"4.22": {previous: "4.21"},
+	}
+
+	for _, release := range syntheticReleases {
+		m := meta[release]
+		parts := strings.Split(release, ".")
+		major, minor := 0, 0
+		if len(parts) >= 2 {
+			fmt.Sscanf(parts[0], "%d", &major)
+			fmt.Sscanf(parts[1], "%d", &minor)
+		}
+
+		develStart := now.AddDate(0, 0, m.gaDays-180)
+		def := models.ReleaseDefinition{
+			Release:              release,
+			Major:                major,
+			Minor:                minor,
+			PreviousRelease:      m.previous,
+			DevelopmentStartDate: &develStart,
+			Product:              "OCP",
+			Status:               "Full Support",
+			Capabilities:         allCaps,
+		}
+		if m.gaDays != 0 {
+			ga := now.AddDate(0, 0, m.gaDays)
+			def.GADate = &ga
+		}
+
+		if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil {
+			return fmt.Errorf("failed to create release definition %s: %w", release, err)
+		}
+	}
+	return nil
+}
+
 func seedProwJobs(dbc *db.DB) error {
 	for _, release := range syntheticReleases {
 		for _, job := range syntheticJobs {
@@ -692,7 +745,7 @@ func syncRegressions(dbc *db.DB) error {
 	provider := pgprovider.NewPostgresProvider(dbc, nil)
 	ctx := context.Background()
 
-	releases, err := provider.QueryReleases(ctx)
+	releases, err := api.GetReleasesFromDB(ctx, dbc)
 	if err != nil {
 		return fmt.Errorf("querying releases: %w", err)
 	}

pkg/api/componentreadiness/component_report.go

@@ -31,8 +31,8 @@ import (
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
 	crtype "github.com/openshift/sippy/pkg/apis/api/componentreport"
 	"github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	"github.com/openshift/sippy/pkg/db"
+	"github.com/openshift/sippy/pkg/db/models"
 	"github.com/openshift/sippy/pkg/util/sets"
 )
 
@@ -82,7 +82,7 @@ func GetComponentReport(
 	reqOptions reqopts.RequestOptions,
 	baseURL string,
 ) (report crtype.ComponentReport, errs []error) {
-	releaseConfigs, err := provider.QueryReleases(ctx)
+	releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc)
 	if err != nil {
 		return report, []error{err}
 	}
@@ -151,7 +151,7 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport)
 	return nil
 }
 
-func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator {
+func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []models.ReleaseDefinition, baseURL string) ComponentReportGenerator {
 	slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match
 	generator := ComponentReportGenerator{
 		dataProvider:   provider,
@@ -175,7 +175,7 @@ type ComponentReportGenerator struct {
 	dbc            *db.DB
 	ReqOptions     reqopts.RequestOptions
 	middlewares    middleware.List
-	releaseConfigs []v1.Release
+	releaseConfigs []models.ReleaseDefinition
 	baseURL        string
 }
 
@@ -278,7 +278,7 @@ func (c *ComponentReportGenerator) initializeMiddleware() {
 	c.middlewares = middleware.List{}
 	// Initialize all our middleware applicable to this request.
 	if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil {
-		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs))
+		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.dbc, c.ReqOptions, c.releaseConfigs))
 	}
 	if c.dbc != nil {
 		c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions))

pkg/api/componentreadiness/dataprovider/bigquery/provider.go

@@ -18,7 +18,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	apiCache "github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
 	"github.com/openshift/sippy/pkg/bigquery/bqlabel"
 	"github.com/openshift/sippy/pkg/util/param"
@@ -170,14 +169,6 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
 	return variants, nil
 }
 
-func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
-	return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions)
-}
-
-func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
-	return apiPkg.GetReleasesFromBigQuery(ctx, p.client)
-}
-
 func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {
 	unnest := ""
 	if nested {

pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go

@@ -8,7 +8,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	"github.com/openshift/sippy/pkg/apis/cache"
-	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 )
 
 // TestStatusQuerier fetches aggregated test pass/fail counts.
@@ -40,12 +39,6 @@ type MetadataQuerier interface {
 	// QueryJobVariants returns all variant names and their possible values.
 	QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error)
 
-	// QueryReleaseDates returns the time ranges for each known release.
-	QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error)
-
-	// QueryReleases returns known release configurations.
-	QueryReleases(ctx context.Context) ([]v1.Release, error)
-
 	// QueryUniqueVariantValues returns distinct values for a variant column
 	// from the past 60 days.
 	QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error)