From f4321802008b273bffb2ace4c101f2bbafab03f4 Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Tue, 23 Jun 2026 15:51:28 -0400 Subject: [PATCH] Move release metadata from BigQuery to PostgreSQL Create a release_definitions table to store release metadata (GA dates, development start dates, previous release, capabilities, etc.) that was previously only available in BigQuery. During the data load cycle, releases fetched from BQ are converted directly from ReleaseRow to ReleaseDefinition and synced to PostgreSQL. All release data consumers now read from PostgreSQL via GetReleasesFromDB and GetReleaseDatesFromDB. The BigQuery release functions (GetReleasesFromBigQuery, GetReleases, QueryReleaseDates) are removed, along with the hardcoded releaseMetadata map from the PostgreSQL data provider and the QueryReleases/QueryReleaseDates methods from the DataProvider interface. Internally, all consumers use models.ReleaseDefinition directly instead of converting through the v1.Release intermediary. Capability constants are defined in pkg/db/models alongside the ReleaseDefinition type. Ref: TRT-2734 Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/automatejira.go | 11 +- cmd/sippy/load.go | 54 +++++---- cmd/sippy/seed_data.go | 55 +++++++++- .../componentreadiness/component_report.go | 10 +- .../dataprovider/bigquery/provider.go | 9 -- .../dataprovider/bigquery/releasedates.go | 44 -------- .../dataprovider/interface.go | 7 -- .../dataprovider/postgres/provider.go | 93 ---------------- .../regressionallowances.go | 6 +- .../regressionallowances_test.go | 4 +- .../releasefallback/releasefallback.go | 38 ++++--- .../releasefallback/releasefallback_test.go | 8 +- .../queryparamparser_test.go | 6 +- pkg/api/componentreadiness/test_details.go | 10 +- pkg/api/componentreadiness/triage.go | 9 +- pkg/api/componentreadiness/triage_test.go | 3 +- .../utils/queryparamparser.go | 8 +- pkg/api/componentreadiness/utils/utils.go | 4 +- .../componentreadiness/utils/utils_test.go | 4 +- pkg/api/job_runs.go | 6 +- pkg/api/releases.go | 79 ++++++++------ pkg/api/releases_test.go | 63 ----------- pkg/api/utils.go | 36 ------ .../jiraautomator/jiraautomator.go | 5 +- pkg/dataloader/crcacheloader/crcacheloader.go | 6 +- .../featuregateloader/featuregateloader.go | 19 ++-- .../loaderwithmetrics/loaderwithmetrics.go | 2 +- .../regressioncacheloader.go | 5 +- .../releasedefloader/releasedefloader.go | 103 ++++++++++++++++++ pkg/dataloader/releaseloader/projects.go | 6 +- pkg/dataloader/releaseloader/releasesync.go | 13 +-- pkg/dataloader/releaseloader/types.go | 6 +- pkg/db/db.go | 1 + pkg/db/models/releases.go | 35 ++++++ pkg/mcp/tools/releases.go | 27 ++--- pkg/sippyserver/metrics/metrics.go | 24 ++-- pkg/sippyserver/server.go | 37 +++---- pkg/util/utils.go | 4 +- pkg/util/utils_test.go | 4 +- .../componentreadiness_test.go | 6 +- 40 files changed, 412 insertions(+), 458 deletions(-) delete mode 100644 pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go create mode 100644 pkg/dataloader/releasedefloader/releasedefloader.go diff --git a/cmd/sippy/automatejira.go b/cmd/sippy/automatejira.go index ed9c930375..d2bdc4e31c 100644 --- a/cmd/sippy/automatejira.go +++ b/cmd/sippy/automatejira.go @@ -152,7 +152,12 @@ func NewAutomateJiraCommand() *cobra.Command { if err != nil { log.WithError(err).Fatal("unable to load views") } - releases, err := api.GetReleases(context.Background(), bigQueryClient, false) + + dbc, err := f.PostgresFlags.GetDBClient() + if err != nil { + log.WithError(err).Fatal("unable to connect to postgres") + } + releases, err := api.GetReleasesFromDB(ctx, dbc) if err != nil { log.WithError(err).Fatal("error querying releases") } @@ -178,10 +183,6 @@ func NewAutomateJiraCommand() *cobra.Command { return errors.WithMessage(err, "error validating options") } - dbc, err := f.PostgresFlags.GetDBClient() - if err != nil { - log.WithError(err).Fatal("unable to connect to postgres") - } j, err := jiraautomator.NewJiraAutomator( jiraClient, bigQueryClient, provider, dbc, cacheOpts, views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount, diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 8a75b65223..a316cba404 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -11,7 +11,6 @@ import ( "cloud.google.com/go/bigquery" "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/dataloader/regressioncacheloader" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -37,10 +36,12 @@ import ( "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/dataloader/prowloader/github" + releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader" "github.com/openshift/sippy/pkg/dataloader/releaseloader" "github.com/openshift/sippy/pkg/dataloader/testownershiploader" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/dailysummary" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/flags" "github.com/openshift/sippy/pkg/github/commenter" ) @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { f.JiraFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading") - fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") + fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)") fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)") fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader") @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command { cacheClient = nil // error hygiene, since we pass this down to quite a few functions } - releaseConfigs := []sippyv1.Release{} - // initializing a bigquery client different from the normal one opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load") bqc, bigqueryErr := bqcachedclient.New( @@ -164,20 +163,23 @@ func NewLoadCommand() *cobra.Command { if f.CacheFlags.EnablePersistentCaching { bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc) } - releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc) - if err != nil { - return errors.Wrapf(err, "error querying releases from bq") - } + } + + // Read release definitions from PG for downstream loader construction. + // On the first run the table may be empty; the release-definitions + // loader will populate it for subsequent runs. + var releaseDefs []models.ReleaseDefinition + if dbErr == nil { + releaseDefs, _ = api.GetReleasesFromDB(context.Background(), dbc) } // Ensure partitions exist for all releases (only when InitDatabase is true) if f.InitDatabase && dbErr == nil { - err = ensurePartitionsForReleases(dbc, releaseConfigs) + err = ensurePartitionsForReleases(dbc, releaseDefs) if err != nil { return errors.Wrapf(err, "error ensuring partitions") } - // Clean up old partitions detached, dropped, err := dbc.CleanupPartitions(false) if err != nil { log.WithError(err).Warning("failed to cleanup old partitions, continuing with load") @@ -201,6 +203,17 @@ func NewLoadCommand() *cobra.Command { var regressionCacheAdded bool for _, l := range f.Loaders { + if l == "release-definitions" { + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading") + } + rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc) + loaders = append(loaders, rdl) + } + // TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob // manifests are updated to use "regression-cache". if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" { @@ -237,7 +250,7 @@ func NewLoadCommand() *cobra.Command { regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, jiraClient) rcl, err := regressioncacheloader.New( - dbc, bqc, config, views.ComponentReadiness, releaseConfigs, + dbc, bqc, config, views.ComponentReadiness, releaseDefs, f.ComponentReadinessFlags.CRTimeRoundingFactor, f.ComponentReadinessFlags.CRTimeRoundingOffset, regressionStore, @@ -252,7 +265,7 @@ func NewLoadCommand() *cobra.Command { if dbErr != nil { return dbErr } - loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseConfigs)) + loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseDefs)) } // Prow Loader @@ -261,7 +274,7 @@ func NewLoadCommand() *cobra.Command { if dbErr != nil { return dbErr } - prowLoader, err := f.prowLoader(ctx, dbc, config, releaseConfigs, promPusher) + prowLoader, err := f.prowLoader(ctx, dbc, config, releaseDefs, promPusher) if err != nil { return err } @@ -331,7 +344,7 @@ func NewLoadCommand() *cobra.Command { // Feature gates if l == "feature-gates" { refreshMatviews = true - fgLoader := featuregateloader.New(dbc, releaseConfigs) + fgLoader := featuregateloader.New(dbc, releaseDefs) loaders = append(loaders, fgLoader) } @@ -418,7 +431,7 @@ func (f *LoadFlags) jobVariantsLoader(ctx context.Context) (dataloader.DataLoade } -func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseConfigs []sippyv1.Release, promPusher *push.Pusher) (dataloader.DataLoader, error) { +func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseDefs []models.ReleaseDefinition, promPusher *push.Pusher) (dataloader.DataLoader, error) { gcsClient, err := gcs.NewGCSClient(ctx, f.GoogleCloudFlags.ServiceAccountCredentialFile, f.GoogleCloudFlags.OAuthClientCredentialFile, @@ -453,8 +466,8 @@ func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1. releases := f.Releases if len(releases) == 0 { // if not specified, use those defined in the Releases table - for _, config := range releaseConfigs { - releases = append(releases, config.Release) // could filter by capability if needed + for _, def := range releaseDefs { + releases = append(releases, def.Release) // could filter by capability if needed } } @@ -504,10 +517,9 @@ func parseProwLoadSince(val string) (time.Time, error) { // ensurePartitionsForReleases creates partitions for all configured releases. // It uses a 7 day lookback window plus 2 days forward from today. // Errors are logged but ignored to prevent blocking the load process. -func ensurePartitionsForReleases(dbc *db.DB, releaseConfigs []sippyv1.Release) error { - // Extract release names from release configs - releases := make([]string, 0, len(releaseConfigs)) - for _, r := range releaseConfigs { +func ensurePartitionsForReleases(dbc *db.DB, releaseDefs []models.ReleaseDefinition) error { + releases := make([]string, 0, len(releaseDefs)) + for _, r := range releaseDefs { releases = append(releases, r.Release) } diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index adc6817edc..5a76339531 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/pflag" "gopkg.in/yaml.v3" + "github.com/openshift/sippy/pkg/api" componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" @@ -411,6 +412,11 @@ func seedSyntheticData(dbc *db.DB) error { return nil } + if err := seedReleaseDefinitions(dbc); err != nil { + return errors.WithMessage(err, "failed to seed release definitions") + } + log.Info("Seeded release definitions") + if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } @@ -447,6 +453,53 @@ func seedSyntheticData(dbc *db.DB) error { return nil } +func seedReleaseDefinitions(dbc *db.DB) error { + now := time.Now().UTC() + allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} + + type relMeta struct { + previous string + gaDays int // negative = days before now; 0 = no GA (in development) + } + meta := map[string]relMeta{ + "4.19": {previous: "4.18", gaDays: -289}, + "4.20": {previous: "4.19", gaDays: -163}, + "4.21": {previous: "4.20", gaDays: -58}, + "4.22": {previous: "4.21"}, + } + + for _, release := range syntheticReleases { + m := meta[release] + parts := strings.Split(release, ".") + major, minor := 0, 0 + if len(parts) >= 2 { + fmt.Sscanf(parts[0], "%d", &major) + fmt.Sscanf(parts[1], "%d", &minor) + } + + develStart := now.AddDate(0, 0, m.gaDays-180) + def := models.ReleaseDefinition{ + Release: release, + Major: major, + Minor: minor, + PreviousRelease: m.previous, + DevelopmentStartDate: &develStart, + Product: "OCP", + Status: "Full Support", + Capabilities: allCaps, + } + if m.gaDays != 0 { + ga := now.AddDate(0, 0, m.gaDays) + def.GADate = &ga + } + + if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil { + return fmt.Errorf("failed to create release definition %s: %w", release, err) + } + } + return nil +} + func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { @@ -694,7 +747,7 @@ func syncRegressions(dbc *db.DB) error { provider := pgprovider.NewPostgresProvider(dbc, nil) ctx := context.Background() - releases, err := provider.QueryReleases(ctx) + releases, err := api.GetReleasesFromDB(ctx, dbc) if err != nil { return fmt.Errorf("querying releases: %w", err) } diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index b0340e36ba..7207cf39e3 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -31,8 +31,8 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/utils" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util/sets" ) @@ -82,7 +82,7 @@ func GetComponentReport( reqOptions reqopts.RequestOptions, baseURL string, ) (report crtype.ComponentReport, errs []error) { - releaseConfigs, err := provider.QueryReleases(ctx) + releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc) if err != nil { return report, []error{err} } @@ -151,7 +151,7 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) return nil } -func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator { +func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []models.ReleaseDefinition, baseURL string) ComponentReportGenerator { slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match generator := ComponentReportGenerator{ dataProvider: provider, @@ -175,7 +175,7 @@ type ComponentReportGenerator struct { dbc *db.DB ReqOptions reqopts.RequestOptions middlewares middleware.List - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition baseURL string } @@ -278,7 +278,7 @@ func (c *ComponentReportGenerator) initializeMiddleware() { c.middlewares = middleware.List{} // Initialize all our middleware applicable to this request. if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil { - c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs)) + c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.dbc, c.ReqOptions, c.releaseConfigs)) } if c.dbc != nil { c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions)) diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go index 85d97523fa..e9a2b73599 100644 --- a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go @@ -18,7 +18,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" apiCache "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/util/param" @@ -170,14 +169,6 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari return variants, nil } -func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions) -} - -func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { - return apiPkg.GetReleasesFromBigQuery(ctx, p.client) -} - func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { unnest := "" if nested { diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go b/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go deleted file mode 100644 index 9f6fea2e4b..0000000000 --- a/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go +++ /dev/null @@ -1,44 +0,0 @@ -package bigquery - -import ( - "context" - - "github.com/openshift/sippy/pkg/api" - "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - "github.com/openshift/sippy/pkg/apis/cache" - "github.com/openshift/sippy/pkg/bigquery" - "github.com/openshift/sippy/pkg/util" -) - -func GetReleaseDatesFromBigQuery(ctx context.Context, client *bigquery.Client, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - queries := &releaseDateQuerier{client: client, reqOptions: reqOptions} - return api.GetDataFromCacheOrGenerate[[]crtest.ReleaseTimeRange](ctx, - client.Cache, - cache.RequestOptions{}, - api.NewCacheSpec(crtest.ReleaseTimeRange{}, "CRReleaseDates~", nil), // global singleton instance - queries.QueryReleaseDates, []crtest.ReleaseTimeRange{}) -} - -type releaseDateQuerier struct { - client *bigquery.Client - reqOptions reqopts.RequestOptions -} - -func (c *releaseDateQuerier) QueryReleaseDates(ctx context.Context) ([]crtest.ReleaseTimeRange, []error) { - releases, err := api.GetReleasesFromBigQuery(ctx, c.client) - if err != nil { - return nil, []error{err} - } - timeRanges := []crtest.ReleaseTimeRange{} - for _, release := range releases { - timeRange := crtest.ReleaseTimeRange{Release: release.Release} - if release.GADate != nil { - prior := util.AdjustReleaseTime(*release.GADate, true, "30", c.reqOptions.CacheOption.CRTimeRoundingFactor, c.reqOptions.CacheOption.CRTimeRoundingOffset) - timeRange.Start = &prior - timeRange.End = release.GADate - } - timeRanges = append(timeRanges, timeRange) - } - return timeRanges, nil -} diff --git a/pkg/api/componentreadiness/dataprovider/interface.go b/pkg/api/componentreadiness/dataprovider/interface.go index 37b40f91e2..f7d84981ab 100644 --- a/pkg/api/componentreadiness/dataprovider/interface.go +++ b/pkg/api/componentreadiness/dataprovider/interface.go @@ -8,7 +8,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" ) // TestStatusQuerier fetches aggregated test pass/fail counts. @@ -40,12 +39,6 @@ type MetadataQuerier interface { // QueryJobVariants returns all variant names and their possible values. QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error) - // QueryReleaseDates returns the time ranges for each known release. - QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) - - // QueryReleases returns known release configurations. - QueryReleases(ctx context.Context) ([]v1.Release, error) - // QueryUniqueVariantValues returns distinct values for a variant column // from the past 60 days. QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 206f0dac4b..2f334a818f 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -17,7 +17,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" ) @@ -132,98 +131,6 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari return variants, nil } -// releaseMetadata holds hardcoded release info for known releases. -// This avoids needing a releases table — we derive release names from prow_jobs -// and fill in metadata from this map. -var releaseMetadata = map[string]struct { - previousRelease string - gaOffsetDays int // 0 = no GA date (in development) - product string // empty = defaults to "OCP" -}{ - "4.17": {previousRelease: "4.16", gaOffsetDays: -540}, - "4.18": {previousRelease: "4.17", gaOffsetDays: -395}, - "4.19": {previousRelease: "4.18", gaOffsetDays: -289}, - "4.20": {previousRelease: "4.19", gaOffsetDays: -163}, - "4.21": {previousRelease: "4.20", gaOffsetDays: -58}, - "4.22": {previousRelease: "4.21"}, - "5.0": {previousRelease: "4.22"}, -} - -func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { - var releaseNames []string - err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`). - Pluck("release", &releaseNames).Error - if err != nil { - return nil, fmt.Errorf("querying releases: %w", err) - } - - caps := map[v1.ReleaseCapability]bool{ - v1.ComponentReadinessCap: true, - v1.FeatureGatesCap: true, - v1.MetricsCap: true, - v1.PayloadTagsCap: true, - v1.SippyClassicCap: true, - } - - now := time.Now().UTC() - var releases []v1.Release - for _, name := range releaseNames { - rel := v1.Release{ - Release: name, - Capabilities: caps, - Product: "OCP", - } - if meta, ok := releaseMetadata[name]; ok { - rel.PreviousRelease = meta.previousRelease - if meta.gaOffsetDays != 0 { - ga := now.AddDate(0, 0, meta.gaOffsetDays) - rel.GADate = &ga - } - if meta.product != "" { - rel.Product = meta.product - } - } - releases = append(releases, rel) - } - return releases, nil -} - -func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - // Derive time ranges from actual data in the DB rather than hardcoded GA dates. - // This ensures fallback queries find data where it actually exists. - type releaseRange struct { - Release string - Start time.Time - End time.Time - } - var ranges []releaseRange - err := p.dbc.DB.WithContext(ctx).Raw(` - SELECT pj.release, - MIN(pjr.timestamp) AS start, - MAX(pjr.timestamp) AS end - FROM prow_job_runs pjr - JOIN prow_jobs pj ON pj.id = pjr.prow_job_id - WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL - GROUP BY pj.release - ORDER BY pj.release DESC - `).Scan(&ranges).Error - if err != nil { - return nil, []error{fmt.Errorf("querying release dates: %w", err)} - } - - var dates []crtest.ReleaseTimeRange - for _, r := range ranges { - start := r.Start - end := r.End - dates = append(dates, crtest.ReleaseTimeRange{ - Release: r.Release, - Start: &start, - End: &end, - }) - } - return dates, nil -} - func (p *PostgresProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { if nested { // Return all variant key names diff --git a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go index 7e79cdb435..16b3d2b95a 100644 --- a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go +++ b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go @@ -11,14 +11,14 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/regressionallowances" log "github.com/sirupsen/logrus" ) var _ middleware.Middleware = &RegressionAllowances{} -func NewRegressionAllowancesMiddleware(reqOptions reqopts.RequestOptions, releaseConfigs []v1.Release) *RegressionAllowances { +func NewRegressionAllowancesMiddleware(reqOptions reqopts.RequestOptions, releaseConfigs []models.ReleaseDefinition) *RegressionAllowances { return &RegressionAllowances{ log: log.WithField("middleware", "RegressionAllowances"), reqOptions: reqOptions, @@ -34,7 +34,7 @@ func NewRegressionAllowancesMiddleware(reqOptions reqopts.RequestOptions, releas type RegressionAllowances struct { log log.FieldLogger reqOptions reqopts.RequestOptions - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition // regressionGetterFunc allows us to unit test without relying on real regression data regressionGetterFunc func(releaseString string, variant crtest.ColumnIdentification, testID string) *regressionallowances.IntentionalRegression diff --git a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go index 19985d50c4..1c98b3da23 100644 --- a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go +++ b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go @@ -8,7 +8,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/regressionallowances" "github.com/stretchr/testify/assert" ) @@ -83,7 +83,7 @@ func Test_PreAnalysis(t *testing.T) { }, } - releaseConfigs := []v1.Release{ + releaseConfigs := []models.ReleaseDefinition{ {Release: "4.19", PreviousRelease: "4.18"}, {Release: "4.18", PreviousRelease: "4.17"}, {Release: "4.17", PreviousRelease: "4.16"}, diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go index 25dc1d83bf..0a31991342 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go @@ -15,7 +15,8 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" apiCache "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util/sets" log "github.com/sirupsen/logrus" @@ -33,11 +34,13 @@ var _ middleware.Middleware = &ReleaseFallback{} func NewReleaseFallbackMiddleware( provider dataprovider.DataProvider, + dbc *db.DB, reqOptions reqopts.RequestOptions, - releaseConfigs []v1.Release, + releaseConfigs []models.ReleaseDefinition, ) *ReleaseFallback { return &ReleaseFallback{ dataProvider: provider, + dbc: dbc, log: log.WithField("middleware", "ReleaseFallback"), reqOptions: reqOptions, releaseConfigs: releaseConfigs, @@ -55,6 +58,7 @@ func NewReleaseFallbackMiddleware( // This is done when we have sufficient test coverage, and a better pass rate. type ReleaseFallback struct { dataProvider dataprovider.DataProvider + dbc *db.DB cachedFallbackTestStatuses *FallbackReleases log log.FieldLogger reqOptions reqopts.RequestOptions @@ -64,7 +68,7 @@ type ReleaseFallback struct { // test ID, but when cache priming for a view, we may have multiple. baseOverrideStatus map[string]map[string][]crstatus.TestJobRunRows baseOverrideMutex sync.Mutex // Mutex to protect the map - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition } func (r *ReleaseFallback) Analyze(testID string, variants map[string]string, report *testdetails.TestComparison) error { @@ -173,7 +177,7 @@ func (r *ReleaseFallback) PostAnalysis(testKey crtest.Identification, testStats func (r *ReleaseFallback) getFallbackBaseQueryStatus(ctx context.Context, allJobVariants crtest.JobVariants, release string, start, end time.Time) []error { - generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs) + generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.dbc, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs) cachedFallbackTestStatuses, errs := api.GetDataFromCacheOrGenerate[*FallbackReleases]( ctx, r.dataProvider.Cache(), r.reqOptions.CacheOption, @@ -193,9 +197,9 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro r.log.Infof("Querying fallback override test statuses for %d test ID options", len(r.reqOptions.TestIDOptions)) // Lookup all release dates, we're going to need them - timeRanges, errs := r.dataProvider.QueryReleaseDates(ctx, r.reqOptions) - if errs != nil { - utils.EnqueueAsync(wg, errCh, errs...) + timeRanges, err := api.GetReleaseDatesFromDB(ctx, r.dbc, r.reqOptions) + if err != nil { + utils.EnqueueAsync(wg, errCh, err) return } @@ -296,6 +300,7 @@ func (r *ReleaseFallback) TestDetailsAnalyze(report *testdetails.Report) error { // each, which can then be used to return the best basis data from those past releases for comparison. type fallbackTestQueryReleasesGenerator struct { dataProvider dataprovider.DataProvider + dbc *db.DB cacheOption apiCache.RequestOptions allJobVariants crtest.JobVariants BaseRelease string @@ -304,19 +309,21 @@ type fallbackTestQueryReleasesGenerator struct { CachedFallbackTestStatuses FallbackReleases lock *sync.Mutex ReqOptions reqopts.RequestOptions - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition } func newFallbackTestQueryReleasesGenerator( provider dataprovider.DataProvider, + dbc *db.DB, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, release string, start, end time.Time, - releaseConfigs []v1.Release, + releaseConfigs []models.ReleaseDefinition, ) fallbackTestQueryReleasesGenerator { generator := fallbackTestQueryReleasesGenerator{ dataProvider: provider, + dbc: dbc, cacheOption: reqOptions.CacheOption, allJobVariants: allJobVariants, BaseRelease: release, @@ -335,7 +342,7 @@ type fallbackTestQueryReleasesGeneratorCacheKey struct { BaseEnd time.Time // VariantDBGroupBy is the only field within VariantOption that is used here VariantDBGroupBy sets.String - // CRTimeRoundingFactor is used by GetReleaseDatesFromBigQuery + // CRTimeRoundingFactor is used by GetReleaseDatesFromDB CRTimeRoundingFactor time.Duration CRTimeRoundingOffset time.Duration // KeyTestNames affects the BuildComponentReportQuery results via filtering logic @@ -360,10 +367,9 @@ func (f *fallbackTestQueryReleasesGenerator) getCacheKey() fallbackTestQueryRele func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context.Context) (*FallbackReleases, []error) { wg := sync.WaitGroup{} f.CachedFallbackTestStatuses = newFallbackReleases() - timeRanges, errs := f.dataProvider.QueryReleaseDates(ctx, f.ReqOptions) - - if errs != nil { - return nil, errs + timeRanges, err := api.GetReleaseDatesFromDB(ctx, f.dbc, f.ReqOptions) + if err != nil { + return nil, []error{err} } selectedTimeRanges := calculateDefaultFallbackReleases(f.BaseRelease, timeRanges, f.releaseConfigs) @@ -402,11 +408,11 @@ func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context return &f.CachedFallbackTestStatuses, nil } -func calculateDefaultFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []v1.Release) []*crtest.ReleaseTimeRange { +func calculateDefaultFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []models.ReleaseDefinition) []*crtest.ReleaseTimeRange { return calculateFallbackReleases(startingRelease, timeRanges, releaseConfigs, defaultFallbackReleases) } -func calculateFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []v1.Release, maxReleases int) []*crtest.ReleaseTimeRange { +func calculateFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []models.ReleaseDefinition, maxReleases int) []*crtest.ReleaseTimeRange { var selectedTimeRanges []*crtest.ReleaseTimeRange fallbackRelease := startingRelease diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go index 3625ebbd57..d410600fd5 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go @@ -9,7 +9,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" ) @@ -83,7 +83,7 @@ func Test_PreAnalysis(t *testing.T) { }, } - releaseConfigs := []v1.Release{ + releaseConfigs := []models.ReleaseDefinition{ {Release: "4.19", PreviousRelease: "4.18"}, {Release: "4.18", PreviousRelease: "4.17"}, {Release: "4.17", PreviousRelease: "4.16"}, @@ -164,7 +164,7 @@ func Test_PreAnalysis(t *testing.T) { } for i, test := range tests { t.Run(test.name, func(t *testing.T) { - rfb := NewReleaseFallbackMiddleware(nil, test.reqOpts, releaseConfigs) + rfb := NewReleaseFallbackMiddleware(nil, nil, test.reqOpts, releaseConfigs) rfb.cachedFallbackTestStatuses = &tests[i].fallbackReleases err := rfb.PreAnalysis(test.testKey, test.testStats) assert.NoError(t, err) @@ -207,7 +207,7 @@ func TestCalculateFallbackReleases(t *testing.T) { allTimeRanges := []crtest.ReleaseTimeRange{release419, release418, release417, release416} expectedTimeRanges := []crtest.ReleaseTimeRange{release419, release418, release417} - releaseConfigs := []v1.Release{ + releaseConfigs := []models.ReleaseDefinition{ {Release: "4.20", PreviousRelease: "4.19"}, {Release: "4.19", PreviousRelease: "4.18"}, {Release: "4.18", PreviousRelease: "4.17"}, diff --git a/pkg/api/componentreadiness/queryparamparser_test.go b/pkg/api/componentreadiness/queryparamparser_test.go index 164280f3ec..c1a39050e3 100644 --- a/pkg/api/componentreadiness/queryparamparser_test.go +++ b/pkg/api/componentreadiness/queryparamparser_test.go @@ -14,7 +14,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" "github.com/stretchr/testify/assert" @@ -31,7 +31,7 @@ var ( func TestParseComponentReportRequest(t *testing.T) { - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.16", Status: "", GADate: util.DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, {Release: "4.15", Status: "", GADate: util.DatePtr(2024, 2, 28, 0, 0, 0, 0, time.UTC)}, } @@ -490,7 +490,7 @@ func TestHATEOASLinkCacheConsistency(t *testing.T) { roundingFactor := 12 * time.Hour roundingOffset := 4 * time.Hour - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.16", Status: "", GADate: util.DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, {Release: "4.17", Status: "", GADate: util.DatePtr(2024, 12, 10, 0, 0, 0, 0, time.UTC)}, } diff --git a/pkg/api/componentreadiness/test_details.go b/pkg/api/componentreadiness/test_details.go index 16eded289a..5729641ccb 100644 --- a/pkg/api/componentreadiness/test_details.go +++ b/pkg/api/componentreadiness/test_details.go @@ -23,10 +23,10 @@ import ( "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) -func GetTestDetails(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []v1.Release, baseURL string) (testdetails.Report, []error) { +func GetTestDetails(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []models.ReleaseDefinition, baseURL string) (testdetails.Report, []error) { generator := NewComponentReportGenerator(provider, reqOptions, dbc, releases, baseURL) if os.Getenv("DEV_MODE") == "1" { return generator.GenerateTestDetailsReport(ctx) @@ -197,9 +197,9 @@ func (c *ComponentReportGenerator) GenerateDetailsReportForTest( } } - timeRanges, errs := c.dataProvider.QueryReleaseDates(ctx, c.ReqOptions) - if errs != nil { - return testdetails.Report{}, errs + timeRanges, err := api.GetReleaseDatesFromDB(ctx, c.dbc, c.ReqOptions) + if err != nil { + return testdetails.Report{}, []error{err} } now := time.Now() diff --git a/pkg/api/componentreadiness/triage.go b/pkg/api/componentreadiness/triage.go index 26c5b875e7..bfb8a2137e 100644 --- a/pkg/api/componentreadiness/triage.go +++ b/pkg/api/componentreadiness/triage.go @@ -19,7 +19,6 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/utils" "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/db/models/jobrunscan" @@ -307,7 +306,7 @@ func DeleteTriage(dbc *gorm.DB, id int) error { // ListRegressions lists all regressions for the provided view OR release. // When view is set, it is resolved to that view's sample release and filtering is by release. -func ListRegressions(dbc *db.DB, release string, views []crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) ([]models.TestRegression, error) { +func ListRegressions(dbc *db.DB, release string, views []crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) ([]models.TestRegression, error) { var regressions []models.TestRegression var err error regressions, err = query.ListRegressions(dbc, release) @@ -324,7 +323,7 @@ func ListRegressions(dbc *db.DB, release string, views []crview.View, releases [ } // GetRegression returns the regression with the matching ID -func GetRegression(dbc *db.DB, id int, views []crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) (*models.TestRegression, error) { +func GetRegression(dbc *db.DB, id int, views []crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) (*models.TestRegression, error) { regression := &models.TestRegression{} res := dbc.DB.Preload("Triages").Preload("JobRuns").Preload("Views").First(regression, id) if res.Error != nil { @@ -796,7 +795,7 @@ func injectHATEOASLinks(triage *models.Triage, baseURL string) { // InjectRegressionHATEOASLinks adds restful links clients can follow for this regression record. // Per-view test_details links use composite keys: test_details:. -func InjectRegressionHATEOASLinks(regression *models.TestRegression, views []crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseAPIURL, baseFrontendURL string) { +func InjectRegressionHATEOASLinks(regression *models.TestRegression, views []crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseAPIURL, baseFrontendURL string) { regression.Links = map[string]string{ "self": fmt.Sprintf(regressionLink, baseAPIURL, regression.ID), } @@ -830,7 +829,7 @@ func FindViewByName(name string, views []crview.View) (crview.View, bool) { // generateTestDetailsURLFromRegression extracts the required data from a regression and view // and calls the GenerateTestDetailsURL function. -func generateTestDetailsURLFromRegression(regression *models.TestRegression, view crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseURL string) (string, error) { +func generateTestDetailsURLFromRegression(regression *models.TestRegression, view crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseURL string) (string, error) { if regression == nil { return "", fmt.Errorf("regression cannot be nil") } diff --git a/pkg/api/componentreadiness/triage_test.go b/pkg/api/componentreadiness/triage_test.go index 30575c5ced..9e4f9df19b 100644 --- a/pkg/api/componentreadiness/triage_test.go +++ b/pkg/api/componentreadiness/triage_test.go @@ -9,7 +9,6 @@ import ( "github.com/lib/pq" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/sets" @@ -612,7 +611,7 @@ func TestCompareTriageObjects(t *testing.T) { func TestInjectRegressionHATEOASLinks(t *testing.T) { ga421 := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.20", GADate: &ga421}, {Release: "4.21", GADate: &ga421}, } diff --git a/pkg/api/componentreadiness/utils/queryparamparser.go b/pkg/api/componentreadiness/utils/queryparamparser.go index d63465a418..4d5105bf2d 100644 --- a/pkg/api/componentreadiness/utils/queryparamparser.go +++ b/pkg/api/componentreadiness/utils/queryparamparser.go @@ -12,7 +12,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/param" ) @@ -20,7 +20,7 @@ import ( // nolint:gocyclo func ParseComponentReportRequest( views []crview.View, - releases []v1.Release, + releases []models.ReleaseDefinition, req *http.Request, allJobVariants crtest.JobVariants, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, @@ -238,7 +238,7 @@ func getRequestedView(req *http.Request, views []crview.View) (*crview.View, err // Translate relative start/end times to actual time.Time: func GetViewReleaseOptions( - releases []v1.Release, + releases []models.ReleaseDefinition, releaseType string, viewRelease reqopts.RelativeRelease, roundingFactor, roundingOffset time.Duration, @@ -408,7 +408,7 @@ func parseAdvancedOptions(req *http.Request) (advancedOption reqopts.Advanced, e return } -func parseDateRange(allReleases []v1.Release, req *http.Request, +func parseDateRange(allReleases []models.ReleaseDefinition, req *http.Request, releaseOpts reqopts.Release, startName string, endName string, roundingFactor, roundingOffset time.Duration, diff --git a/pkg/api/componentreadiness/utils/utils.go b/pkg/api/componentreadiness/utils/utils.go index 2a5556c5c8..5f147121d8 100644 --- a/pkg/api/componentreadiness/utils/utils.go +++ b/pkg/api/componentreadiness/utils/utils.go @@ -16,10 +16,10 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) -func PreviousRelease(release string, releaseConfigs []sippyv1.Release) (string, error) { +func PreviousRelease(release string, releaseConfigs []models.ReleaseDefinition) (string, error) { for _, config := range releaseConfigs { if config.Release == release { if config.PreviousRelease != "" { diff --git a/pkg/api/componentreadiness/utils/utils_test.go b/pkg/api/componentreadiness/utils/utils_test.go index f3318a27f4..ee608cb0e4 100644 --- a/pkg/api/componentreadiness/utils/utils_test.go +++ b/pkg/api/componentreadiness/utils/utils_test.go @@ -8,7 +8,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func TestGenerateTestDetailsURL(t *testing.T) { // Define releases with GA dates for all tests ga419 := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) ga420 := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ { Release: "4.19", GADate: &ga419, diff --git a/pkg/api/job_runs.go b/pkg/api/job_runs.go index 3e1fa1bdd4..8da730af38 100644 --- a/pkg/api/job_runs.go +++ b/pkg/api/job_runs.go @@ -369,8 +369,7 @@ func JobRunRiskAnalysis( compareRelease := jobRun.ProwJob.Release neverStableJob := false if compareRelease == "Presubmits" { - // Get latest release from the DB: - ar, err := GetReleases(ctx, bqc, false) + ar, err := GetReleasesFromDB(ctx, dbc) if err != nil { return apitype.ProwJobRunRiskAnalysis{}, err } @@ -419,8 +418,7 @@ func JobRunRiskAnalysis( } if totalJobRuns < 20 { - // go back to the prior release and get more jobIds to compare against - releases, err := GetReleases(ctx, bqc, false) + releases, err := GetReleasesFromDB(ctx, dbc) if err != nil { logger.WithError(err).Error("Failed to get releases for prior release lookup") } else { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index 287f951df2..f155b42484 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -15,6 +15,8 @@ import ( "gorm.io/gorm" apitype "github.com/openshift/sippy/pkg/apis/api" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" @@ -23,6 +25,7 @@ import ( "github.com/openshift/sippy/pkg/db/query" "github.com/openshift/sippy/pkg/filter" "github.com/openshift/sippy/pkg/testidentification" + "github.com/openshift/sippy/pkg/util" ) func PrintPullRequestsReport(w http.ResponseWriter, req *http.Request, dbClient *db.DB) { @@ -445,9 +448,9 @@ func releaseFilter(req *http.Request, dbc *gorm.DB) *gorm.DB { return dbc } -// GetReleasesFromBigQuery gets all releases defined in the Releases table in BigQuery -func GetReleasesFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.Release, error) { - releases := []sippyv1.Release{} +// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table. +func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) { + var rows []sippyv1.ReleaseRow queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable) @@ -455,7 +458,7 @@ func GetReleasesFromBigQuery(ctx context.Context, client *bqcachedclient.Client) it, err := q.Read(ctx) if err != nil { log.WithError(err).Error("error querying releases data from bigquery") - return releases, err + return rows, err } for { @@ -466,40 +469,15 @@ func GetReleasesFromBigQuery(ctx context.Context, client *bqcachedclient.Client) } if err != nil { log.WithError(err).Error("error parsing release row from bigquery") - return releases, err + return rows, err } - releases = append(releases, transformRelease(r)) + rows = append(rows, r) } - return releases, nil -} - -// transformRelease converts the BQ release row to v1.Release type -func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { - release := sippyv1.Release{ - Release: r.Release, - Status: r.ReleaseStatus.String(), - PreviousRelease: r.PreviousRelease.StringVal, - Capabilities: make(map[sippyv1.ReleaseCapability]bool), - Product: r.Product.StringVal, - } - if r.GADate.Valid { - gaDate := r.GADate.Date.In(time.UTC) - release.GADate = &gaDate - } - if r.DevelStartDate.IsValid() { - develStartDate := r.DevelStartDate.In(time.UTC) - release.DevelopmentStartDate = &develStartDate - } - if r.Capabilities != nil { - for _, capability := range r.Capabilities { - release.Capabilities[capability] = true - } - } - return release + return rows, nil } // BuildReleasesResponse creates the API response structure for releases -func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { +func BuildReleasesResponse(releases []models.ReleaseDefinition, lastUpdated time.Time) apitype.Releases { gaDateMap := make(map[string]time.Time) dateMap := make(map[string]apitype.ReleaseDates) response := apitype.Releases{ @@ -521,11 +499,15 @@ func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) ap releaseDate.DevelopmentStart = release.DevelopmentStartDate response.Dates[release.Release] = releaseDate } + caps := make(map[sippyv1.ReleaseCapability]bool, len(release.Capabilities)) + for _, cap := range release.Capabilities { + caps[sippyv1.ReleaseCapability(cap)] = true + } response.ReleaseAttrs[release.Release] = apitype.Release{ Name: release.Release, PreviousRelease: release.PreviousRelease, ReleaseDates: releaseDate, - Capabilities: release.Capabilities, + Capabilities: caps, Product: release.Product, } } @@ -533,6 +515,35 @@ func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) ap return response } +// GetReleaseDatesFromDB derives CR time ranges from release_definitions GA dates. +func GetReleaseDatesFromDB(ctx context.Context, dbc *db.DB, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, error) { + defs, err := GetReleasesFromDB(ctx, dbc) + if err != nil { + return nil, err + } + var timeRanges []crtest.ReleaseTimeRange + for _, def := range defs { + tr := crtest.ReleaseTimeRange{Release: def.Release} + if def.GADate != nil { + prior := util.AdjustReleaseTime(*def.GADate, true, "30", reqOptions.CacheOption.CRTimeRoundingFactor, reqOptions.CacheOption.CRTimeRoundingOffset) + tr.Start = &prior + tr.End = def.GADate + } + timeRanges = append(timeRanges, tr) + } + return timeRanges, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table. +func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]models.ReleaseDefinition, error) { + var defs []models.ReleaseDefinition + err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error + if err != nil { + return nil, fmt.Errorf("querying release definitions: %w", err) + } + return defs, nil +} + // PayloadForJobRun returns the payload release tag that was used for a given job run. func PayloadForJobRun(ctx context.Context, bigQueryClient *bqcachedclient.Client, jobRunID string) ([]apitype.JobPayload, error) { // Calculate date range: 6 months ago through today diff --git a/pkg/api/releases_test.go b/pkg/api/releases_test.go index 62967569cb..a9063535f4 100644 --- a/pkg/api/releases_test.go +++ b/pkg/api/releases_test.go @@ -1,73 +1,10 @@ package api import ( - "testing" - "time" - - "cloud.google.com/go/bigquery" - "cloud.google.com/go/civil" - - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - - "github.com/stretchr/testify/assert" - apitype "github.com/openshift/sippy/pkg/apis/api" "github.com/openshift/sippy/pkg/db/models" ) -func TestTransformRelease(t *testing.T) { - - devStart420, _ := time.Parse(time.RFC3339, "2025-04-18T00:00:00.00Z") - devStart419, _ := time.Parse(time.RFC3339, "2024-11-25T00:00:00.00Z") - gaDate419, _ := time.Parse(time.RFC3339, "2025-05-09T00:00:00.00Z") - - tests := []struct { - name string - releaseRow sippyv1.ReleaseRow - expectedRelease sippyv1.Release - }{ - { - name: "release without devel start", - releaseRow: sippyv1.ReleaseRow{Release: "4.20", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}}, - expectedRelease: sippyv1.Release{Release: "4.20", Status: "Development"}, - }, - { - name: "release with devel start", - releaseRow: sippyv1.ReleaseRow{Release: "4.20", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, DevelStartDate: civil.Date{ - Year: 2025, - Month: 4, - Day: 18, - }}, - expectedRelease: sippyv1.Release{Release: "4.20", Status: "Development", DevelopmentStartDate: &devStart420}, - }, - { - name: "release with ga date", - releaseRow: sippyv1.ReleaseRow{Release: "4.19", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, DevelStartDate: civil.Date{ - Year: 2024, - Month: 11, - Day: 25, - }, GADate: bigquery.NullDate{ - Date: civil.Date{ - Year: 2025, - Month: 5, - Day: 9}, - Valid: true, - }}, - expectedRelease: sippyv1.Release{Release: "4.19", Status: "Development", DevelopmentStartDate: &devStart419, GADate: &gaDate419}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - release := transformRelease(tc.releaseRow) - assert.Equal(t, tc.expectedRelease.Release, release.Release, "unexpected release") - assert.Equal(t, tc.expectedRelease.Status, release.Status, "unexpected status") - assert.Equal(t, tc.expectedRelease.GADate, release.GADate, "unexpected status") - assert.Equal(t, tc.expectedRelease.DevelopmentStartDate, release.DevelopmentStartDate, "unexpected devel start") - }) - } -} - func buildFakeReleaseHealthReport(osVersion string) apitype.ReleaseHealthReport { return apitype.ReleaseHealthReport{ ReleaseTag: models.ReleaseTag{ diff --git a/pkg/api/utils.go b/pkg/api/utils.go index e65df3d923..fbeabe2677 100644 --- a/pkg/api/utils.go +++ b/pkg/api/utils.go @@ -1,52 +1,16 @@ package api import ( - "context" "fmt" "net/http" "net/url" "strings" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - log "github.com/sirupsen/logrus" - "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - bqclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/util/sets" ) -type releaseGenerator struct { - client *bqclient.Client -} - -func (r *releaseGenerator) ListReleases(ctx context.Context) ([]v1.Release, []error) { - releases, err := GetReleasesFromBigQuery(ctx, r.client) - if err != nil { - log.WithError(err).Error("error getting releases from bigquery") - return releases, []error{err} - } - return releases, nil -} - -// GetReleases gets all the releases defined in the BQ Releases table. -func GetReleases(ctx context.Context, bqc *bqclient.Client, forceRefresh bool) ([]v1.Release, error) { - releaseGen := releaseGenerator{bqc} - - var err error - rels, errs := GetDataFromCacheOrGenerate[[]v1.Release]( - ctx, - bqc.Cache, - cache.RequestOptions{ForceRefresh: forceRefresh}, - NewCacheSpec(v1.Release{}, "Releases~", nil), // no cache options needed here, global list - releaseGen.ListReleases, - []v1.Release{}) - if len(errs) > 0 { - err = errs[0] - } - return rels, err -} - // VariantsStringToSet converts comma separated variant string into a set; also validates that the variants are known func VariantsStringToSet(allJobVariants crtest.JobVariants, variantsString string) (sets.String, error) { variantSet := sets.String{} diff --git a/pkg/componentreadiness/jiraautomator/jiraautomator.go b/pkg/componentreadiness/jiraautomator/jiraautomator.go index 369d93dcb3..5e76dbefeb 100644 --- a/pkg/componentreadiness/jiraautomator/jiraautomator.go +++ b/pkg/componentreadiness/jiraautomator/jiraautomator.go @@ -24,6 +24,7 @@ import ( bqclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" log "github.com/sirupsen/logrus" @@ -56,7 +57,7 @@ type JiraAutomator struct { dbc *db.DB cacheOptions cache.RequestOptions views []crview.View - releases []v1.Release + releases []models.ReleaseDefinition sippyURL string // columnThresholds defines a threshold for the number of red cells in a column. // When the number of red cells of a column is over this threshold, a jira card will be created for the @@ -75,7 +76,7 @@ func NewJiraAutomator( dbc *db.DB, cacheOptions cache.RequestOptions, views []crview.View, - releases []v1.Release, + releases []models.ReleaseDefinition, sippyURL, jiraAccount string, includeComponents sets.String, columnThresholds map[Variant]int, diff --git a/pkg/dataloader/crcacheloader/crcacheloader.go b/pkg/dataloader/crcacheloader/crcacheloader.go index bf0902347d..aed0810b42 100644 --- a/pkg/dataloader/crcacheloader/crcacheloader.go +++ b/pkg/dataloader/crcacheloader/crcacheloader.go @@ -18,9 +18,9 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/config/v1" - apiv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" log "github.com/sirupsen/logrus" ) @@ -33,7 +33,7 @@ type ComponentReadinessCacheLoader struct { dbc *db.DB errs []error views *sippytypes.SippyViews - releases []apiv1.Release + releases []models.ReleaseDefinition cacheClient cache.Cache bqClient *bigquery.Client dataProvider dataprovider.DataProvider @@ -48,7 +48,7 @@ func New( bqClient *bigquery.Client, config *v1.SippyConfig, views *sippytypes.SippyViews, - releases []apiv1.Release, + releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration) *ComponentReadinessCacheLoader { return &ComponentReadinessCacheLoader{ diff --git a/pkg/dataloader/featuregateloader/featuregateloader.go b/pkg/dataloader/featuregateloader/featuregateloader.go index ab2ef4cc8f..0edb4ecdf4 100644 --- a/pkg/dataloader/featuregateloader/featuregateloader.go +++ b/pkg/dataloader/featuregateloader/featuregateloader.go @@ -10,7 +10,6 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" @@ -21,15 +20,15 @@ import ( ) type FeatureGateLoader struct { - dbc *db.DB - errs []error - releaseConfigs []v1.Release + dbc *db.DB + errs []error + releaseDefs []models.ReleaseDefinition } -func New(dbc *db.DB, configs []v1.Release) *FeatureGateLoader { +func New(dbc *db.DB, releaseDefs []models.ReleaseDefinition) *FeatureGateLoader { return &FeatureGateLoader{ - dbc: dbc, - releaseConfigs: configs, + dbc: dbc, + releaseDefs: releaseDefs, } } @@ -68,9 +67,9 @@ func (l *FeatureGateLoader) Errors() []error { func (l *FeatureGateLoader) getTargetReleases() []string { var targetReleases []string - for _, release := range l.releaseConfigs { - if release.Capabilities[v1.FeatureGatesCap] { - targetReleases = append(targetReleases, release.Release) + for _, def := range l.releaseDefs { + if def.HasCapability(models.CapFeatureGates) { + targetReleases = append(targetReleases, def.Release) } } diff --git a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go index fc2d5424c5..485ab53664 100644 --- a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go +++ b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go @@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics { return loader } -var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} +var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} // sortLoaders guarantees that the loaders run in a predictable and proper order func (l *LoaderWithMetrics) sortLoaders() { diff --git a/pkg/dataloader/regressioncacheloader/regressioncacheloader.go b/pkg/dataloader/regressioncacheloader/regressioncacheloader.go index 43e5f56f52..95d82d79cb 100644 --- a/pkg/dataloader/regressioncacheloader/regressioncacheloader.go +++ b/pkg/dataloader/regressioncacheloader/regressioncacheloader.go @@ -19,7 +19,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" "github.com/openshift/sippy/pkg/apis/cache" configv1 "github.com/openshift/sippy/pkg/apis/config/v1" - apiv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" @@ -47,7 +46,7 @@ type RegressionCacheLoader struct { // Cache priming deps bqClient *bigquery.Client config *configv1.SippyConfig - releases []apiv1.Release + releases []models.ReleaseDefinition crTimeRoundingFactor time.Duration crTimeRoundingOffset time.Duration @@ -60,7 +59,7 @@ func New( bqClient *bigquery.Client, config *configv1.SippyConfig, views []crview.View, - releases []apiv1.Release, + releases []models.ReleaseDefinition, crTimeRoundingFactor time.Duration, crTimeRoundingOffset time.Duration, regressionStore componentreadiness.RegressionStore, diff --git a/pkg/dataloader/releasedefloader/releasedefloader.go b/pkg/dataloader/releasedefloader/releasedefloader.go new file mode 100644 index 0000000000..cc589d4ade --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader.go @@ -0,0 +1,103 @@ +package releasedefloader + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" + + "github.com/openshift/sippy/pkg/api" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL. +type ReleaseDefinitionLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + errs []error +} + +func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader { + return &ReleaseDefinitionLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + } +} + +func (l *ReleaseDefinitionLoader) Name() string { + return "release-definitions" +} + +func (l *ReleaseDefinitionLoader) Load() { + releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient) + if err != nil { + l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err)) + return + } + defs := make([]models.ReleaseDefinition, 0, len(releaseRows)) + for _, row := range releaseRows { + defs = append(defs, ReleaseRowToDefinition(row)) + } + if err := syncReleaseDefinitions(l.dbc, defs); err != nil { + l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err)) + } +} + +func (l *ReleaseDefinitionLoader) Errors() []error { + return l.errs +} + +func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error { + for _, def := range defs { + err := dbc.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "release"}}, + DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}), + }).Create(&def).Error + if err != nil { + return fmt.Errorf("upserting release definition %s: %w", def.Release, err) + } + } + log.Infof("Synced %d release definitions to postgres", len(defs)) + return nil +} + +// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model. +func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition { + caps := make(pq.StringArray, 0, len(r.Capabilities)) + for _, cap := range r.Capabilities { + caps = append(caps, string(cap)) + } + sort.Strings(caps) + + def := models.ReleaseDefinition{ + Release: r.Release, + Major: r.Major, + Minor: r.Minor, + PreviousRelease: r.PreviousRelease.StringVal, + Product: r.Product.StringVal, + Status: r.ReleaseStatus.StringVal, + Capabilities: caps, + } + if r.Patch.Valid { + p := int(r.Patch.Int64) + def.Patch = &p + } + if r.GADate.Valid { + ga := r.GADate.Date.In(time.UTC) + def.GADate = &ga + } + if r.DevelStartDate.IsValid() { + ds := r.DevelStartDate.In(time.UTC) + def.DevelopmentStartDate = &ds + } + return def +} diff --git a/pkg/dataloader/releaseloader/projects.go b/pkg/dataloader/releaseloader/projects.go index 4f794719e2..7077b1c51d 100644 --- a/pkg/dataloader/releaseloader/projects.go +++ b/pkg/dataloader/releaseloader/projects.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) func (ocp *OCPProject) GetName() string { @@ -19,7 +19,7 @@ func (ocp *OCPProject) GetRcDomain(architecture string) (domain string) { return architecture + ".ocp.releases.ci.openshift.org" } -func (ocp *OCPProject) IsProjectRelease(release v1.Release) bool { +func (ocp *OCPProject) IsProjectRelease(release models.ReleaseDefinition) bool { return release.Product == "OCP" } @@ -46,7 +46,7 @@ func (okd *OKDProject) GetRcDomain(architecture string) (domain string) { return architecture + ".origin.releases.ci.openshift.org" } -func (okd *OKDProject) IsProjectRelease(release v1.Release) bool { +func (okd *OKDProject) IsProjectRelease(release models.ReleaseDefinition) bool { return release.Product == "OKD" } diff --git a/pkg/dataloader/releaseloader/releasesync.go b/pkg/dataloader/releaseloader/releasesync.go index 82c1cb8202..e3be121996 100644 --- a/pkg/dataloader/releaseloader/releasesync.go +++ b/pkg/dataloader/releaseloader/releasesync.go @@ -12,7 +12,6 @@ import ( "strings" "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -39,21 +38,21 @@ type ReleaseLoader struct { db *db.DB bqClient *bqcachedclient.Client httpClient *http.Client - releases map[string]v1.Release + releases map[string]models.ReleaseDefinition architectures []string projects []PayloadProject errors []error } -func New(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client, releases, architectures []string, releaseConfigs []v1.Release) *ReleaseLoader { - configForRelease := make(map[string]v1.Release, len(releaseConfigs)) +func New(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client, releases, architectures []string, releaseConfigs []models.ReleaseDefinition) *ReleaseLoader { + configForRelease := make(map[string]models.ReleaseDefinition, len(releaseConfigs)) for _, config := range releaseConfigs { - if config.Capabilities[v1.PayloadTagsCap] { + if config.HasCapability(models.CapPayloadTags) { configForRelease[config.Release] = config } } if len(releases) > 0 { - filteredRCs := make(map[string]v1.Release, len(releases)) + filteredRCs := make(map[string]models.ReleaseDefinition, len(releases)) for _, release := range releases { if config, ok := configForRelease[release]; ok { filteredRCs[release] = config @@ -514,7 +513,7 @@ func (rs *ReleaseStream) baseReleaseStreamURL() string { } // buildReleaseStreams builds relevant release streams for specified releases that belong to the project. -func buildReleaseStreams(releases map[string]v1.Release, architectures []string, project PayloadProject) []ReleaseStream { +func buildReleaseStreams(releases map[string]models.ReleaseDefinition, architectures []string, project PayloadProject) []ReleaseStream { releaseStreams := make([]ReleaseStream, 0, len(releases)*len(project.GetStreams())) for release, config := range releases { if project.IsProjectRelease(config) { diff --git a/pkg/dataloader/releaseloader/types.go b/pkg/dataloader/releaseloader/types.go index c0936cec21..3f0831519e 100644 --- a/pkg/dataloader/releaseloader/types.go +++ b/pkg/dataloader/releaseloader/types.go @@ -3,12 +3,12 @@ package releaseloader import ( "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) type ReleaseStream struct { Name string - Release v1.Release + Release models.ReleaseDefinition Stream string Architecture string Domain string @@ -116,7 +116,7 @@ type PayloadProject interface { GetRcDomain(architecture string) (domain string) // IsProjectRelease returns true if the release (from Releases table) belongs to this project - IsProjectRelease(release v1.Release) bool + IsProjectRelease(release models.ReleaseDefinition) bool // FullReleaseStream builds a full releaseStream name to look for on the release-controller, or empty string if n/a FullReleaseStream(release, stream, architecture string) string diff --git a/pkg/db/db.go b/pkg/db/db.go index 73de1a1ff8..752686eff8 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { // List of all models to migrate modelsToMigrate := []any{ + &models.ReleaseDefinition{}, &models.ReleaseTag{}, &models.ReleasePullRequest{}, &models.ReleaseRepository{}, diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index 01c578e628..0bb91ae28d 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -6,6 +6,41 @@ import ( "github.com/lib/pq" ) +// ReleaseDefinition stores release metadata synced from BigQuery's Releases table. +// This is distinct from ReleaseTag, which tracks individual payload tags. +type ReleaseDefinition struct { + Model + + Release string `json:"release" gorm:"uniqueIndex;column:release"` + Major int `json:"major" gorm:"column:major"` + Minor int `json:"minor" gorm:"column:minor"` + Patch *int `json:"patch,omitempty" gorm:"column:patch"` + PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` + GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date"` + DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date"` + Product string `json:"product" gorm:"column:product"` + Status string `json:"status" gorm:"column:status"` + Capabilities pq.StringArray `json:"capabilities" gorm:"type:text[];column:capabilities"` +} + +const ( + CapComponentReadiness = "componentReadiness" + CapSippyClassic = "sippyClassic" + CapMetrics = "metrics" + CapPullRequests = "pullRequests" + CapFeatureGates = "featureGates" + CapPayloadTags = "payloadTags" +) + +func (rd *ReleaseDefinition) HasCapability(cap string) bool { + for _, c := range rd.Capabilities { + if c == cap { + return true + } + } + return false +} + type ReleaseTag struct { Model diff --git a/pkg/mcp/tools/releases.go b/pkg/mcp/tools/releases.go index 7ef6e6834d..be65331ff2 100644 --- a/pkg/mcp/tools/releases.go +++ b/pkg/mcp/tools/releases.go @@ -40,30 +40,27 @@ func (rt *ReleasesTool) GetHandler() func(ctx context.Context, request mcp.CallT return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { log.Debug("Handling get_releases tool call") - // Get releases from BigQuery (never force refresh for MCP) - releases, err := api.GetReleases(ctx, rt.deps.BigQueryClient, false) + if rt.deps.DBClient == nil { + return rt.CreateErrorResponse(fmt.Errorf("no database available for releases")) + } + + releases, err := api.GetReleasesFromDB(ctx, rt.deps.DBClient) if err != nil { log.WithError(err).Error("error querying releases") return rt.CreateErrorResponse(fmt.Errorf("error querying releases: %w", err)) } - // Get last updated time from database if available var lastUpdated time.Time - if rt.deps.DBClient != nil && rt.deps.DBClient.DB != nil { - type LastUpdatedQuery struct { - Max time.Time - } - var result LastUpdatedQuery - // Assume our last update is the last time we inserted a prow job run. - if err := rt.deps.DBClient.DB.Raw("SELECT MAX(created_at) FROM prow_job_runs").Scan(&result).Error; err == nil { - lastUpdated = result.Max - } + type LastUpdatedQuery struct { + Max time.Time + } + var result LastUpdatedQuery + // Assume our last update is the last time we inserted a prow job run. + if err := rt.deps.DBClient.DB.Raw("SELECT MAX(created_at) FROM prow_job_runs").Scan(&result).Error; err == nil { + lastUpdated = result.Max } - // Build response using shared function response := api.BuildReleasesResponse(releases, lastUpdated) - - // Return JSON response return rt.CreateJSONResponse(response) } } diff --git a/pkg/sippyserver/metrics/metrics.go b/pkg/sippyserver/metrics/metrics.go index e6563d00e0..614ba3401d 100644 --- a/pkg/sippyserver/metrics/metrics.go +++ b/pkg/sippyserver/metrics/metrics.go @@ -17,8 +17,8 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" @@ -102,7 +102,7 @@ var ( }, []string{"release", "compare_release", "platform", "backend", "upgrade_type", "master_nodes_updated", "network", "topology", "architecture", "feature_set", "os", "releaseStatus"}) ) -func getReleaseStatus(releases []v1.Release, release string) string { +func getReleaseStatus(releases []models.ReleaseDefinition, release string) string { releaseStatus := releaseStatusEOL for _, r := range releases { if r.Release == release && len(r.Status) != 0 { @@ -119,10 +119,10 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP start := time.Now() log.Info("beginning refresh metrics") - var releases []v1.Release - if crProvider != nil { + var releases []models.ReleaseDefinition + if dbc != nil { var err error - releases, err = crProvider.QueryReleases(ctx) + releases, err = api.GetReleasesFromDB(ctx, dbc) if err != nil { return err } @@ -186,7 +186,7 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP } func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, - cacheOptions cache.RequestOptions, views []crview.View, releases []v1.Release) { + cacheOptions cache.RequestOptions, views []crview.View, releases []models.ReleaseDefinition) { for _, view := range views { if view.Metrics.Enabled { err := updateComponentReadinessMetricsForView(ctx, provider, dbc, cacheOptions, view, releases) @@ -200,7 +200,7 @@ func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider } // updateComponentReadinessTrackingForView queries the report for the given view, and then updates metrics. -func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []v1.Release) error { +func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []models.ReleaseDefinition) error { logger := log.WithField("view", view.Name) logger.Info("generating report for view") @@ -291,9 +291,9 @@ func refreshBuildClusterMetrics(dbc *db.DB, reportEnd time.Time) error { return nil } -func refreshPayloadMetrics(dbc *db.DB, reportEnd time.Time, releases []v1.Release) { +func refreshPayloadMetrics(dbc *db.DB, reportEnd time.Time, releases []models.ReleaseDefinition) { for _, r := range releases { - if !r.Capabilities[v1.MetricsCap] { + if !r.HasCapability(models.CapMetrics) { continue } results, err := api.ReleaseHealthReports(dbc, r.Release, reportEnd) @@ -339,7 +339,7 @@ func refreshPayloadMetrics(dbc *db.DB, reportEnd time.Time, releases []v1.Releas // refreshDisruptionMetrics queries our BigQuery views for current release vs two weeks ago, and previous release GA. // Metrics are published for the delta for each NURP which can then be alerted on if certain thresholds are exceeded. // The previous GA view should have its release and GA date updated on each release GA. -func refreshDisruptionMetrics(client *bqclient.Client, releases []v1.Release) error { +func refreshDisruptionMetrics(client *bqclient.Client, releases []models.ReleaseDefinition) error { if client == nil || client.BQ == nil { log.Warningf("not generating disruption metrics as we don't have a bigquery client") return nil @@ -382,11 +382,11 @@ type promReportType struct { period string } -func buildPromReportTypes(releases []v1.Release) []promReportType { +func buildPromReportTypes(releases []models.ReleaseDefinition) []promReportType { var promReportTypes []promReportType for _, release := range releases { - if !release.Capabilities[v1.MetricsCap] { + if !release.HasCapability(models.CapMetrics) { continue } promReportTypes = append(promReportTypes, promReportType{release: release.Release, period: string(sippyprocessingv1.TwoDayReport)}) diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 4899ea25a4..9115fe6a58 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -51,7 +51,6 @@ import ( "github.com/openshift/sippy/pkg/api/jobrunintervals" apitype "github.com/openshift/sippy/pkg/apis/api" "github.com/openshift/sippy/pkg/apis/cache" - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" sippybq "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/dailysummary" @@ -189,17 +188,12 @@ type Server struct { rateLimiters map[string]*rateLimiter } -// getReleases returns release data, preferring the BigQuery client with caching -// when available, falling back to the data provider for mock mode. -func (s *Server) getReleases(ctx context.Context, forceRefresh ...bool) ([]sippyv1.Release, error) { - if s.bigQueryClient != nil { - refresh := len(forceRefresh) > 0 && forceRefresh[0] - return api.GetReleases(ctx, s.bigQueryClient, refresh) +// getReleases returns release data from PostgreSQL. +func (s *Server) getReleases(ctx context.Context) ([]models.ReleaseDefinition, error) { + if s.db == nil { + return nil, fmt.Errorf("no database available for releases") } - if s.crDataProvider != nil { - return s.crDataProvider.QueryReleases(ctx) - } - return nil, fmt.Errorf("no data source available for releases") + return api.GetReleasesFromDB(ctx, s.db) } type rateLimiter struct { @@ -907,8 +901,8 @@ func (s *Server) jsonTestRunsAndOutputsFromBigQuery(w http.ResponseWriter, req * outputs, err := api.GetTestRunsAndOutputsFromBigQuery(req.Context(), s.bigQueryClient, testID, prowJobRunIDList, prowJobNames, includeSuccess, startDate, endDate) if err != nil { - log.WithError(err).Error("error querying test runs from bigquery") - failureResponse(w, http.StatusInternalServerError, "error querying test runs from bigquery") + log.WithError(err).Error("error querying test runs from database") + failureResponse(w, http.StatusInternalServerError, "error querying test runs from database") return } @@ -922,11 +916,11 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re } outputs, errs := componentreadiness.GetComponentTestVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying test variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying test variants from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -939,11 +933,11 @@ func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Re } outputs, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying job variants from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1034,7 +1028,7 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo baseURL, ) if len(errs) > 0 { - return componentreport.ComponentReport{}, fmt.Errorf("error querying component from big query: %v", errs) + return componentreport.ComponentReport{}, fmt.Errorf("error querying component from database: %v", errs) } // Add any warnings from parsing to the report @@ -1080,11 +1074,11 @@ func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWrite baseURL := api.GetBaseURL(req) outputs, errs := componentreadiness.GetTestDetails(req.Context(), s.crDataProvider, s.db, reqOptions, allReleases, baseURL) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying component test details from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying component test details from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1153,8 +1147,7 @@ func (s *Server) jsonTestDetailsReportFromDB(w http.ResponseWriter, req *http.Re } func (s *Server) jsonReleasesReportFromDB(w http.ResponseWriter, req *http.Request) { - forceRefresh := req.URL.Query().Get("forceRefresh") != "" - releases, err := s.getReleases(req.Context(), forceRefresh) + releases, err := s.getReleases(req.Context()) if err != nil { log.WithError(err).Error("error querying releases") failureResponse(w, http.StatusInternalServerError, "error querying releases") diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 92fe0ad58e..d5a4c449d8 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) // TruncateAligned rounds t down to the nearest multiple of factor, aligned to a @@ -125,7 +125,7 @@ var releaseRelativeRE = regexp.MustCompile(`^(now|ga|end)(?:-([0-9]+)([d]))?$`) // For isStart=false we would round up to 23:59:59. // // endTime must be specified if your timeStr uses the end directive. (end-90d) Otherwise it is not required or used. -func ParseCRReleaseTime(allReleases []v1.Release, release, timeStr string, isStart bool, endTime *time.Time, crTimeRoundingFactor, crTimeRoundingOffset time.Duration) (time.Time, error) { +func ParseCRReleaseTime(allReleases []models.ReleaseDefinition, release, timeStr string, isStart bool, endTime *time.Time, crTimeRoundingFactor, crTimeRoundingOffset time.Duration) (time.Time, error) { var relTime time.Time diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 4f1110c823..ccffa66b90 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -4,13 +4,13 @@ import ( "testing" "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestParseCRReleaseTime(t *testing.T) { - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.16", Status: "", GADate: DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, } diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/componentreadiness_test.go index 770744a1cf..2ecb477400 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/componentreadiness_test.go @@ -62,9 +62,9 @@ func TestRegressionCacheLoader(t *testing.T) { require.NoError(t, err, "error parsing seed views") require.Greater(t, len(sippyViews.ComponentReadiness), 0, "no views found in seed-views.yaml") - // Get release configs from BigQuery - releaseConfigs, err := api.GetReleasesFromBigQuery(ctx, bqClient) - require.NoError(t, err, "error getting releases from bigquery") + // Get release configs from the database + releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc) + require.NoError(t, err, "error getting releases from database") // Build a regression store regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, nil)