diff --git a/cmd/sippy/automatejira.go b/cmd/sippy/automatejira.go index ed9c930375..d2bdc4e31c 100644 --- a/cmd/sippy/automatejira.go +++ b/cmd/sippy/automatejira.go @@ -152,7 +152,12 @@ func NewAutomateJiraCommand() *cobra.Command { if err != nil { log.WithError(err).Fatal("unable to load views") } - releases, err := api.GetReleases(context.Background(), bigQueryClient, false) + + dbc, err := f.PostgresFlags.GetDBClient() + if err != nil { + log.WithError(err).Fatal("unable to connect to postgres") + } + releases, err := api.GetReleasesFromDB(ctx, dbc) if err != nil { log.WithError(err).Fatal("error querying releases") } @@ -178,10 +183,6 @@ func NewAutomateJiraCommand() *cobra.Command { return errors.WithMessage(err, "error validating options") } - dbc, err := f.PostgresFlags.GetDBClient() - if err != nil { - log.WithError(err).Fatal("unable to connect to postgres") - } j, err := jiraautomator.NewJiraAutomator( jiraClient, bigQueryClient, provider, dbc, cacheOpts, views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount, diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 8a75b65223..a316cba404 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -11,7 +11,6 @@ import ( "cloud.google.com/go/bigquery" "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/dataloader/regressioncacheloader" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -37,10 +36,12 @@ import ( "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/dataloader/prowloader/github" + releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader" "github.com/openshift/sippy/pkg/dataloader/releaseloader" "github.com/openshift/sippy/pkg/dataloader/testownershiploader" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/dailysummary" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/flags" "github.com/openshift/sippy/pkg/github/commenter" ) @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { f.JiraFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading") - fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") + fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)") fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)") fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader") @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command { cacheClient = nil // error hygiene, since we pass this down to quite a few functions } - releaseConfigs := []sippyv1.Release{} - // initializing a bigquery client different from the normal one opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load") bqc, bigqueryErr := bqcachedclient.New( @@ -164,20 +163,23 @@ func NewLoadCommand() *cobra.Command { if f.CacheFlags.EnablePersistentCaching { bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc) } - releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc) - if err != nil { - return errors.Wrapf(err, "error querying releases from bq") - } + } + + // Read release definitions from PG for downstream loader construction. + // On the first run the table may be empty; the release-definitions + // loader will populate it for subsequent runs. + var releaseDefs []models.ReleaseDefinition + if dbErr == nil { + releaseDefs, _ = api.GetReleasesFromDB(context.Background(), dbc) } // Ensure partitions exist for all releases (only when InitDatabase is true) if f.InitDatabase && dbErr == nil { - err = ensurePartitionsForReleases(dbc, releaseConfigs) + err = ensurePartitionsForReleases(dbc, releaseDefs) if err != nil { return errors.Wrapf(err, "error ensuring partitions") } - // Clean up old partitions detached, dropped, err := dbc.CleanupPartitions(false) if err != nil { log.WithError(err).Warning("failed to cleanup old partitions, continuing with load") @@ -201,6 +203,17 @@ func NewLoadCommand() *cobra.Command { var regressionCacheAdded bool for _, l := range f.Loaders { + if l == "release-definitions" { + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading") + } + rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc) + loaders = append(loaders, rdl) + } + // TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob // manifests are updated to use "regression-cache". if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" { @@ -237,7 +250,7 @@ func NewLoadCommand() *cobra.Command { regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, jiraClient) rcl, err := regressioncacheloader.New( - dbc, bqc, config, views.ComponentReadiness, releaseConfigs, + dbc, bqc, config, views.ComponentReadiness, releaseDefs, f.ComponentReadinessFlags.CRTimeRoundingFactor, f.ComponentReadinessFlags.CRTimeRoundingOffset, regressionStore, @@ -252,7 +265,7 @@ func NewLoadCommand() *cobra.Command { if dbErr != nil { return dbErr } - loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseConfigs)) + loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseDefs)) } // Prow Loader @@ -261,7 +274,7 @@ func NewLoadCommand() *cobra.Command { if dbErr != nil { return dbErr } - prowLoader, err := f.prowLoader(ctx, dbc, config, releaseConfigs, promPusher) + prowLoader, err := f.prowLoader(ctx, dbc, config, releaseDefs, promPusher) if err != nil { return err } @@ -331,7 +344,7 @@ func NewLoadCommand() *cobra.Command { // Feature gates if l == "feature-gates" { refreshMatviews = true - fgLoader := featuregateloader.New(dbc, releaseConfigs) + fgLoader := featuregateloader.New(dbc, releaseDefs) loaders = append(loaders, fgLoader) } @@ -418,7 +431,7 @@ func (f *LoadFlags) jobVariantsLoader(ctx context.Context) (dataloader.DataLoade } -func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseConfigs []sippyv1.Release, promPusher *push.Pusher) (dataloader.DataLoader, error) { +func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseDefs []models.ReleaseDefinition, promPusher *push.Pusher) (dataloader.DataLoader, error) { gcsClient, err := gcs.NewGCSClient(ctx, f.GoogleCloudFlags.ServiceAccountCredentialFile, f.GoogleCloudFlags.OAuthClientCredentialFile, @@ -453,8 +466,8 @@ func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1. releases := f.Releases if len(releases) == 0 { // if not specified, use those defined in the Releases table - for _, config := range releaseConfigs { - releases = append(releases, config.Release) // could filter by capability if needed + for _, def := range releaseDefs { + releases = append(releases, def.Release) // could filter by capability if needed } } @@ -504,10 +517,9 @@ func parseProwLoadSince(val string) (time.Time, error) { // ensurePartitionsForReleases creates partitions for all configured releases. // It uses a 7 day lookback window plus 2 days forward from today. // Errors are logged but ignored to prevent blocking the load process. -func ensurePartitionsForReleases(dbc *db.DB, releaseConfigs []sippyv1.Release) error { - // Extract release names from release configs - releases := make([]string, 0, len(releaseConfigs)) - for _, r := range releaseConfigs { +func ensurePartitionsForReleases(dbc *db.DB, releaseDefs []models.ReleaseDefinition) error { + releases := make([]string, 0, len(releaseDefs)) + for _, r := range releaseDefs { releases = append(releases, r.Release) } diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index adc6817edc..5a76339531 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/pflag" "gopkg.in/yaml.v3" + "github.com/openshift/sippy/pkg/api" componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" @@ -411,6 +412,11 @@ func seedSyntheticData(dbc *db.DB) error { return nil } + if err := seedReleaseDefinitions(dbc); err != nil { + return errors.WithMessage(err, "failed to seed release definitions") + } + log.Info("Seeded release definitions") + if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } @@ -447,6 +453,53 @@ func seedSyntheticData(dbc *db.DB) error { return nil } +func seedReleaseDefinitions(dbc *db.DB) error { + now := time.Now().UTC() + allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} + + type relMeta struct { + previous string + gaDays int // negative = days before now; 0 = no GA (in development) + } + meta := map[string]relMeta{ + "4.19": {previous: "4.18", gaDays: -289}, + "4.20": {previous: "4.19", gaDays: -163}, + "4.21": {previous: "4.20", gaDays: -58}, + "4.22": {previous: "4.21"}, + } + + for _, release := range syntheticReleases { + m := meta[release] + parts := strings.Split(release, ".") + major, minor := 0, 0 + if len(parts) >= 2 { + fmt.Sscanf(parts[0], "%d", &major) + fmt.Sscanf(parts[1], "%d", &minor) + } + + develStart := now.AddDate(0, 0, m.gaDays-180) + def := models.ReleaseDefinition{ + Release: release, + Major: major, + Minor: minor, + PreviousRelease: m.previous, + DevelopmentStartDate: &develStart, + Product: "OCP", + Status: "Full Support", + Capabilities: allCaps, + } + if m.gaDays != 0 { + ga := now.AddDate(0, 0, m.gaDays) + def.GADate = &ga + } + + if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil { + return fmt.Errorf("failed to create release definition %s: %w", release, err) + } + } + return nil +} + func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { @@ -694,7 +747,7 @@ func syncRegressions(dbc *db.DB) error { provider := pgprovider.NewPostgresProvider(dbc, nil) ctx := context.Background() - releases, err := provider.QueryReleases(ctx) + releases, err := api.GetReleasesFromDB(ctx, dbc) if err != nil { return fmt.Errorf("querying releases: %w", err) } diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index b0340e36ba..7207cf39e3 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -31,8 +31,8 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/utils" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util/sets" ) @@ -82,7 +82,7 @@ func GetComponentReport( reqOptions reqopts.RequestOptions, baseURL string, ) (report crtype.ComponentReport, errs []error) { - releaseConfigs, err := provider.QueryReleases(ctx) + releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc) if err != nil { return report, []error{err} } @@ -151,7 +151,7 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) return nil } -func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator { +func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []models.ReleaseDefinition, baseURL string) ComponentReportGenerator { slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match generator := ComponentReportGenerator{ dataProvider: provider, @@ -175,7 +175,7 @@ type ComponentReportGenerator struct { dbc *db.DB ReqOptions reqopts.RequestOptions middlewares middleware.List - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition baseURL string } @@ -278,7 +278,7 @@ func (c *ComponentReportGenerator) initializeMiddleware() { c.middlewares = middleware.List{} // Initialize all our middleware applicable to this request. if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil { - c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs)) + c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.dbc, c.ReqOptions, c.releaseConfigs)) } if c.dbc != nil { c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions)) diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go index 85d97523fa..e9a2b73599 100644 --- a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go @@ -18,7 +18,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" apiCache "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/util/param" @@ -170,14 +169,6 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari return variants, nil } -func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions) -} - -func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { - return apiPkg.GetReleasesFromBigQuery(ctx, p.client) -} - func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { unnest := "" if nested { diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go b/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go deleted file mode 100644 index 9f6fea2e4b..0000000000 --- a/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go +++ /dev/null @@ -1,44 +0,0 @@ -package bigquery - -import ( - "context" - - "github.com/openshift/sippy/pkg/api" - "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - "github.com/openshift/sippy/pkg/apis/cache" - "github.com/openshift/sippy/pkg/bigquery" - "github.com/openshift/sippy/pkg/util" -) - -func GetReleaseDatesFromBigQuery(ctx context.Context, client *bigquery.Client, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - queries := &releaseDateQuerier{client: client, reqOptions: reqOptions} - return api.GetDataFromCacheOrGenerate[[]crtest.ReleaseTimeRange](ctx, - client.Cache, - cache.RequestOptions{}, - api.NewCacheSpec(crtest.ReleaseTimeRange{}, "CRReleaseDates~", nil), // global singleton instance - queries.QueryReleaseDates, []crtest.ReleaseTimeRange{}) -} - -type releaseDateQuerier struct { - client *bigquery.Client - reqOptions reqopts.RequestOptions -} - -func (c *releaseDateQuerier) QueryReleaseDates(ctx context.Context) ([]crtest.ReleaseTimeRange, []error) { - releases, err := api.GetReleasesFromBigQuery(ctx, c.client) - if err != nil { - return nil, []error{err} - } - timeRanges := []crtest.ReleaseTimeRange{} - for _, release := range releases { - timeRange := crtest.ReleaseTimeRange{Release: release.Release} - if release.GADate != nil { - prior := util.AdjustReleaseTime(*release.GADate, true, "30", c.reqOptions.CacheOption.CRTimeRoundingFactor, c.reqOptions.CacheOption.CRTimeRoundingOffset) - timeRange.Start = &prior - timeRange.End = release.GADate - } - timeRanges = append(timeRanges, timeRange) - } - return timeRanges, nil -} diff --git a/pkg/api/componentreadiness/dataprovider/interface.go b/pkg/api/componentreadiness/dataprovider/interface.go index 37b40f91e2..f7d84981ab 100644 --- a/pkg/api/componentreadiness/dataprovider/interface.go +++ b/pkg/api/componentreadiness/dataprovider/interface.go @@ -8,7 +8,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" ) // TestStatusQuerier fetches aggregated test pass/fail counts. @@ -40,12 +39,6 @@ type MetadataQuerier interface { // QueryJobVariants returns all variant names and their possible values. QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error) - // QueryReleaseDates returns the time ranges for each known release. - QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) - - // QueryReleases returns known release configurations. - QueryReleases(ctx context.Context) ([]v1.Release, error) - // QueryUniqueVariantValues returns distinct values for a variant column // from the past 60 days. QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 206f0dac4b..2f334a818f 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -17,7 +17,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" ) @@ -132,98 +131,6 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari return variants, nil } -// releaseMetadata holds hardcoded release info for known releases. -// This avoids needing a releases table — we derive release names from prow_jobs -// and fill in metadata from this map. -var releaseMetadata = map[string]struct { - previousRelease string - gaOffsetDays int // 0 = no GA date (in development) - product string // empty = defaults to "OCP" -}{ - "4.17": {previousRelease: "4.16", gaOffsetDays: -540}, - "4.18": {previousRelease: "4.17", gaOffsetDays: -395}, - "4.19": {previousRelease: "4.18", gaOffsetDays: -289}, - "4.20": {previousRelease: "4.19", gaOffsetDays: -163}, - "4.21": {previousRelease: "4.20", gaOffsetDays: -58}, - "4.22": {previousRelease: "4.21"}, - "5.0": {previousRelease: "4.22"}, -} - -func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { - var releaseNames []string - err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`). - Pluck("release", &releaseNames).Error - if err != nil { - return nil, fmt.Errorf("querying releases: %w", err) - } - - caps := map[v1.ReleaseCapability]bool{ - v1.ComponentReadinessCap: true, - v1.FeatureGatesCap: true, - v1.MetricsCap: true, - v1.PayloadTagsCap: true, - v1.SippyClassicCap: true, - } - - now := time.Now().UTC() - var releases []v1.Release - for _, name := range releaseNames { - rel := v1.Release{ - Release: name, - Capabilities: caps, - Product: "OCP", - } - if meta, ok := releaseMetadata[name]; ok { - rel.PreviousRelease = meta.previousRelease - if meta.gaOffsetDays != 0 { - ga := now.AddDate(0, 0, meta.gaOffsetDays) - rel.GADate = &ga - } - if meta.product != "" { - rel.Product = meta.product - } - } - releases = append(releases, rel) - } - return releases, nil -} - -func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - // Derive time ranges from actual data in the DB rather than hardcoded GA dates. - // This ensures fallback queries find data where it actually exists. - type releaseRange struct { - Release string - Start time.Time - End time.Time - } - var ranges []releaseRange - err := p.dbc.DB.WithContext(ctx).Raw(` - SELECT pj.release, - MIN(pjr.timestamp) AS start, - MAX(pjr.timestamp) AS end - FROM prow_job_runs pjr - JOIN prow_jobs pj ON pj.id = pjr.prow_job_id - WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL - GROUP BY pj.release - ORDER BY pj.release DESC - `).Scan(&ranges).Error - if err != nil { - return nil, []error{fmt.Errorf("querying release dates: %w", err)} - } - - var dates []crtest.ReleaseTimeRange - for _, r := range ranges { - start := r.Start - end := r.End - dates = append(dates, crtest.ReleaseTimeRange{ - Release: r.Release, - Start: &start, - End: &end, - }) - } - return dates, nil -} - func (p *PostgresProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { if nested { // Return all variant key names diff --git a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go index 7e79cdb435..16b3d2b95a 100644 --- a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go +++ b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go @@ -11,14 +11,14 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/regressionallowances" log "github.com/sirupsen/logrus" ) var _ middleware.Middleware = &RegressionAllowances{} -func NewRegressionAllowancesMiddleware(reqOptions reqopts.RequestOptions, releaseConfigs []v1.Release) *RegressionAllowances { +func NewRegressionAllowancesMiddleware(reqOptions reqopts.RequestOptions, releaseConfigs []models.ReleaseDefinition) *RegressionAllowances { return &RegressionAllowances{ log: log.WithField("middleware", "RegressionAllowances"), reqOptions: reqOptions, @@ -34,7 +34,7 @@ func NewRegressionAllowancesMiddleware(reqOptions reqopts.RequestOptions, releas type RegressionAllowances struct { log log.FieldLogger reqOptions reqopts.RequestOptions - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition // regressionGetterFunc allows us to unit test without relying on real regression data regressionGetterFunc func(releaseString string, variant crtest.ColumnIdentification, testID string) *regressionallowances.IntentionalRegression diff --git a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go index 19985d50c4..1c98b3da23 100644 --- a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go +++ b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances_test.go @@ -8,7 +8,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/regressionallowances" "github.com/stretchr/testify/assert" ) @@ -83,7 +83,7 @@ func Test_PreAnalysis(t *testing.T) { }, } - releaseConfigs := []v1.Release{ + releaseConfigs := []models.ReleaseDefinition{ {Release: "4.19", PreviousRelease: "4.18"}, {Release: "4.18", PreviousRelease: "4.17"}, {Release: "4.17", PreviousRelease: "4.16"}, diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go index 25dc1d83bf..0a31991342 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go @@ -15,7 +15,8 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" apiCache "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util/sets" log "github.com/sirupsen/logrus" @@ -33,11 +34,13 @@ var _ middleware.Middleware = &ReleaseFallback{} func NewReleaseFallbackMiddleware( provider dataprovider.DataProvider, + dbc *db.DB, reqOptions reqopts.RequestOptions, - releaseConfigs []v1.Release, + releaseConfigs []models.ReleaseDefinition, ) *ReleaseFallback { return &ReleaseFallback{ dataProvider: provider, + dbc: dbc, log: log.WithField("middleware", "ReleaseFallback"), reqOptions: reqOptions, releaseConfigs: releaseConfigs, @@ -55,6 +58,7 @@ func NewReleaseFallbackMiddleware( // This is done when we have sufficient test coverage, and a better pass rate. type ReleaseFallback struct { dataProvider dataprovider.DataProvider + dbc *db.DB cachedFallbackTestStatuses *FallbackReleases log log.FieldLogger reqOptions reqopts.RequestOptions @@ -64,7 +68,7 @@ type ReleaseFallback struct { // test ID, but when cache priming for a view, we may have multiple. baseOverrideStatus map[string]map[string][]crstatus.TestJobRunRows baseOverrideMutex sync.Mutex // Mutex to protect the map - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition } func (r *ReleaseFallback) Analyze(testID string, variants map[string]string, report *testdetails.TestComparison) error { @@ -173,7 +177,7 @@ func (r *ReleaseFallback) PostAnalysis(testKey crtest.Identification, testStats func (r *ReleaseFallback) getFallbackBaseQueryStatus(ctx context.Context, allJobVariants crtest.JobVariants, release string, start, end time.Time) []error { - generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs) + generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.dbc, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs) cachedFallbackTestStatuses, errs := api.GetDataFromCacheOrGenerate[*FallbackReleases]( ctx, r.dataProvider.Cache(), r.reqOptions.CacheOption, @@ -193,9 +197,9 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro r.log.Infof("Querying fallback override test statuses for %d test ID options", len(r.reqOptions.TestIDOptions)) // Lookup all release dates, we're going to need them - timeRanges, errs := r.dataProvider.QueryReleaseDates(ctx, r.reqOptions) - if errs != nil { - utils.EnqueueAsync(wg, errCh, errs...) + timeRanges, err := api.GetReleaseDatesFromDB(ctx, r.dbc, r.reqOptions) + if err != nil { + utils.EnqueueAsync(wg, errCh, err) return } @@ -296,6 +300,7 @@ func (r *ReleaseFallback) TestDetailsAnalyze(report *testdetails.Report) error { // each, which can then be used to return the best basis data from those past releases for comparison. type fallbackTestQueryReleasesGenerator struct { dataProvider dataprovider.DataProvider + dbc *db.DB cacheOption apiCache.RequestOptions allJobVariants crtest.JobVariants BaseRelease string @@ -304,19 +309,21 @@ type fallbackTestQueryReleasesGenerator struct { CachedFallbackTestStatuses FallbackReleases lock *sync.Mutex ReqOptions reqopts.RequestOptions - releaseConfigs []v1.Release + releaseConfigs []models.ReleaseDefinition } func newFallbackTestQueryReleasesGenerator( provider dataprovider.DataProvider, + dbc *db.DB, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, release string, start, end time.Time, - releaseConfigs []v1.Release, + releaseConfigs []models.ReleaseDefinition, ) fallbackTestQueryReleasesGenerator { generator := fallbackTestQueryReleasesGenerator{ dataProvider: provider, + dbc: dbc, cacheOption: reqOptions.CacheOption, allJobVariants: allJobVariants, BaseRelease: release, @@ -335,7 +342,7 @@ type fallbackTestQueryReleasesGeneratorCacheKey struct { BaseEnd time.Time // VariantDBGroupBy is the only field within VariantOption that is used here VariantDBGroupBy sets.String - // CRTimeRoundingFactor is used by GetReleaseDatesFromBigQuery + // CRTimeRoundingFactor is used by GetReleaseDatesFromDB CRTimeRoundingFactor time.Duration CRTimeRoundingOffset time.Duration // KeyTestNames affects the BuildComponentReportQuery results via filtering logic @@ -360,10 +367,9 @@ func (f *fallbackTestQueryReleasesGenerator) getCacheKey() fallbackTestQueryRele func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context.Context) (*FallbackReleases, []error) { wg := sync.WaitGroup{} f.CachedFallbackTestStatuses = newFallbackReleases() - timeRanges, errs := f.dataProvider.QueryReleaseDates(ctx, f.ReqOptions) - - if errs != nil { - return nil, errs + timeRanges, err := api.GetReleaseDatesFromDB(ctx, f.dbc, f.ReqOptions) + if err != nil { + return nil, []error{err} } selectedTimeRanges := calculateDefaultFallbackReleases(f.BaseRelease, timeRanges, f.releaseConfigs) @@ -402,11 +408,11 @@ func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context return &f.CachedFallbackTestStatuses, nil } -func calculateDefaultFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []v1.Release) []*crtest.ReleaseTimeRange { +func calculateDefaultFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []models.ReleaseDefinition) []*crtest.ReleaseTimeRange { return calculateFallbackReleases(startingRelease, timeRanges, releaseConfigs, defaultFallbackReleases) } -func calculateFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []v1.Release, maxReleases int) []*crtest.ReleaseTimeRange { +func calculateFallbackReleases(startingRelease string, timeRanges []crtest.ReleaseTimeRange, releaseConfigs []models.ReleaseDefinition, maxReleases int) []*crtest.ReleaseTimeRange { var selectedTimeRanges []*crtest.ReleaseTimeRange fallbackRelease := startingRelease diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go index 3625ebbd57..d410600fd5 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go @@ -9,7 +9,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" ) @@ -83,7 +83,7 @@ func Test_PreAnalysis(t *testing.T) { }, } - releaseConfigs := []v1.Release{ + releaseConfigs := []models.ReleaseDefinition{ {Release: "4.19", PreviousRelease: "4.18"}, {Release: "4.18", PreviousRelease: "4.17"}, {Release: "4.17", PreviousRelease: "4.16"}, @@ -164,7 +164,7 @@ func Test_PreAnalysis(t *testing.T) { } for i, test := range tests { t.Run(test.name, func(t *testing.T) { - rfb := NewReleaseFallbackMiddleware(nil, test.reqOpts, releaseConfigs) + rfb := NewReleaseFallbackMiddleware(nil, nil, test.reqOpts, releaseConfigs) rfb.cachedFallbackTestStatuses = &tests[i].fallbackReleases err := rfb.PreAnalysis(test.testKey, test.testStats) assert.NoError(t, err) @@ -207,7 +207,7 @@ func TestCalculateFallbackReleases(t *testing.T) { allTimeRanges := []crtest.ReleaseTimeRange{release419, release418, release417, release416} expectedTimeRanges := []crtest.ReleaseTimeRange{release419, release418, release417} - releaseConfigs := []v1.Release{ + releaseConfigs := []models.ReleaseDefinition{ {Release: "4.20", PreviousRelease: "4.19"}, {Release: "4.19", PreviousRelease: "4.18"}, {Release: "4.18", PreviousRelease: "4.17"}, diff --git a/pkg/api/componentreadiness/queryparamparser_test.go b/pkg/api/componentreadiness/queryparamparser_test.go index 164280f3ec..c1a39050e3 100644 --- a/pkg/api/componentreadiness/queryparamparser_test.go +++ b/pkg/api/componentreadiness/queryparamparser_test.go @@ -14,7 +14,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" "github.com/stretchr/testify/assert" @@ -31,7 +31,7 @@ var ( func TestParseComponentReportRequest(t *testing.T) { - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.16", Status: "", GADate: util.DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, {Release: "4.15", Status: "", GADate: util.DatePtr(2024, 2, 28, 0, 0, 0, 0, time.UTC)}, } @@ -490,7 +490,7 @@ func TestHATEOASLinkCacheConsistency(t *testing.T) { roundingFactor := 12 * time.Hour roundingOffset := 4 * time.Hour - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.16", Status: "", GADate: util.DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, {Release: "4.17", Status: "", GADate: util.DatePtr(2024, 12, 10, 0, 0, 0, 0, time.UTC)}, } diff --git a/pkg/api/componentreadiness/test_details.go b/pkg/api/componentreadiness/test_details.go index 16eded289a..5729641ccb 100644 --- a/pkg/api/componentreadiness/test_details.go +++ b/pkg/api/componentreadiness/test_details.go @@ -23,10 +23,10 @@ import ( "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) -func GetTestDetails(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []v1.Release, baseURL string) (testdetails.Report, []error) { +func GetTestDetails(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []models.ReleaseDefinition, baseURL string) (testdetails.Report, []error) { generator := NewComponentReportGenerator(provider, reqOptions, dbc, releases, baseURL) if os.Getenv("DEV_MODE") == "1" { return generator.GenerateTestDetailsReport(ctx) @@ -197,9 +197,9 @@ func (c *ComponentReportGenerator) GenerateDetailsReportForTest( } } - timeRanges, errs := c.dataProvider.QueryReleaseDates(ctx, c.ReqOptions) - if errs != nil { - return testdetails.Report{}, errs + timeRanges, err := api.GetReleaseDatesFromDB(ctx, c.dbc, c.ReqOptions) + if err != nil { + return testdetails.Report{}, []error{err} } now := time.Now() diff --git a/pkg/api/componentreadiness/triage.go b/pkg/api/componentreadiness/triage.go index 26c5b875e7..bfb8a2137e 100644 --- a/pkg/api/componentreadiness/triage.go +++ b/pkg/api/componentreadiness/triage.go @@ -19,7 +19,6 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/utils" "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/db/models/jobrunscan" @@ -307,7 +306,7 @@ func DeleteTriage(dbc *gorm.DB, id int) error { // ListRegressions lists all regressions for the provided view OR release. // When view is set, it is resolved to that view's sample release and filtering is by release. -func ListRegressions(dbc *db.DB, release string, views []crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) ([]models.TestRegression, error) { +func ListRegressions(dbc *db.DB, release string, views []crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) ([]models.TestRegression, error) { var regressions []models.TestRegression var err error regressions, err = query.ListRegressions(dbc, release) @@ -324,7 +323,7 @@ func ListRegressions(dbc *db.DB, release string, views []crview.View, releases [ } // GetRegression returns the regression with the matching ID -func GetRegression(dbc *db.DB, id int, views []crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) (*models.TestRegression, error) { +func GetRegression(dbc *db.DB, id int, views []crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, req *http.Request) (*models.TestRegression, error) { regression := &models.TestRegression{} res := dbc.DB.Preload("Triages").Preload("JobRuns").Preload("Views").First(regression, id) if res.Error != nil { @@ -796,7 +795,7 @@ func injectHATEOASLinks(triage *models.Triage, baseURL string) { // InjectRegressionHATEOASLinks adds restful links clients can follow for this regression record. // Per-view test_details links use composite keys: test_details:. -func InjectRegressionHATEOASLinks(regression *models.TestRegression, views []crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseAPIURL, baseFrontendURL string) { +func InjectRegressionHATEOASLinks(regression *models.TestRegression, views []crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseAPIURL, baseFrontendURL string) { regression.Links = map[string]string{ "self": fmt.Sprintf(regressionLink, baseAPIURL, regression.ID), } @@ -830,7 +829,7 @@ func FindViewByName(name string, views []crview.View) (crview.View, bool) { // generateTestDetailsURLFromRegression extracts the required data from a regression and view // and calls the GenerateTestDetailsURL function. -func generateTestDetailsURLFromRegression(regression *models.TestRegression, view crview.View, releases []v1.Release, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseURL string) (string, error) { +func generateTestDetailsURLFromRegression(regression *models.TestRegression, view crview.View, releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, baseURL string) (string, error) { if regression == nil { return "", fmt.Errorf("regression cannot be nil") } diff --git a/pkg/api/componentreadiness/triage_test.go b/pkg/api/componentreadiness/triage_test.go index 30575c5ced..9e4f9df19b 100644 --- a/pkg/api/componentreadiness/triage_test.go +++ b/pkg/api/componentreadiness/triage_test.go @@ -9,7 +9,6 @@ import ( "github.com/lib/pq" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/sets" @@ -612,7 +611,7 @@ func TestCompareTriageObjects(t *testing.T) { func TestInjectRegressionHATEOASLinks(t *testing.T) { ga421 := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.20", GADate: &ga421}, {Release: "4.21", GADate: &ga421}, } diff --git a/pkg/api/componentreadiness/utils/queryparamparser.go b/pkg/api/componentreadiness/utils/queryparamparser.go index d63465a418..4d5105bf2d 100644 --- a/pkg/api/componentreadiness/utils/queryparamparser.go +++ b/pkg/api/componentreadiness/utils/queryparamparser.go @@ -12,7 +12,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/param" ) @@ -20,7 +20,7 @@ import ( // nolint:gocyclo func ParseComponentReportRequest( views []crview.View, - releases []v1.Release, + releases []models.ReleaseDefinition, req *http.Request, allJobVariants crtest.JobVariants, crTimeRoundingFactor, crTimeRoundingOffset time.Duration, @@ -238,7 +238,7 @@ func getRequestedView(req *http.Request, views []crview.View) (*crview.View, err // Translate relative start/end times to actual time.Time: func GetViewReleaseOptions( - releases []v1.Release, + releases []models.ReleaseDefinition, releaseType string, viewRelease reqopts.RelativeRelease, roundingFactor, roundingOffset time.Duration, @@ -408,7 +408,7 @@ func parseAdvancedOptions(req *http.Request) (advancedOption reqopts.Advanced, e return } -func parseDateRange(allReleases []v1.Release, req *http.Request, +func parseDateRange(allReleases []models.ReleaseDefinition, req *http.Request, releaseOpts reqopts.Release, startName string, endName string, roundingFactor, roundingOffset time.Duration, diff --git a/pkg/api/componentreadiness/utils/utils.go b/pkg/api/componentreadiness/utils/utils.go index 2a5556c5c8..5f147121d8 100644 --- a/pkg/api/componentreadiness/utils/utils.go +++ b/pkg/api/componentreadiness/utils/utils.go @@ -16,10 +16,10 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) -func PreviousRelease(release string, releaseConfigs []sippyv1.Release) (string, error) { +func PreviousRelease(release string, releaseConfigs []models.ReleaseDefinition) (string, error) { for _, config := range releaseConfigs { if config.Release == release { if config.PreviousRelease != "" { diff --git a/pkg/api/componentreadiness/utils/utils_test.go b/pkg/api/componentreadiness/utils/utils_test.go index f3318a27f4..ee608cb0e4 100644 --- a/pkg/api/componentreadiness/utils/utils_test.go +++ b/pkg/api/componentreadiness/utils/utils_test.go @@ -8,7 +8,7 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func TestGenerateTestDetailsURL(t *testing.T) { // Define releases with GA dates for all tests ga419 := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) ga420 := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ { Release: "4.19", GADate: &ga419, diff --git a/pkg/api/job_runs.go b/pkg/api/job_runs.go index 3e1fa1bdd4..8da730af38 100644 --- a/pkg/api/job_runs.go +++ b/pkg/api/job_runs.go @@ -369,8 +369,7 @@ func JobRunRiskAnalysis( compareRelease := jobRun.ProwJob.Release neverStableJob := false if compareRelease == "Presubmits" { - // Get latest release from the DB: - ar, err := GetReleases(ctx, bqc, false) + ar, err := GetReleasesFromDB(ctx, dbc) if err != nil { return apitype.ProwJobRunRiskAnalysis{}, err } @@ -419,8 +418,7 @@ func JobRunRiskAnalysis( } if totalJobRuns < 20 { - // go back to the prior release and get more jobIds to compare against - releases, err := GetReleases(ctx, bqc, false) + releases, err := GetReleasesFromDB(ctx, dbc) if err != nil { logger.WithError(err).Error("Failed to get releases for prior release lookup") } else { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index 287f951df2..f155b42484 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -15,6 +15,8 @@ import ( "gorm.io/gorm" apitype "github.com/openshift/sippy/pkg/apis/api" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" @@ -23,6 +25,7 @@ import ( "github.com/openshift/sippy/pkg/db/query" "github.com/openshift/sippy/pkg/filter" "github.com/openshift/sippy/pkg/testidentification" + "github.com/openshift/sippy/pkg/util" ) func PrintPullRequestsReport(w http.ResponseWriter, req *http.Request, dbClient *db.DB) { @@ -445,9 +448,9 @@ func releaseFilter(req *http.Request, dbc *gorm.DB) *gorm.DB { return dbc } -// GetReleasesFromBigQuery gets all releases defined in the Releases table in BigQuery -func GetReleasesFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.Release, error) { - releases := []sippyv1.Release{} +// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table. +func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) { + var rows []sippyv1.ReleaseRow queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable) @@ -455,7 +458,7 @@ func GetReleasesFromBigQuery(ctx context.Context, client *bqcachedclient.Client) it, err := q.Read(ctx) if err != nil { log.WithError(err).Error("error querying releases data from bigquery") - return releases, err + return rows, err } for { @@ -466,40 +469,15 @@ func GetReleasesFromBigQuery(ctx context.Context, client *bqcachedclient.Client) } if err != nil { log.WithError(err).Error("error parsing release row from bigquery") - return releases, err + return rows, err } - releases = append(releases, transformRelease(r)) + rows = append(rows, r) } - return releases, nil -} - -// transformRelease converts the BQ release row to v1.Release type -func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { - release := sippyv1.Release{ - Release: r.Release, - Status: r.ReleaseStatus.String(), - PreviousRelease: r.PreviousRelease.StringVal, - Capabilities: make(map[sippyv1.ReleaseCapability]bool), - Product: r.Product.StringVal, - } - if r.GADate.Valid { - gaDate := r.GADate.Date.In(time.UTC) - release.GADate = &gaDate - } - if r.DevelStartDate.IsValid() { - develStartDate := r.DevelStartDate.In(time.UTC) - release.DevelopmentStartDate = &develStartDate - } - if r.Capabilities != nil { - for _, capability := range r.Capabilities { - release.Capabilities[capability] = true - } - } - return release + return rows, nil } // BuildReleasesResponse creates the API response structure for releases -func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { +func BuildReleasesResponse(releases []models.ReleaseDefinition, lastUpdated time.Time) apitype.Releases { gaDateMap := make(map[string]time.Time) dateMap := make(map[string]apitype.ReleaseDates) response := apitype.Releases{ @@ -521,11 +499,15 @@ func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) ap releaseDate.DevelopmentStart = release.DevelopmentStartDate response.Dates[release.Release] = releaseDate } + caps := make(map[sippyv1.ReleaseCapability]bool, len(release.Capabilities)) + for _, cap := range release.Capabilities { + caps[sippyv1.ReleaseCapability(cap)] = true + } response.ReleaseAttrs[release.Release] = apitype.Release{ Name: release.Release, PreviousRelease: release.PreviousRelease, ReleaseDates: releaseDate, - Capabilities: release.Capabilities, + Capabilities: caps, Product: release.Product, } } @@ -533,6 +515,35 @@ func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) ap return response } +// GetReleaseDatesFromDB derives CR time ranges from release_definitions GA dates. +func GetReleaseDatesFromDB(ctx context.Context, dbc *db.DB, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, error) { + defs, err := GetReleasesFromDB(ctx, dbc) + if err != nil { + return nil, err + } + var timeRanges []crtest.ReleaseTimeRange + for _, def := range defs { + tr := crtest.ReleaseTimeRange{Release: def.Release} + if def.GADate != nil { + prior := util.AdjustReleaseTime(*def.GADate, true, "30", reqOptions.CacheOption.CRTimeRoundingFactor, reqOptions.CacheOption.CRTimeRoundingOffset) + tr.Start = &prior + tr.End = def.GADate + } + timeRanges = append(timeRanges, tr) + } + return timeRanges, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table. +func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]models.ReleaseDefinition, error) { + var defs []models.ReleaseDefinition + err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error + if err != nil { + return nil, fmt.Errorf("querying release definitions: %w", err) + } + return defs, nil +} + // PayloadForJobRun returns the payload release tag that was used for a given job run. func PayloadForJobRun(ctx context.Context, bigQueryClient *bqcachedclient.Client, jobRunID string) ([]apitype.JobPayload, error) { // Calculate date range: 6 months ago through today diff --git a/pkg/api/releases_test.go b/pkg/api/releases_test.go index 62967569cb..a9063535f4 100644 --- a/pkg/api/releases_test.go +++ b/pkg/api/releases_test.go @@ -1,73 +1,10 @@ package api import ( - "testing" - "time" - - "cloud.google.com/go/bigquery" - "cloud.google.com/go/civil" - - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - - "github.com/stretchr/testify/assert" - apitype "github.com/openshift/sippy/pkg/apis/api" "github.com/openshift/sippy/pkg/db/models" ) -func TestTransformRelease(t *testing.T) { - - devStart420, _ := time.Parse(time.RFC3339, "2025-04-18T00:00:00.00Z") - devStart419, _ := time.Parse(time.RFC3339, "2024-11-25T00:00:00.00Z") - gaDate419, _ := time.Parse(time.RFC3339, "2025-05-09T00:00:00.00Z") - - tests := []struct { - name string - releaseRow sippyv1.ReleaseRow - expectedRelease sippyv1.Release - }{ - { - name: "release without devel start", - releaseRow: sippyv1.ReleaseRow{Release: "4.20", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}}, - expectedRelease: sippyv1.Release{Release: "4.20", Status: "Development"}, - }, - { - name: "release with devel start", - releaseRow: sippyv1.ReleaseRow{Release: "4.20", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, DevelStartDate: civil.Date{ - Year: 2025, - Month: 4, - Day: 18, - }}, - expectedRelease: sippyv1.Release{Release: "4.20", Status: "Development", DevelopmentStartDate: &devStart420}, - }, - { - name: "release with ga date", - releaseRow: sippyv1.ReleaseRow{Release: "4.19", ReleaseStatus: bigquery.NullString{Valid: true, StringVal: "Development"}, DevelStartDate: civil.Date{ - Year: 2024, - Month: 11, - Day: 25, - }, GADate: bigquery.NullDate{ - Date: civil.Date{ - Year: 2025, - Month: 5, - Day: 9}, - Valid: true, - }}, - expectedRelease: sippyv1.Release{Release: "4.19", Status: "Development", DevelopmentStartDate: &devStart419, GADate: &gaDate419}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - release := transformRelease(tc.releaseRow) - assert.Equal(t, tc.expectedRelease.Release, release.Release, "unexpected release") - assert.Equal(t, tc.expectedRelease.Status, release.Status, "unexpected status") - assert.Equal(t, tc.expectedRelease.GADate, release.GADate, "unexpected status") - assert.Equal(t, tc.expectedRelease.DevelopmentStartDate, release.DevelopmentStartDate, "unexpected devel start") - }) - } -} - func buildFakeReleaseHealthReport(osVersion string) apitype.ReleaseHealthReport { return apitype.ReleaseHealthReport{ ReleaseTag: models.ReleaseTag{ diff --git a/pkg/api/utils.go b/pkg/api/utils.go index e65df3d923..fbeabe2677 100644 --- a/pkg/api/utils.go +++ b/pkg/api/utils.go @@ -1,52 +1,16 @@ package api import ( - "context" "fmt" "net/http" "net/url" "strings" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - log "github.com/sirupsen/logrus" - "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - bqclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/util/sets" ) -type releaseGenerator struct { - client *bqclient.Client -} - -func (r *releaseGenerator) ListReleases(ctx context.Context) ([]v1.Release, []error) { - releases, err := GetReleasesFromBigQuery(ctx, r.client) - if err != nil { - log.WithError(err).Error("error getting releases from bigquery") - return releases, []error{err} - } - return releases, nil -} - -// GetReleases gets all the releases defined in the BQ Releases table. -func GetReleases(ctx context.Context, bqc *bqclient.Client, forceRefresh bool) ([]v1.Release, error) { - releaseGen := releaseGenerator{bqc} - - var err error - rels, errs := GetDataFromCacheOrGenerate[[]v1.Release]( - ctx, - bqc.Cache, - cache.RequestOptions{ForceRefresh: forceRefresh}, - NewCacheSpec(v1.Release{}, "Releases~", nil), // no cache options needed here, global list - releaseGen.ListReleases, - []v1.Release{}) - if len(errs) > 0 { - err = errs[0] - } - return rels, err -} - // VariantsStringToSet converts comma separated variant string into a set; also validates that the variants are known func VariantsStringToSet(allJobVariants crtest.JobVariants, variantsString string) (sets.String, error) { variantSet := sets.String{} diff --git a/pkg/componentreadiness/jiraautomator/jiraautomator.go b/pkg/componentreadiness/jiraautomator/jiraautomator.go index 369d93dcb3..5e76dbefeb 100644 --- a/pkg/componentreadiness/jiraautomator/jiraautomator.go +++ b/pkg/componentreadiness/jiraautomator/jiraautomator.go @@ -24,6 +24,7 @@ import ( bqclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" log "github.com/sirupsen/logrus" @@ -56,7 +57,7 @@ type JiraAutomator struct { dbc *db.DB cacheOptions cache.RequestOptions views []crview.View - releases []v1.Release + releases []models.ReleaseDefinition sippyURL string // columnThresholds defines a threshold for the number of red cells in a column. // When the number of red cells of a column is over this threshold, a jira card will be created for the @@ -75,7 +76,7 @@ func NewJiraAutomator( dbc *db.DB, cacheOptions cache.RequestOptions, views []crview.View, - releases []v1.Release, + releases []models.ReleaseDefinition, sippyURL, jiraAccount string, includeComponents sets.String, columnThresholds map[Variant]int, diff --git a/pkg/dataloader/crcacheloader/crcacheloader.go b/pkg/dataloader/crcacheloader/crcacheloader.go index bf0902347d..aed0810b42 100644 --- a/pkg/dataloader/crcacheloader/crcacheloader.go +++ b/pkg/dataloader/crcacheloader/crcacheloader.go @@ -18,9 +18,9 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/config/v1" - apiv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" log "github.com/sirupsen/logrus" ) @@ -33,7 +33,7 @@ type ComponentReadinessCacheLoader struct { dbc *db.DB errs []error views *sippytypes.SippyViews - releases []apiv1.Release + releases []models.ReleaseDefinition cacheClient cache.Cache bqClient *bigquery.Client dataProvider dataprovider.DataProvider @@ -48,7 +48,7 @@ func New( bqClient *bigquery.Client, config *v1.SippyConfig, views *sippytypes.SippyViews, - releases []apiv1.Release, + releases []models.ReleaseDefinition, crTimeRoundingFactor, crTimeRoundingOffset time.Duration) *ComponentReadinessCacheLoader { return &ComponentReadinessCacheLoader{ diff --git a/pkg/dataloader/featuregateloader/featuregateloader.go b/pkg/dataloader/featuregateloader/featuregateloader.go index ab2ef4cc8f..0edb4ecdf4 100644 --- a/pkg/dataloader/featuregateloader/featuregateloader.go +++ b/pkg/dataloader/featuregateloader/featuregateloader.go @@ -10,7 +10,6 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" @@ -21,15 +20,15 @@ import ( ) type FeatureGateLoader struct { - dbc *db.DB - errs []error - releaseConfigs []v1.Release + dbc *db.DB + errs []error + releaseDefs []models.ReleaseDefinition } -func New(dbc *db.DB, configs []v1.Release) *FeatureGateLoader { +func New(dbc *db.DB, releaseDefs []models.ReleaseDefinition) *FeatureGateLoader { return &FeatureGateLoader{ - dbc: dbc, - releaseConfigs: configs, + dbc: dbc, + releaseDefs: releaseDefs, } } @@ -68,9 +67,9 @@ func (l *FeatureGateLoader) Errors() []error { func (l *FeatureGateLoader) getTargetReleases() []string { var targetReleases []string - for _, release := range l.releaseConfigs { - if release.Capabilities[v1.FeatureGatesCap] { - targetReleases = append(targetReleases, release.Release) + for _, def := range l.releaseDefs { + if def.HasCapability(models.CapFeatureGates) { + targetReleases = append(targetReleases, def.Release) } } diff --git a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go index fc2d5424c5..485ab53664 100644 --- a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go +++ b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go @@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics { return loader } -var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} +var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} // sortLoaders guarantees that the loaders run in a predictable and proper order func (l *LoaderWithMetrics) sortLoaders() { diff --git a/pkg/dataloader/regressioncacheloader/regressioncacheloader.go b/pkg/dataloader/regressioncacheloader/regressioncacheloader.go index 43e5f56f52..95d82d79cb 100644 --- a/pkg/dataloader/regressioncacheloader/regressioncacheloader.go +++ b/pkg/dataloader/regressioncacheloader/regressioncacheloader.go @@ -19,7 +19,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" "github.com/openshift/sippy/pkg/apis/cache" configv1 "github.com/openshift/sippy/pkg/apis/config/v1" - apiv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" @@ -47,7 +46,7 @@ type RegressionCacheLoader struct { // Cache priming deps bqClient *bigquery.Client config *configv1.SippyConfig - releases []apiv1.Release + releases []models.ReleaseDefinition crTimeRoundingFactor time.Duration crTimeRoundingOffset time.Duration @@ -60,7 +59,7 @@ func New( bqClient *bigquery.Client, config *configv1.SippyConfig, views []crview.View, - releases []apiv1.Release, + releases []models.ReleaseDefinition, crTimeRoundingFactor time.Duration, crTimeRoundingOffset time.Duration, regressionStore componentreadiness.RegressionStore, diff --git a/pkg/dataloader/releasedefloader/releasedefloader.go b/pkg/dataloader/releasedefloader/releasedefloader.go new file mode 100644 index 0000000000..cc589d4ade --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader.go @@ -0,0 +1,103 @@ +package releasedefloader + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" + + "github.com/openshift/sippy/pkg/api" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL. +type ReleaseDefinitionLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + errs []error +} + +func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader { + return &ReleaseDefinitionLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + } +} + +func (l *ReleaseDefinitionLoader) Name() string { + return "release-definitions" +} + +func (l *ReleaseDefinitionLoader) Load() { + releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient) + if err != nil { + l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err)) + return + } + defs := make([]models.ReleaseDefinition, 0, len(releaseRows)) + for _, row := range releaseRows { + defs = append(defs, ReleaseRowToDefinition(row)) + } + if err := syncReleaseDefinitions(l.dbc, defs); err != nil { + l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err)) + } +} + +func (l *ReleaseDefinitionLoader) Errors() []error { + return l.errs +} + +func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error { + for _, def := range defs { + err := dbc.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "release"}}, + DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}), + }).Create(&def).Error + if err != nil { + return fmt.Errorf("upserting release definition %s: %w", def.Release, err) + } + } + log.Infof("Synced %d release definitions to postgres", len(defs)) + return nil +} + +// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model. +func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition { + caps := make(pq.StringArray, 0, len(r.Capabilities)) + for _, cap := range r.Capabilities { + caps = append(caps, string(cap)) + } + sort.Strings(caps) + + def := models.ReleaseDefinition{ + Release: r.Release, + Major: r.Major, + Minor: r.Minor, + PreviousRelease: r.PreviousRelease.StringVal, + Product: r.Product.StringVal, + Status: r.ReleaseStatus.StringVal, + Capabilities: caps, + } + if r.Patch.Valid { + p := int(r.Patch.Int64) + def.Patch = &p + } + if r.GADate.Valid { + ga := r.GADate.Date.In(time.UTC) + def.GADate = &ga + } + if r.DevelStartDate.IsValid() { + ds := r.DevelStartDate.In(time.UTC) + def.DevelopmentStartDate = &ds + } + return def +} diff --git a/pkg/dataloader/releaseloader/projects.go b/pkg/dataloader/releaseloader/projects.go index 4f794719e2..7077b1c51d 100644 --- a/pkg/dataloader/releaseloader/projects.go +++ b/pkg/dataloader/releaseloader/projects.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) func (ocp *OCPProject) GetName() string { @@ -19,7 +19,7 @@ func (ocp *OCPProject) GetRcDomain(architecture string) (domain string) { return architecture + ".ocp.releases.ci.openshift.org" } -func (ocp *OCPProject) IsProjectRelease(release v1.Release) bool { +func (ocp *OCPProject) IsProjectRelease(release models.ReleaseDefinition) bool { return release.Product == "OCP" } @@ -46,7 +46,7 @@ func (okd *OKDProject) GetRcDomain(architecture string) (domain string) { return architecture + ".origin.releases.ci.openshift.org" } -func (okd *OKDProject) IsProjectRelease(release v1.Release) bool { +func (okd *OKDProject) IsProjectRelease(release models.ReleaseDefinition) bool { return release.Product == "OKD" } diff --git a/pkg/dataloader/releaseloader/releasesync.go b/pkg/dataloader/releaseloader/releasesync.go index 82c1cb8202..e3be121996 100644 --- a/pkg/dataloader/releaseloader/releasesync.go +++ b/pkg/dataloader/releaseloader/releasesync.go @@ -12,7 +12,6 @@ import ( "strings" "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -39,21 +38,21 @@ type ReleaseLoader struct { db *db.DB bqClient *bqcachedclient.Client httpClient *http.Client - releases map[string]v1.Release + releases map[string]models.ReleaseDefinition architectures []string projects []PayloadProject errors []error } -func New(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client, releases, architectures []string, releaseConfigs []v1.Release) *ReleaseLoader { - configForRelease := make(map[string]v1.Release, len(releaseConfigs)) +func New(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client, releases, architectures []string, releaseConfigs []models.ReleaseDefinition) *ReleaseLoader { + configForRelease := make(map[string]models.ReleaseDefinition, len(releaseConfigs)) for _, config := range releaseConfigs { - if config.Capabilities[v1.PayloadTagsCap] { + if config.HasCapability(models.CapPayloadTags) { configForRelease[config.Release] = config } } if len(releases) > 0 { - filteredRCs := make(map[string]v1.Release, len(releases)) + filteredRCs := make(map[string]models.ReleaseDefinition, len(releases)) for _, release := range releases { if config, ok := configForRelease[release]; ok { filteredRCs[release] = config @@ -514,7 +513,7 @@ func (rs *ReleaseStream) baseReleaseStreamURL() string { } // buildReleaseStreams builds relevant release streams for specified releases that belong to the project. -func buildReleaseStreams(releases map[string]v1.Release, architectures []string, project PayloadProject) []ReleaseStream { +func buildReleaseStreams(releases map[string]models.ReleaseDefinition, architectures []string, project PayloadProject) []ReleaseStream { releaseStreams := make([]ReleaseStream, 0, len(releases)*len(project.GetStreams())) for release, config := range releases { if project.IsProjectRelease(config) { diff --git a/pkg/dataloader/releaseloader/types.go b/pkg/dataloader/releaseloader/types.go index c0936cec21..3f0831519e 100644 --- a/pkg/dataloader/releaseloader/types.go +++ b/pkg/dataloader/releaseloader/types.go @@ -3,12 +3,12 @@ package releaseloader import ( "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) type ReleaseStream struct { Name string - Release v1.Release + Release models.ReleaseDefinition Stream string Architecture string Domain string @@ -116,7 +116,7 @@ type PayloadProject interface { GetRcDomain(architecture string) (domain string) // IsProjectRelease returns true if the release (from Releases table) belongs to this project - IsProjectRelease(release v1.Release) bool + IsProjectRelease(release models.ReleaseDefinition) bool // FullReleaseStream builds a full releaseStream name to look for on the release-controller, or empty string if n/a FullReleaseStream(release, stream, architecture string) string diff --git a/pkg/db/db.go b/pkg/db/db.go index 73de1a1ff8..752686eff8 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { // List of all models to migrate modelsToMigrate := []any{ + &models.ReleaseDefinition{}, &models.ReleaseTag{}, &models.ReleasePullRequest{}, &models.ReleaseRepository{}, diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index 01c578e628..0bb91ae28d 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -6,6 +6,41 @@ import ( "github.com/lib/pq" ) +// ReleaseDefinition stores release metadata synced from BigQuery's Releases table. +// This is distinct from ReleaseTag, which tracks individual payload tags. +type ReleaseDefinition struct { + Model + + Release string `json:"release" gorm:"uniqueIndex;column:release"` + Major int `json:"major" gorm:"column:major"` + Minor int `json:"minor" gorm:"column:minor"` + Patch *int `json:"patch,omitempty" gorm:"column:patch"` + PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` + GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date"` + DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date"` + Product string `json:"product" gorm:"column:product"` + Status string `json:"status" gorm:"column:status"` + Capabilities pq.StringArray `json:"capabilities" gorm:"type:text[];column:capabilities"` +} + +const ( + CapComponentReadiness = "componentReadiness" + CapSippyClassic = "sippyClassic" + CapMetrics = "metrics" + CapPullRequests = "pullRequests" + CapFeatureGates = "featureGates" + CapPayloadTags = "payloadTags" +) + +func (rd *ReleaseDefinition) HasCapability(cap string) bool { + for _, c := range rd.Capabilities { + if c == cap { + return true + } + } + return false +} + type ReleaseTag struct { Model diff --git a/pkg/mcp/tools/releases.go b/pkg/mcp/tools/releases.go index 7ef6e6834d..be65331ff2 100644 --- a/pkg/mcp/tools/releases.go +++ b/pkg/mcp/tools/releases.go @@ -40,30 +40,27 @@ func (rt *ReleasesTool) GetHandler() func(ctx context.Context, request mcp.CallT return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { log.Debug("Handling get_releases tool call") - // Get releases from BigQuery (never force refresh for MCP) - releases, err := api.GetReleases(ctx, rt.deps.BigQueryClient, false) + if rt.deps.DBClient == nil { + return rt.CreateErrorResponse(fmt.Errorf("no database available for releases")) + } + + releases, err := api.GetReleasesFromDB(ctx, rt.deps.DBClient) if err != nil { log.WithError(err).Error("error querying releases") return rt.CreateErrorResponse(fmt.Errorf("error querying releases: %w", err)) } - // Get last updated time from database if available var lastUpdated time.Time - if rt.deps.DBClient != nil && rt.deps.DBClient.DB != nil { - type LastUpdatedQuery struct { - Max time.Time - } - var result LastUpdatedQuery - // Assume our last update is the last time we inserted a prow job run. - if err := rt.deps.DBClient.DB.Raw("SELECT MAX(created_at) FROM prow_job_runs").Scan(&result).Error; err == nil { - lastUpdated = result.Max - } + type LastUpdatedQuery struct { + Max time.Time + } + var result LastUpdatedQuery + // Assume our last update is the last time we inserted a prow job run. + if err := rt.deps.DBClient.DB.Raw("SELECT MAX(created_at) FROM prow_job_runs").Scan(&result).Error; err == nil { + lastUpdated = result.Max } - // Build response using shared function response := api.BuildReleasesResponse(releases, lastUpdated) - - // Return JSON response return rt.CreateJSONResponse(response) } } diff --git a/pkg/sippyserver/metrics/metrics.go b/pkg/sippyserver/metrics/metrics.go index e6563d00e0..614ba3401d 100644 --- a/pkg/sippyserver/metrics/metrics.go +++ b/pkg/sippyserver/metrics/metrics.go @@ -17,8 +17,8 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness" "github.com/openshift/sippy/pkg/apis/cache" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" @@ -102,7 +102,7 @@ var ( }, []string{"release", "compare_release", "platform", "backend", "upgrade_type", "master_nodes_updated", "network", "topology", "architecture", "feature_set", "os", "releaseStatus"}) ) -func getReleaseStatus(releases []v1.Release, release string) string { +func getReleaseStatus(releases []models.ReleaseDefinition, release string) string { releaseStatus := releaseStatusEOL for _, r := range releases { if r.Release == release && len(r.Status) != 0 { @@ -119,10 +119,10 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP start := time.Now() log.Info("beginning refresh metrics") - var releases []v1.Release - if crProvider != nil { + var releases []models.ReleaseDefinition + if dbc != nil { var err error - releases, err = crProvider.QueryReleases(ctx) + releases, err = api.GetReleasesFromDB(ctx, dbc) if err != nil { return err } @@ -186,7 +186,7 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP } func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, - cacheOptions cache.RequestOptions, views []crview.View, releases []v1.Release) { + cacheOptions cache.RequestOptions, views []crview.View, releases []models.ReleaseDefinition) { for _, view := range views { if view.Metrics.Enabled { err := updateComponentReadinessMetricsForView(ctx, provider, dbc, cacheOptions, view, releases) @@ -200,7 +200,7 @@ func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider } // updateComponentReadinessTrackingForView queries the report for the given view, and then updates metrics. -func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []v1.Release) error { +func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []models.ReleaseDefinition) error { logger := log.WithField("view", view.Name) logger.Info("generating report for view") @@ -291,9 +291,9 @@ func refreshBuildClusterMetrics(dbc *db.DB, reportEnd time.Time) error { return nil } -func refreshPayloadMetrics(dbc *db.DB, reportEnd time.Time, releases []v1.Release) { +func refreshPayloadMetrics(dbc *db.DB, reportEnd time.Time, releases []models.ReleaseDefinition) { for _, r := range releases { - if !r.Capabilities[v1.MetricsCap] { + if !r.HasCapability(models.CapMetrics) { continue } results, err := api.ReleaseHealthReports(dbc, r.Release, reportEnd) @@ -339,7 +339,7 @@ func refreshPayloadMetrics(dbc *db.DB, reportEnd time.Time, releases []v1.Releas // refreshDisruptionMetrics queries our BigQuery views for current release vs two weeks ago, and previous release GA. // Metrics are published for the delta for each NURP which can then be alerted on if certain thresholds are exceeded. // The previous GA view should have its release and GA date updated on each release GA. -func refreshDisruptionMetrics(client *bqclient.Client, releases []v1.Release) error { +func refreshDisruptionMetrics(client *bqclient.Client, releases []models.ReleaseDefinition) error { if client == nil || client.BQ == nil { log.Warningf("not generating disruption metrics as we don't have a bigquery client") return nil @@ -382,11 +382,11 @@ type promReportType struct { period string } -func buildPromReportTypes(releases []v1.Release) []promReportType { +func buildPromReportTypes(releases []models.ReleaseDefinition) []promReportType { var promReportTypes []promReportType for _, release := range releases { - if !release.Capabilities[v1.MetricsCap] { + if !release.HasCapability(models.CapMetrics) { continue } promReportTypes = append(promReportTypes, promReportType{release: release.Release, period: string(sippyprocessingv1.TwoDayReport)}) diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 4899ea25a4..9115fe6a58 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -51,7 +51,6 @@ import ( "github.com/openshift/sippy/pkg/api/jobrunintervals" apitype "github.com/openshift/sippy/pkg/apis/api" "github.com/openshift/sippy/pkg/apis/cache" - sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" sippybq "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/dailysummary" @@ -189,17 +188,12 @@ type Server struct { rateLimiters map[string]*rateLimiter } -// getReleases returns release data, preferring the BigQuery client with caching -// when available, falling back to the data provider for mock mode. -func (s *Server) getReleases(ctx context.Context, forceRefresh ...bool) ([]sippyv1.Release, error) { - if s.bigQueryClient != nil { - refresh := len(forceRefresh) > 0 && forceRefresh[0] - return api.GetReleases(ctx, s.bigQueryClient, refresh) +// getReleases returns release data from PostgreSQL. +func (s *Server) getReleases(ctx context.Context) ([]models.ReleaseDefinition, error) { + if s.db == nil { + return nil, fmt.Errorf("no database available for releases") } - if s.crDataProvider != nil { - return s.crDataProvider.QueryReleases(ctx) - } - return nil, fmt.Errorf("no data source available for releases") + return api.GetReleasesFromDB(ctx, s.db) } type rateLimiter struct { @@ -907,8 +901,8 @@ func (s *Server) jsonTestRunsAndOutputsFromBigQuery(w http.ResponseWriter, req * outputs, err := api.GetTestRunsAndOutputsFromBigQuery(req.Context(), s.bigQueryClient, testID, prowJobRunIDList, prowJobNames, includeSuccess, startDate, endDate) if err != nil { - log.WithError(err).Error("error querying test runs from bigquery") - failureResponse(w, http.StatusInternalServerError, "error querying test runs from bigquery") + log.WithError(err).Error("error querying test runs from database") + failureResponse(w, http.StatusInternalServerError, "error querying test runs from database") return } @@ -922,11 +916,11 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re } outputs, errs := componentreadiness.GetComponentTestVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying test variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying test variants from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -939,11 +933,11 @@ func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Re } outputs, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying job variants from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1034,7 +1028,7 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo baseURL, ) if len(errs) > 0 { - return componentreport.ComponentReport{}, fmt.Errorf("error querying component from big query: %v", errs) + return componentreport.ComponentReport{}, fmt.Errorf("error querying component from database: %v", errs) } // Add any warnings from parsing to the report @@ -1080,11 +1074,11 @@ func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWrite baseURL := api.GetBaseURL(req) outputs, errs := componentreadiness.GetTestDetails(req.Context(), s.crDataProvider, s.db, reqOptions, allReleases, baseURL) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying component test details from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying component test details from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1153,8 +1147,7 @@ func (s *Server) jsonTestDetailsReportFromDB(w http.ResponseWriter, req *http.Re } func (s *Server) jsonReleasesReportFromDB(w http.ResponseWriter, req *http.Request) { - forceRefresh := req.URL.Query().Get("forceRefresh") != "" - releases, err := s.getReleases(req.Context(), forceRefresh) + releases, err := s.getReleases(req.Context()) if err != nil { log.WithError(err).Error("error querying releases") failureResponse(w, http.StatusInternalServerError, "error querying releases") diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 92fe0ad58e..d5a4c449d8 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" ) // TruncateAligned rounds t down to the nearest multiple of factor, aligned to a @@ -125,7 +125,7 @@ var releaseRelativeRE = regexp.MustCompile(`^(now|ga|end)(?:-([0-9]+)([d]))?$`) // For isStart=false we would round up to 23:59:59. // // endTime must be specified if your timeStr uses the end directive. (end-90d) Otherwise it is not required or used. -func ParseCRReleaseTime(allReleases []v1.Release, release, timeStr string, isStart bool, endTime *time.Time, crTimeRoundingFactor, crTimeRoundingOffset time.Duration) (time.Time, error) { +func ParseCRReleaseTime(allReleases []models.ReleaseDefinition, release, timeStr string, isStart bool, endTime *time.Time, crTimeRoundingFactor, crTimeRoundingOffset time.Duration) (time.Time, error) { var relTime time.Time diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 4f1110c823..ccffa66b90 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -4,13 +4,13 @@ import ( "testing" "time" - v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestParseCRReleaseTime(t *testing.T) { - releases := []v1.Release{ + releases := []models.ReleaseDefinition{ {Release: "4.16", Status: "", GADate: DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, } diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/componentreadiness_test.go index 770744a1cf..2ecb477400 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/componentreadiness_test.go @@ -62,9 +62,9 @@ func TestRegressionCacheLoader(t *testing.T) { require.NoError(t, err, "error parsing seed views") require.Greater(t, len(sippyViews.ComponentReadiness), 0, "no views found in seed-views.yaml") - // Get release configs from BigQuery - releaseConfigs, err := api.GetReleasesFromBigQuery(ctx, bqClient) - require.NoError(t, err, "error getting releases from bigquery") + // Get release configs from the database + releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc) + require.NoError(t, err, "error getting releases from database") // Build a regression store regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, nil)