Skip to content

Move release metadata from BigQuery to PostgreSQL #3674

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mstaeble wants to merge 1 commit into from
Closed

Move release metadata from BigQuery to PostgreSQL #3674

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions cmd/sippy/automatejira.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,12 @@ func NewAutomateJiraCommand() *cobra.Command {
if err != nil {
log.WithError(err).Fatal("unable to load views")
}
releases, err := api.GetReleases(context.Background(), bigQueryClient, false)

dbc, err := f.PostgresFlags.GetDBClient()
if err != nil {
log.WithError(err).Fatal("unable to connect to postgres")
}
releases, err := api.GetReleasesFromDB(ctx, dbc)
if err != nil {
log.WithError(err).Fatal("error querying releases")
}
Expand All @@ -178,10 +183,6 @@ func NewAutomateJiraCommand() *cobra.Command {
return errors.WithMessage(err, "error validating options")
}

dbc, err := f.PostgresFlags.GetDBClient()
if err != nil {
log.WithError(err).Fatal("unable to connect to postgres")
}
j, err := jiraautomator.NewJiraAutomator(
jiraClient, bigQueryClient, provider, dbc, cacheOpts,
views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount,
Expand Down
54 changes: 33 additions & 21 deletions cmd/sippy/load.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"cloud.google.com/go/bigquery"
"github.com/openshift/sippy/pkg/api"
"github.com/openshift/sippy/pkg/api/componentreadiness"
sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
"github.com/openshift/sippy/pkg/dataloader/regressioncacheloader"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
Expand All @@ -37,10 +36,12 @@ import (
"github.com/openshift/sippy/pkg/dataloader/prowloader"
"github.com/openshift/sippy/pkg/dataloader/prowloader/gcs"
"github.com/openshift/sippy/pkg/dataloader/prowloader/github"
releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader"
"github.com/openshift/sippy/pkg/dataloader/releaseloader"
"github.com/openshift/sippy/pkg/dataloader/testownershiploader"
"github.com/openshift/sippy/pkg/db"
"github.com/openshift/sippy/pkg/db/dailysummary"
"github.com/openshift/sippy/pkg/db/models"
"github.com/openshift/sippy/pkg/flags"
"github.com/openshift/sippy/pkg/github/commenter"
)
Expand Down Expand Up @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) {
f.JiraFlags.BindFlags(fs)

fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading")
fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading")
fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading")
fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)")
fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)")
fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader")
Expand Down Expand Up @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command {
cacheClient = nil // error hygiene, since we pass this down to quite a few functions
}

releaseConfigs := []sippyv1.Release{}

// initializing a bigquery client different from the normal one
opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load")
bqc, bigqueryErr := bqcachedclient.New(
Expand All @@ -164,20 +163,23 @@ func NewLoadCommand() *cobra.Command {
if f.CacheFlags.EnablePersistentCaching {
bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc)
}
releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc)
if err != nil {
return errors.Wrapf(err, "error querying releases from bq")
}
}

// Read release definitions from PG for downstream loader construction.
// On the first run the table may be empty; the release-definitions
// loader will populate it for subsequent runs.
var releaseDefs []models.ReleaseDefinition
if dbErr == nil {
releaseDefs, _ = api.GetReleasesFromDB(context.Background(), dbc)
}

// Ensure partitions exist for all releases (only when InitDatabase is true)
if f.InitDatabase && dbErr == nil {
err = ensurePartitionsForReleases(dbc, releaseConfigs)
err = ensurePartitionsForReleases(dbc, releaseDefs)
if err != nil {
return errors.Wrapf(err, "error ensuring partitions")
}

// Clean up old partitions
detached, dropped, err := dbc.CleanupPartitions(false)
if err != nil {
log.WithError(err).Warning("failed to cleanup old partitions, continuing with load")
Expand All @@ -201,6 +203,17 @@ func NewLoadCommand() *cobra.Command {

var regressionCacheAdded bool
for _, l := range f.Loaders {
if l == "release-definitions" {
if bigqueryErr != nil {
return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading")
}
if dbErr != nil {
return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading")
}
rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc)
loaders = append(loaders, rdl)
}

// TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob
// manifests are updated to use "regression-cache".
if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" {
Expand Down Expand Up @@ -237,7 +250,7 @@ func NewLoadCommand() *cobra.Command {
regressionStore := componentreadiness.NewPostgresRegressionStore(dbc, jiraClient)

rcl, err := regressioncacheloader.New(
dbc, bqc, config, views.ComponentReadiness, releaseConfigs,
dbc, bqc, config, views.ComponentReadiness, releaseDefs,
f.ComponentReadinessFlags.CRTimeRoundingFactor,
f.ComponentReadinessFlags.CRTimeRoundingOffset,
regressionStore,
Expand All @@ -252,7 +265,7 @@ func NewLoadCommand() *cobra.Command {
if dbErr != nil {
return dbErr
}
loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseConfigs))
loaders = append(loaders, releaseloader.New(ctx, dbc, bqc, f.Releases, f.Architectures, releaseDefs))
}

// Prow Loader
Expand All @@ -261,7 +274,7 @@ func NewLoadCommand() *cobra.Command {
if dbErr != nil {
return dbErr
}
prowLoader, err := f.prowLoader(ctx, dbc, config, releaseConfigs, promPusher)
prowLoader, err := f.prowLoader(ctx, dbc, config, releaseDefs, promPusher)
if err != nil {
return err
}
Expand Down Expand Up @@ -331,7 +344,7 @@ func NewLoadCommand() *cobra.Command {
// Feature gates
if l == "feature-gates" {
refreshMatviews = true
fgLoader := featuregateloader.New(dbc, releaseConfigs)
fgLoader := featuregateloader.New(dbc, releaseDefs)
loaders = append(loaders, fgLoader)
}

Expand Down Expand Up @@ -418,7 +431,7 @@ func (f *LoadFlags) jobVariantsLoader(ctx context.Context) (dataloader.DataLoade

}

func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseConfigs []sippyv1.Release, promPusher *push.Pusher) (dataloader.DataLoader, error) {
func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.SippyConfig, releaseDefs []models.ReleaseDefinition, promPusher *push.Pusher) (dataloader.DataLoader, error) {
gcsClient, err := gcs.NewGCSClient(ctx,
f.GoogleCloudFlags.ServiceAccountCredentialFile,
f.GoogleCloudFlags.OAuthClientCredentialFile,
Expand Down Expand Up @@ -453,8 +466,8 @@ func (f *LoadFlags) prowLoader(ctx context.Context, dbc *db.DB, sippyConfig *v1.

releases := f.Releases
if len(releases) == 0 { // if not specified, use those defined in the Releases table
for _, config := range releaseConfigs {
releases = append(releases, config.Release) // could filter by capability if needed
for _, def := range releaseDefs {
releases = append(releases, def.Release) // could filter by capability if needed
}
}

Expand Down Expand Up @@ -504,10 +517,9 @@ func parseProwLoadSince(val string) (time.Time, error) {
// ensurePartitionsForReleases creates partitions for all configured releases.
// It uses a 7 day lookback window plus 2 days forward from today.
// Errors are logged but ignored to prevent blocking the load process.
func ensurePartitionsForReleases(dbc *db.DB, releaseConfigs []sippyv1.Release) error {
// Extract release names from release configs
releases := make([]string, 0, len(releaseConfigs))
for _, r := range releaseConfigs {
func ensurePartitionsForReleases(dbc *db.DB, releaseDefs []models.ReleaseDefinition) error {
releases := make([]string, 0, len(releaseDefs))
for _, r := range releaseDefs {
releases = append(releases, r.Release)
}

Expand Down
55 changes: 54 additions & 1 deletion cmd/sippy/seed_data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/spf13/pflag"
"gopkg.in/yaml.v3"

"github.com/openshift/sippy/pkg/api"
componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness"
pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
Expand Down Expand Up @@ -411,6 +412,11 @@ func seedSyntheticData(dbc *db.DB) error {
return nil
}

if err := seedReleaseDefinitions(dbc); err != nil {
return errors.WithMessage(err, "failed to seed release definitions")
}
log.Info("Seeded release definitions")

if err := createTestSuite(dbc, "synthetic"); err != nil {
return errors.WithMessage(err, "failed to create test suite")
}
Expand Down Expand Up @@ -447,6 +453,53 @@ func seedSyntheticData(dbc *db.DB) error {
return nil
}

func seedReleaseDefinitions(dbc *db.DB) error {
now := time.Now().UTC()
allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic}

type relMeta struct {
previous string
gaDays int // negative = days before now; 0 = no GA (in development)
}
meta := map[string]relMeta{
"4.19": {previous: "4.18", gaDays: -289},
"4.20": {previous: "4.19", gaDays: -163},
"4.21": {previous: "4.20", gaDays: -58},
"4.22": {previous: "4.21"},
}

for _, release := range syntheticReleases {
m := meta[release]
parts := strings.Split(release, ".")
major, minor := 0, 0
if len(parts) >= 2 {
fmt.Sscanf(parts[0], "%d", &major)
fmt.Sscanf(parts[1], "%d", &minor)
}

develStart := now.AddDate(0, 0, m.gaDays-180)
def := models.ReleaseDefinition{
Release: release,
Major: major,
Minor: minor,
PreviousRelease: m.previous,
DevelopmentStartDate: &develStart,
Product: "OCP",
Status: "Full Support",
Capabilities: allCaps,
}
if m.gaDays != 0 {
ga := now.AddDate(0, 0, m.gaDays)
def.GADate = &ga
}

if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil {
return fmt.Errorf("failed to create release definition %s: %w", release, err)
}
}
return nil
}

func seedProwJobs(dbc *db.DB) error {
for _, release := range syntheticReleases {
for _, job := range syntheticJobs {
Expand Down Expand Up @@ -694,7 +747,7 @@ func syncRegressions(dbc *db.DB) error {
provider := pgprovider.NewPostgresProvider(dbc, nil)
ctx := context.Background()

releases, err := provider.QueryReleases(ctx)
releases, err := api.GetReleasesFromDB(ctx, dbc)
if err != nil {
return fmt.Errorf("querying releases: %w", err)
}
Expand Down
10 changes: 5 additions & 5 deletions pkg/api/componentreadiness/component_report.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ import (
"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
crtype "github.com/openshift/sippy/pkg/apis/api/componentreport"
"github.com/openshift/sippy/pkg/apis/cache"
v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
"github.com/openshift/sippy/pkg/db"
"github.com/openshift/sippy/pkg/db/models"
"github.com/openshift/sippy/pkg/util/sets"
)

Expand Down Expand Up @@ -82,7 +82,7 @@ func GetComponentReport(
reqOptions reqopts.RequestOptions,
baseURL string,
) (report crtype.ComponentReport, errs []error) {
releaseConfigs, err := provider.QueryReleases(ctx)
releaseConfigs, err := api.GetReleasesFromDB(ctx, dbc)
if err != nil {
return report, []error{err}
}
Expand Down Expand Up @@ -151,7 +151,7 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport)
return nil
}

func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator {
func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []models.ReleaseDefinition, baseURL string) ComponentReportGenerator {
slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match
generator := ComponentReportGenerator{
dataProvider: provider,
Expand All @@ -175,7 +175,7 @@ type ComponentReportGenerator struct {
dbc *db.DB
ReqOptions reqopts.RequestOptions
middlewares middleware.List
releaseConfigs []v1.Release
releaseConfigs []models.ReleaseDefinition
baseURL string
}

Expand Down Expand Up @@ -278,7 +278,7 @@ func (c *ComponentReportGenerator) initializeMiddleware() {
c.middlewares = middleware.List{}
// Initialize all our middleware applicable to this request.
if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil {
c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs))
c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.dbc, c.ReqOptions, c.releaseConfigs))
}
if c.dbc != nil {
c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions))
Expand Down
9 changes: 0 additions & 9 deletions pkg/api/componentreadiness/dataprovider/bigquery/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
apiCache "github.com/openshift/sippy/pkg/apis/cache"
v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
"github.com/openshift/sippy/pkg/bigquery/bqlabel"
"github.com/openshift/sippy/pkg/util/param"
Expand Down Expand Up @@ -170,14 +169,6 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari
return variants, nil
}

func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) {
return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions)
}

func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) {
return apiPkg.GetReleasesFromBigQuery(ctx, p.client)
}

func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) {
unnest := ""
if nested {
Expand Down
44 changes: 0 additions & 44 deletions pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go
View file
Open in desktop

This file was deleted.

7 changes: 0 additions & 7 deletions pkg/api/componentreadiness/dataprovider/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
"github.com/openshift/sippy/pkg/apis/cache"
v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
)

// TestStatusQuerier fetches aggregated test pass/fail counts.
Expand Down Expand Up @@ -40,12 +39,6 @@ type MetadataQuerier interface {
// QueryJobVariants returns all variant names and their possible values.
QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error)

// QueryReleaseDates returns the time ranges for each known release.
QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error)

// QueryReleases returns known release configurations.
QueryReleases(ctx context.Context) ([]v1.Release, error)

// QueryUniqueVariantValues returns distinct values for a variant column
// from the past 60 days.
QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error)
Expand Down
Loading