chore: optimize stack fetching in deployment operator (#362)

zreigz · web-flow · commit 5a2c8def576f · 2025-02-07T09:28:18.000+01:00
* optimize stack fetching in deployment operator

* bump helm/kind-action

* update go client

* update go client

* fix linter
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -16,7 +16,7 @@ jobs:
           go-version-file: go.mod
           cache: false
       - name: Create kind cluster
-        uses: helm/kind-action@v1.10.0
+        uses: helm/kind-action@v1.12.0
       - run: kind get clusters
       - run: go test -v -race ./pkg/cache/... -tags="e2e"
       - run: go test -v -race ./pkg/applier/filters/... -tags="e2e"
diff --git a/go.mod b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/opencost/opencost/core v0.0.0-20241216191657-30e5d9a27f41
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/pkg/errors v0.9.1
-	github.com/pluralsh/console/go/client v1.28.2
+	github.com/pluralsh/console/go/client v1.28.3
 	github.com/pluralsh/controller-reconcile-helper v0.1.0
 	github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34
 	github.com/pluralsh/polly v0.1.10
diff --git a/go.sum b/go.sum
@@ -1199,8 +1199,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pluralsh/console/go/client v1.28.2 h1:fNK0xPuGtbGKK1RTbGyRXP9HN5nCHoZUddRxYIdoWUE=
-github.com/pluralsh/console/go/client v1.28.2/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k=
+github.com/pluralsh/console/go/client v1.28.3 h1:KyHpOoVHhMRMAZp99k7LqZqX6+FrOwy9sGVXaDSi9FA=
+github.com/pluralsh/console/go/client v1.28.3/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k=
 github.com/pluralsh/controller-reconcile-helper v0.1.0 h1:BV3dYZFH5rn8ZvZjtpkACSv/GmLEtRftNQj/Y4ddHEo=
 github.com/pluralsh/controller-reconcile-helper v0.1.0/go.mod h1:RxAbvSB4/jkvx616krCdNQXPbpGJXW3J1L3rASxeFOA=
 github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 h1:ab2PN+6if/Aq3/sJM0AVdy1SYuMAnq4g20VaKhTm/Bw=
diff --git a/pkg/client/console.go b/pkg/client/console.go
@@ -71,7 +71,7 @@ type Client interface {
 	CompleteStackRun(id string, attributes console.StackRunAttributes) error
 	UpdateStackRun(id string, attributes console.StackRunAttributes) error
 	UpdateStackRunStep(id string, attributes console.RunStepAttributes) error
-	ListClusterStackRuns(after *string, first *int64) (*console.ListClusterStackIds_ClusterStackRuns, error)
+	ListClusterStackRuns(after *string, first *int64) (*console.ListClusterMinimalStacks_ClusterStackRuns, error)
 	GetUser(email string) (*console.UserFragment, error)
 	GetGroup(name string) (*console.GroupFragment, error)
 	SaveUpgradeInsights(attributes []*console.UpgradeInsightAttributes, addons []*console.CloudAddonAttributes) (*console.SaveUpgradeInsights, error)
diff --git a/pkg/client/stack.go b/pkg/client/stack.go
@@ -72,8 +72,8 @@ func (c *client) GetStackRun(id string) (*gqlclient.StackRunMinimalFragment, err
 	return restore.StackRun, nil
 }
 
-func (c *client) ListClusterStackRuns(after *string, first *int64) (*gqlclient.ListClusterStackIds_ClusterStackRuns, error) {
-	resp, err := c.consoleClient.ListClusterStackIds(c.ctx, after, first, nil, nil)
+func (c *client) ListClusterStackRuns(after *string, first *int64) (*gqlclient.ListClusterMinimalStacks_ClusterStackRuns, error) {
+	resp, err := c.consoleClient.ListClusterMinimalStacks(c.ctx, after, first, nil, nil)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/controller/service/reconciler_scraper.go b/pkg/controller/service/reconciler_scraper.go
@@ -2,10 +2,10 @@ package service
 
 import (
 	"context"
-	"github.com/pluralsh/deployment-operator/pkg/client"
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
+	"github.com/pluralsh/deployment-operator/pkg/client"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
diff --git a/pkg/controller/stacks/reconciler.go b/pkg/controller/stacks/reconciler.go
@@ -88,10 +88,10 @@ func (r *StackReconciler) ShutdownQueue() {
 	r.stackQueue.ShutDown()
 }
 
-func (r *StackReconciler) ListStacks(ctx context.Context) *algorithms.Pager[*console.StackRunIDEdgeFragment] {
+func (r *StackReconciler) ListStacks(ctx context.Context) *algorithms.Pager[*console.MinimalStackRunEdgeFragment] {
 	logger := log.FromContext(ctx)
 	logger.Info("create stack run pager")
-	fetch := func(page *string, size int64) ([]*console.StackRunIDEdgeFragment, *algorithms.PageInfo, error) {
+	fetch := func(page *string, size int64) ([]*console.MinimalStackRunEdgeFragment, *algorithms.PageInfo, error) {
 		resp, err := r.consoleClient.ListClusterStackRuns(page, &size)
 		if err != nil {
 			logger.Error(err, "failed to fetch stack run")
@@ -104,7 +104,7 @@ func (r *StackReconciler) ListStacks(ctx context.Context) *algorithms.Pager[*con
 		}
 		return resp.Edges, pageInfo, nil
 	}
-	return algorithms.NewPager[*console.StackRunIDEdgeFragment](common.DefaultPageSize, fetch)
+	return algorithms.NewPager[*console.MinimalStackRunEdgeFragment](common.DefaultPageSize, fetch)
 }
 
 func (r *StackReconciler) Poll(ctx context.Context) error {
@@ -120,6 +120,7 @@ func (r *StackReconciler) Poll(ctx context.Context) error {
 		}
 		for _, stack := range stacks {
 			logger.Info("sending update for", "stack run", stack.Node.ID)
+			r.stackCache.Add(stack.Node.ID, stack.Node)
 			r.stackQueue.Add(stack.Node.ID)
 		}
 	}
diff --git a/pkg/test/mocks/Client_mock.go b/pkg/test/mocks/Client_mock.go

Original file line number	Diff line number	Diff line change
`@@ -72,8 +72,8 @@ func (c client) GetStackRun(id string) (gqlclient.StackRunMinimalFragment, err`
`72`	`72`	`return restore.StackRun, nil`
`73`	`73`	`}`
`74`	`74`
`75`		`-func (c client) ListClusterStackRuns(after string, first int64) (gqlclient.ListClusterStackIds_ClusterStackRuns, error) {`
`76`		`- resp, err := c.consoleClient.ListClusterStackIds(c.ctx, after, first, nil, nil)`
	`75`	`+func (c client) ListClusterStackRuns(after string, first int64) (gqlclient.ListClusterMinimalStacks_ClusterStackRuns, error) {`
	`76`	`+ resp, err := c.consoleClient.ListClusterMinimalStacks(c.ctx, after, first, nil, nil)`
`77`	`77`	`if err != nil {`
`78`	`78`	`return nil, err`
`79`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -88,10 +88,10 @@ func (r *StackReconciler) ShutdownQueue() {`
`88`	`88`	`r.stackQueue.ShutDown()`
`89`	`89`	`}`
`90`	`90`
`91`		`-func (r StackReconciler) ListStacks(ctx context.Context) algorithms.Pager[*console.StackRunIDEdgeFragment] {`
	`91`	`+func (r StackReconciler) ListStacks(ctx context.Context) algorithms.Pager[*console.MinimalStackRunEdgeFragment] {`
`92`	`92`	`logger := log.FromContext(ctx)`
`93`	`93`	`logger.Info("create stack run pager")`
`94`		`- fetch := func(page string, size int64) ([]console.StackRunIDEdgeFragment, *algorithms.PageInfo, error) {`
	`94`	`+ fetch := func(page string, size int64) ([]console.MinimalStackRunEdgeFragment, *algorithms.PageInfo, error) {`
`95`	`95`	`resp, err := r.consoleClient.ListClusterStackRuns(page, &size)`
`96`	`96`	`if err != nil {`
`97`	`97`	`logger.Error(err, "failed to fetch stack run")`
`@@ -104,7 +104,7 @@ func (r StackReconciler) ListStacks(ctx context.Context) algorithms.Pager[*con`
`104`	`104`	`}`
`105`	`105`	`return resp.Edges, pageInfo, nil`
`106`	`106`	`}`
`107`		`- return algorithms.NewPager[*console.StackRunIDEdgeFragment](common.DefaultPageSize, fetch)`
	`107`	`+ return algorithms.NewPager[*console.MinimalStackRunEdgeFragment](common.DefaultPageSize, fetch)`
`108`	`108`	`}`
`109`	`109`
`110`	`110`	`func (r *StackReconciler) Poll(ctx context.Context) error {`
`@@ -120,6 +120,7 @@ func (r *StackReconciler) Poll(ctx context.Context) error {`
`120`	`120`	`}`
`121`	`121`	`for _, stack := range stacks {`
`122`	`122`	`logger.Info("sending update for", "stack run", stack.Node.ID)`
	`123`	`+ r.stackCache.Add(stack.Node.ID, stack.Node)`
`123`	`124`	`r.stackQueue.Add(stack.Node.ID)`
`124`	`125`	`}`
`125`	`126`	`}`