opt

image-scanner/App.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"github.com/caarlos0/env"
 	"github.com/devtron-labs/image-scanner/pkg/middleware"
+	"github.com/devtron-labs/image-scanner/pkg/recovery"
 	"net/http"
 	"os"
 	"time"
@@ -41,17 +42,19 @@ type App struct {
 	db               *pg.DB
 	natsSubscription *pubsub.NatSubscriptionImpl
 	pubSubClient     *client.PubSubClientServiceImpl
+	recoveryManager  *recovery.RecoveryManager
 }
 
 func NewApp(Router *api.Router, Logger *zap.SugaredLogger,
 	db *pg.DB, natsSubscription *pubsub.NatSubscriptionImpl,
-	pubSubClient *client.PubSubClientServiceImpl) *App {
+	pubSubClient *client.PubSubClientServiceImpl, recoveryManager *recovery.RecoveryManager) *App {
 	return &App{
 		Router:           Router,
 		Logger:           Logger,
 		db:               db,
 		natsSubscription: natsSubscription,
 		pubSubClient:     pubSubClient,
+		recoveryManager:  recoveryManager,
 	}
 }
 
@@ -73,6 +76,13 @@ func (app *App) Start() {
 	app.Router.Router.Use(middleware.PrometheusMiddleware)
 	app.Router.Router.Use(middlewares.Recovery)
 	app.server = server
+
+	// Start the recovery manager if enabled
+	if app.recoveryManager != nil {
+		app.Logger.Infow("starting recovery manager")
+		app.recoveryManager.Start()
+	}
+
 	err = server.ListenAndServe()
 	if err != nil && !errors.Is(err, http.ErrServerClosed) {
 		app.Logger.Errorw("error in startup", "err", err)
@@ -84,6 +94,13 @@ func (app *App) Stop() {
 	app.Logger.Infow("image scanner shutdown initiating")
 	timeoutContext, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
+
+	// Stop the recovery manager if it exists
+	if app.recoveryManager != nil {
+		app.Logger.Infow("stopping recovery manager")
+		app.recoveryManager.Stop()
+	}
+
 	app.Logger.Infow("closing router")
 	err := app.server.Shutdown(timeoutContext)
 	if err != nil {

image-scanner/Wire.go

@@ -27,6 +27,7 @@ import (
 	"github.com/devtron-labs/image-scanner/pkg/grafeasService"
 	"github.com/devtron-labs/image-scanner/pkg/klarService"
 	"github.com/devtron-labs/image-scanner/pkg/logger"
+	"github.com/devtron-labs/image-scanner/pkg/recovery"
 	"github.com/devtron-labs/image-scanner/pkg/roundTripper"
 	"github.com/devtron-labs/image-scanner/pkg/security"
 	"github.com/devtron-labs/image-scanner/pkg/sql"
@@ -98,6 +99,11 @@ func InitializeApp() (*App, error) {
 		monitoring.NewMonitoringRouter,
 
 		roundTripper.NewRoundTripperServiceImpl,
+
+		// Recovery Manager
+		recovery.NewRecoveryManager,
+		api.NewRecoveryHandlerImpl,
+		wire.Bind(new(api.RecoveryHandler), new(*api.RecoveryHandlerImpl)),
 		wire.Bind(new(roundTripper.RoundTripperService), new(*roundTripper.RoundTripperServiceImpl)),
 	)
 	return &App{}, nil

image-scanner/api/RecoveryHandler.go

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	"net/http"
+
+	"github.com/devtron-labs/image-scanner/pkg/recovery"
+	"go.uber.org/zap"
+)
+
+type RecoveryHandler interface {
+	GetRecoveryStatus(w http.ResponseWriter, r *http.Request)
+	StartRecovery(w http.ResponseWriter, r *http.Request)
+	StopRecovery(w http.ResponseWriter, r *http.Request)
+}
+
+type RecoveryHandlerImpl struct {
+	Logger          *zap.SugaredLogger
+	RecoveryManager *recovery.RecoveryManager
+}
+
+func NewRecoveryHandlerImpl(logger *zap.SugaredLogger, recoveryManager *recovery.RecoveryManager) *RecoveryHandlerImpl {
+	return &RecoveryHandlerImpl{
+		Logger:          logger,
+		RecoveryManager: recoveryManager,
+	}
+}
+
+func (impl *RecoveryHandlerImpl) GetRecoveryStatus(w http.ResponseWriter, r *http.Request) {
+	metrics := impl.RecoveryManager.GetMetrics()
+	WriteJsonResp(w, nil, metrics, http.StatusOK)
+}
+
+func (impl *RecoveryHandlerImpl) StartRecovery(w http.ResponseWriter, r *http.Request) {
+	impl.RecoveryManager.Start()
+	WriteJsonResp(w, nil, map[string]string{"status": "recovery process started"}, http.StatusOK)
+}
+
+func (impl *RecoveryHandlerImpl) StopRecovery(w http.ResponseWriter, r *http.Request) {
+	impl.RecoveryManager.Stop()
+	WriteJsonResp(w, nil, map[string]string{"status": "recovery process stopped"}, http.StatusOK)
+}

image-scanner/api/RecoveryHandler_test.go

@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/devtron-labs/image-scanner/pkg/recovery"
+	"github.com/devtron-labs/image-scanner/pkg/security"
+	"go.uber.org/zap"
+)
+
+func TestRecoveryHandler(t *testing.T) {
+	// Create a logger
+	logger, _ := zap.NewDevelopment()
+	sugaredLogger := logger.Sugar()
+
+	// Create a config
+	config := &security.ImageScanConfig{
+		EnableProgressingScanCheck: true,
+		RecoveryBatchSize:          10,
+		RecoveryBatchDelaySeconds:  1,
+		RecoveryMaxWorkers:         3,
+		RecoveryStartDelaySeconds:  1,
+	}
+
+	// Create a recovery manager
+	rm := recovery.NewRecoveryManager(
+		sugaredLogger,
+		config,
+		nil,
+		nil,
+		nil,
+		nil,
+	)
+
+	// Create a recovery handler
+	handler := NewRecoveryHandlerImpl(sugaredLogger, rm)
+
+	// Test GetRecoveryStatus
+	t.Run("GetRecoveryStatus", func(t *testing.T) {
+		req, err := http.NewRequest("GET", "/recovery/status", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rr := httptest.NewRecorder()
+		handlerFunc := http.HandlerFunc(handler.GetRecoveryStatus)
+		handlerFunc.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != http.StatusOK {
+			t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
+		}
+
+		var response struct {
+			Code   int                      `json:"code"`
+			Status string                   `json:"status"`
+			Result recovery.RecoveryMetrics `json:"result"`
+		}
+		err = json.Unmarshal(rr.Body.Bytes(), &response)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if response.Code != 200 {
+			t.Errorf("expected response code 200, got %d", response.Code)
+		}
+	})
+
+	// Test StartRecovery
+	t.Run("StartRecovery", func(t *testing.T) {
+		req, err := http.NewRequest("POST", "/recovery/start", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rr := httptest.NewRecorder()
+		handlerFunc := http.HandlerFunc(handler.StartRecovery)
+		handlerFunc.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != http.StatusOK {
+			t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
+		}
+
+		// Check that recovery is running
+		metrics := rm.GetMetrics()
+		if !metrics.IsRunning {
+			t.Error("Recovery should be running after StartRecovery")
+		}
+	})
+
+	// Test StopRecovery
+	t.Run("StopRecovery", func(t *testing.T) {
+		req, err := http.NewRequest("POST", "/recovery/stop", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rr := httptest.NewRecorder()
+		handlerFunc := http.HandlerFunc(handler.StopRecovery)
+		handlerFunc.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != http.StatusOK {
+			t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
+		}
+
+		// Wait a bit for the goroutine to stop
+		time.Sleep(100 * time.Millisecond)
+
+		// Check that recovery is stopped
+		metrics := rm.GetMetrics()
+		if metrics.IsRunning {
+			t.Error("Recovery should be stopped after StopRecovery")
+		}
+	})
+}

image-scanner/api/Router.go

@@ -29,11 +29,12 @@ type Router struct {
 	logger           *zap.SugaredLogger
 	Router           *mux.Router
 	restHandler      RestHandler
+	recoveryHandler  RecoveryHandler
 	monitoringRouter *monitoring.MonitoringRouter
 }
 
-func NewRouter(logger *zap.SugaredLogger, restHandler RestHandler, monitoringRouter *monitoring.MonitoringRouter) *Router {
-	return &Router{logger: logger, Router: mux.NewRouter(), restHandler: restHandler, monitoringRouter: monitoringRouter}
+func NewRouter(logger *zap.SugaredLogger, restHandler RestHandler, recoveryHandler RecoveryHandler, monitoringRouter *monitoring.MonitoringRouter) *Router {
+	return &Router{logger: logger, Router: mux.NewRouter(), restHandler: restHandler, recoveryHandler: recoveryHandler, monitoringRouter: monitoringRouter}
 }
 
 func (r Router) Init() {
@@ -59,4 +60,8 @@ func (r Router) Init() {
 	r.Router.Path("/scanner/image").HandlerFunc(r.restHandler.ScanForVulnerability).Methods("POST")
 	r.Router.Path("/scanner/save-result").HandlerFunc(r.restHandler.RegisterAndSaveScannedResult).Methods("POST")
 
+	// Recovery endpoints
+	r.Router.Path("/recovery/status").HandlerFunc(r.recoveryHandler.GetRecoveryStatus).Methods("GET")
+	r.Router.Path("/recovery/start").HandlerFunc(r.recoveryHandler.StartRecovery).Methods("POST")
+	r.Router.Path("/recovery/stop").HandlerFunc(r.recoveryHandler.StopRecovery).Methods("POST")
 }

image-scanner/config.md

@@ -4,7 +4,11 @@
 | Variable Name       | Value                                  | Description                   |
 |---------------------|----------------------------------------|-------------------------------|
 | CLAIR_ADDR          | clair-dcd.devtroncd:6060               | For connecting to Clair if it's enabled |
-| ENABLE_PROGRESSING_SCAN_CHECK | "true"                                | Flag to enable/disable checking for progressing scans at startup (set to "false" to improve startup performance) |
+| ENABLE_PROGRESSING_SCAN_CHECK | "true"                                | Flag to enable/disable checking for progressing scans (set to "false" to disable recovery) |
+| RECOVERY_BATCH_SIZE | "10"                                  | Number of scans to process in each batch during recovery |
+| RECOVERY_BATCH_DELAY_SECONDS | "5"                                   | Delay between processing batches in seconds |
+| RECOVERY_MAX_WORKERS | "3"                                   | Maximum number of concurrent workers for recovery |
+| RECOVERY_START_DELAY_SECONDS | "10"                                  | Delay before starting recovery process after startup |
 | CLIENT_ID           | client-2                               | Client ID                        |
 | NATS_SERVER_HOST    | nats://devtron-nats.devtroncd:4222    | For connecting to NATS         |
 | PG_LOG_QUERY        | "false"                                | PostgreSQL Query Logging (false to disable) |

image-scanner/env_gen.json

@@ -1 +1 @@
-[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_PROGRESSING_SCAN_CHECK","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}]
+[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_PROGRESSING_SCAN_CHECK","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RECOVERY_BATCH_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RECOVERY_BATCH_SIZE","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RECOVERY_MAX_WORKERS","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RECOVERY_START_DELAY_SECONDS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}]

image-scanner/env_gen.md

@@ -32,6 +32,10 @@
  | PG_QUERY_DUR_THRESHOLD | int64 |5000 |  |  | false |
  | PG_USER | string | |  |  | false |
  | PROJECT_ID | string |projects/devtron-project-id |  |  | false |
+ | RECOVERY_BATCH_DELAY_SECONDS | int |5 |  |  | false |
+ | RECOVERY_BATCH_SIZE | int |10 |  |  | false |
+ | RECOVERY_MAX_WORKERS | int |3 |  |  | false |
+ | RECOVERY_START_DELAY_SECONDS | int |10 |  |  | false |
  | SCANNER_TYPE | string | |  |  | false |
  | SERVER_HTTP_PORT | int |8080 |  |  | false |
  | STREAM_CONFIG_JSON | string | |  |  | false |