diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/common-lib/constants/constants.go b/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/common-lib/constants/constants.go +++ b/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/common-lib/utils/SqlUtil.go b/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/common-lib/utils/SqlUtil.go +++ b/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/image-scanner/Wire.go b/image-scanner/Wire.go index 45422689a..82cf642fe 100644 --- a/image-scanner/Wire.go +++ b/image-scanner/Wire.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/common-lib/monitoring" client "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/image-scanner/api" + "github.com/devtron-labs/image-scanner/pkg/asyncProvider" "github.com/devtron-labs/image-scanner/pkg/clairService" "github.com/devtron-labs/image-scanner/pkg/grafeasService" "github.com/devtron-labs/image-scanner/pkg/klarService" @@ -38,6 +39,7 @@ import ( func InitializeApp() (*App, error) { wire.Build( + asyncProvider.AsyncWireSet, NewApp, api.NewRouter, logger.NewSugardLogger, diff --git a/image-scanner/config.md b/image-scanner/config.md index 9fbe61721..959e3afa0 100644 --- a/image-scanner/config.md +++ b/image-scanner/config.md @@ -1,9 +1,10 @@ -# IMAGESCANER CONFIGMAP +# IMAGESCANER CONFIGMAP | Variable Name | Value | Description | |---------------------|----------------------------------------|-------------------------------| | CLAIR_ADDR | clair-dcd.devtroncd:6060 | For connecting to Clair if it's enabled | +| ENABLE_PROGRESSING_SCAN_CHECK | "true" | Flag to enable/disable checking for progressing scans at startup (set to "false" to improve startup performance) | | CLIENT_ID | client-2 | Client ID | | NATS_SERVER_HOST | nats://devtron-nats.devtroncd:4222 | For connecting to NATS | | PG_LOG_QUERY | "false" | PostgreSQL Query Logging (false to disable) | diff --git a/image-scanner/env_gen.json b/image-scanner/env_gen.json index 1d5fbcfc1..4c0437777 100644 --- a/image-scanner/env_gen.json +++ b/image-scanner/env_gen.json @@ -1 +1 @@ -[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_PROGRESSING_SCAN_CHECK","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/image-scanner/env_gen.md b/image-scanner/env_gen.md index 7d8c12043..eac53ee9c 100644 --- a/image-scanner/env_gen.md +++ b/image-scanner/env_gen.md @@ -8,6 +8,7 @@ | CLAIR_TIMEOUT | int |30 | | | false | | CONSUMER_CONFIG_JSON | string | | | | false | | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | ENABLE_PROGRESSING_SCAN_CHECK | bool |true | | | false | | ENABLE_STATSVIZ | bool |false | | | false | | IMAGE_SCAN_ASYNC_TIMEOUT | int |3 | | | false | | IMAGE_SCAN_TIMEOUT | int |10 | | | false | diff --git a/image-scanner/pkg/asyncProvider/AsyncProvider.go b/image-scanner/pkg/asyncProvider/AsyncProvider.go new file mode 100644 index 000000000..4531c0b01 --- /dev/null +++ b/image-scanner/pkg/asyncProvider/AsyncProvider.go @@ -0,0 +1,11 @@ +package asyncProvider + +import ( + "github.com/devtron-labs/common-lib/async" + "github.com/devtron-labs/common-lib/constants" + "go.uber.org/zap" +) + +func NewAsyncRunnable(logger *zap.SugaredLogger) *async.Runnable { + return async.NewAsyncRunnable(logger, constants.ImageScanner) +} diff --git a/image-scanner/pkg/asyncProvider/wire_AsyncProvider.go b/image-scanner/pkg/asyncProvider/wire_AsyncProvider.go new file mode 100644 index 000000000..1f0df311c --- /dev/null +++ b/image-scanner/pkg/asyncProvider/wire_AsyncProvider.go @@ -0,0 +1,9 @@ +package asyncProvider + +import ( + "github.com/google/wire" +) + +var AsyncWireSet = wire.NewSet( + NewAsyncRunnable, +) diff --git a/image-scanner/pkg/security/ImageScanService.go b/image-scanner/pkg/security/ImageScanService.go index be0d2ceac..d1c026f0d 100644 --- a/image-scanner/pkg/security/ImageScanService.go +++ b/image-scanner/pkg/security/ImageScanService.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/Knetic/govaluate" "github.com/caarlos0/env" + "github.com/devtron-labs/common-lib/async" bean2 "github.com/devtron-labs/common-lib/imageScan/bean" "github.com/devtron-labs/image-scanner/common" cliUtil "github.com/devtron-labs/image-scanner/internals/step-lib/util/cli-util" @@ -84,6 +85,7 @@ type ImageScanServiceImpl struct { DockerArtifactStoreRepository repository.DockerArtifactStoreRepository RegistryIndexMappingRepository repository.RegistryIndexMappingRepository CliCommandEnv []string + asyncRunnable *async.Runnable } func NewImageScanServiceImpl(logger *zap.SugaredLogger, scanHistoryRepository repository.ImageScanHistoryRepository, @@ -96,7 +98,8 @@ func NewImageScanServiceImpl(logger *zap.SugaredLogger, scanHistoryRepository re scanToolStepRepository repository.ScanToolStepRepository, scanStepConditionMappingRepository repository.ScanStepConditionMappingRepository, imageScanConfig *ImageScanConfig, - dockerArtifactStoreRepository repository.DockerArtifactStoreRepository, registryIndexMappingRepository repository.RegistryIndexMappingRepository) *ImageScanServiceImpl { + dockerArtifactStoreRepository repository.DockerArtifactStoreRepository, registryIndexMappingRepository repository.RegistryIndexMappingRepository, + asyncRunnable *async.Runnable) *ImageScanServiceImpl { imageScanService := &ImageScanServiceImpl{Logger: logger, ScanHistoryRepository: scanHistoryRepository, ScanResultRepository: scanResultRepository, ScanObjectMetaRepository: scanObjectMetaRepository, CveStoreRepository: cveStoreRepository, ImageScanDeployInfoRepository: imageScanDeployInfoRepository, @@ -110,8 +113,15 @@ func NewImageScanServiceImpl(logger *zap.SugaredLogger, scanHistoryRepository re DockerArtifactStoreRepository: dockerArtifactStoreRepository, RegistryIndexMappingRepository: registryIndexMappingRepository, CliCommandEnv: os.Environ(), + asyncRunnable: asyncRunnable, + } + // Only check for progressing scans if the flag is enabled + if imageScanConfig.EnableProgressingScanCheck { + logger.Infow("checking for progressing scans at startup") + asyncRunnable.Execute(imageScanService.HandleProgressingScans) + } else { + logger.Infow("skipping progressing scans check at startup as it is disabled") } - imageScanService.HandleProgressingScans() return imageScanService } @@ -125,10 +135,11 @@ func GetImageScannerConfig() (*ImageScanConfig, error) { } type ImageScanConfig struct { - ScannerType string `env:"SCANNER_TYPE" envDefault:""` - ScanTryCount int `env:"IMAGE_SCAN_TRY_COUNT" envDefault:"1"` - ScanImageTimeout int `env:"IMAGE_SCAN_TIMEOUT" envDefault:"10"` // Time is considered in minutes - ScanImageAsyncTimeout int `env:"IMAGE_SCAN_ASYNC_TIMEOUT" envDefault:"3"` // Time is considered in minutes + ScannerType string `env:"SCANNER_TYPE" envDefault:""` + ScanTryCount int `env:"IMAGE_SCAN_TRY_COUNT" envDefault:"1"` + ScanImageTimeout int `env:"IMAGE_SCAN_TIMEOUT" envDefault:"10"` // Time is considered in minutes + ScanImageAsyncTimeout int `env:"IMAGE_SCAN_ASYNC_TIMEOUT" envDefault:"3"` // Time is considered in minutes + EnableProgressingScanCheck bool `env:"ENABLE_PROGRESSING_SCAN_CHECK" envDefault:"true"` // Flag to enable/disable checking for progressing scans at startup } func (impl *ImageScanServiceImpl) GetImageToBeScannedAndFetchCliEnv(scanEvent *bean2.ImageScanEvent) (string, error) { @@ -954,63 +965,63 @@ func (impl *ImageScanServiceImpl) HandleProgressingScans() { impl.Logger.Errorw("error in getting all scans by running state", "err", err) return } - - var executionHistoryDirPath string - flagForDeleting := false - // Create Folder for output data for execution history only if any pending scans are there due to pod died if len(scanHistories) > 0 { - flagForDeleting = true + var executionHistoryDirPath string + // Create Folder for output data for execution history only if any pending scans are there due to pod died executionHistoryDirPath = impl.CreateFolderForOutputData(scanHistories[0].ImageScanExecutionHistoryId) - } - wg := &sync.WaitGroup{} - wg.Add(len(scanHistories)) - imagescanExecutionHistories, err := impl.ScanHistoryRepository.FindAll() - if err != nil { - impl.Logger.Errorw("error in getting scan histories on start up", "err", err) - return - } - imageScanToolMetadatas, err := impl.ScanToolMetadataRepository.FindAllActiveTools() - if err != nil { - impl.Logger.Errorw("error in getting all active tools", "err", err) - } - imageScanExecutionHistoryMap := make(map[int]*repository.ImageScanExecutionHistory) - imageScanToolsMap := make(map[int]*repository.ScanToolMetadata) - - for _, imageScanExecutionHistory := range imagescanExecutionHistories { - imageScanExecutionHistoryMap[imageScanExecutionHistory.Id] = imageScanExecutionHistory - } - for _, imageScanToolMetaData := range imageScanToolMetadatas { - imageScanToolsMap[imageScanToolMetaData.Id] = imageScanToolMetaData - } - - //System doing image scanning for all pending scans - for _, scanHistory := range scanHistories { - scanEvent := bean2.ImageScanEvent{} - scanEventJson := imageScanExecutionHistoryMap[scanHistory.ImageScanExecutionHistoryId].SourceMetadataJson - if len(scanEventJson) == 0 { - return + imageScanExecutionHistoryIds := make([]int, 0, len(scanHistories)) + for _, scanHistory := range scanHistories { + imageScanExecutionHistoryIds = append(imageScanExecutionHistoryIds, scanHistory.ImageScanExecutionHistoryId) } - scanTool := imageScanToolsMap[scanHistory.ScanToolId] - err = json.Unmarshal([]byte(scanEventJson), &scanEvent) + wg := &sync.WaitGroup{} + wg.Add(len(scanHistories)) + imagescanExecutionHistories, err := impl.ScanHistoryRepository.FindByIds(imageScanExecutionHistoryIds) if err != nil { - impl.Logger.Errorw("error in un-marshaling", "err", err) + impl.Logger.Errorw("error in getting scan histories on start up", "err", err) return } - imageScanRenderDto, err := impl.GetImageScanRenderDto(scanEvent.DockerRegistryId, &scanEvent) + imageScanToolMetadatas, err := impl.ScanToolMetadataRepository.FindAllActiveTools() if err != nil { - impl.Logger.Errorw("service error, GetImageScanRenderDto", "dockerRegistryId", scanEvent.DockerRegistryId, "err", err) + impl.Logger.Errorw("error in getting all active tools", "err", err) return } - _, _, err = impl.ScanImageForTool(scanTool, scanHistory.ImageScanExecutionHistoryId, executionHistoryDirPath, wg, 1, context.Background(), imageScanRenderDto) - if err != nil { - impl.Logger.Errorw("error in scanning image", "err", err) - return + imageScanExecutionHistoryMap := make(map[int]*repository.ImageScanExecutionHistory) + imageScanToolsMap := make(map[int]*repository.ScanToolMetadata) + + for _, imageScanExecutionHistory := range imagescanExecutionHistories { + imageScanExecutionHistoryMap[imageScanExecutionHistory.Id] = imageScanExecutionHistory } - } - wg.Wait() + for _, imageScanToolMetaData := range imageScanToolMetadatas { + imageScanToolsMap[imageScanToolMetaData.Id] = imageScanToolMetaData + } + + //System doing image scanning for all pending scans + for _, scanHistory := range scanHistories { + scanEvent := bean2.ImageScanEvent{} + scanEventJson := imageScanExecutionHistoryMap[scanHistory.ImageScanExecutionHistoryId].SourceMetadataJson + if len(scanEventJson) == 0 { + return + } + scanTool := imageScanToolsMap[scanHistory.ScanToolId] + err = json.Unmarshal([]byte(scanEventJson), &scanEvent) + if err != nil { + impl.Logger.Errorw("error in un-marshaling", "err", err) + return + } + imageScanRenderDto, err := impl.GetImageScanRenderDto(scanEvent.DockerRegistryId, &scanEvent) + if err != nil { + impl.Logger.Errorw("service error, GetImageScanRenderDto", "dockerRegistryId", scanEvent.DockerRegistryId, "err", err) + return + } + _, _, err = impl.ScanImageForTool(scanTool, scanHistory.ImageScanExecutionHistoryId, executionHistoryDirPath, wg, 1, context.Background(), imageScanRenderDto) + if err != nil { + impl.Logger.Errorw("error in scanning image", "err", err) + return + } + } + wg.Wait() - //deleting executionDirectoryPath - if flagForDeleting { + //deleting executionDirectoryPath err = os.Remove(executionHistoryDirPath) if err != nil { impl.Logger.Errorw("error in deleting executionHistoryDirectory", "executionHistoryDirPath", executionHistoryDirPath, "err", err) diff --git a/image-scanner/pkg/sql/repository/ImageScanHistoryRepository.go b/image-scanner/pkg/sql/repository/ImageScanHistoryRepository.go index c29c1925e..fe6b38fb5 100644 --- a/image-scanner/pkg/sql/repository/ImageScanHistoryRepository.go +++ b/image-scanner/pkg/sql/repository/ImageScanHistoryRepository.go @@ -61,6 +61,7 @@ type ImageScanHistoryRepository interface { Save(tx *pg.Tx, model *ImageScanExecutionHistory) error FindAll() ([]*ImageScanExecutionHistory, error) FindOne(id int) (*ImageScanExecutionHistory, error) + FindByIds(ids []int) ([]*ImageScanExecutionHistory, error) FindByImageDigest(image string) (*ImageScanExecutionHistory, error) FindByImageDigests(digest []string) ([]*ImageScanExecutionHistory, error) Update(model *ImageScanExecutionHistory) error @@ -103,6 +104,15 @@ func (impl ImageScanHistoryRepositoryImpl) FindOne(id int) (*ImageScanExecutionH Where("id = ?", id).Select() return &model, err } +func (impl ImageScanHistoryRepositoryImpl) FindByIds(ids []int) ([]*ImageScanExecutionHistory, error) { + models := make([]*ImageScanExecutionHistory, 0) + if len(ids) == 0 { + return models, nil + } + err := impl.dbConnection.Model(&models). + Where("id in (?)", pg.In(ids)).Select() + return models, err +} func (impl ImageScanHistoryRepositoryImpl) FindByImageDigest(image string) (*ImageScanExecutionHistory, error) { var model ImageScanExecutionHistory diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/async/async.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/async/async.go new file mode 100644 index 000000000..425af98c4 --- /dev/null +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/async/async.go @@ -0,0 +1,73 @@ +package async + +import ( + "fmt" + "github.com/devtron-labs/common-lib/constants" + "github.com/devtron-labs/common-lib/pubsub-lib/metrics" + "github.com/devtron-labs/common-lib/utils/runTime" + "go.uber.org/zap" + "runtime/debug" +) + +type Runnable struct { + logger *zap.SugaredLogger + serviceName constants.ServiceName +} + +type RunAsyncMetaData struct { + Method string + Path string +} + +func NewAsyncRunnable(logger *zap.SugaredLogger, serviceName constants.ServiceName) *Runnable { + return &Runnable{ + logger: logger, + serviceName: serviceName, + } +} + +func NewRunAsyncMetaData() *RunAsyncMetaData { + return &RunAsyncMetaData{} +} + +func CallerMethod(methodName string) NewUpdateMetaData { + return func(m *RunAsyncMetaData) { + m.Method = methodName + } +} + +func CallerPath(pathName string) NewUpdateMetaData { + return func(m *RunAsyncMetaData) { + m.Path = pathName + } +} + +type NewUpdateMetaData func(*RunAsyncMetaData) + +func (impl *Runnable) Execute(runnableFunc func()) { + impl.run(runnableFunc, + CallerMethod(runTime.GetCallerFunctionName()), + CallerPath(fmt.Sprintf("%s:%d", runTime.GetCallerFileName(), runTime.GetCallerLineNumber())), + ) +} + +func (impl *Runnable) run(fn func(), metadataOpts ...NewUpdateMetaData) { + metaData := NewRunAsyncMetaData() + for _, metadataOpt := range metadataOpts { + // updating meta data + if metadataOpt != nil { + metadataOpt(metaData) + } + } + go func() { + defer func() { + if r := recover(); r != nil { + metrics.IncPanicRecoveryCount("go-routine", impl.serviceName.ToString(), metaData.Method, metaData.Path) + impl.logger.Errorw(fmt.Sprintf("%s %s", constants.GoRoutinePanicMsgLogPrefix, "go-routine recovered from panic"), "err", r, "stack", string(debug.Stack())) + } + }() + if fn != nil { + fn() + } + }() +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/runTime/GetCallerFrames.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/runTime/GetCallerFrames.go new file mode 100644 index 000000000..ff54cba4a --- /dev/null +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/runTime/GetCallerFrames.go @@ -0,0 +1,50 @@ +package runTime + +import ( + "runtime" +) + +// GetCallerFileName - returns the file name of the invoked func +func GetCallerFileName() string { + // Skip GetCurrentFunctionName + return getFrame(1).File +} + +// GetCallerLineNumber - returns the line number of the invoked func +func GetCallerLineNumber() int { + // Skip GetCurrentFunctionName + return getFrame(1).Line +} + +// GetCallerFunctionName - returns the function name of the invoked func +func GetCallerFunctionName() string { + // Skip GetCallerFunctionName and the function to get the caller of + return getFrame(2).Function +} + +// getFrame returns the runtime.Frame for the targetFrameIndex from the runtime caller stack +// +// examples: +// 1. getFrame(0) -> returns current method frame +// 2. getFrame(1) -> returns caller method frame +// 3. getFrame(2) -> returns caller's caller method frame +func getFrame(targetFrameIndex int) runtime.Frame { + // Set size to targetFrameIndex + 2 to ensure we have room for function runtime.Callers and getFrame + programCounters := make([]uintptr, targetFrameIndex+2) + // We need the frame at index targetFrameIndex + 2, since we never want function runtime.Callers and getFrame + n := runtime.Callers(2, programCounters) + + frame := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(programCounters[:n]) + for more, frameIndex := true, 0; more && frameIndex <= targetFrameIndex; frameIndex++ { + var frameCandidate runtime.Frame + frameCandidate, more = frames.Next() + if frameIndex == targetFrameIndex { + frame = frameCandidate + } + } + } + + return frame +} diff --git a/image-scanner/wire_gen.go b/image-scanner/wire_gen.go index d3a947ce0..0c48cbb0b 100644 --- a/image-scanner/wire_gen.go +++ b/image-scanner/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -10,6 +10,7 @@ import ( "github.com/devtron-labs/common-lib/monitoring" "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/image-scanner/api" + "github.com/devtron-labs/image-scanner/pkg/asyncProvider" "github.com/devtron-labs/image-scanner/pkg/clairService" "github.com/devtron-labs/image-scanner/pkg/grafeasService" "github.com/devtron-labs/image-scanner/pkg/klarService" @@ -56,7 +57,8 @@ func InitializeApp() (*App, error) { } dockerArtifactStoreRepositoryImpl := repository.NewDockerArtifactStoreRepositoryImpl(db, sugaredLogger) registryIndexMappingRepositoryImpl := repository.NewRegistryIndexMappingRepositoryImpl(db, sugaredLogger) - imageScanServiceImpl := security.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, ciArtifactRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, scanToolMetadataRepositoryImpl, scanStepConditionRepositoryImpl, scanToolStepRepositoryImpl, scanStepConditionMappingRepositoryImpl, imageScanConfig, dockerArtifactStoreRepositoryImpl, registryIndexMappingRepositoryImpl) + runnable := asyncProvider.NewAsyncRunnable(sugaredLogger) + imageScanServiceImpl := security.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, ciArtifactRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, scanToolMetadataRepositoryImpl, scanStepConditionRepositoryImpl, scanToolStepRepositoryImpl, scanStepConditionMappingRepositoryImpl, imageScanConfig, dockerArtifactStoreRepositoryImpl, registryIndexMappingRepositoryImpl, runnable) klarConfig, err := klarService.GetKlarConfig() if err != nil { return nil, err diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil { diff --git a/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go index f327d712b..0baf2387e 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -34,6 +34,7 @@ const ( Kubelink ServiceName = "KUBELINK" GitSensor ServiceName = "GITSENSOR" Kubewatch ServiceName = "KUBEWATCH" + ImageScanner ServiceName = "IMAGE_SCANNER" ) // metrics name constants diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go index 3bdec67be..bc5dbeba1 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer } else { status = SUCCESS } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) } // Log pg query if enabled logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) if logNetworkFailure { - log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) if logFailureQuery { - log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError) } if logThresholdQueries { - log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query) } } var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", -}, []string{"status", "serviceName", "functionName", "errorType"}) +}, []string{"status", "serviceName", "errorType"}) func getErrorType(err error) ErrorType { if err == nil {