Skip to content

sync: Main sync develop #272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vikramdevtron merged 8 commits into from
May 19, 2025
Merged

sync: Main sync develop #272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3e4a8ca
pending scan bypass
prkhrkat May 15, 2025
bbc8a3a
pending scan bypass
prkhrkat May 15, 2025
813ff96
Merge pull request #266 from devtron-labs/image-scanner-oomkill
prkhrkat May 15, 2025
1f57945
fix: Remove funtionName from labels from pg_query_duration_seconds_* …
Ash-exp May 16, 2025
a8cc2e4
chore: dependency version updated to 'origin/fix-pg-metrix-labels'
Ash-exp May 16, 2025
57bedf3
Merge pull request #268 from devtron-labs/fix-pg-metrix-labels
Ash-exp May 16, 2025
2c18801
fix: optimised handle progressing flows (#269)
Shivam-nagar23 May 19, 2025
47c92c3
main sync develop
vikramdevtron May 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions common-lib/constants/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const (
Kubelink ServiceName = "KUBELINK"
GitSensor ServiceName = "GITSENSOR"
Kubewatch ServiceName = "KUBEWATCH"
ImageScanner ServiceName = "IMAGE_SCANNER"
)

// metrics name constants
Expand Down
12 changes: 6 additions & 6 deletions common-lib/utils/SqlUtil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,31 +85,31 @@ func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQuer
} else {
status = SUCCESS
}
PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds())
PgQueryDuration.WithLabelValues(status, cfg.ServiceName, getErrorType(pgError).String()).Observe(queryDuration.Seconds())
}

// Log pg query if enabled
logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold
logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError)
if logNetworkFailure {
log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError)
log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError)
}
logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError)
if logFailureQuery {
log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError)
log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query, "pgError", pgError)
}
if logThresholdQueries {
log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query)
log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query)
}
if cfg.LogAllQuery {
log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query)
log.Println("query time", "duration", queryDuration.Seconds(), "functionName", event.FuncName, "query", event.Query)
}
}

var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "pg_query_duration_seconds",
Help: "Duration of PG queries",
}, []string{"status", "serviceName", "functionName", "errorType"})
}, []string{"status", "serviceName", "errorType"})

func getErrorType(err error) ErrorType {
if err == nil {
Expand Down
1 change: 1 addition & 0 deletions git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions image-scanner/Wire.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/devtron-labs/common-lib/monitoring"
client "github.com/devtron-labs/common-lib/pubsub-lib"
"github.com/devtron-labs/image-scanner/api"
"github.com/devtron-labs/image-scanner/pkg/asyncProvider"
"github.com/devtron-labs/image-scanner/pkg/clairService"
"github.com/devtron-labs/image-scanner/pkg/grafeasService"
"github.com/devtron-labs/image-scanner/pkg/klarService"
Expand All @@ -38,6 +39,7 @@ import (

func InitializeApp() (*App, error) {
wire.Build(
asyncProvider.AsyncWireSet,
NewApp,
api.NewRouter,
logger.NewSugardLogger,
Expand Down
3 changes: 2 additions & 1 deletion image-scanner/config.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# IMAGESCANER CONFIGMAP
# IMAGESCANER CONFIGMAP


| Variable Name | Value | Description |
|---------------------|----------------------------------------|-------------------------------|
| CLAIR_ADDR | clair-dcd.devtroncd:6060 | For connecting to Clair if it's enabled |
| ENABLE_PROGRESSING_SCAN_CHECK | "true" | Flag to enable/disable checking for progressing scans at startup (set to "false" to improve startup performance) |
| CLIENT_ID | client-2 | Client ID |
| NATS_SERVER_HOST | nats://devtron-nats.devtroncd:4222 | For connecting to NATS |
| PG_LOG_QUERY | "false" | PostgreSQL Query Logging (false to disable) |
Expand Down
2 changes: 1 addition & 1 deletion image-scanner/env_gen.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}]
[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_PROGRESSING_SCAN_CHECK","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}]
1 change: 1 addition & 0 deletions image-scanner/env_gen.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
| CLAIR_TIMEOUT | int |30 | | | false |
| CONSUMER_CONFIG_JSON | string | | | | false |
| DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false |
| ENABLE_PROGRESSING_SCAN_CHECK | bool |true | | | false |
| ENABLE_STATSVIZ | bool |false | | | false |
| IMAGE_SCAN_ASYNC_TIMEOUT | int |3 | | | false |
| IMAGE_SCAN_TIMEOUT | int |10 | | | false |
Expand Down
11 changes: 11 additions & 0 deletions image-scanner/pkg/asyncProvider/AsyncProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package asyncProvider

import (
"github.com/devtron-labs/common-lib/async"
"github.com/devtron-labs/common-lib/constants"
"go.uber.org/zap"
)

func NewAsyncRunnable(logger *zap.SugaredLogger) *async.Runnable {
return async.NewAsyncRunnable(logger, constants.ImageScanner)
}
9 changes: 9 additions & 0 deletions image-scanner/pkg/asyncProvider/wire_AsyncProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package asyncProvider

import (
"github.com/google/wire"
)

var AsyncWireSet = wire.NewSet(
NewAsyncRunnable,
)
Loading
Loading