diff --git a/authenticator/go.mod b/authenticator/go.mod index 52f24a648..49fa3a726 100644 --- a/authenticator/go.mod +++ b/authenticator/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.6 require ( github.com/caarlos0/env/v6 v6.7.2 github.com/coreos/go-oidc/v3 v3.11.0 - github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/sirupsen/logrus v1.8.1 diff --git a/authenticator/go.sum b/authenticator/go.sum index 21f1a3941..0760f2fc9 100644 --- a/authenticator/go.sum +++ b/authenticator/go.sum @@ -97,8 +97,8 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/authenticator/vendor/github.com/golang-jwt/jwt/v4/parser.go b/authenticator/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5..0fc510a0a 100644 --- a/authenticator/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/authenticator/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/authenticator/vendor/modules.txt b/authenticator/vendor/modules.txt index b83684d74..bdf55e44b 100644 --- a/authenticator/vendor/modules.txt +++ b/authenticator/vendor/modules.txt @@ -19,7 +19,7 @@ github.com/go-logr/logr ## explicit; go 1.15 github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/protobuf v1.5.3 diff --git a/chart-sync/go.mod b/chart-sync/go.mod index a11a3509b..7aeb5e1fe 100644 --- a/chart-sync/go.mod +++ b/chart-sync/go.mod @@ -5,7 +5,7 @@ go 1.22.4 toolchain go1.22.6 replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 ) diff --git a/chart-sync/go.sum b/chart-sync/go.sum index 395e0b556..1f09ee85d 100644 --- a/chart-sync/go.sum +++ b/chart-sync/go.sum @@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= diff --git a/chart-sync/internals/sql/DockerArtifactStoreRepository.go b/chart-sync/internals/sql/DockerArtifactStoreRepository.go index e2d873a6a..5094af2b4 100644 --- a/chart-sync/internals/sql/DockerArtifactStoreRepository.go +++ b/chart-sync/internals/sql/DockerArtifactStoreRepository.go @@ -57,6 +57,7 @@ type DockerArtifactStore struct { Cert string `sql:"cert" json:"cert,omitempty"` Active bool `sql:"active,notnull" json:"active"` RemoteConnectionConfigId int `sql:"remote_connection_config_id"` + CredentialsType string `sql:"credentials_type,notnull"` OCIRegistryConfig []*OCIRegistryConfig RemoteConnectionConfig *RemoteConnectionConfig AuditLog diff --git a/chart-sync/pkg/registry/adapter.go b/chart-sync/pkg/registry/adapter.go index cf778fc77..b81419cb4 100644 --- a/chart-sync/pkg/registry/adapter.go +++ b/chart-sync/pkg/registry/adapter.go @@ -46,5 +46,6 @@ func NewToRegistryConfig(store *sql.DockerArtifactStore) (*registry.Configuratio RegistryType: string(store.RegistryType), IsPublicRegistry: store.OCIRegistryConfig[0].IsPublic, RemoteConnectionConfig: remoteConnectionConfig, + CredentialsType: store.CredentialsType, }, nil } diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/InternalErrorCode.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/InternalErrorCode.go new file mode 100644 index 000000000..0e807b971 --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/InternalErrorCode.go @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + */ + +package constants + +const ( + UserCreateDBFailed string = "6001" + UserCreatePolicyFailed string = "6002" + UserUpdateDBFailed string = "6003" + UserUpdatePolicyFailed string = "6004" + UserNoTokenProvided string = "6005" + UserNotFoundForToken string = "6006" + UserCreateFetchRoleFailed string = "6007" + UserUpdateFetchRoleFailed string = "6008" +) diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go new file mode 100644 index 000000000..f327d712b --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package constants + +const ( + PanicLogIdentifier = "DEVTRON_PANIC_RECOVER" + GoRoutinePanicMsgLogPrefix = "GO_ROUTINE_PANIC_LOG:" +) + +// service names constant + +type ServiceName string + +func (m ServiceName) ToString() string { + return string(m) +} + +const ( + Orchestrator ServiceName = "ORCHESTRATOR" + Kubelink ServiceName = "KUBELINK" + GitSensor ServiceName = "GITSENSOR" + Kubewatch ServiceName = "KUBEWATCH" +) + +// metrics name constants +const ( + NATS_PUBLISH_COUNT = "nats_publish_count" + NATS_CONSUMPTION_COUNT = "nats_consumption_count" + NATS_CONSUMING_COUNT = "nats_consuming_count" + NATS_EVENT_CONSUMPTION_TIME = "nats_event_consumption_time" + NATS_EVENT_PUBLISH_TIME = "nats_event_publish_time" + NATS_EVENT_DELIVERY_COUNT = "nats_event_delivery_count" + PANIC_RECOVERY_COUNT = "panic_recovery_count" + REVERSE_PROXY_PANIC_RECOVERY_COUNT = "reverse_proxy_panic_recovery_count" +) + +// metrics labels constant +const ( + PANIC_TYPE = "panic_type" + HOST = "host" + METHOD = "method" + PATH = "path" + TOPIC = "topic" + STATUS = "status" +) + +// multiple history rows for one source event +type SourceType int + +const ( + SourceTypeImage SourceType = 1 + SourceTypeCode SourceType = 2 + SourceTypeSbom SourceType = 3 // can be used in future for direct sbom scanning +) + +type SourceSubType int + +const ( + SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) + SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) +) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go index cd25e430d..0bee1dbc0 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go @@ -20,6 +20,7 @@ type Configuration struct { RegistryType string IsPublicRegistry bool RemoteConnectionConfig *bean.RemoteConnectionConfigBean + CredentialsType string } type RegistryConnectionType string @@ -53,3 +54,7 @@ const ( const ( REGISTRY_CREDENTIAL_BASE_PATH = "/home/devtron/registry-credentials" ) + +const ( + HTTP = "http" +) diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go index ac062c8ea..12ae9806c 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go @@ -14,6 +14,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "os" "strings" ) @@ -176,4 +177,11 @@ func GetTlsConfig(config *Configuration) (*tls.Config, error) { return tlsConfig, nil } -// TODO: add support for certFile, keyFile on UI? +func IsPlainHttp(URL string) bool { + parsedURL, err := url.Parse(URL) + if err != nil { + return false + } + plainHttp := parsedURL.Scheme == HTTP + return plainHttp +} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go index 645af368c..1fa08fded 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go @@ -1,6 +1,7 @@ package registry import ( + "github.com/devtron-labs/common-lib/constants" "go.uber.org/zap" "helm.sh/helm/v3/pkg/registry" ) @@ -45,7 +46,8 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r } clientOptions := []registry.ClientOption{registry.ClientOptHTTPClient(httpClient)} - if config.RegistryConnectionType == INSECURE_CONNECTION { + + if IsPlainHttp(config.RegistryUrl) { clientOptions = append(clientOptions, registry.ClientOptPlainHTTP()) } @@ -55,7 +57,7 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r return nil, err } - if config != nil && !config.IsPublicRegistry { + if config != nil && !config.IsPublicRegistry && !(config.CredentialsType == string(constants.CredentialsTypeAnonymous)) { registryClient, err = GetLoggedInClient(registryClient, config) if err != nil { return nil, err diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go index 933b97f4f..9f8524e48 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/chart-sync/vendor/modules.txt b/chart-sync/vendor/modules.txt index 2532c081c..60dc43afd 100644 --- a/chart-sync/vendor/modules.txt +++ b/chart-sync/vendor/modules.txt @@ -93,8 +93,9 @@ github.com/containerd/platforms # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 +github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/helmLib/registry @@ -790,4 +791,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/chart-sync/wire_gen.go b/chart-sync/wire_gen.go index 874f69de6..0022efdf4 100644 --- a/chart-sync/wire_gen.go +++ b/chart-sync/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/ci-runner/go.mod b/ci-runner/go.mod index f4f194725..5ce4c2f21 100644 --- a/ci-runner/go.mod +++ b/ci-runner/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.21.8 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 require ( github.com/Knetic/govaluate v3.0.0+incompatible @@ -70,7 +70,7 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/go-pg/pg v6.15.1+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect diff --git a/ci-runner/go.sum b/ci-runner/go.sum index bfeb99e28..b04d8e57d 100644 --- a/ci-runner/go.sum +++ b/ci-runner/go.sum @@ -95,8 +95,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= @@ -140,8 +140,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= diff --git a/ci-runner/helper/DockerHelper.go b/ci-runner/helper/DockerHelper.go index 853afa54b..84f573ecb 100644 --- a/ci-runner/helper/DockerHelper.go +++ b/ci-runner/helper/DockerHelper.go @@ -31,6 +31,7 @@ import ( "github.com/caarlos0/env" cicxt "github.com/devtron-labs/ci-runner/executor/context" bean2 "github.com/devtron-labs/ci-runner/helper/bean" + "github.com/devtron-labs/common-lib/constants" "github.com/devtron-labs/ci-runner/util" "github.com/devtron-labs/common-lib/utils" @@ -176,6 +177,7 @@ func (impl *DockerHelperImpl) StartDockerDaemonAndDockerLogin(commonWorkflowRequ SecretKey: commonWorkflowRequest.SecretKey, DockerRegistryURL: commonWorkflowRequest.IntermediateDockerRegistryUrl, DockerRegistryType: commonWorkflowRequest.DockerRegistryType, + CredentialsType: commonWorkflowRequest.CredentialsType, }) if err != nil { return err @@ -210,7 +212,7 @@ const CacheModeMax = "max" const CacheModeMin = "min" type DockerCredentials struct { - DockerUsername, DockerPassword, AwsRegion, AccessKey, SecretKey, DockerRegistryURL, DockerRegistryType string + DockerUsername, DockerPassword, AwsRegion, AccessKey, SecretKey, DockerRegistryURL, DockerRegistryType, CredentialsType string } type EnvironmentVariables struct { @@ -224,6 +226,9 @@ func (impl *DockerHelperImpl) GetCommandToExecute(cmd string) *exec.Cmd { } func (impl *DockerHelperImpl) DockerLogin(ciContext cicxt.CiContext, dockerCredentials *DockerCredentials) error { + if dockerCredentials.CredentialsType == string(constants.CredentialsTypeAnonymous) { + return nil + } performDockerLogin := func() error { username := dockerCredentials.DockerUsername pwd := dockerCredentials.DockerPassword diff --git a/ci-runner/helper/EventHelper.go b/ci-runner/helper/EventHelper.go index f892a7b31..f8ebce402 100644 --- a/ci-runner/helper/EventHelper.go +++ b/ci-runner/helper/EventHelper.go @@ -102,6 +102,7 @@ type CommonWorkflowRequest struct { DockerImageTag string `json:"dockerImageTag"` DockerRegistryId string `json:"dockerRegistryId"` DockerRegistryType string `json:"dockerRegistryType"` + CredentialsType string `json:"credentialsType"` DockerRegistryURL string `json:"dockerRegistryURL"` DockerRegistryConnectionConfig *bean.RemoteConnectionConfigBean `json:"dockerRegistryConnectionConfig"` DockerConnection string `json:"dockerConnection"` diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/ci-runner/vendor/github.com/golang-jwt/jwt/v4/parser.go b/ci-runner/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5..0fc510a0a 100644 --- a/ci-runner/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/ci-runner/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/ci-runner/vendor/modules.txt b/ci-runner/vendor/modules.txt index beb803bcc..ca02b07a5 100644 --- a/ci-runner/vendor/modules.txt +++ b/ci-runner/vendor/modules.txt @@ -248,7 +248,7 @@ github.com/cespare/xxhash/v2 # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/constants @@ -342,7 +342,7 @@ github.com/go-resty/resty/v2 ## explicit; go 1.15 github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da @@ -991,4 +991,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/common-lib/constants/constants.go b/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/common-lib/constants/constants.go +++ b/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/common-lib/go.mod b/common-lib/go.mod index f7c071b1e..c1840a8a1 100644 --- a/common-lib/go.mod +++ b/common-lib/go.mod @@ -10,10 +10,12 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.9.23 github.com/arl/statsviz v0.6.0 github.com/aws/aws-sdk-go v1.44.116 + github.com/aws/aws-sdk-go-v2 v1.36.1 github.com/aws/aws-sdk-go-v2/config v1.29.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.59 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.61 github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1 + github.com/aws/smithy-go v1.22.2 github.com/caarlos0/env v3.5.0+incompatible github.com/docker/docker v27.2.0+incompatible github.com/gammazero/workerpool v1.1.3 @@ -49,7 +51,6 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.36.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect @@ -63,7 +64,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect - github.com/aws/smithy-go v1.22.2 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -110,7 +110,7 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect diff --git a/common-lib/go.sum b/common-lib/go.sum index 40a7e1230..4de4c690c 100644 --- a/common-lib/go.sum +++ b/common-lib/go.sum @@ -190,8 +190,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= diff --git a/common-lib/helmLib/registry/bean.go b/common-lib/helmLib/registry/bean.go index cd25e430d..0bee1dbc0 100644 --- a/common-lib/helmLib/registry/bean.go +++ b/common-lib/helmLib/registry/bean.go @@ -20,6 +20,7 @@ type Configuration struct { RegistryType string IsPublicRegistry bool RemoteConnectionConfig *bean.RemoteConnectionConfigBean + CredentialsType string } type RegistryConnectionType string @@ -53,3 +54,7 @@ const ( const ( REGISTRY_CREDENTIAL_BASE_PATH = "/home/devtron/registry-credentials" ) + +const ( + HTTP = "http" +) diff --git a/common-lib/helmLib/registry/common.go b/common-lib/helmLib/registry/common.go index ac062c8ea..12ae9806c 100644 --- a/common-lib/helmLib/registry/common.go +++ b/common-lib/helmLib/registry/common.go @@ -14,6 +14,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "os" "strings" ) @@ -176,4 +177,11 @@ func GetTlsConfig(config *Configuration) (*tls.Config, error) { return tlsConfig, nil } -// TODO: add support for certFile, keyFile on UI? +func IsPlainHttp(URL string) bool { + parsedURL, err := url.Parse(URL) + if err != nil { + return false + } + plainHttp := parsedURL.Scheme == HTTP + return plainHttp +} diff --git a/common-lib/helmLib/registry/defaultSettings.go b/common-lib/helmLib/registry/defaultSettings.go index 645af368c..1fa08fded 100644 --- a/common-lib/helmLib/registry/defaultSettings.go +++ b/common-lib/helmLib/registry/defaultSettings.go @@ -1,6 +1,7 @@ package registry import ( + "github.com/devtron-labs/common-lib/constants" "go.uber.org/zap" "helm.sh/helm/v3/pkg/registry" ) @@ -45,7 +46,8 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r } clientOptions := []registry.ClientOption{registry.ClientOptHTTPClient(httpClient)} - if config.RegistryConnectionType == INSECURE_CONNECTION { + + if IsPlainHttp(config.RegistryUrl) { clientOptions = append(clientOptions, registry.ClientOptPlainHTTP()) } @@ -55,7 +57,7 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r return nil, err } - if config != nil && !config.IsPublicRegistry { + if config != nil && !config.IsPublicRegistry && !(config.CredentialsType == string(constants.CredentialsTypeAnonymous)) { registryClient, err = GetLoggedInClient(registryClient, config) if err != nil { return nil, err diff --git a/common-lib/informer/bean.go b/common-lib/informer/bean.go new file mode 100644 index 000000000..38aed85af --- /dev/null +++ b/common-lib/informer/bean.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package informer + +const ( + ClusterModifyEventSecretType = "cluster.request/modify" + ClusterActionAdd = "add" + ClusterActionUpdate = "update" + ClusterActionDelete = "delete" + SecretFieldAction = "action" + SecretFieldClusterId = "cluster_id" +) + +const ( + WorkflowTypeLabelKey = "workflowType" + CiWorkflowName = "ci" + CdWorkflowName = "cd" +) + +const ( + DevtronOwnerInstanceLabelKey = "devtron.ai/owner-instance" +) + +const ( + PodDeletedMessage = "pod deleted" +) diff --git a/common-lib/telemetry/PosthogClient.go b/common-lib/telemetry/PosthogClient.go index 033adf411..79a8c72f8 100644 --- a/common-lib/telemetry/PosthogClient.go +++ b/common-lib/telemetry/PosthogClient.go @@ -1,12 +1,30 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package telemetry import ( + "context" "encoding/base64" "encoding/json" + "go.opentelemetry.io/otel" "time" "github.com/devtron-labs/common-lib/utils/http" - cache "github.com/patrickmn/go-cache" + "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "go.uber.org/zap" ) @@ -16,26 +34,13 @@ type PosthogClient struct { cache *cache.Cache } -var ( - PosthogApiKey string = "" - PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight - HeartbeatCronExpr string = "0 0/6 * * *" - CacheExpiry int = 1440 - PosthogEncodedApiKey string = "" - IsOptOut bool = false -) - -const ( - TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" - TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" - ResponseApiKey string = "PosthogApiKey" - ResponseUrlKey string = "PosthogEndpoint" -) +func (p *PosthogClient) GetCache() *cache.Cache { + return p.cache +} func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { if PosthogApiKey == "" { - encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(TelemetryApiKeyEndpoint, logger) + encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(context.Background(), TelemetryApiKeyEndpoint, logger) if err != nil { logger.Errorw("exception caught while getting api key", "err", err) } else { @@ -59,14 +64,16 @@ func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { return pgClient, nil } -func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { +func getPosthogApiKey(ctx context.Context, encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "telemetry.getPosthogApiKey") + defer span.End() decodedPosthogApiKeyUrl, err := base64.StdEncoding.DecodeString(encodedPosthogApiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, decode error", "err", err) return "", "", "", err } apiKeyUrl := string(decodedPosthogApiKeyUrl) - response, err := http.HttpRequest(apiKeyUrl) + response, err := http.HttpRequest(newCtx, apiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, http call", "err", err) return "", "", "", err @@ -78,7 +85,7 @@ func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) return "", "", "", err } var datamap map[string]string - if err := json.Unmarshal(posthogInfoByte, &datamap); err != nil { + if err = json.Unmarshal(posthogInfoByte, &datamap); err != nil { logger.Errorw("error while unmarshal data", "err", err) return "", "", "", err } diff --git a/common-lib/telemetry/bean.go b/common-lib/telemetry/bean.go new file mode 100644 index 000000000..b85621d0f --- /dev/null +++ b/common-lib/telemetry/bean.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +var ( + PosthogApiKey string = "" + PosthogEndpoint string = "https://app.posthog.com" + SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight + HeartbeatCronExpr string = "0 0/6 * * *" + CacheExpiry int = 1440 + PosthogEncodedApiKey string = "" + IsOptOut bool = false +) + +const ( + TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" + TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" + ResponseApiKey string = "PosthogApiKey" + ResponseUrlKey string = "PosthogEndpoint" +) diff --git a/common-lib/utils/CommonUtils.go b/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/common-lib/utils/CommonUtils.go +++ b/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/common-lib/utils/SqlUtil.go b/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/common-lib/utils/bean/bean.go b/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/common-lib/utils/bean/bean.go +++ b/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/common-lib/utils/http/HttpUtil.go b/common-lib/utils/http/HttpUtil.go index 933b97f4f..9f8524e48 100644 --- a/common-lib/utils/http/HttpUtil.go +++ b/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/common-lib/utils/sql/UtilStructs.go b/common-lib/utils/sql/AuditLog.go similarity index 100% rename from common-lib/utils/sql/UtilStructs.go rename to common-lib/utils/sql/AuditLog.go diff --git a/common-lib/vendor/github.com/golang-jwt/jwt/v4/parser.go b/common-lib/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5..0fc510a0a 100644 --- a/common-lib/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/common-lib/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/common-lib/vendor/modules.txt b/common-lib/vendor/modules.txt index f0aa5cab0..3a9fcc451 100644 --- a/common-lib/vendor/modules.txt +++ b/common-lib/vendor/modules.txt @@ -397,7 +397,7 @@ github.com/go-playground/universal-translator ## explicit; go 1.15 github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da diff --git a/git-sensor/go.mod b/git-sensor/go.mod index 7a75158fd..0c78a685e 100644 --- a/git-sensor/go.mod +++ b/git-sensor/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.22.4 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 require ( github.com/caarlos0/env v3.5.0+incompatible diff --git a/git-sensor/go.sum b/git-sensor/go.sum index fa222cf31..c7937a6b2 100644 --- a/git-sensor/go.sum +++ b/git-sensor/go.sum @@ -27,8 +27,8 @@ github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e/go.mod h1:1TqULGlTey+VNhAu/ag7NJuUvByJemkqodsc9L5PHJk= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/git-sensor/internals/sql/WebhookEventDataMappingRepository.go b/git-sensor/internals/sql/WebhookEventDataMappingRepository.go index 4923541a6..a7b6bf94b 100644 --- a/git-sensor/internals/sql/WebhookEventDataMappingRepository.go +++ b/git-sensor/internals/sql/WebhookEventDataMappingRepository.go @@ -17,7 +17,6 @@ package sql import ( - "github.com/devtron-labs/git-sensor/util" "github.com/go-pg/pg" "github.com/go-pg/pg/orm" "time" @@ -61,15 +60,7 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetCiPipelineMaterialWebhookDa Where("webhook_data_id =? ", webhookParsedDataId). Where("is_active = TRUE "). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return &mapping, nil + return &mapping, err } func (impl WebhookEventDataMappingRepositoryImpl) SaveCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *CiPipelineMaterialWebhookDataMapping) error { @@ -89,15 +80,7 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetMatchedCiPipelineMaterialWe Where("is_active = TRUE "). Where("condition_matched = TRUE "). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return pipelineMaterials, nil + return pipelineMaterials, err } func (impl WebhookEventDataMappingRepositoryImpl) InactivateWebhookDataMappingForPipelineMaterials(ciPipelineMaterialIds []int) error { @@ -127,15 +110,7 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetWebhookPayloadDataForPipeli Offset(offset). Order("updated_on " + sortOrder). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return mappings, nil + return mappings, err } func (impl WebhookEventDataMappingRepositoryImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId int, webhookParsedDataId int) (*CiPipelineMaterialWebhookDataMapping, error) { @@ -149,13 +124,5 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetWebhookPayloadFilterDataFor Where("webhook_data_id =? ", webhookParsedDataId). Where("is_active = TRUE "). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return &mapping, nil + return &mapping, err } diff --git a/git-sensor/internals/sql/WebhookEventParsedDataRepository.go b/git-sensor/internals/sql/WebhookEventParsedDataRepository.go index 609b653c3..51890dab6 100644 --- a/git-sensor/internals/sql/WebhookEventParsedDataRepository.go +++ b/git-sensor/internals/sql/WebhookEventParsedDataRepository.go @@ -17,7 +17,6 @@ package sql import ( - "github.com/devtron-labs/git-sensor/util" "github.com/go-pg/pg" "time" ) @@ -54,13 +53,7 @@ func NewWebhookEventParsedDataRepositoryImpl(dbConnection *pg.DB) *WebhookEventP func (impl WebhookEventParsedDataRepositoryImpl) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*WebhookEventParsedData, error) { var webhookEventParsedData WebhookEventParsedData err := impl.dbConnection.Model(&webhookEventParsedData).Where("event_id =? ", eventId).Where("unique_id =? ", uniqueId).Select() - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - return &webhookEventParsedData, nil + return &webhookEventParsedData, err } func (impl WebhookEventParsedDataRepositoryImpl) SaveWebhookParsedEventData(webhookEventParsedData *WebhookEventParsedData) error { @@ -88,12 +81,5 @@ func (impl WebhookEventParsedDataRepositoryImpl) GetWebhookEventParsedDataById(i err := impl.dbConnection.Model(&webhookEventParsedData). Where("id = ? ", id). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - return &webhookEventParsedData, nil + return &webhookEventParsedData, err } diff --git a/git-sensor/internals/sql/mocks/WebhookEventDataMappingRepository.go b/git-sensor/internals/sql/mocks/WebhookEventDataMappingRepository.go new file mode 100644 index 000000000..79701b03a --- /dev/null +++ b/git-sensor/internals/sql/mocks/WebhookEventDataMappingRepository.go @@ -0,0 +1,201 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + sql "github.com/devtron-labs/git-sensor/internals/sql" + mock "github.com/stretchr/testify/mock" +) + +// WebhookEventDataMappingRepository is an autogenerated mock type for the WebhookEventDataMappingRepository type +type WebhookEventDataMappingRepository struct { + mock.Mock +} + +// GetCiPipelineMaterialWebhookDataMapping provides a mock function with given fields: ciPipelineMaterialId, webhookParsedDataId +func (_m *WebhookEventDataMappingRepository) GetCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId int, webhookParsedDataId int) (*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId, webhookParsedDataId) + + if len(ret) == 0 { + panic("no return value specified for GetCiPipelineMaterialWebhookDataMapping") + } + + var r0 *sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId, webhookParsedDataId) + } + if rf, ok := ret.Get(0).(func(int, int) *sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetMatchedCiPipelineMaterialWebhookDataMappingForPipelineMaterial provides a mock function with given fields: ciPipelineMaterialId +func (_m *WebhookEventDataMappingRepository) GetMatchedCiPipelineMaterialWebhookDataMappingForPipelineMaterial(ciPipelineMaterialId int) ([]*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId) + + if len(ret) == 0 { + panic("no return value specified for GetMatchedCiPipelineMaterialWebhookDataMappingForPipelineMaterial") + } + + var r0 []*sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int) ([]*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId) + } + if rf, ok := ret.Get(0).(func(int) []*sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(ciPipelineMaterialId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookPayloadDataForPipelineMaterialId provides a mock function with given fields: ciPipelineMaterialId, limit, offset, eventTimeSortOrder +func (_m *WebhookEventDataMappingRepository) GetWebhookPayloadDataForPipelineMaterialId(ciPipelineMaterialId int, limit int, offset int, eventTimeSortOrder string) ([]*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookPayloadDataForPipelineMaterialId") + } + + var r0 []*sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int, int, int, string) ([]*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + } + if rf, ok := ret.Get(0).(func(int, int, int, string) []*sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int, int, int, string) error); ok { + r1 = rf(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookPayloadFilterDataForPipelineMaterialId provides a mock function with given fields: ciPipelineMaterialId, webhookParsedDataId +func (_m *WebhookEventDataMappingRepository) GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId int, webhookParsedDataId int) (*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId, webhookParsedDataId) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookPayloadFilterDataForPipelineMaterialId") + } + + var r0 *sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId, webhookParsedDataId) + } + if rf, ok := ret.Get(0).(func(int, int) *sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// InactivateWebhookDataMappingForPipelineMaterials provides a mock function with given fields: ciPipelineMaterialIds +func (_m *WebhookEventDataMappingRepository) InactivateWebhookDataMappingForPipelineMaterials(ciPipelineMaterialIds []int) error { + ret := _m.Called(ciPipelineMaterialIds) + + if len(ret) == 0 { + panic("no return value specified for InactivateWebhookDataMappingForPipelineMaterials") + } + + var r0 error + if rf, ok := ret.Get(0).(func([]int) error); ok { + r0 = rf(ciPipelineMaterialIds) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SaveCiPipelineMaterialWebhookDataMapping provides a mock function with given fields: ciPipelineMaterialWebhookDataMapping +func (_m *WebhookEventDataMappingRepository) SaveCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *sql.CiPipelineMaterialWebhookDataMapping) error { + ret := _m.Called(ciPipelineMaterialWebhookDataMapping) + + if len(ret) == 0 { + panic("no return value specified for SaveCiPipelineMaterialWebhookDataMapping") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.CiPipelineMaterialWebhookDataMapping) error); ok { + r0 = rf(ciPipelineMaterialWebhookDataMapping) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateCiPipelineMaterialWebhookDataMapping provides a mock function with given fields: ciPipelineMaterialWebhookDataMapping +func (_m *WebhookEventDataMappingRepository) UpdateCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *sql.CiPipelineMaterialWebhookDataMapping) error { + ret := _m.Called(ciPipelineMaterialWebhookDataMapping) + + if len(ret) == 0 { + panic("no return value specified for UpdateCiPipelineMaterialWebhookDataMapping") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.CiPipelineMaterialWebhookDataMapping) error); ok { + r0 = rf(ciPipelineMaterialWebhookDataMapping) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewWebhookEventDataMappingRepository creates a new instance of WebhookEventDataMappingRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWebhookEventDataMappingRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *WebhookEventDataMappingRepository { + mock := &WebhookEventDataMappingRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/git-sensor/internals/sql/mocks/WebhookEventParsedDataRepository.go b/git-sensor/internals/sql/mocks/WebhookEventParsedDataRepository.go new file mode 100644 index 000000000..7500db93c --- /dev/null +++ b/git-sensor/internals/sql/mocks/WebhookEventParsedDataRepository.go @@ -0,0 +1,153 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + sql "github.com/devtron-labs/git-sensor/internals/sql" + mock "github.com/stretchr/testify/mock" +) + +// WebhookEventParsedDataRepository is an autogenerated mock type for the WebhookEventParsedDataRepository type +type WebhookEventParsedDataRepository struct { + mock.Mock +} + +// GetWebhookEventParsedDataById provides a mock function with given fields: id +func (_m *WebhookEventParsedDataRepository) GetWebhookEventParsedDataById(id int) (*sql.WebhookEventParsedData, error) { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookEventParsedDataById") + } + + var r0 *sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func(int) (*sql.WebhookEventParsedData, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(int) *sql.WebhookEventParsedData); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookEventParsedDataByIds provides a mock function with given fields: ids, limit +func (_m *WebhookEventParsedDataRepository) GetWebhookEventParsedDataByIds(ids []int, limit int) ([]*sql.WebhookEventParsedData, error) { + ret := _m.Called(ids, limit) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookEventParsedDataByIds") + } + + var r0 []*sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func([]int, int) ([]*sql.WebhookEventParsedData, error)); ok { + return rf(ids, limit) + } + if rf, ok := ret.Get(0).(func([]int, int) []*sql.WebhookEventParsedData); ok { + r0 = rf(ids, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func([]int, int) error); ok { + r1 = rf(ids, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookParsedEventDataByEventIdAndUniqueId provides a mock function with given fields: eventId, uniqueId +func (_m *WebhookEventParsedDataRepository) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) { + ret := _m.Called(eventId, uniqueId) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookParsedEventDataByEventIdAndUniqueId") + } + + var r0 *sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func(int, string) (*sql.WebhookEventParsedData, error)); ok { + return rf(eventId, uniqueId) + } + if rf, ok := ret.Get(0).(func(int, string) *sql.WebhookEventParsedData); ok { + r0 = rf(eventId, uniqueId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(eventId, uniqueId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SaveWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventParsedDataRepository) SaveWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for SaveWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventParsedDataRepository) UpdateWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for UpdateWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewWebhookEventParsedDataRepository creates a new instance of WebhookEventParsedDataRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWebhookEventParsedDataRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *WebhookEventParsedDataRepository { + mock := &WebhookEventParsedDataRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/git-sensor/pkg/RepoManages.go b/git-sensor/pkg/RepoManages.go index 9f73f4674..5aec548f4 100644 --- a/git-sensor/pkg/RepoManages.go +++ b/git-sensor/pkg/RepoManages.go @@ -385,7 +385,7 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq gitCtx = gitCtx.WithCredentials(userName, password). WithTLSData(gitProvider.CaCert, gitProvider.TlsKey, gitProvider.TlsCert, gitProvider.EnableTLSVerification) - checkoutPath, _, _, err := impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) + checkoutPath, err := impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) if err != nil { return material, err } @@ -899,28 +899,28 @@ func (impl RepoManagerImpl) GetWebhookAndCiDataById(id int, ciPipelineMaterialId impl.logger.Debugw("Getting webhook data ", "id", id) webhookDataFromDb, err := impl.webhookEventParsedDataRepository.GetWebhookEventParsedDataById(id) - - if err != nil { - impl.logger.Errorw("error in getting webhook data for Id ", "Id", id, "err", err) - return nil, err - } - - filterData, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId, id) if err != nil { - impl.logger.Errorw("error in getting webhook payload filter data for webhookParsedId ", "Id", id, "err", err) + impl.logger.Errorw("error in getting webhook data for id ", "id", id, "err", err) return nil, err } - webhookData := impl.webhookEventBeanConverter.ConvertFromWebhookParsedDataSqlBean(webhookDataFromDb) webhookAndCiData := &git.WebhookAndCiData{ WebhookData: webhookData, } + filterData, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId, id) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("error in getting webhook payload filter data for webhookParsedId ", "ciPipelineMaterialId", ciPipelineMaterialId, "id", id, "err", err) + return nil, err + } else if util2.IsErrNoRows(err) { + impl.logger.Warnw("no webhook payload filter data found for webhookParsedId", "ciPipelineMaterialId", ciPipelineMaterialId, "id", id) + return webhookAndCiData, nil + } + if filterData != nil { webhookAndCiData.ExtraEnvironmentVariables = util.BuildExtraEnvironmentVariablesForCi(filterData.FilterResults, webhookDataFromDb.CiEnvVariableData) } - return webhookAndCiData, nil } @@ -1070,18 +1070,25 @@ func (impl RepoManagerImpl) GetWebhookPayloadDataForPipelineMaterialId(request * } func (impl RepoManagerImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(request *git.WebhookPayloadFilterDataRequest) (*git.WebhookPayloadFilterDataResponse, error) { - impl.logger.Debugw("Getting webhook payload filter data ", "request", request) + impl.logger.Debugw("Getting webhook payload filter data", "request", request) - mapping, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(request.CiPipelineMaterialId, request.ParsedDataId) + parsedData, err := impl.webhookEventParsedDataRepository.GetWebhookEventParsedDataById(request.ParsedDataId) if err != nil { - impl.logger.Errorw("error in getting webhook filter payload data ", "err", err) + impl.logger.Errorw("error in getting parsed webhook data by id", "id", request.ParsedDataId, "err", err) return nil, err } - parsedData, err := impl.webhookEventParsedDataRepository.GetWebhookEventParsedDataById(request.ParsedDataId) - if err != nil { - impl.logger.Errorw("error in getting parsed webhook data ", "err", err) + webhookPayloadFilterDataResponse := &git.WebhookPayloadFilterDataResponse{ + PayloadId: parsedData.PayloadDataId, + } + + mapping, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(request.CiPipelineMaterialId, request.ParsedDataId) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("error in getting webhook filter payload data", "ciPipelineMaterialId", request.CiPipelineMaterialId, "parsedDataId", request.ParsedDataId, "err", err) return nil, err + } else if util2.IsErrNoRows(err) { + impl.logger.Warnw("no webhook filter payload data found for parsedDataId", "ciPipelineMaterialId", request.CiPipelineMaterialId, "parsedDataId", request.ParsedDataId) + return webhookPayloadFilterDataResponse, nil } filterResults := mapping.FilterResults @@ -1099,11 +1106,7 @@ func (impl RepoManagerImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(req webhookPayloadFilterDataSelectorResponses = append(webhookPayloadFilterDataSelectorResponses, webhookPayloadFilterDataSelectorResponse) } } - - webhookPayloadFilterDataResponse := &git.WebhookPayloadFilterDataResponse{ - PayloadId: parsedData.PayloadDataId, - SelectorsData: webhookPayloadFilterDataSelectorResponses, - } + webhookPayloadFilterDataResponse.SelectorsData = webhookPayloadFilterDataSelectorResponses return webhookPayloadFilterDataResponse, nil } diff --git a/git-sensor/pkg/RepoManages_test.go b/git-sensor/pkg/RepoManages_test.go new file mode 100644 index 000000000..859f0cc2d --- /dev/null +++ b/git-sensor/pkg/RepoManages_test.go @@ -0,0 +1,196 @@ +package pkg + +import ( + "fmt" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/internals/sql/mocks" + "github.com/devtron-labs/git-sensor/pkg/git" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "testing" +) + +func TestRepoManagerImpl_GetWebhookAndCiDataById(t *testing.T) { + type args struct { + id int + ciPipelineMaterialId int + } + tests := []struct { + name string + args args + want *git.WebhookAndCiData + webhookEventParsedDataMockExecution func(*mocks.WebhookEventParsedDataRepository) + webhookEventDataMappingMockExecution func(*mocks.WebhookEventDataMappingRepository) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEvent_ParsedData_And_DataMappings_Found", + args: args{ + id: 1, + ciPipelineMaterialId: 1, + }, + want: &git.WebhookAndCiData{ + ExtraEnvironmentVariables: map[string]string{ + "VAR1": "envValue1", + "VAR2": "envValue2", + "VAR3": "filterValue3", + "VAR4": "filterValue4", + }, + WebhookData: &git.WebhookData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + wantErr: assert.NoError, + webhookEventParsedDataMockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookEventParsedDataById", 1). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "VAR1": "envValue1", + "VAR2": "envValue2", + }, + }, nil).Once() + }, + webhookEventDataMappingMockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetWebhookPayloadFilterDataForPipelineMaterialId", 1, 1). + Return(&sql.CiPipelineMaterialWebhookDataMapping{ + Id: 1, + CiPipelineMaterialId: 1, + FilterResults: []*sql.CiPipelineMaterialWebhookDataMappingFilterResult{ + { + MatchedGroups: map[string]string{ + "VAR1": "filterValue1", + "VAR2": "filterValue2", + }, + }, + { + MatchedGroups: map[string]string{ + "VAR3": "filterValue3", + "VAR4": "filterValue4", + }, + }, + }, + }, nil).Once() + }, + }, + { + name: "WebhookEvent_ParsedData_Found_But_DataMappings_NotFound", + args: args{ + id: 1, + ciPipelineMaterialId: 1, + }, + want: &git.WebhookAndCiData{ + WebhookData: &git.WebhookData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + wantErr: assert.NoError, + webhookEventParsedDataMockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookEventParsedDataById", 1). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "VAR1": "envValue1", + "VAR2": "envValue2", + }, + }, nil).Once() + }, + webhookEventDataMappingMockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetWebhookPayloadFilterDataForPipelineMaterialId", 1, 1). + Return(nil, pg.ErrNoRows).Once() + }, + }, + { + name: "WebhookEvent_ParsedData_And_DataMappings_NotFound", + args: args{ + id: 1, + ciPipelineMaterialId: 1, + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, pg.ErrNoRows.Error(), msgAndArgs...) + }, + webhookEventParsedDataMockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookEventParsedDataById", 1). + Return(nil, pg.ErrNoRows).Once() + }, + webhookEventDataMappingMockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + AssertNotCalled(t, "GetWebhookPayloadFilterDataForPipelineMaterialId", 1, 1) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getRepoManagerImpl(t, tt.webhookEventParsedDataMockExecution, tt.webhookEventDataMappingMockExecution) + got, err := impl.GetWebhookAndCiDataById(tt.args.id, tt.args.ciPipelineMaterialId) + if !tt.wantErr(t, err, fmt.Sprintf("GetWebhookAndCiDataById(%v, %v)", tt.args.id, tt.args.ciPipelineMaterialId)) { + return + } + assert.Equalf(t, tt.want, got, "GetWebhookAndCiDataById(%v, %v)", tt.args.id, tt.args.ciPipelineMaterialId) + }) + } +} + +func getRepoManagerImpl( + t *testing.T, + webhookEventParsedDataMockExecution func(*mocks.WebhookEventParsedDataRepository), + webhookEventDataMappingMockExecution func(*mocks.WebhookEventDataMappingRepository), +) *RepoManagerImpl { + sugaredLogger := getLogger() + webhookEventDataMappingRepository := getWebhookEventDataMappingRepository(t) + if webhookEventDataMappingMockExecution != nil { + webhookEventDataMappingMockExecution(webhookEventDataMappingRepository) + } + webhookEventParsedDataRepositoryMock := getWebhookEventParsedDataRepositoryMock(t) + if webhookEventParsedDataMockExecution != nil { + webhookEventParsedDataMockExecution(webhookEventParsedDataRepositoryMock) + } + webhookEventBeanConverterImpl := git.NewWebhookEventBeanConverterImpl() + return NewRepoManagerImpl( + sugaredLogger, + nil, nil, nil, nil, nil, nil, nil, nil, + webhookEventParsedDataRepositoryMock, + webhookEventDataMappingRepository, + webhookEventBeanConverterImpl, nil, nil, + ) +} + +func getLogger() *zap.SugaredLogger { + return logger.NewSugaredLogger() +} + +func getWebhookEventParsedDataRepositoryMock(t *testing.T) *mocks.WebhookEventParsedDataRepository { + return mocks.NewWebhookEventParsedDataRepository(t) +} + +func getWebhookEventDataMappingRepository(t *testing.T) *mocks.WebhookEventDataMappingRepository { + return mocks.NewWebhookEventDataMappingRepository(t) +} diff --git a/git-sensor/pkg/git/RepositoryManager.go b/git-sensor/pkg/git/RepositoryManager.go index 7aa672f9d..b2a7eb146 100644 --- a/git-sensor/pkg/git/RepositoryManager.go +++ b/git-sensor/pkg/git/RepositoryManager.go @@ -45,7 +45,7 @@ type RepositoryManager interface { InitRepoAndGetSshPrivateKeyPath(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, string, error) FetchRepo(gitCtx GitContext, location string) (errMsg string, err error) BackupGitMaterialBeforeUpdate(detailedExistingMaterial *sql.GitMaterial) error - GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) + GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, err error) GetCheckoutLocation(gitCtx GitContext, material *sql.GitMaterial, url, checkoutPath string) string TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase // Clean cleans a directory @@ -122,24 +122,38 @@ func (impl *RepositoryManagerImpl) getBackupDirForMaterial(material *sql.GitMate return path.Join(BACKUP_BASE_DIR, strconv.Itoa(material.Id), time.Now().Format("02-Jan-2006 15:04:05")) } -func (impl *RepositoryManagerImpl) GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) { +func (impl *RepositoryManagerImpl) GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, err error) { //gitRegex := `/(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|\#[-\d\w._]+?)$/` httpsRegex := `^https.*` httpsMatched, err := regexp.MatchString(httpsRegex, material.Url) if httpsMatched { locationWithoutProtocol := strings.ReplaceAll(material.Url, "https://", "") checkoutPath := path.Join(impl.getBaseDirForMaterial(material), locationWithoutProtocol) - return checkoutPath, httpsMatched, false, nil + return checkoutPath, nil + } + httpRegex := `^http.*` + httpMatch, err := regexp.MatchString(httpRegex, material.Url) + if httpMatch { + locationWithoutProtocol := strings.ReplaceAll(material.Url, "http://", "") + checkoutPath := path.Join(impl.getBaseDirForMaterial(material), locationWithoutProtocol) + return checkoutPath, nil } sshRegex := `^git@.*` sshMatched, err := regexp.MatchString(sshRegex, material.Url) if sshMatched { checkoutPath := path.Join(impl.getBaseDirForMaterial(material), material.Url) - return checkoutPath, httpsMatched, sshMatched, nil + return checkoutPath, nil + } + // this regex is used to generic ssh providers like gogs where format is @:/.git + scpLikeSSHRegex := `^[\w-]+@[\w.-]+:[\w./-]+\.git$` + rootMatched, err := regexp.MatchString(scpLikeSSHRegex, material.Url) + if rootMatched { + checkoutPath := path.Join(impl.getBaseDirForMaterial(material), material.Url) + return checkoutPath, nil } - return "", httpsMatched, sshMatched, fmt.Errorf("unsupported format url %s", material.Url) + return "", fmt.Errorf("unsupported format url %s", material.Url) } func (impl *RepositoryManagerImpl) GetCheckoutLocation(gitCtx GitContext, material *sql.GitMaterial, url, checkoutPath string) string { diff --git a/git-sensor/pkg/git/Util.go b/git-sensor/pkg/git/Util.go index 7d4101734..2b96e6eee 100644 --- a/git-sensor/pkg/git/Util.go +++ b/git-sensor/pkg/git/Util.go @@ -60,6 +60,7 @@ func GetProjectName(url string) string { url = url[strings.LastIndex(url, "/")+1:] return strings.TrimSuffix(url, ".git") } + func GetCheckoutPath(url string, cloneLocation string) string { //url= https://github.com/devtron-labs/git-sensor.git cloneLocation= git-base/1/github.com/prakash100198 //then this function returns git-base/1/github.com/prakash100198/SampleGoLangProject/.git @@ -83,11 +84,13 @@ func GetUserNamePassword(gitProvider *sql.GitProvider) (userName, password strin return "", "", fmt.Errorf("unsupported %s", gitProvider.AuthMode) } } + func getSSHPrivateKeyFolderAndFilePath(gitProviderId int) (string, string) { sshPrivateKeyFolderPath := path.Join(SSH_PRIVATE_KEY_DIR, strconv.Itoa(gitProviderId)) sshPrivateKeyFilePath := path.Join(sshPrivateKeyFolderPath, SSH_PRIVATE_KEY_FILE_NAME) return sshPrivateKeyFolderPath, sshPrivateKeyFilePath } + func GetOrCreateSshPrivateKeyOnDisk(gitProviderId int, sshPrivateKeyContent string) (privateKeyPath string, err error) { sshPrivateKeyFolderPath, sshPrivateKeyFilePath := getSSHPrivateKeyFolderAndFilePath(gitProviderId) @@ -232,6 +235,7 @@ func processFileStatOutputNameOnly(commitDiff string) (FileStats, error) { return filestat, nil } + func IsRepoShallowCloned(checkoutPath string) bool { return strings.Contains(checkoutPath, "/.git") } @@ -240,3 +244,7 @@ func getTestBaseDir() string { dir, _ := os.UserHomeDir() return path.Join(dir, "/tmp") } + +var ( + ErrWebhookEventParsedDataNotFound = fmt.Errorf("webhook event parsed data not found") +) diff --git a/git-sensor/pkg/git/Watcher.go b/git-sensor/pkg/git/Watcher.go index 58bd3264d..c0ff1d51a 100644 --- a/git-sensor/pkg/git/Watcher.go +++ b/git-sensor/pkg/git/Watcher.go @@ -195,7 +195,7 @@ func (impl *GitWatcherImpl) pollAndUpdateGitMaterial(materialReq *sql.GitMateria // Helper function to handle SSH key creation and retry fetching material func (impl *GitWatcherImpl) handleSshKeyCreationAndRetry(gitCtx GitContext, material *sql.GitMaterial, location string, gitProvider *sql.GitProvider) (updated bool, repo *GitRepository, errMsg string, err error) { if strings.Contains(material.CheckoutLocation, "/.git") { - location, _, _, err = impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) + location, err = impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) if err != nil { impl.logger.Errorw("error in getting clone location ", "material", material, "errMsg", errMsg, "err", err) return false, nil, errMsg, err diff --git a/git-sensor/pkg/git/WebhookEventService.go b/git-sensor/pkg/git/WebhookEventService.go index 8d44fcdfb..40a49d0da 100644 --- a/git-sensor/pkg/git/WebhookEventService.go +++ b/git-sensor/pkg/git/WebhookEventService.go @@ -22,6 +22,7 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/internals/util" + util2 "github.com/devtron-labs/git-sensor/util" _ "github.com/robfig/cron/v3" "go.uber.org/zap" "regexp" @@ -94,17 +95,18 @@ func (impl WebhookEventServiceImpl) GetAllGitHostWebhookEventByGitHostName(gitHo func (impl WebhookEventServiceImpl) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) { impl.logger.Debugw("fetching webhook event parsed data for ", "eventId", eventId, "uniqueId", uniqueId) - if len(uniqueId) == 0 { - return nil, nil + impl.logger.Warn("uniqueId is blank. so skipping fetching webhook event parsed data...") + return nil, ErrWebhookEventParsedDataNotFound } - webhookEventParsedData, err := impl.webhookEventParsedDataRepository.GetWebhookParsedEventDataByEventIdAndUniqueId(eventId, uniqueId) - if err != nil { - impl.logger.Errorw("getting error while fetching webhook event parsed data ", "err", err) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("getting error while fetching webhook event parsed data ", "eventId", eventId, "uniqueId", uniqueId, "err", err) return nil, err + } else if util2.IsErrNoRows(err) { + impl.logger.Warnw("webhook event parsed data not found", "eventId", eventId, "uniqueId", uniqueId) + return nil, ErrWebhookEventParsedDataNotFound } - return webhookEventParsedData, nil } @@ -332,14 +334,20 @@ func (impl WebhookEventServiceImpl) NotifyForAutoCi(material *CiPipelineMaterial return err } -func (impl WebhookEventServiceImpl) HandleMaterialWebhookMappingIntoDb(ciPipelineMaterialId int, webhookParsedDataId int, conditionMatched bool, filterResults []*sql.CiPipelineMaterialWebhookDataMappingFilterResult) error { - impl.logger.Debug("Handling Material webhook mapping into DB") - +func (impl WebhookEventServiceImpl) getCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId int, webhookParsedDataId int) (*sql.CiPipelineMaterialWebhookDataMapping, bool, error) { + var isNewMapping bool mapping, err := impl.webhookEventDataMappingRepository.GetCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId, webhookParsedDataId) - if err != nil { - impl.logger.Errorw("err in getting ci-pipeline vs webhook data mapping", "err", err) - return err + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("err in getting ci-pipeline vs webhook data mapping", "ciPipelineMaterialId", ciPipelineMaterialId, "webhookParsedDataId", webhookParsedDataId, "err", err) + return mapping, isNewMapping, err + } else if util2.IsErrNoRows(err) { + isNewMapping = true } + return mapping, isNewMapping, nil +} + +func (impl WebhookEventServiceImpl) HandleMaterialWebhookMappingIntoDb(ciPipelineMaterialId int, webhookParsedDataId int, conditionMatched bool, filterResults []*sql.CiPipelineMaterialWebhookDataMappingFilterResult) error { + impl.logger.Debugw("Handling Material webhook mapping into DB", "ciPipelineMaterialId", ciPipelineMaterialId, "webhookParsedDataId", webhookParsedDataId) ciPipelineMaterialWebhookDataMapping := &sql.CiPipelineMaterialWebhookDataMapping{ CiPipelineMaterialId: ciPipelineMaterialId, @@ -349,8 +357,12 @@ func (impl WebhookEventServiceImpl) HandleMaterialWebhookMappingIntoDb(ciPipelin UpdatedOn: time.Now(), } - isNewMapping := mapping == nil - + // get mapping + mapping, isNewMapping, err := impl.getCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId, webhookParsedDataId) + if err != nil { + impl.logger.Errorw("err in getting ci-pipeline vs webhook data mapping", "ciPipelineMaterialId", ciPipelineMaterialId, "webhookParsedDataId", webhookParsedDataId, "err", err) + return err + } if isNewMapping { // insert into DB impl.logger.Debug("Saving mapping into DB") diff --git a/git-sensor/pkg/git/WebhookEventService_test.go b/git-sensor/pkg/git/WebhookEventService_test.go new file mode 100644 index 000000000..994dc2900 --- /dev/null +++ b/git-sensor/pkg/git/WebhookEventService_test.go @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package git + +import ( + "fmt" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/internals/sql/mocks" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "os" + "testing" + "time" +) + +func TestWebhookEventServiceImpl_GetWebhookParsedEventDataByEventIdAndUniqueId(t *testing.T) { + + type args struct { + eventId int + uniqueId string + } + + tests := []struct { + name string + args args + want *sql.WebhookEventParsedData + mockExecution func(*mocks.WebhookEventParsedDataRepository) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEventParsedData_Found", + args: args{ + eventId: 1, + uniqueId: "uniqueId", + }, + want: &sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, + wantErr: assert.NoError, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + }, + }, + { + name: "WebhookEventParsedData_NotFound", + args: args{ + eventId: 1, + uniqueId: "uniqueId", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, ErrWebhookEventParsedDataNotFound.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, pg.ErrNoRows).Once() + }, + }, + { + name: "Empty_UniqueId", + args: args{ + eventId: 1, + uniqueId: "", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, ErrWebhookEventParsedDataNotFound.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl.AssertNotCalled(t, "GetWebhookParsedEventDataByEventIdAndUniqueId", 2) + }, + }, + { + name: "PG_Timeout_Error", + args: args{ + eventId: 1, + uniqueId: "uniqueId", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, os.ErrDeadlineExceeded).Once() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getWebhookEventServiceImpl(t, tt.mockExecution, nil) + got, err := impl.GetWebhookParsedEventDataByEventIdAndUniqueId(tt.args.eventId, tt.args.uniqueId) + if !tt.wantErr(t, err, fmt.Sprintf("GetWebhookParsedEventDataByEventIdAndUniqueId(%v, %v)", tt.args.eventId, tt.args.uniqueId)) { + return + } + assert.Equalf(t, tt.want, got, "GetWebhookParsedEventDataByEventIdAndUniqueId(%v, %v)", tt.args.eventId, tt.args.uniqueId) + }) + } +} + +func TestWebhookEventServiceImpl_getCiPipelineMaterialWebhookDataMapping(t *testing.T) { + type args struct { + ciPipelineMaterialId int + webhookParsedDataId int + } + tests := []struct { + name string + args args + want *sql.CiPipelineMaterialWebhookDataMapping + isNewEntry bool + mockExecution func(*mocks.WebhookEventDataMappingRepository) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEventDataMapping_Found", + args: args{ + ciPipelineMaterialId: 1, + webhookParsedDataId: 1, + }, + want: &sql.CiPipelineMaterialWebhookDataMapping{ + Id: 1, + CiPipelineMaterialId: 1, + WebhookDataId: 1, + ConditionMatched: true, + IsActive: true, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, + isNewEntry: false, + wantErr: assert.NoError, + mockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetCiPipelineMaterialWebhookDataMapping", 1, 1). + Return(&sql.CiPipelineMaterialWebhookDataMapping{ + Id: 1, + CiPipelineMaterialId: 1, + WebhookDataId: 1, + ConditionMatched: true, + IsActive: true, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + }, + }, + { + name: "WebhookEventDataMapping_NotFound", + args: args{ + ciPipelineMaterialId: 1, + webhookParsedDataId: 1, + }, + want: nil, + isNewEntry: true, + wantErr: assert.NoError, + mockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetCiPipelineMaterialWebhookDataMapping", 1, 1). + Return(nil, pg.ErrNoRows).Once() + }, + }, + { + name: "PG_Timeout_Error", + args: args{ + ciPipelineMaterialId: 1, + webhookParsedDataId: 1, + }, + want: nil, + isNewEntry: false, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetCiPipelineMaterialWebhookDataMapping", 1, 1). + Return(nil, os.ErrDeadlineExceeded).Once() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getWebhookEventServiceImpl(t, nil, tt.mockExecution) + got, isNewMapping, err := impl.getCiPipelineMaterialWebhookDataMapping(tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId) + if !tt.wantErr(t, err, fmt.Sprintf("getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId)) { + return + } + if !tt.wantErr(t, err, fmt.Sprintf("getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId)) { + return + } + assert.Equalf(t, tt.isNewEntry, isNewMapping, "getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId) + assert.Equalf(t, tt.want, got, "getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId) + }) + } +} + +func getWebhookEventServiceImpl( + t *testing.T, + webhookEventParsedDataMockExecution func(*mocks.WebhookEventParsedDataRepository), + webhookEventDataMappingMockExecution func(*mocks.WebhookEventDataMappingRepository), +) *WebhookEventServiceImpl { + sugaredLogger := getLogger() + webhookEventDataMappingRepository := getWebhookEventDataMappingRepository(t) + if webhookEventDataMappingMockExecution != nil { + webhookEventDataMappingMockExecution(webhookEventDataMappingRepository) + } + webhookEventParsedDataRepositoryMock := getWebhookEventParsedDataRepositoryMock(t) + if webhookEventParsedDataMockExecution != nil { + webhookEventParsedDataMockExecution(webhookEventParsedDataRepositoryMock) + } + return NewWebhookEventServiceImpl( + sugaredLogger, + nil, + webhookEventParsedDataRepositoryMock, + webhookEventDataMappingRepository, nil, nil, nil, nil, + ) +} + +func getLogger() *zap.SugaredLogger { + return logger.NewSugaredLogger() +} + +func getWebhookEventParsedDataRepositoryMock(t *testing.T) *mocks.WebhookEventParsedDataRepository { + return mocks.NewWebhookEventParsedDataRepository(t) +} + +func getWebhookEventDataMappingRepository(t *testing.T) *mocks.WebhookEventDataMappingRepository { + return mocks.NewWebhookEventDataMappingRepository(t) +} diff --git a/git-sensor/pkg/git/WebhookHandler.go b/git-sensor/pkg/git/WebhookHandler.go index 19af9cf60..ab4bf726e 100644 --- a/git-sensor/pkg/git/WebhookHandler.go +++ b/git-sensor/pkg/git/WebhookHandler.go @@ -17,8 +17,10 @@ package git import ( + "errors" "github.com/devtron-labs/git-sensor/internals/sql" "go.uber.org/zap" + "slices" "strings" "time" ) @@ -82,7 +84,7 @@ func (impl WebhookHandlerImpl) HandleWebhookEvent(webhookEvent *WebhookEvent) er for _, event := range events { if len(event.EventTypesCsv) > 0 { eventTypes := strings.Split(event.EventTypesCsv, ",") - if !contains(eventTypes, eventType) { + if !slices.Contains(eventTypes, eventType) { continue } } @@ -100,42 +102,43 @@ func (impl WebhookHandlerImpl) HandleWebhookEvent(webhookEvent *WebhookEvent) er webhookEventParsedData.EventId = eventId webhookEventParsedData.EventActionType = event.ActionType webhookEventParsedData.PayloadDataId = payloadId - - // fetch webhook parsed data from DB if unique id is not blank - webhookParsedEventGetData, err := impl.webhookEventService.GetWebhookParsedEventDataByEventIdAndUniqueId(eventId, webhookEventParsedData.UniqueId) - if err != nil { - impl.logger.Errorw("error in getting parsed webhook event data", "err", err) - return err + if dbErr := impl.upsertWebhookEventParsedData(eventId, webhookEventParsedData); dbErr != nil { + impl.logger.Errorw("error in upserting webhook event parsed data", "eventId", eventId, "err", dbErr) + // intentionally not returning error here, as we want to continue processing other events } - - // save or update in DB - if webhookParsedEventGetData != nil { - webhookEventParsedData.Id = webhookParsedEventGetData.Id - webhookEventParsedData.CreatedOn = webhookParsedEventGetData.CreatedOn - webhookEventParsedData.UpdatedOn = time.Now() - impl.webhookEventService.UpdateWebhookParsedEventData(webhookEventParsedData) - } else { - webhookEventParsedData.CreatedOn = time.Now() - impl.webhookEventService.SaveWebhookParsedEventData(webhookEventParsedData) - } - // match ci trigger condition and notify err = impl.webhookEventService.MatchCiTriggerConditionAndNotify(event, webhookEventParsedData, fullDataMap) if err != nil { impl.logger.Errorw("error in matching ci trigger condition for webhook after db save", "err", err) return err } - } - return nil } -func contains(s []string, str string) bool { - for _, v := range s { - if v == str { - return true +func (impl WebhookHandlerImpl) upsertWebhookEventParsedData(eventId int, webhookEventParsedData *sql.WebhookEventParsedData) error { + // fetch webhook parsed data from DB if unique id is not blank + webhookParsedEventGetData, err := impl.webhookEventService.GetWebhookParsedEventDataByEventIdAndUniqueId(eventId, webhookEventParsedData.UniqueId) + if err != nil && !errors.Is(err, ErrWebhookEventParsedDataNotFound) { + impl.logger.Errorw("error in getting parsed webhook event data", "eventId", eventId, "uniqueId", webhookEventParsedData.UniqueId, "err", err) + return err + } else if errors.Is(err, ErrWebhookEventParsedDataNotFound) { + impl.logger.Infow("webhook event parsed data not found in db, creating a new one", "eventId", eventId, "uniqueId", webhookEventParsedData.UniqueId) + // save in DB + webhookEventParsedData.CreatedOn = time.Now() + if err = impl.webhookEventService.SaveWebhookParsedEventData(webhookEventParsedData); err != nil { + impl.logger.Errorw("error in saving webhook event parsed data", "err", err) + return err + } + } else { + // update in DB + webhookEventParsedData.Id = webhookParsedEventGetData.Id + webhookEventParsedData.CreatedOn = webhookParsedEventGetData.CreatedOn + webhookEventParsedData.UpdatedOn = time.Now() + if err = impl.webhookEventService.UpdateWebhookParsedEventData(webhookEventParsedData); err != nil { + impl.logger.Errorw("error in updating webhook event parsed data", "err", err) + return err } } - return false + return nil } diff --git a/git-sensor/pkg/git/WebhookHandler_test.go b/git-sensor/pkg/git/WebhookHandler_test.go new file mode 100644 index 000000000..6434b1620 --- /dev/null +++ b/git-sensor/pkg/git/WebhookHandler_test.go @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package git + +import ( + "fmt" + "github.com/devtron-labs/git-sensor/internals/sql" + mocks "github.com/devtron-labs/git-sensor/pkg/git/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "os" + "testing" + "time" +) + +func TestWebhookHandlerImpl_upsertWebhookEventParsedData(t *testing.T) { + type args struct { + eventId int + webhookEventParsedData *sql.WebhookEventParsedData + } + tests := []struct { + name string + args args + mockExecution func(service *mocks.WebhookEventService) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEventParsedData_Upsert_Success", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + }, + CreatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + service.On("UpdateWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(nil).Once() + service.AssertNotCalled(t, "SaveWebhookParsedEventData", mock.Anything) + }, + wantErr: assert.NoError, + }, + { + name: "WebhookEventParsedData_Upsert_Error", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + }, + CreatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + service.On("UpdateWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(os.ErrDeadlineExceeded).Once() + service.AssertNotCalled(t, "SaveWebhookParsedEventData", mock.Anything) + }, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + }, + { + name: "WebhookEventParsedData_Save_Success", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, ErrWebhookEventParsedDataNotFound).Once() + service.On("SaveWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(nil).Once() + service.AssertNotCalled(t, "UpdateWebhookParsedEventData", mock.Anything) + }, + wantErr: assert.NoError, + }, + { + name: "WebhookEventParsedData_Save_Error", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, ErrWebhookEventParsedDataNotFound).Once() + service.On("SaveWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(os.ErrDeadlineExceeded).Once() + service.AssertNotCalled(t, "UpdateWebhookParsedEventData", mock.Anything) + }, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + }, + { + name: "GetWebhookParsedEventDataByEventIdAndUniqueId_PG_Connection_Error", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, os.ErrDeadlineExceeded).Once() + service.AssertNotCalled(t, "SaveWebhookParsedEventData", mock.Anything) + service.AssertNotCalled(t, "UpdateWebhookParsedEventData", mock.Anything) + }, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getNewWebhookHandlerImplImpl(t, tt.mockExecution) + tt.wantErr(t, impl.upsertWebhookEventParsedData(tt.args.eventId, tt.args.webhookEventParsedData), fmt.Sprintf("upsertWebhookEventParsedData(%v, %v)", tt.args.eventId, tt.args.webhookEventParsedData)) + }) + } +} + +func getNewWebhookHandlerImplImpl( + t *testing.T, + WebhookEventServiceMockExecution func(service *mocks.WebhookEventService), +) *WebhookHandlerImpl { + sugaredLogger := getLogger() + webhookEventService := getWebhookEventServiceMock(t) + if WebhookEventServiceMockExecution != nil { + WebhookEventServiceMockExecution(webhookEventService) + } + return NewWebhookHandlerImpl( + sugaredLogger, + webhookEventService, + nil, + ) +} + +func getWebhookEventServiceMock(t *testing.T) *mocks.WebhookEventService { + return mocks.NewWebhookEventService(t) +} diff --git a/git-sensor/pkg/git/mocks/WebhookEventService.go b/git-sensor/pkg/git/mocks/WebhookEventService.go new file mode 100644 index 000000000..98435275d --- /dev/null +++ b/git-sensor/pkg/git/mocks/WebhookEventService.go @@ -0,0 +1,171 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + sql "github.com/devtron-labs/git-sensor/internals/sql" + mock "github.com/stretchr/testify/mock" +) + +// WebhookEventService is an autogenerated mock type for the WebhookEventService type +type WebhookEventService struct { + mock.Mock +} + +// GetAllGitHostWebhookEventByGitHostId provides a mock function with given fields: gitHostId, gitHostName +func (_m *WebhookEventService) GetAllGitHostWebhookEventByGitHostId(gitHostId int, gitHostName string) ([]*sql.GitHostWebhookEvent, error) { + ret := _m.Called(gitHostId, gitHostName) + + if len(ret) == 0 { + panic("no return value specified for GetAllGitHostWebhookEventByGitHostId") + } + + var r0 []*sql.GitHostWebhookEvent + var r1 error + if rf, ok := ret.Get(0).(func(int, string) ([]*sql.GitHostWebhookEvent, error)); ok { + return rf(gitHostId, gitHostName) + } + if rf, ok := ret.Get(0).(func(int, string) []*sql.GitHostWebhookEvent); ok { + r0 = rf(gitHostId, gitHostName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.GitHostWebhookEvent) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(gitHostId, gitHostName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllGitHostWebhookEventByGitHostName provides a mock function with given fields: gitHostName +func (_m *WebhookEventService) GetAllGitHostWebhookEventByGitHostName(gitHostName string) ([]*sql.GitHostWebhookEvent, error) { + ret := _m.Called(gitHostName) + + if len(ret) == 0 { + panic("no return value specified for GetAllGitHostWebhookEventByGitHostName") + } + + var r0 []*sql.GitHostWebhookEvent + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*sql.GitHostWebhookEvent, error)); ok { + return rf(gitHostName) + } + if rf, ok := ret.Get(0).(func(string) []*sql.GitHostWebhookEvent); ok { + r0 = rf(gitHostName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.GitHostWebhookEvent) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(gitHostName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookParsedEventDataByEventIdAndUniqueId provides a mock function with given fields: eventId, uniqueId +func (_m *WebhookEventService) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) { + ret := _m.Called(eventId, uniqueId) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookParsedEventDataByEventIdAndUniqueId") + } + + var r0 *sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func(int, string) (*sql.WebhookEventParsedData, error)); ok { + return rf(eventId, uniqueId) + } + if rf, ok := ret.Get(0).(func(int, string) *sql.WebhookEventParsedData); ok { + r0 = rf(eventId, uniqueId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(eventId, uniqueId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MatchCiTriggerConditionAndNotify provides a mock function with given fields: event, webhookEventParsedData, fullDataMap +func (_m *WebhookEventService) MatchCiTriggerConditionAndNotify(event *sql.GitHostWebhookEvent, webhookEventParsedData *sql.WebhookEventParsedData, fullDataMap map[string]string) error { + ret := _m.Called(event, webhookEventParsedData, fullDataMap) + + if len(ret) == 0 { + panic("no return value specified for MatchCiTriggerConditionAndNotify") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.GitHostWebhookEvent, *sql.WebhookEventParsedData, map[string]string) error); ok { + r0 = rf(event, webhookEventParsedData, fullDataMap) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SaveWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventService) SaveWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for SaveWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventService) UpdateWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for UpdateWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewWebhookEventService creates a new instance of WebhookEventService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWebhookEventService(t interface { + mock.TestingT + Cleanup(func()) +}) *WebhookEventService { + mock := &WebhookEventService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/git-sensor/vendor/modules.txt b/git-sensor/vendor/modules.txt index d6e624b38..175b9c02a 100644 --- a/git-sensor/vendor/modules.txt +++ b/git-sensor/vendor/modules.txt @@ -66,7 +66,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -488,4 +488,4 @@ gopkg.in/yaml.v3 # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/image-scanner/go.mod b/image-scanner/go.mod index 21e7f9cf3..bda783a4a 100644 --- a/image-scanner/go.mod +++ b/image-scanner/go.mod @@ -74,4 +74,4 @@ require ( mellium.im/sasl v0.3.2 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/image-scanner/go.sum b/image-scanner/go.sum index 745a595dc..e55cd9a1c 100644 --- a/image-scanner/go.sum +++ b/image-scanner/go.sum @@ -280,8 +280,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/image-scanner/vendor/modules.txt b/image-scanner/vendor/modules.txt index 8cce1acda..4213760b5 100644 --- a/image-scanner/vendor/modules.txt +++ b/image-scanner/vendor/modules.txt @@ -72,7 +72,7 @@ github.com/cespare/xxhash/v2 github.com/coreos/clair/api/v3/clairpb github.com/coreos/clair/database github.com/coreos/clair/ext/versionfmt -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -440,4 +440,4 @@ google.golang.org/protobuf/types/known/wrapperspb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/kubelink/go.mod b/kubelink/go.mod index a31002666..b551b5717 100644 --- a/kubelink/go.mod +++ b/kubelink/go.mod @@ -177,7 +177,7 @@ require ( ) replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 // https://github.com/kubernetes/kubernetes/issues/79384#issuecomment-505627280 k8s.io/api => k8s.io/api v0.29.0 diff --git a/kubelink/go.sum b/kubelink/go.sum index ffcfa7992..483110a2d 100644 --- a/kubelink/go.sum +++ b/kubelink/go.sum @@ -79,8 +79,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/distribution/v3 v3.0.0-beta.1 h1:X+ELTxPuZ1Xe5MsD3kp2wfGUhc8I+MPfRis8dZ818Ic= diff --git a/kubelink/grpc/applist.pb.go b/kubelink/grpc/applist.pb.go index aab851bcb..c3bf6d220 100644 --- a/kubelink/grpc/applist.pb.go +++ b/kubelink/grpc/applist.pb.go @@ -4263,6 +4263,7 @@ type RegistryCredential struct { Connection string `protobuf:"bytes,11,opt,name=Connection,proto3" json:"Connection,omitempty"` RegistryName string `protobuf:"bytes,12,opt,name=RegistryName,proto3" json:"RegistryName,omitempty"` RegistryCertificate string `protobuf:"bytes,13,opt,name=RegistryCertificate,proto3" json:"RegistryCertificate,omitempty"` + CredentialsType string `protobuf:"bytes,14,opt,name=CredentialsType,proto3" json:"CredentialsType,omitempty"` } func (x *RegistryCredential) Reset() { @@ -4388,6 +4389,13 @@ func (x *RegistryCredential) GetRegistryCertificate() string { return "" } +func (x *RegistryCredential) GetCredentialsType() string { + if x != nil { + return x.CredentialsType + } + return "" +} + type ProxyConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -5333,7 +5341,7 @@ var file_grpc_applist_proto_rawDesc = []byte{ 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x12, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xeb, 0x03, 0x0a, 0x12, 0x52, 0x65, + 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x95, 0x04, 0x0a, 0x12, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x55, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x55, @@ -5364,179 +5372,181 @@ var file_grpc_applist_proto_rawDesc = []byte{ 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x13, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0x29, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, - 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, - 0x72, 0x6c, 0x22, 0xa1, 0x01, 0x0a, 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2a, 0x0a, 0x10, 0x53, 0x53, 0x48, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x10, 0x53, 0x53, 0x48, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, 0x73, 0x73, 0x77, - 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, - 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, - 0x68, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x53, 0x53, 0x48, 0x41, - 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, 0x22, 0xd5, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x4f, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x17, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x52, 0x16, 0x52, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, - 0x6f, 0x64, 0x12, 0x2e, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x3a, 0x0a, 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x53, 0x53, - 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x53, - 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x6f, - 0x0a, 0x13, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, 0x67, 0x65, - 0x64, 0x49, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, - 0x67, 0x65, 0x64, 0x49, 0x6e, 0x12, 0x38, 0x0a, 0x0a, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x4f, 0x43, 0x49, 0x52, - 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x52, 0x0a, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, - 0x4f, 0x0a, 0x17, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x50, 0x75, - 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, - 0x2a, 0x38, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, - 0x4f, 0x58, 0x59, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x48, 0x10, 0x01, 0x12, 0x0a, - 0x0a, 0x06, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x10, 0x02, 0x32, 0xd9, 0x0f, 0x0a, 0x12, 0x41, - 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x39, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x0f, 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, - 0x64, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x14, - 0x4c, 0x69, 0x73, 0x74, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x0f, 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, - 0x2f, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, - 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, - 0x12, 0x2f, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, - 0x00, 0x12, 0x31, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x56, 0x32, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x09, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, - 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x55, 0x6e, - 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, 0x62, 0x65, - 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x48, - 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x46, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, - 0x65, 0x6e, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, - 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, - 0x48, 0x65, 0x6c, 0x6d, 0x41, 0x70, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, - 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x0d, 0x47, 0x65, - 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x11, 0x2e, 0x41, 0x70, - 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, - 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x00, 0x12, 0x40, - 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x6e, 0x69, - 0x66, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, - 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x43, 0x0a, 0x10, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, - 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x19, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, - 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x13, 0x47, 0x65, - 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x12, 0x18, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, - 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, - 0x1b, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, - 0x69, 0x74, 0x68, 0x43, 0x68, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x2e, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x54, 0x79, 0x70, + 0x65, 0x22, 0x29, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x72, 0x6c, 0x22, 0xa1, 0x01, 0x0a, + 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x2a, 0x0a, 0x10, 0x53, 0x53, 0x48, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x53, 0x53, 0x48, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0b, + 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, + 0x0a, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, + 0x12, 0x1e, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, + 0x22, 0xd5, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4f, 0x0a, 0x16, 0x52, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x52, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x2e, 0x0a, 0x0b, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3a, 0x0a, 0x0f, + 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, + 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, + 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x6f, 0x0a, 0x13, 0x4f, 0x43, 0x49, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1e, 0x0a, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x64, 0x49, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x64, 0x49, 0x6e, 0x12, + 0x38, 0x0a, 0x0a, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, + 0x79, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x0a, 0x50, + 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x4f, 0x0a, 0x17, 0x4f, 0x43, 0x49, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, + 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x2a, 0x38, 0x0a, 0x16, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x10, 0x00, 0x12, + 0x07, 0x0a, 0x03, 0x53, 0x53, 0x48, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x49, 0x52, 0x45, + 0x43, 0x54, 0x10, 0x02, 0x32, 0xd9, 0x0f, 0x0a, 0x12, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x10, 0x4c, + 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x0f, 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x10, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, 0x64, 0x41, 0x70, 0x70, 0x4c, 0x69, + 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x6c, + 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x0f, + 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x14, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x2f, 0x0a, 0x0c, 0x47, 0x65, 0x74, + 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, + 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x2f, 0x0a, 0x0c, 0x47, 0x65, + 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, + 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, + 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x31, 0x0a, 0x0e, 0x47, + 0x65, 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x56, 0x32, 0x12, 0x11, 0x2e, + 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x34, + 0x0a, 0x09, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, + 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, + 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x55, 0x6e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, + 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x14, + 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x41, 0x70, + 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x73, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, + 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x0e, + 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x10, 0x55, 0x6e, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, + 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, + 0x65, 0x72, 0x1a, 0x19, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x3c, 0x0a, 0x12, 0x49, 0x73, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x12, 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, - 0x0f, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x12, 0x17, 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, - 0x0d, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, - 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x4d, 0x0a, 0x11, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, - 0x74, 0x42, 0x75, 0x6c, 0x6b, 0x12, 0x1a, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1a, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x5a, 0x0a, 0x1d, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, - 0x41, 0x6e, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, - 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x57, 0x69, 0x74, 0x68, 0x43, 0x68, 0x61, 0x72, 0x74, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x1d, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, - 0x68, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x19, 0x2e, 0x48, - 0x65, 0x6c, 0x6d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x65, - 0x73, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x43, 0x68, 0x61, 0x72, - 0x74, 0x4e, 0x6f, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x52, 0x0a, 0x1d, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x68, 0x61, 0x72, - 0x74, 0x12, 0x16, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x43, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x12, 0x16, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x13, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x1a, 0x14, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x1a, 0x50, 0x75, 0x73, 0x68, - 0x48, 0x65, 0x6c, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x54, 0x6f, 0x4f, 0x43, 0x49, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x4f, 0x43, - 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x23, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x46, 0x6f, 0x72, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x15, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x46, - 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x3d, - 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x44, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x73, 0x12, 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x12, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, - 0x65, 0x64, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x57, 0x0a, - 0x23, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, - 0x65, 0x65, 0x55, 0x73, 0x69, 0x6e, 0x67, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x12, 0x17, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x65, 0x76, 0x74, 0x72, 0x6f, 0x6e, 0x2d, 0x6c, 0x61, 0x62, - 0x73, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x6c, 0x69, 0x6e, 0x6b, 0x2f, 0x62, 0x65, 0x61, 0x6e, 0x2f, - 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, + 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x18, 0x2e, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x49, + 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x1b, 0x55, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x68, 0x61, + 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x12, 0x49, 0x73, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x12, + 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, + 0x69, 0x65, 0x72, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0f, 0x52, 0x6f, 0x6c, 0x6c, 0x62, + 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x17, 0x2e, 0x52, 0x6f, 0x6c, + 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0d, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x11, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x42, 0x75, 0x6c, 0x6b, 0x12, + 0x1a, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x42, 0x75, + 0x6c, 0x6b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1d, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x41, 0x6e, 0x64, 0x52, 0x65, 0x74, + 0x72, 0x69, 0x65, 0x76, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, + 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x68, + 0x61, 0x72, 0x74, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x1d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x75, 0x73, 0x74, 0x6f, + 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x19, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1a, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x43, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x39, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x49, 0x6e, + 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x43, 0x68, 0x61, 0x72, 0x74, 0x4e, 0x6f, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x1d, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, + 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, 0x2e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, + 0x0a, 0x13, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, + 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x1a, 0x14, 0x2e, 0x4f, 0x43, 0x49, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x49, 0x0a, 0x1a, 0x50, 0x75, 0x73, 0x68, 0x48, 0x65, 0x6c, 0x6d, 0x43, 0x68, + 0x61, 0x72, 0x74, 0x54, 0x6f, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, + 0x12, 0x13, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, + 0x23, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x46, 0x6f, 0x72, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x47, + 0x65, 0x74, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, + 0x15, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, + 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x2e, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, + 0x72, 0x1a, 0x12, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, 0x64, 0x41, 0x70, 0x70, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x23, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x55, 0x73, 0x69, 0x6e, + 0x67, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x17, + 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, + 0x65, 0x76, 0x74, 0x72, 0x6f, 0x6e, 0x2d, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x6b, 0x75, 0x62, 0x65, + 0x6c, 0x69, 0x6e, 0x6b, 0x2f, 0x62, 0x65, 0x61, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/kubelink/grpc/applist.proto b/kubelink/grpc/applist.proto index a5df8249e..4e2584ab2 100644 --- a/kubelink/grpc/applist.proto +++ b/kubelink/grpc/applist.proto @@ -474,6 +474,7 @@ message RegistryCredential { string Connection = 11; string RegistryName = 12; string RegistryCertificate = 13; + string CredentialsType = 14; } enum RemoteConnectionMethod { diff --git a/kubelink/pkg/k8sInformer/K8sInformer.go b/kubelink/pkg/k8sInformer/K8sInformer.go index 807eb6924..67ff10b3f 100644 --- a/kubelink/pkg/k8sInformer/K8sInformer.go +++ b/kubelink/pkg/k8sInformer/K8sInformer.go @@ -20,7 +20,9 @@ import ( "errors" "fmt" "github.com/devtron-labs/common-lib/async" + informerBean "github.com/devtron-labs/common-lib/informer" k8sUtils "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/kubelink/bean" globalConfig "github.com/devtron-labs/kubelink/config" "github.com/devtron-labs/kubelink/converter" @@ -41,16 +43,6 @@ import ( "time" ) -const ( - HELM_RELEASE_SECRET_TYPE = "helm.sh/release.v1" - CLUSTER_MODIFY_EVENT_SECRET_TYPE = "cluster.request/modify" - DEFAULT_CLUSTER = "default_cluster" - INFORMER_ALREADY_EXIST_MESSAGE = "INFORMER_ALREADY_EXIST" - ADD = "add" - UPDATE = "update" - DELETE = "delete" -) - type K8sInformer interface { GetAllReleaseByClusterId(clusterId int) []*client.DeployedAppDetail CheckReleaseExists(clusterId int32, releaseIdentifier string) bool @@ -118,13 +110,13 @@ func (impl *K8sInformerImpl) registerInformersForAllClusters() error { func (impl *K8sInformerImpl) OnStateChange(clusterId int, action string) { impl.logger.Infow("syncing informer on cluster config update/delete", "action", action, "clusterId", clusterId) switch action { - case UPDATE: + case informerBean.ClusterActionUpdate: err := impl.syncInformer(clusterId) if err != nil && !errors.Is(err, InformerAlreadyExistError) { impl.logger.Errorw("error in updating informer for cluster", "id", clusterId, "err", err) return } - case DELETE: + case informerBean.ClusterActionDelete: deleteClusterInfo, err := impl.clusterRepository.FindByIdWithActiveFalse(clusterId) if err != nil { impl.logger.Errorw("Error in fetching cluster by id", "cluster-id ", clusterId) @@ -143,7 +135,7 @@ func (impl *K8sInformerImpl) BuildInformerForAllClusters(clusterInfos []*bean.Cl if len(clusterInfos) == 0 { clusterInfo := &bean.ClusterInfo{ ClusterId: 1, - ClusterName: DEFAULT_CLUSTER, + ClusterName: commonBean.DEFAULT_CLUSTER, InsecureSkipTLSVerify: true, } err := impl.startInformer(*clusterInfo) @@ -197,7 +189,7 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { } // for default cluster adding an extra informer, this informer will add informer on new clusters - if clusterInfo.ClusterName == DEFAULT_CLUSTER { + if clusterInfo.ClusterName == commonBean.DEFAULT_CLUSTER { impl.logger.Debugw("starting informer, reading new cluster request for default cluster") labelOptions := kubeinformers.WithTweakListOptions(func(opts *metav1.ListOptions) { //kubectl get secret --field-selector type==cluster.request/modify --all-namespaces @@ -211,7 +203,7 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { startTime := time.Now() impl.logger.Debugw("CLUSTER_ADD_INFORMER: cluster secret add event received", "obj", obj, "time", time.Now()) if secretObject, ok := obj.(*coreV1.Secret); ok { - if secretObject.Type != CLUSTER_MODIFY_EVENT_SECRET_TYPE { + if secretObject.Type != informerBean.ClusterModifyEventSecretType { return } data := secretObject.Data @@ -223,14 +215,14 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { impl.logger.Errorw("error in converting cluster id to int", "clusterId", id, "err", err) return } - if string(action) == ADD { + if string(action) == informerBean.ClusterActionAdd { err = impl.startInformerAndPopulateCache(idInt) if err != nil && !errors.Is(err, InformerAlreadyExistError) { impl.logger.Errorw("error in adding informer for cluster", "id", idInt, "err", err) return } } - if string(action) == UPDATE { + if string(action) == informerBean.ClusterActionUpdate { err = impl.syncInformer(idInt) if err != nil && !errors.Is(err, InformerAlreadyExistError) { impl.logger.Errorw("error in updating informer for cluster", "id", clusterInfo.ClusterId, "name", clusterInfo.ClusterName, "err", err) @@ -244,7 +236,7 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { startTime := time.Now() impl.logger.Debugw("CLUSTER_UPDATE_INFORMER: cluster secret update event received", "oldObj", oldObj, "newObj", newObj, "time", time.Now()) if secretObject, ok := newObj.(*coreV1.Secret); ok { - if secretObject.Type != CLUSTER_MODIFY_EVENT_SECRET_TYPE { + if secretObject.Type != informerBean.ClusterModifyEventSecretType { return } data := secretObject.Data @@ -256,14 +248,14 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { impl.logger.Errorw("error in converting cluster id to int", "clusterId", id, "err", err) return } - if string(action) == ADD { + if string(action) == informerBean.ClusterActionAdd { err = impl.startInformerAndPopulateCache(idInt) if err != nil && !errors.Is(err, InformerAlreadyExistError) { impl.logger.Errorw("error in adding informer for cluster", "clusterId", idInt, "err", err) return } } - if string(action) == UPDATE { + if string(action) == informerBean.ClusterActionAdd { impl.OnStateChange(idInt, string(action)) impl.informOtherListeners(idInt, string(action)) } @@ -274,7 +266,7 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { startTime := time.Now() impl.logger.Debugw("CLUSTER_DELETE_INFORMER: secret delete event received", "obj", obj, "time", time.Now()) if secretObject, ok := obj.(*coreV1.Secret); ok { - if secretObject.Type != CLUSTER_MODIFY_EVENT_SECRET_TYPE { + if secretObject.Type != informerBean.ClusterModifyEventSecretType { return } data := secretObject.Data @@ -285,7 +277,7 @@ func (impl *K8sInformerImpl) startInformer(clusterInfo bean.ClusterInfo) error { impl.logger.Errorw("error in converting cluster id to int", "clusterId", id, "err", err) return } - if string(action) == DELETE { + if string(action) == informerBean.ClusterActionDelete { impl.OnStateChange(idInt, string(action)) impl.informOtherListeners(idInt, string(action)) } diff --git a/kubelink/pkg/k8sInformer/error.go b/kubelink/pkg/k8sInformer/error.go index c5eabd5b3..9fcf1ea8f 100644 --- a/kubelink/pkg/k8sInformer/error.go +++ b/kubelink/pkg/k8sInformer/error.go @@ -26,3 +26,8 @@ var ( // InformerAlreadyExistError is returned when an informer already exists InformerAlreadyExistError = errors.New(INFORMER_ALREADY_EXIST_MESSAGE) ) + +const ( + HELM_RELEASE_SECRET_TYPE = "helm.sh/release.v1" + INFORMER_ALREADY_EXIST_MESSAGE = "INFORMER_ALREADY_EXIST" +) diff --git a/kubelink/pkg/service/helmApplicationService/adapter/adapter.go b/kubelink/pkg/service/helmApplicationService/adapter/adapter.go index 5c887d78b..207676c23 100644 --- a/kubelink/pkg/service/helmApplicationService/adapter/adapter.go +++ b/kubelink/pkg/service/helmApplicationService/adapter/adapter.go @@ -26,6 +26,7 @@ func NewRegistryConfig(credential *client.RegistryCredential) (*registry.Configu RegistryCertificateString: credential.RegistryCertificate, RegistryType: credential.RegistryType, IsPublicRegistry: credential.IsPublic, + CredentialsType: credential.CredentialsType, } if credential.Connection == registry.SECURE_WITH_CERT { diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index ea7622fd0..586395f1b 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -737,8 +737,7 @@ func (impl *HelmAppServiceImpl) installRelease(ctx context.Context, request *cli chartSpec.CreateNamespace = false impl.logger.Debugw("Installing release", "name", releaseIdentifier.ReleaseName, "namespace", releaseIdentifier.ReleaseNamespace, "dry-run", dryRun) - switch impl.helmReleaseConfig.RunHelmInstallInAsyncMode { - case false: + if !runInstallInAsyncMode(request.InstallAppVersionHistoryId, impl.helmReleaseConfig.RunHelmInstallInAsyncMode) { impl.logger.Debugw("Installing release", "name", releaseIdentifier.ReleaseName, "namespace", releaseIdentifier.ReleaseNamespace, "dry-run", dryRun) rel, err := helmClientObj.InstallChart(context.Background(), chartSpec) if err != nil { @@ -749,9 +748,8 @@ func (impl *HelmAppServiceImpl) installRelease(ctx context.Context, request *cli } return nil, err } - return rel, nil - case true: + } else { go func() { helmInstallMessage := commonHelmService.HelmReleaseStatusConfig{ InstallAppVersionHistoryId: int(request.InstallAppVersionHistoryId), @@ -852,8 +850,7 @@ func (impl *HelmAppServiceImpl) UpgradeReleaseWithChartInfo(ctx context.Context, chartSpec.DependencyUpdate = true chartSpec.UpgradeCRDs = true - switch impl.helmReleaseConfig.RunHelmInstallInAsyncMode { - case false: + if !runInstallInAsyncMode(request.InstallAppVersionHistoryId, impl.helmReleaseConfig.RunHelmInstallInAsyncMode) { impl.logger.Debug("Upgrading release with chart info") _, err = helmClientObj.UpgradeReleaseWithChartInfo(context.Background(), chartSpec) if UpgradeErr, ok := err.(*driver.StorageDriverError); ok { @@ -872,7 +869,7 @@ func (impl *HelmAppServiceImpl) UpgradeReleaseWithChartInfo(ctx context.Context, } } } - case true: + } else { go func() { impl.logger.Debug("Upgrading release with chart info") _, err = helmClientObj.UpgradeReleaseWithChartInfo(context.Background(), chartSpec) diff --git a/kubelink/pkg/service/helmApplicationService/utils.go b/kubelink/pkg/service/helmApplicationService/utils.go index b2162c1d6..86a02f037 100644 --- a/kubelink/pkg/service/helmApplicationService/utils.go +++ b/kubelink/pkg/service/helmApplicationService/utils.go @@ -33,3 +33,10 @@ const ( func IsReleaseNotFoundInCacheError(err error) bool { return errors.Is(err, k8sInformer.ErrorCacheMissReleaseNotFound) } + +func runInstallInAsyncMode(installedAppVersionHistoryId int32, isAsyncEnabled bool) bool { + if installedAppVersionHistoryId == 0 { + return false + } + return isAsyncEnabled +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go index cd25e430d..0bee1dbc0 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go @@ -20,6 +20,7 @@ type Configuration struct { RegistryType string IsPublicRegistry bool RemoteConnectionConfig *bean.RemoteConnectionConfigBean + CredentialsType string } type RegistryConnectionType string @@ -53,3 +54,7 @@ const ( const ( REGISTRY_CREDENTIAL_BASE_PATH = "/home/devtron/registry-credentials" ) + +const ( + HTTP = "http" +) diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go index ac062c8ea..12ae9806c 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go @@ -14,6 +14,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "os" "strings" ) @@ -176,4 +177,11 @@ func GetTlsConfig(config *Configuration) (*tls.Config, error) { return tlsConfig, nil } -// TODO: add support for certFile, keyFile on UI? +func IsPlainHttp(URL string) bool { + parsedURL, err := url.Parse(URL) + if err != nil { + return false + } + plainHttp := parsedURL.Scheme == HTTP + return plainHttp +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go index 645af368c..1fa08fded 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go @@ -1,6 +1,7 @@ package registry import ( + "github.com/devtron-labs/common-lib/constants" "go.uber.org/zap" "helm.sh/helm/v3/pkg/registry" ) @@ -45,7 +46,8 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r } clientOptions := []registry.ClientOption{registry.ClientOptHTTPClient(httpClient)} - if config.RegistryConnectionType == INSECURE_CONNECTION { + + if IsPlainHttp(config.RegistryUrl) { clientOptions = append(clientOptions, registry.ClientOptPlainHTTP()) } @@ -55,7 +57,7 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r return nil, err } - if config != nil && !config.IsPublicRegistry { + if config != nil && !config.IsPublicRegistry && !(config.CredentialsType == string(constants.CredentialsTypeAnonymous)) { registryClient, err = GetLoggedInClient(registryClient, config) if err != nil { return nil, err diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/informer/bean.go b/kubelink/vendor/github.com/devtron-labs/common-lib/informer/bean.go new file mode 100644 index 000000000..38aed85af --- /dev/null +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/informer/bean.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package informer + +const ( + ClusterModifyEventSecretType = "cluster.request/modify" + ClusterActionAdd = "add" + ClusterActionUpdate = "update" + ClusterActionDelete = "delete" + SecretFieldAction = "action" + SecretFieldClusterId = "cluster_id" +) + +const ( + WorkflowTypeLabelKey = "workflowType" + CiWorkflowName = "ci" + CdWorkflowName = "cd" +) + +const ( + DevtronOwnerInstanceLabelKey = "devtron.ai/owner-instance" +) + +const ( + PodDeletedMessage = "pod deleted" +) diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go index 933b97f4f..9f8524e48 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/kubelink/vendor/modules.txt b/kubelink/vendor/modules.txt index 8eb002ce5..c42934f0e 100644 --- a/kubelink/vendor/modules.txt +++ b/kubelink/vendor/modules.txt @@ -127,13 +127,14 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/helmLib/registry +github.com/devtron-labs/common-lib/informer github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/monitoring github.com/devtron-labs/common-lib/monitoring/pprof @@ -1354,7 +1355,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 # go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 # k8s.io/api => k8s.io/api v0.29.0 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.0 diff --git a/kubewatch/go.mod b/kubewatch/go.mod index 05ad8ae4c..19ce53672 100644 --- a/kubewatch/go.mod +++ b/kubewatch/go.mod @@ -101,7 +101,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -245,4 +245,4 @@ replace ( k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/kubewatch/go.sum b/kubewatch/go.sum index 57dab83dd..01abb87c0 100644 --- a/kubewatch/go.sum +++ b/kubewatch/go.sum @@ -719,8 +719,8 @@ github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -847,8 +847,8 @@ github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= diff --git a/kubewatch/pkg/informer/bean/bean.go b/kubewatch/pkg/informer/bean/bean.go index bc05d4c32..43c5ead05 100644 --- a/kubewatch/pkg/informer/bean/bean.go +++ b/kubewatch/pkg/informer/bean/bean.go @@ -16,6 +16,8 @@ package bean +import "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + type ClusterInfo struct { ClusterId int `json:"clusterId"` ClusterName string `json:"clusterName"` @@ -24,25 +26,16 @@ type ClusterInfo struct { } const ( - CLUSTER_MODIFY_EVENT_SECRET_TYPE = "cluster.request/modify" - CLUSTER_MODIFY_EVENT_FIELD_SELECTOR = "type==cluster.request/modify" - INFORMER_ALREADY_EXIST_MESSAGE = "INFORMER_ALREADY_EXIST" - CLUSTER_ACTION_ADD = "add" - CLUSTER_ACTION_UPDATE = "update" - CLUSTER_ACTION_DELETE = "delete" - SECRET_FIELD_ACTION = "action" - SECRET_FIELD_CLUSTER_ID = "cluster_id" - - POD_DELETED_MESSAGE = "pod deleted" - EXIT_CODE_143_ERROR = "Error (exit code 143)" - CI_WORKFLOW_NAME = "ci" - CD_WORKFLOW_NAME = "cd" - WORKFLOW_LABEL_SELECTOR = "devtron.ai/purpose==workflow" - WORKFLOW_TYPE_LABEL_KEY = "workflowType" - JobKind = "Job" - NodeNoLongerExists = "PodGC: node no longer exists" - UPDATE_EVENT = "update_event" - DELETE_EVENT = "delete_event" + ClusterModifyEventFieldSelector = "type==cluster.request/modify" + InformerAlreadyExistMessage = "INFORMER_ALREADY_EXIST" + + ExitCode143Error = "Error (exit code 143)" + NodeNoLongerExists = "PodGC: node no longer exists" + UpdateEvent = "update_event" + DeleteEvent = "delete_event" + + WorkflowLabelSelector = "devtron.ai/purpose==workflow" + JobKind = "Job" ) type ClusterLabels struct { @@ -56,3 +49,12 @@ func NewClusterLabels(clusterName string, clusterId int) *ClusterLabels { ClusterId: clusterId, } } + +const ( + DevtronOwnerInstance = "devtronOwnerInstance" +) + +type CiCdStatus struct { + DevtronOwnerInstance string `json:"devtronOwnerInstance"` + *v1alpha1.WorkflowStatus +} diff --git a/kubewatch/pkg/informer/cluster/argoCD/informer.go b/kubewatch/pkg/informer/cluster/argoCD/informer.go index 7ade5dfc3..044a6a6c8 100644 --- a/kubewatch/pkg/informer/cluster/argoCD/informer.go +++ b/kubewatch/pkg/informer/cluster/argoCD/informer.go @@ -73,10 +73,10 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste restConfig := impl.k8sUtil.GetK8sConfigForCluster(clusterInfo) applicationInformer := impl.informerClient.GetSharedInformerClient(resourceBean.ApplicationResourceType) clusterLabels := informerBean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id) - acdInformer, err := applicationInformer.GetSharedInformer(clusterInfo.Id, impl.appConfig.GetACDNamespace(), restConfig) + acdInformer, err := applicationInformer.GetSharedInformer(clusterLabels, impl.appConfig.GetACDNamespace(), restConfig) if err != nil { impl.logger.Errorw("error in registering acd informer", "err", err, "clusterId", clusterInfo.Id) - middleware.IncUnregisteredInformers(clusterLabels, middleware.ARGO_CD) + middleware.IncUnregisteredInformers(clusterLabels, middleware.ARGO_CD_INFORMER) return err } stopChannel, err := impl.getStopChannel(clusterLabels) diff --git a/kubewatch/pkg/informer/cluster/argoWf/cd/informer.go b/kubewatch/pkg/informer/cluster/argoWf/cd/informer.go index 48c035d4f..b8aa91d54 100644 --- a/kubewatch/pkg/informer/cluster/argoWf/cd/informer.go +++ b/kubewatch/pkg/informer/cluster/argoWf/cd/informer.go @@ -65,10 +65,10 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste restConfig := impl.k8sUtil.GetK8sConfigForCluster(clusterInfo) cdWfInformer := impl.informerClient.GetSharedInformerClient(resourceBean.CdWorkflowResourceType) clusterLabels := informerBean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id) - workflowInformer, err := cdWfInformer.GetSharedInformer(clusterInfo.Id, impl.appConfig.GetCdWfNamespace(), restConfig) + workflowInformer, err := cdWfInformer.GetSharedInformer(clusterLabels, impl.appConfig.GetCdWfNamespace(), restConfig) if err != nil { impl.logger.Errorw("error in starting workflow informer", "err", err) - middleware.IncUnregisteredInformers(clusterLabels, middleware.CD_STAGE_ARGO_WORLFLOW) + middleware.IncUnregisteredInformers(clusterLabels, middleware.CD_STAGE_ARGO_WORLFLOW_INFORMER) } stopChan, err := impl.getCdArgoWfStopChannel(clusterLabels) if err != nil { diff --git a/kubewatch/pkg/informer/cluster/argoWf/ci/informer.go b/kubewatch/pkg/informer/cluster/argoWf/ci/informer.go index 587e9d600..306a6b3b6 100644 --- a/kubewatch/pkg/informer/cluster/argoWf/ci/informer.go +++ b/kubewatch/pkg/informer/cluster/argoWf/ci/informer.go @@ -65,10 +65,10 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste restConfig := impl.k8sUtil.GetK8sConfigForCluster(clusterInfo) ciWfInformer := impl.informerClient.GetSharedInformerClient(resourceBean.CiWorkflowResourceType) clusterLabels := informerBean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id) - workflowInformer, err := ciWfInformer.GetSharedInformer(clusterInfo.Id, impl.appConfig.GetCiWfNamespace(), restConfig) + workflowInformer, err := ciWfInformer.GetSharedInformer(clusterLabels, impl.appConfig.GetCiWfNamespace(), restConfig) if err != nil { impl.logger.Errorw("error in starting workflow informer", "err", err) - middleware.IncUnregisteredInformers(clusterLabels, middleware.CI_STAGE_ARGO_WORKFLOW) + middleware.IncUnregisteredInformers(clusterLabels, middleware.CI_STAGE_ARGO_WORKFLOW_INFORMER) } stopChan, err := impl.getCiArgoWfStopChannel(clusterLabels) if err != nil { diff --git a/kubewatch/pkg/informer/cluster/argoWf/util.go b/kubewatch/pkg/informer/cluster/argoWf/util.go new file mode 100644 index 000000000..bc774d7f0 --- /dev/null +++ b/kubewatch/pkg/informer/cluster/argoWf/util.go @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package argoWf + +import ( + "fmt" + informerBean "github.com/devtron-labs/common-lib/informer" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" +) + +func GetNatsTopicForWorkflow(workflowType string) (string, error) { + switch workflowType { + case informerBean.CdWorkflowName: + return pubsub.CD_WORKFLOW_STATUS_UPDATE, nil + case informerBean.CiWorkflowName: + return pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, nil + } + return "", fmt.Errorf("no topic mapped to workflow type %s", workflowType) +} diff --git a/kubewatch/pkg/informer/cluster/helper.go b/kubewatch/pkg/informer/cluster/helper.go index 5105afdf4..23883624e 100644 --- a/kubewatch/pkg/informer/cluster/helper.go +++ b/kubewatch/pkg/informer/cluster/helper.go @@ -19,8 +19,9 @@ package cluster import ( "errors" "fmt" + informerBean "github.com/devtron-labs/common-lib/informer" repository "github.com/devtron-labs/kubewatch/pkg/cluster" - informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" + "github.com/devtron-labs/kubewatch/pkg/informer/bean" informerErr "github.com/devtron-labs/kubewatch/pkg/informer/errors" "github.com/devtron-labs/kubewatch/pkg/middleware" "github.com/go-pg/pg" @@ -30,7 +31,7 @@ import ( "time" ) -func (impl *InformerImpl) getStopChannel(informerFactory kubeinformers.SharedInformerFactory, clusterLabels *informerBean.ClusterLabels) (chan struct{}, error) { +func (impl *InformerImpl) getStopChannel(informerFactory kubeinformers.SharedInformerFactory, clusterLabels *bean.ClusterLabels) (chan struct{}, error) { stopChannel := make(chan struct{}) stopper, found := impl.getClusterInformerStopper() if found { @@ -42,11 +43,11 @@ func (impl *InformerImpl) getStopChannel(informerFactory kubeinformers.SharedInf return stopChannel, nil } -func (impl *InformerImpl) getClusterInformerStopper() (*informerBean.FactoryStopper, bool) { +func (impl *InformerImpl) getClusterInformerStopper() (*bean.FactoryStopper, bool) { return impl.clusterInformerStopper, impl.clusterInformerStopper.HasInformer() } -func (impl *InformerImpl) setClusterInformerStopper(stopper *informerBean.FactoryStopper) { +func (impl *InformerImpl) setClusterInformerStopper(stopper *bean.FactoryStopper) { impl.clusterInformerStopper = stopper } @@ -60,7 +61,7 @@ func (impl *InformerImpl) stopDevtronClusterWatcher() error { } func (impl *InformerImpl) startClientInformers(clusterInfo *repository.Cluster) error { - for supportedClient := range informerBean.SupportedClientMap { + for supportedClient := range bean.SupportedClientMap { clientAdvisor, err := impl.GetClient(supportedClient, clusterInfo) if err != nil { impl.logger.Errorw("error in getting client advisor", "supportedClient", supportedClient, "err", err) @@ -78,7 +79,7 @@ func (impl *InformerImpl) startClientInformers(clusterInfo *repository.Cluster) } func (impl *InformerImpl) stopInformersForCluster(clusterId int) error { - for supportedClient := range informerBean.SupportedClientMap { + for supportedClient := range bean.SupportedClientMap { clientAdvisor, err := impl.GetClientStopper(supportedClient) if err != nil { impl.logger.Errorw("error in getting client advisor", "supportedClient", supportedClient, "err", err) @@ -100,7 +101,7 @@ func (impl *InformerImpl) startInformerForCluster(clusterInfo *repository.Cluste }() if len(clusterInfo.ErrorInConnecting) > 0 { impl.logger.Debugw("cluster is not reachable", "clusterId", clusterInfo.Id, "clusterName", clusterInfo.ClusterName) - middleware.IncUnreachableCluster(informerBean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id)) + middleware.IncUnreachableCluster(bean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id)) } // FIXME: If cluster is not reachable, we won't be able to start the informer for it. // But once orchestrator is able to connect to the cluster, we should start the informer using it's event. @@ -113,23 +114,23 @@ func (impl *InformerImpl) startInformerForCluster(clusterInfo *repository.Cluste } func (impl *InformerImpl) handleClusterChangeEvent(secretObject *coreV1.Secret) error { - if secretObject.Type != informerBean.CLUSTER_MODIFY_EVENT_SECRET_TYPE { + if secretObject.Type != informerBean.ClusterModifyEventSecretType { return nil } data := secretObject.Data - action := data[informerBean.SECRET_FIELD_ACTION] - id := string(data[informerBean.SECRET_FIELD_CLUSTER_ID]) + action := data[informerBean.SecretFieldAction] + id := string(data[informerBean.SecretFieldClusterId]) clusterId, convErr := strconv.Atoi(id) if convErr != nil { impl.logger.Errorw("error in converting cluster id to int", "clusterId", id, "err", convErr) return convErr } - if string(action) == informerBean.CLUSTER_ACTION_ADD { + if string(action) == informerBean.ClusterActionAdd { if err := impl.reloadInformerForCluster(clusterId); err != nil { impl.logger.Errorw("error in starting informer for cluster", "clusterId", clusterId, "err", err) return err } - } else if string(action) == informerBean.CLUSTER_ACTION_UPDATE { + } else if string(action) == informerBean.ClusterActionUpdate { if err := impl.syncMultiClusterInformer(clusterId); err != nil { impl.logger.Errorw("error in updating informer for cluster", "id", clusterId, "err", err) return err @@ -139,17 +140,17 @@ func (impl *InformerImpl) handleClusterChangeEvent(secretObject *coreV1.Secret) } func (impl *InformerImpl) handleClusterDeleteEvent(secretObject *coreV1.Secret) error { - if secretObject.Type != informerBean.CLUSTER_MODIFY_EVENT_SECRET_TYPE { + if secretObject.Type != informerBean.ClusterModifyEventSecretType { return nil } data := secretObject.Data - action := data[informerBean.SECRET_FIELD_ACTION] - id := string(data[informerBean.SECRET_FIELD_CLUSTER_ID]) + action := data[informerBean.SecretFieldAction] + id := string(data[informerBean.SecretFieldClusterId]) clusterId, err := strconv.Atoi(id) if err != nil { return err } - if string(action) == informerBean.CLUSTER_ACTION_DELETE { + if string(action) == informerBean.ClusterActionDelete { if err = impl.handleClusterDelete(clusterId); err != nil { impl.logger.Errorw("error in handling cluster delete event", "clusterId", clusterId, "err", err) return err diff --git a/kubewatch/pkg/informer/cluster/informer.go b/kubewatch/pkg/informer/cluster/informer.go index 9a3b5d02b..8e7f0439a 100644 --- a/kubewatch/pkg/informer/cluster/informer.go +++ b/kubewatch/pkg/informer/cluster/informer.go @@ -95,7 +95,7 @@ func (impl *InformerImpl) StartDevtronClusterWatcher() error { } impl.logger.Debug("starting informer, reading new cluster request for default cluster", "clusterId", clusterInfo.Id, "clusterName", clusterInfo.ClusterName) labelOptions := kubeinformers.WithTweakListOptions(func(opts *metav1.ListOptions) { - opts.FieldSelector = informerBean.CLUSTER_MODIFY_EVENT_FIELD_SELECTOR + opts.FieldSelector = informerBean.ClusterModifyEventFieldSelector }) // addFunc is called when a new secret is created addFunc := func(secretObject *coreV1.Secret) { @@ -125,7 +125,7 @@ func (impl *InformerImpl) StartDevtronClusterWatcher() error { secretFactory, err := informerFactory.GetSharedInformerFactory(restConfig, clusterLabels, eventHandler, labelOptions) if err != nil { impl.logger.Errorw("error in registering default cluster secret informer", "err", err) - middleware.IncUnregisteredInformers(clusterLabels, middleware.DEFAULT_CLUSTER_SECRET) + middleware.IncUnregisteredInformers(clusterLabels, middleware.DEFAULT_CLUSTER_SECRET_INFORMER) return err } stopChannel, err := impl.getStopChannel(secretFactory, clusterLabels) diff --git a/kubewatch/pkg/informer/cluster/systemExec/helper.go b/kubewatch/pkg/informer/cluster/systemExec/helper.go index 3080f51a4..7ca088360 100644 --- a/kubewatch/pkg/informer/cluster/systemExec/helper.go +++ b/kubewatch/pkg/informer/cluster/systemExec/helper.go @@ -21,7 +21,8 @@ import ( "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/workflow/common" - informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" + informerBean "github.com/devtron-labs/common-lib/informer" + "github.com/devtron-labs/kubewatch/pkg/informer/bean" "github.com/devtron-labs/kubewatch/pkg/informer/errors" "golang.org/x/exp/maps" coreV1 "k8s.io/api/core/v1" @@ -32,7 +33,7 @@ import ( "sort" ) -func (impl *InformerImpl) getSystemWfStopper(clusterId int) (*informerBean.FactoryStopper, bool) { +func (impl *InformerImpl) getSystemWfStopper(clusterId int) (*bean.FactoryStopper, bool) { stopper, ok := impl.systemWfInformerStopper[clusterId] if ok { return stopper, stopper.HasInformer() @@ -44,7 +45,7 @@ func (impl *InformerImpl) getStoppableClusterIds() []int { return maps.Keys(impl.systemWfInformerStopper) } -func (impl *InformerImpl) getStopChannel(informerFactory kubeinformers.SharedInformerFactory, clusterLabels *informerBean.ClusterLabels) (chan struct{}, error) { +func (impl *InformerImpl) getStopChannel(informerFactory kubeinformers.SharedInformerFactory, clusterLabels *bean.ClusterLabels) (chan struct{}, error) { stopChannel := make(chan struct{}) stopper, ok := impl.systemWfInformerStopper[clusterLabels.ClusterId] if ok && stopper.HasInformer() { @@ -59,7 +60,7 @@ func (impl *InformerImpl) getStopChannel(informerFactory kubeinformers.SharedInf func (impl *InformerImpl) checkIfPodDeletedAndUpdateMessage(podName, namespace string, nodeStatus v1alpha1.NodeStatus, config *rest.Config) (v1alpha1.NodeStatus, bool) { - if (nodeStatus.Phase == v1alpha1.NodeFailed || nodeStatus.Phase == v1alpha1.NodeError) && (nodeStatus.Message == informerBean.EXIT_CODE_143_ERROR || nodeStatus.Message == informerBean.NodeNoLongerExists) { + if (nodeStatus.Phase == v1alpha1.NodeFailed || nodeStatus.Phase == v1alpha1.NodeError) && (nodeStatus.Message == bean.ExitCode143Error || nodeStatus.Message == bean.NodeNoLongerExists) { clusterClient, k8sErr := impl.k8sUtil.GetK8sClientForConfig(config) if k8sErr != nil { return nodeStatus, false @@ -68,13 +69,13 @@ func (impl *InformerImpl) checkIfPodDeletedAndUpdateMessage(podName, namespace s if err != nil { impl.logger.Errorw("error in getting pod from clusterClient", "podName", podName, "namespace", namespace, "err", err) if isResourceNotFoundErr(err) { - nodeStatus.Message = informerBean.POD_DELETED_MESSAGE + nodeStatus.Message = informerBean.PodDeletedMessage return nodeStatus, true } return nodeStatus, false } if pod.DeletionTimestamp != nil { - nodeStatus.Message = informerBean.POD_DELETED_MESSAGE + nodeStatus.Message = informerBean.PodDeletedMessage return nodeStatus, true } } @@ -178,7 +179,7 @@ func (impl *InformerImpl) inferFailedReason(eventType string, pod *coreV1.Pod) ( // case2: we get the above event[n-1] and last[n] event with pod phase as failed and the main container state as terminated. // we want to handle the below case in last[n] event only,last event is always caught by DELETE_EVENT informer. - if eventType == informerBean.DELETE_EVENT { + if eventType == bean.DeleteEvent { // we should mark this case as an error if ctr.Name == common.MainContainerName { return v1alpha1.NodeFailed, getFailedReasonFromPodConditions(pod.Status.Conditions) diff --git a/kubewatch/pkg/informer/cluster/systemExec/informer.go b/kubewatch/pkg/informer/cluster/systemExec/informer.go index a198c0f9e..56189d89a 100644 --- a/kubewatch/pkg/informer/cluster/systemExec/informer.go +++ b/kubewatch/pkg/informer/cluster/systemExec/informer.go @@ -19,10 +19,12 @@ package systemExec import ( "encoding/json" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + informerBean "github.com/devtron-labs/common-lib/informer" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" repository "github.com/devtron-labs/kubewatch/pkg/cluster" "github.com/devtron-labs/kubewatch/pkg/config" - informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" + "github.com/devtron-labs/kubewatch/pkg/informer/bean" + "github.com/devtron-labs/kubewatch/pkg/informer/cluster/argoWf" "github.com/devtron-labs/kubewatch/pkg/middleware" "github.com/devtron-labs/kubewatch/pkg/resource" resourceBean "github.com/devtron-labs/kubewatch/pkg/resource/bean" @@ -41,7 +43,7 @@ type InformerImpl struct { k8sUtil utils.K8sUtil pubSubClient *pubsub.PubSubClientServiceImpl informerClient resource.InformerClient - systemWfInformerStopper map[int]*informerBean.FactoryStopper + systemWfInformerStopper map[int]*bean.FactoryStopper } func NewInformerImpl(logger *zap.SugaredLogger, @@ -55,7 +57,7 @@ func NewInformerImpl(logger *zap.SugaredLogger, k8sUtil: k8sUtil, pubSubClient: pubSubClient, informerClient: informerClient, - systemWfInformerStopper: make(map[int]*informerBean.FactoryStopper), + systemWfInformerStopper: make(map[int]*bean.FactoryStopper), } } @@ -71,14 +73,17 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste impl.logger.Infow("starting system executor informer for cluster", "clusterId", clusterInfo.Id, "clusterName", clusterInfo.ClusterName) restConfig := impl.k8sUtil.GetK8sConfigForCluster(clusterInfo) labelOptions := kubeinformers.WithTweakListOptions(func(opts *metav1.ListOptions) { - opts.LabelSelector = informerBean.WORKFLOW_LABEL_SELECTOR + opts.LabelSelector = bean.WorkflowLabelSelector }) + clusterLabels := bean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id) // updateFunc is called when an existing pod is updated updateFunc := func(oldPodObj, newPodObj *coreV1.Pod) { - // atleast one of the pod version will be not nil + // at least one of the pod version will be not nil if !foundAnyUpdateInPodStatus(oldPodObj, newPodObj) { - podName := newPodObj.Name - logArgs := []interface{}{"podName", podName, "newPodStatusObj", newPodObj.Status} + logArgs := make([]any, 0) + if newPodObj != nil { + logArgs = append(logArgs, "podName", newPodObj.Name, "newPodStatusObj", newPodObj.Status) + } if oldPodObj != nil { logArgs = append(logArgs, "oldPodStatusObj", oldPodObj.Status) } @@ -88,21 +93,29 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste if newPodObj != nil { var workflowType string - if newPodObj.Labels != nil { - if val, ok := newPodObj.Labels[informerBean.WORKFLOW_TYPE_LABEL_KEY]; ok { - workflowType = val - } + podLabels := newPodObj.GetLabels() + if val, ok := podLabels[informerBean.WorkflowTypeLabelKey]; ok { + workflowType = val } impl.logger.Debugw("event received in pods update informer", "time", time.Now(), "podObjStatus", newPodObj.Status) - nodeStatus := impl.assessNodeStatus(informerBean.UPDATE_EVENT, newPodObj) + nodeStatus := impl.assessNodeStatus(bean.UpdateEvent, newPodObj) workflowStatus := getWorkflowStatus(newPodObj, nodeStatus, workflowType) if workflowStatus.Message == "" && workflowStatus.Phase == v1alpha1.WorkflowFailed { impl.logger.Debugw("skipping the failed workflow update event as message is empty", "workflow", workflowStatus) return } + if val, ok := podLabels[informerBean.DevtronOwnerInstanceLabelKey]; ok { + workflowStatus.DevtronOwnerInstance = val + } else { + impl.logger.Warnw("devtron administrator instance label is not found in the pod. not a devtron workflow", "podLabels", podLabels) + middleware.IncNonAdministrativeEvents(clusterLabels, middleware.RESOURCE_K8S_JOB) + // return statement is skipped intentionally for backward compatibility + // TODO Asutosh: remove this return statement in future + // return + } wfJson, err := json.Marshal(workflowStatus) if err != nil { - impl.logger.Errorw("error occurred while marshalling workflowJson", "err", err) + impl.logger.Errorw("error occurred while marshalling workflowJson", "workflowStatus", workflowStatus, "err", err) return } impl.logger.Debugw("sending system executor workflow update event", "workflow", string(wfJson)) @@ -110,38 +123,46 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste log.Println("don't publish") return } - topic, err := getTopic(workflowType) + topic, err := argoWf.GetNatsTopicForWorkflow(workflowType) if err != nil { - impl.logger.Errorw("error while getting Topic") + impl.logger.Errorw("error while getting topic", "workflowType", workflowType, "err", err) return } err = impl.pubSubClient.Publish(topic, string(wfJson)) if err != nil { - impl.logger.Errorw("error while publishing Request", "err", err) + impl.logger.Errorw("error while publishing request", "topic", topic, "wfJson", wfJson, "err", err) return } - impl.logger.Debugw("system executor workflow update sent", "workflowType", workflowType) + impl.logger.Debugw("system executor workflow update sent", "topic", topic, "workflowType", workflowType) } } // deleteFunc is called when an existing pod is deleted deleteFunc := func(podObj *coreV1.Pod) { var workflowType string - if podObj.Labels != nil { - if val, ok := podObj.Labels[informerBean.WORKFLOW_TYPE_LABEL_KEY]; ok { - workflowType = val - } + podLabels := podObj.GetLabels() + if val, ok := podLabels[informerBean.WorkflowTypeLabelKey]; ok { + workflowType = val } impl.logger.Debugw("event received in Pods delete informer", "time", time.Now(), "podObjStatus", podObj.Status) - nodeStatus := impl.assessNodeStatus(informerBean.DELETE_EVENT, podObj) + nodeStatus := impl.assessNodeStatus(bean.DeleteEvent, podObj) nodeStatus, reTriggerRequired := impl.checkIfPodDeletedAndUpdateMessage(podObj.Name, podObj.Namespace, nodeStatus, restConfig) if !reTriggerRequired { - //not sending this deleted event if it's not a re-trigger case + // not sending this deleted event if it's not a re-trigger case return } workflowStatus := getWorkflowStatus(podObj, nodeStatus, workflowType) + if val, ok := podLabels[informerBean.DevtronOwnerInstanceLabelKey]; ok { + workflowStatus.DevtronOwnerInstance = val + } else { + impl.logger.Warnw("devtron administrator instance label is not found in the pod. not a devtron workflow", "podLabels", podLabels) + middleware.IncNonAdministrativeEvents(clusterLabels, middleware.RESOURCE_K8S_JOB) + // return statement is skipped intentionally for backward compatibility + // TODO Asutosh: remove this return statement in future + // return + } wfJson, err := json.Marshal(workflowStatus) if err != nil { - impl.logger.Errorw("error occurred while marshalling workflowJson", "err", err) + impl.logger.Errorw("error occurred while marshalling workflowJson", "workflowStatus", workflowStatus, "err", err) return } impl.logger.Debugw("sending system executor cd workflow delete event", "workflow", string(wfJson)) @@ -149,27 +170,26 @@ func (impl *InformerImpl) StartInformerForCluster(clusterInfo *repository.Cluste log.Println("don't publish") return } - topic, err := getTopic(workflowType) + topic, err := argoWf.GetNatsTopicForWorkflow(workflowType) if err != nil { - impl.logger.Errorw("error while getting topic") + impl.logger.Errorw("error while getting topic", "workflowType", workflowType, "err", err) return } err = impl.pubSubClient.Publish(topic, string(wfJson)) if err != nil { - impl.logger.Errorw("error while publishing request", "err", err) + impl.logger.Errorw("error while publishing request", "topic", topic, "wfJson", wfJson, "err", err) return } - impl.logger.Debugw("workflow update sent", "workflowType", workflowType) + impl.logger.Debugw("workflow update sent", "topic", topic, "workflowType", workflowType) } podInformerFactory := impl.informerClient.GetPodInformerFactory() eventHandler := resourceBean.NewEventHandlers[coreV1.Pod](). UpdateFuncHandler(updateFunc). DeleteFuncHandler(deleteFunc) - clusterLabels := informerBean.NewClusterLabels(clusterInfo.ClusterName, clusterInfo.Id) podFactory, err := podInformerFactory.GetSharedInformerFactory(restConfig, clusterLabels, eventHandler, labelOptions) if err != nil { impl.logger.Errorw("error in adding event handler for cluster pod informer", "err", err) - middleware.IncUnregisteredInformers(clusterLabels, middleware.SYSTEM_EXECUTOR) + middleware.IncUnregisteredInformers(clusterLabels, middleware.SYSTEM_EXECUTOR_INFORMER) return err } stopChannel, err := impl.getStopChannel(podFactory, clusterLabels) diff --git a/kubewatch/pkg/informer/cluster/systemExec/util.go b/kubewatch/pkg/informer/cluster/systemExec/util.go index 763116d8f..1c5865bdc 100644 --- a/kubewatch/pkg/informer/cluster/systemExec/util.go +++ b/kubewatch/pkg/informer/cluster/systemExec/util.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/workflow/common" - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" coreV1 "k8s.io/api/core/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" @@ -28,16 +27,6 @@ import ( "k8s.io/utils/pointer" ) -func getTopic(workflowType string) (string, error) { - switch workflowType { - case informerBean.CD_WORKFLOW_NAME: - return pubsub.CD_WORKFLOW_STATUS_UPDATE, nil - case informerBean.CI_WORKFLOW_NAME: - return pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, nil - } - return "", fmt.Errorf("no topic mapped to workflow type %s", workflowType) -} - func getLatestFinishedAt(pod *coreV1.Pod) metav1.Time { var latest metav1.Time for _, ctr := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) { @@ -93,8 +82,10 @@ func isResourceNotFoundErr(err error) bool { return false } -func getWorkflowStatus(podObj *coreV1.Pod, nodeStatus v1alpha1.NodeStatus, templateName string) *v1alpha1.WorkflowStatus { - workflowStatus := &v1alpha1.WorkflowStatus{} +func getWorkflowStatus(podObj *coreV1.Pod, nodeStatus v1alpha1.NodeStatus, templateName string) *informerBean.CiCdStatus { + workflowStatus := &informerBean.CiCdStatus{ + WorkflowStatus: &v1alpha1.WorkflowStatus{}, + } workflowPhase := v1alpha1.WorkflowPhase(nodeStatus.Phase) if workflowPhase == v1alpha1.WorkflowPending { workflowPhase = v1alpha1.WorkflowRunning diff --git a/kubewatch/pkg/informer/errors/errors.go b/kubewatch/pkg/informer/errors/errors.go index 83049511b..68c70ede3 100644 --- a/kubewatch/pkg/informer/errors/errors.go +++ b/kubewatch/pkg/informer/errors/errors.go @@ -21,4 +21,4 @@ import ( "github.com/devtron-labs/kubewatch/pkg/informer/bean" ) -var AlreadyExists = errors.New(bean.INFORMER_ALREADY_EXIST_MESSAGE) +var AlreadyExists = errors.New(bean.InformerAlreadyExistMessage) diff --git a/kubewatch/pkg/middleware/instrument.go b/kubewatch/pkg/middleware/instrument.go index 7b580c1d9..65c874c88 100644 --- a/kubewatch/pkg/middleware/instrument.go +++ b/kubewatch/pkg/middleware/instrument.go @@ -25,8 +25,9 @@ import ( // metrics name constants const ( - KUBEWATCH_UNREACHABLE_CLIENT_COUNT = "Kubewatch_unreachable_client_count" - KUBEWATCH_UNREGISTERED_INFORMER_COUNT = "Kubewatch_unregistered_informer_count" + KUBEWATCH_UNREACHABLE_CLIENT_COUNT = "Kubewatch_unreachable_client_count" + KUBEWATCH_UNREGISTERED_INFORMER_COUNT = "Kubewatch_unregistered_informer_count" + KUBEWATCH_NON_ADMINISTRATIVE_EVENTS_COUNT = "Kubewatch_non_administrative_events_count" ) // metrics labels constants @@ -34,12 +35,36 @@ const ( CLUSTER_NAME = "clusterName" CLUSTER_ID = "clusterId" INFORMER_NAME = "informerName" + RESOURCE_TYPE = "resourceType" +) + +// ResourceMetrics type +type ResourceMetrics string - CI_STAGE_ARGO_WORKFLOW = "CIStageArgoWorkflow" - CD_STAGE_ARGO_WORLFLOW = "CDStageArgoWorkflow" - ARGO_CD = "ArgoCD" - DEFAULT_CLUSTER_SECRET = "DefaultClusterSecret" - SYSTEM_EXECUTOR = "SystemExecutor" +func (m ResourceMetrics) String() string { + return string(m) +} + +// resource labels values constants +const ( + RESOURCE_ARGO_WORKFLOW ResourceMetrics = "ArgoWorkflow" + RESOURCE_K8S_JOB ResourceMetrics = "K8sJob" +) + +// InformerMetrics type +type InformerMetrics string + +func (m InformerMetrics) String() string { + return string(m) +} + +// informer labels values constants +const ( + CI_STAGE_ARGO_WORKFLOW_INFORMER InformerMetrics = "CIStageArgoWorkflow" + CD_STAGE_ARGO_WORLFLOW_INFORMER InformerMetrics = "CDStageArgoWorkflow" + ARGO_CD_INFORMER InformerMetrics = "ArgoCD" + DEFAULT_CLUSTER_SECRET_INFORMER InformerMetrics = "DefaultClusterSecret" + SYSTEM_EXECUTOR_INFORMER InformerMetrics = "SystemExecutor" ) var UnreachableCluster = promauto.NewCounterVec( @@ -53,15 +78,27 @@ var UnreachableCluster = promauto.NewCounterVec( var UnregisteredInformers = promauto.NewCounterVec( prometheus.CounterOpts{ Name: KUBEWATCH_UNREGISTERED_INFORMER_COUNT, - Help: "How many informers are unregistered, with cluster name and cluster id.", + Help: "How many informers are unregistered, with informer name, cluster name and cluster id.", }, []string{CLUSTER_NAME, CLUSTER_ID, INFORMER_NAME}, ) +var NonAdministrativeEvents = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: KUBEWATCH_NON_ADMINISTRATIVE_EVENTS_COUNT, + Help: "How many events are non-administrative (not devtron managed), with resource type, cluster name and cluster id.", + }, + []string{CLUSTER_NAME, CLUSTER_ID, RESOURCE_TYPE}, +) + func IncUnreachableCluster(clusterLabels *bean.ClusterLabels) { UnreachableCluster.WithLabelValues(clusterLabels.ClusterName, strconv.Itoa(clusterLabels.ClusterId)).Inc() } -func IncUnregisteredInformers(clusterLabels *bean.ClusterLabels, informerName string) { - UnregisteredInformers.WithLabelValues(clusterLabels.ClusterName, strconv.Itoa(clusterLabels.ClusterId), informerName).Inc() +func IncUnregisteredInformers(clusterLabels *bean.ClusterLabels, informerName InformerMetrics) { + UnregisteredInformers.WithLabelValues(clusterLabels.ClusterName, strconv.Itoa(clusterLabels.ClusterId), informerName.String()).Inc() +} + +func IncNonAdministrativeEvents(clusterLabels *bean.ClusterLabels, resourceType ResourceMetrics) { + NonAdministrativeEvents.WithLabelValues(clusterLabels.ClusterName, strconv.Itoa(clusterLabels.ClusterId), resourceType.String()).Inc() } diff --git a/kubewatch/pkg/resource/application/handler.go b/kubewatch/pkg/resource/application/handler.go index fe2519b07..614262585 100644 --- a/kubewatch/pkg/resource/application/handler.go +++ b/kubewatch/pkg/resource/application/handler.go @@ -22,6 +22,7 @@ import ( "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" applicationInformer "github.com/argoproj/argo-cd/v2/pkg/client/informers/externalversions/application/v1alpha1" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" "go.uber.org/zap" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -42,7 +43,7 @@ func NewInformerImpl(logger *zap.SugaredLogger, } } -func (impl *InformerImpl) GetSharedInformer(clusterId int, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) { +func (impl *InformerImpl) GetSharedInformer(clusterLabels *informerBean.ClusterLabels, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) { startTime := time.Now() defer func() { impl.logger.Debugw("registered application informer", "namespace", namespace, "time", time.Since(startTime)) @@ -66,7 +67,7 @@ func (impl *InformerImpl) GetSharedInformer(clusterId int, namespace string, k8s if newApp.Status.History != nil && len(newApp.Status.History) > 0 { if oldApp.Status.History == nil || len(oldApp.Status.History) == 0 { impl.logger.Debug("new deployment detected") - impl.sendAppUpdate(clusterId, newApp, statusTime) + impl.sendAppUpdate(clusterLabels.ClusterId, newApp, statusTime) } else { impl.logger.Debugf("old deployment detected for update: %s, status:%s", oldApp.Name, oldApp.Status.Health.Status) oldRevision := oldApp.Status.Sync.Revision @@ -76,7 +77,7 @@ func (impl *InformerImpl) GetSharedInformer(clusterId int, namespace string, k8s newSyncStatus := string(newApp.Status.Sync.Status) oldSyncStatus := string(oldApp.Status.Sync.Status) if (oldRevision != newRevision) || (oldStatus != newStatus) || (newSyncStatus != oldSyncStatus) { - impl.sendAppUpdate(clusterId, newApp, statusTime) + impl.sendAppUpdate(clusterLabels.ClusterId, newApp, statusTime) impl.logger.Debug("send update app:" + oldApp.Name + ", oldRevision: " + oldRevision + ", newRevision:" + newRevision + ", oldStatus: " + oldStatus + ", newStatus: " + newStatus + ", newSyncStatus: " + newSyncStatus + ", oldSyncStatus: " + oldSyncStatus) @@ -98,7 +99,7 @@ func (impl *InformerImpl) GetSharedInformer(clusterId int, namespace string, k8s if app, ok := obj.(*applicationBean.Application); ok { statusTime := time.Now() impl.logger.Debugf("app delete detected: %s, status:%s", app.Name, app.Status.Health.Status) - impl.sendAppDelete(clusterId, app, statusTime) + impl.sendAppDelete(clusterLabels.ClusterId, app, statusTime) } }, }) diff --git a/kubewatch/pkg/resource/sharedInformer.go b/kubewatch/pkg/resource/sharedInformer.go index 754239ec8..a8b52254c 100644 --- a/kubewatch/pkg/resource/sharedInformer.go +++ b/kubewatch/pkg/resource/sharedInformer.go @@ -17,6 +17,7 @@ package resource import ( + informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" "github.com/devtron-labs/kubewatch/pkg/resource/application" "github.com/devtron-labs/kubewatch/pkg/resource/bean" "github.com/devtron-labs/kubewatch/pkg/resource/workflow" @@ -25,7 +26,7 @@ import ( ) type SharedInformer interface { - GetSharedInformer(clusterId int, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) + GetSharedInformer(clusterLabels *informerBean.ClusterLabels, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) } func (impl *InformerClientImpl) GetSharedInformerClient(sharedInformerType bean.SharedInformerType) SharedInformer { diff --git a/kubewatch/pkg/resource/unimplemented.go b/kubewatch/pkg/resource/unimplemented.go index 31eb68f65..34ce00a38 100644 --- a/kubewatch/pkg/resource/unimplemented.go +++ b/kubewatch/pkg/resource/unimplemented.go @@ -18,6 +18,7 @@ package resource import ( "fmt" + informerBean "github.com/devtron-labs/kubewatch/pkg/informer/bean" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" ) @@ -29,6 +30,6 @@ func NewUnimplementedImpl() *UnimplementedImpl { return &UnimplementedImpl{} } -func (impl *UnimplementedImpl) GetSharedInformer(clusterId int, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) { +func (impl *UnimplementedImpl) GetSharedInformer(clusterLabels *informerBean.ClusterLabels, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) { return nil, fmt.Errorf("informer not implemented") } diff --git a/kubewatch/pkg/resource/workflow/handler.go b/kubewatch/pkg/resource/workflow/handler.go index f3880447b..07091d271 100644 --- a/kubewatch/pkg/resource/workflow/handler.go +++ b/kubewatch/pkg/resource/workflow/handler.go @@ -19,8 +19,12 @@ package workflow import ( "encoding/json" "github.com/argoproj/argo-workflows/v3/workflow/util" + informerBean "github.com/devtron-labs/common-lib/informer" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/kubewatch/pkg/config" + "github.com/devtron-labs/kubewatch/pkg/informer/bean" + "github.com/devtron-labs/kubewatch/pkg/informer/cluster/argoWf" + "github.com/devtron-labs/kubewatch/pkg/middleware" "go.uber.org/zap" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/dynamic" @@ -30,31 +34,31 @@ import ( ) type InformerImpl struct { - logger *zap.SugaredLogger - client *pubsub.PubSubClientServiceImpl - appConfig *config.AppConfig - natsTopicName string + logger *zap.SugaredLogger + client *pubsub.PubSubClientServiceImpl + appConfig *config.AppConfig + workflowType string } func NewCiInformerImpl(logger *zap.SugaredLogger, client *pubsub.PubSubClientServiceImpl, appConfig *config.AppConfig) *InformerImpl { return &InformerImpl{ - logger: logger, - client: client, - appConfig: appConfig, - natsTopicName: pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, + logger: logger, + client: client, + appConfig: appConfig, + workflowType: informerBean.CiWorkflowName, } } func NewCdInformerImpl(logger *zap.SugaredLogger, client *pubsub.PubSubClientServiceImpl, appConfig *config.AppConfig) *InformerImpl { return &InformerImpl{ - logger: logger, - client: client, - appConfig: appConfig, - natsTopicName: pubsub.CD_WORKFLOW_STATUS_UPDATE, + logger: logger, + client: client, + appConfig: appConfig, + workflowType: informerBean.CdWorkflowName, } } -func (impl *InformerImpl) GetSharedInformer(clusterId int, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) { +func (impl *InformerImpl) GetSharedInformer(clusterLabels *bean.ClusterLabels, namespace string, k8sConfig *rest.Config) (cache.SharedIndexInformer, error) { startTime := time.Now() defer func() { impl.logger.Debugw("registered workflow informer", "namespace", namespace, "time", time.Since(startTime)) @@ -73,29 +77,56 @@ func (impl *InformerImpl) GetSharedInformer(clusterId int, namespace string, k8s _, err = workflowInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) {}, UpdateFunc: func(oldWf, newWf interface{}) { - impl.logger.Info("workflow update detected") - if workflow, ok := newWf.(*unstructured.Unstructured).Object["status"]; ok { - wfJson, err := json.Marshal(workflow) - if err != nil { - impl.logger.Errorw("error occurred while marshalling workflow", "err", err) + impl.logger.Infow("workflow update detected", "workflowType", impl.workflowType) + if workflowObject, ok := newWf.(*unstructured.Unstructured); ok { + workflow, found, jsonErr := unstructured.NestedMap(workflowObject.Object, "status") + if jsonErr != nil { + impl.logger.Errorw("error in getting workflow status", "wfObject", workflowObject.Object, "err", jsonErr) return - } - impl.logger.Debugw("sending workflow update event ", "wfJson", string(wfJson)) - var reqBody = wfJson - if impl.appConfig.GetExternalConfig().External { - err = publishEventsOnRest(reqBody, impl.natsTopicName, impl.appConfig.GetExternalConfig()) - } else { - if impl.client == nil { - impl.logger.Warn("don't publish") + } else if found { + workflowLabels := workflowObject.GetLabels() + if val, ok := workflowLabels[informerBean.WorkflowTypeLabelKey]; ok && impl.workflowType != val { + impl.logger.Warnw("workflow type label is not matching with the workflow type", "workflowType", impl.workflowType, "workflowTypeLabel", val) + // return statement is skipped intentionally for backward compatibility + // TODO Asutosh: Use this as a labelSelector to filter out the workflows in future. return } - err = impl.client.Publish(impl.natsTopicName, string(reqBody)) - } - if err != nil { - impl.logger.Errorw("Error while publishing Request", "err ", err) - return + if val, ok := workflowLabels[informerBean.DevtronOwnerInstanceLabelKey]; ok { + workflow[bean.DevtronOwnerInstance] = val + } else { + impl.logger.Warnw("devtron administrator instance label is not found in the workflow. not a devtron workflow", "workflowLabels", workflowLabels) + middleware.IncNonAdministrativeEvents(clusterLabels, middleware.RESOURCE_ARGO_WORKFLOW) + // return statement is skipped intentionally for backward compatibility + // TODO Asutosh: remove this return statement in future + // return + } + wfJson, err := json.Marshal(workflow) + if err != nil { + impl.logger.Errorw("error occurred while marshalling workflow", "err", err) + return + } + natsTopicName, err := argoWf.GetNatsTopicForWorkflow(impl.workflowType) + if err != nil { + impl.logger.Errorw("error in getting nats topic for workflow", "workflowType", impl.workflowType, "err", err) + return + } + impl.logger.Debugw("sending workflow update event", "natsTopicName", natsTopicName, "wfJson", string(wfJson)) + var reqBody = wfJson + if impl.appConfig.GetExternalConfig().External { + err = publishEventsOnRest(reqBody, natsTopicName, impl.appConfig.GetExternalConfig()) + } else { + if impl.client == nil { + impl.logger.Warn("don't publish") + return + } + err = impl.client.Publish(natsTopicName, string(reqBody)) + } + if err != nil { + impl.logger.Errorw("error while publishing request", "natsTopicName", natsTopicName, "err", err) + return + } + impl.logger.Debug("workflow update sent") } - impl.logger.Debug("workflow update sent") } }, DeleteFunc: func(wf interface{}) {}, diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/informer/bean.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/informer/bean.go new file mode 100644 index 000000000..38aed85af --- /dev/null +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/informer/bean.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package informer + +const ( + ClusterModifyEventSecretType = "cluster.request/modify" + ClusterActionAdd = "add" + ClusterActionUpdate = "update" + ClusterActionDelete = "delete" + SecretFieldAction = "action" + SecretFieldClusterId = "cluster_id" +) + +const ( + WorkflowTypeLabelKey = "workflowType" + CiWorkflowName = "ci" + CdWorkflowName = "cd" +) + +const ( + DevtronOwnerInstanceLabelKey = "devtron.ai/owner-instance" +) + +const ( + PodDeletedMessage = "pod deleted" +) diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go index 933b97f4f..9f8524e48 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/kubewatch/vendor/github.com/golang-jwt/jwt/v4/parser.go b/kubewatch/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5..0fc510a0a 100644 --- a/kubewatch/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/kubewatch/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/kubewatch/vendor/modules.txt b/kubewatch/vendor/modules.txt index 8ee66bd9a..cd55c9f2b 100644 --- a/kubewatch/vendor/modules.txt +++ b/kubewatch/vendor/modules.txt @@ -215,12 +215,13 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util +github.com/devtron-labs/common-lib/informer github.com/devtron-labs/common-lib/monitoring github.com/devtron-labs/common-lib/monitoring/pprof github.com/devtron-labs/common-lib/monitoring/statsViz @@ -393,7 +394,7 @@ github.com/gobwas/glob/util/strings ## explicit; go 1.15 github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da @@ -1760,4 +1761,4 @@ upper.io/db.v3/postgresql # k8s.io/mount-utils => k8s.io/mount-utils v0.29.7 # k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.7 # k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/lens/go.mod b/lens/go.mod index 2ecf38715..d2fc852ea 100644 --- a/lens/go.mod +++ b/lens/go.mod @@ -59,4 +59,4 @@ require ( github.com/onsi/gomega v1.18.1 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 diff --git a/lens/go.sum b/lens/go.sum index 0bea68c17..32b42e9f8 100644 --- a/lens/go.sum +++ b/lens/go.sum @@ -22,8 +22,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f h1:o8eUXiQxP6k8liKkIwdOVjibZvd+jNvz3U0jqyqXdr8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 h1:g8ry9IOQzcrX4bkzE/jTCRlixarb0FzTt64w8qNd2wA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/lens/vendor/modules.txt b/lens/vendor/modules.txt index 96f70618a..349dd66cb 100644 --- a/lens/vendor/modules.txt +++ b/lens/vendor/modules.txt @@ -7,7 +7,7 @@ github.com/caarlos0/env # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -289,4 +289,4 @@ google.golang.org/protobuf/types/known/timestamppb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250415183144-214ff7e2424f +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425061517-45edc2765147