diff --git a/chart-sync/go.mod b/chart-sync/go.mod index 8e87d039c..90c0a484a 100644 --- a/chart-sync/go.mod +++ b/chart-sync/go.mod @@ -5,7 +5,7 @@ go 1.22.4 toolchain go1.22.6 replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 ) diff --git a/chart-sync/go.sum b/chart-sync/go.sum index f7733a6f2..f938021a0 100644 --- a/chart-sync/go.sum +++ b/chart-sync/go.sum @@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= diff --git a/chart-sync/internals/sql/DockerArtifactStoreRepository.go b/chart-sync/internals/sql/DockerArtifactStoreRepository.go index e2d873a6a..5094af2b4 100644 --- a/chart-sync/internals/sql/DockerArtifactStoreRepository.go +++ b/chart-sync/internals/sql/DockerArtifactStoreRepository.go @@ -57,6 +57,7 @@ type DockerArtifactStore struct { Cert string `sql:"cert" json:"cert,omitempty"` Active bool `sql:"active,notnull" json:"active"` RemoteConnectionConfigId int `sql:"remote_connection_config_id"` + CredentialsType string `sql:"credentials_type,notnull"` OCIRegistryConfig []*OCIRegistryConfig RemoteConnectionConfig *RemoteConnectionConfig AuditLog diff --git a/chart-sync/pkg/registry/adapter.go b/chart-sync/pkg/registry/adapter.go index cf778fc77..b81419cb4 100644 --- a/chart-sync/pkg/registry/adapter.go +++ b/chart-sync/pkg/registry/adapter.go @@ -46,5 +46,6 @@ func NewToRegistryConfig(store *sql.DockerArtifactStore) (*registry.Configuratio RegistryType: string(store.RegistryType), IsPublicRegistry: store.OCIRegistryConfig[0].IsPublic, RemoteConnectionConfig: remoteConnectionConfig, + CredentialsType: store.CredentialsType, }, nil } diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/InternalErrorCode.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/InternalErrorCode.go new file mode 100644 index 000000000..0e807b971 --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/InternalErrorCode.go @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + */ + +package constants + +const ( + UserCreateDBFailed string = "6001" + UserCreatePolicyFailed string = "6002" + UserUpdateDBFailed string = "6003" + UserUpdatePolicyFailed string = "6004" + UserNoTokenProvided string = "6005" + UserNotFoundForToken string = "6006" + UserCreateFetchRoleFailed string = "6007" + UserUpdateFetchRoleFailed string = "6008" +) diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go new file mode 100644 index 000000000..f327d712b --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package constants + +const ( + PanicLogIdentifier = "DEVTRON_PANIC_RECOVER" + GoRoutinePanicMsgLogPrefix = "GO_ROUTINE_PANIC_LOG:" +) + +// service names constant + +type ServiceName string + +func (m ServiceName) ToString() string { + return string(m) +} + +const ( + Orchestrator ServiceName = "ORCHESTRATOR" + Kubelink ServiceName = "KUBELINK" + GitSensor ServiceName = "GITSENSOR" + Kubewatch ServiceName = "KUBEWATCH" +) + +// metrics name constants +const ( + NATS_PUBLISH_COUNT = "nats_publish_count" + NATS_CONSUMPTION_COUNT = "nats_consumption_count" + NATS_CONSUMING_COUNT = "nats_consuming_count" + NATS_EVENT_CONSUMPTION_TIME = "nats_event_consumption_time" + NATS_EVENT_PUBLISH_TIME = "nats_event_publish_time" + NATS_EVENT_DELIVERY_COUNT = "nats_event_delivery_count" + PANIC_RECOVERY_COUNT = "panic_recovery_count" + REVERSE_PROXY_PANIC_RECOVERY_COUNT = "reverse_proxy_panic_recovery_count" +) + +// metrics labels constant +const ( + PANIC_TYPE = "panic_type" + HOST = "host" + METHOD = "method" + PATH = "path" + TOPIC = "topic" + STATUS = "status" +) + +// multiple history rows for one source event +type SourceType int + +const ( + SourceTypeImage SourceType = 1 + SourceTypeCode SourceType = 2 + SourceTypeSbom SourceType = 3 // can be used in future for direct sbom scanning +) + +type SourceSubType int + +const ( + SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) + SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) +) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go index cd25e430d..0bee1dbc0 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go @@ -20,6 +20,7 @@ type Configuration struct { RegistryType string IsPublicRegistry bool RemoteConnectionConfig *bean.RemoteConnectionConfigBean + CredentialsType string } type RegistryConnectionType string @@ -53,3 +54,7 @@ const ( const ( REGISTRY_CREDENTIAL_BASE_PATH = "/home/devtron/registry-credentials" ) + +const ( + HTTP = "http" +) diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go index ac062c8ea..12ae9806c 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go @@ -14,6 +14,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "os" "strings" ) @@ -176,4 +177,11 @@ func GetTlsConfig(config *Configuration) (*tls.Config, error) { return tlsConfig, nil } -// TODO: add support for certFile, keyFile on UI? +func IsPlainHttp(URL string) bool { + parsedURL, err := url.Parse(URL) + if err != nil { + return false + } + plainHttp := parsedURL.Scheme == HTTP + return plainHttp +} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go index 645af368c..1fa08fded 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go @@ -1,6 +1,7 @@ package registry import ( + "github.com/devtron-labs/common-lib/constants" "go.uber.org/zap" "helm.sh/helm/v3/pkg/registry" ) @@ -45,7 +46,8 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r } clientOptions := []registry.ClientOption{registry.ClientOptHTTPClient(httpClient)} - if config.RegistryConnectionType == INSECURE_CONNECTION { + + if IsPlainHttp(config.RegistryUrl) { clientOptions = append(clientOptions, registry.ClientOptPlainHTTP()) } @@ -55,7 +57,7 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r return nil, err } - if config != nil && !config.IsPublicRegistry { + if config != nil && !config.IsPublicRegistry && !(config.CredentialsType == string(constants.CredentialsTypeAnonymous)) { registryClient, err = GetLoggedInClient(registryClient, config) if err != nil { return nil, err diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/chart-sync/vendor/modules.txt b/chart-sync/vendor/modules.txt index 8486d7228..b1ae7035a 100644 --- a/chart-sync/vendor/modules.txt +++ b/chart-sync/vendor/modules.txt @@ -93,8 +93,9 @@ github.com/containerd/platforms # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 +github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/helmLib/registry @@ -790,4 +791,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/chart-sync/wire_gen.go b/chart-sync/wire_gen.go index 874f69de6..0022efdf4 100644 --- a/chart-sync/wire_gen.go +++ b/chart-sync/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/ci-runner/go.mod b/ci-runner/go.mod index 492785b93..9e199f46f 100644 --- a/ci-runner/go.mod +++ b/ci-runner/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.21.8 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c require ( github.com/Knetic/govaluate v3.0.0+incompatible diff --git a/ci-runner/go.sum b/ci-runner/go.sum index efb262a73..85a6e8806 100644 --- a/ci-runner/go.sum +++ b/ci-runner/go.sum @@ -95,8 +95,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/ci-runner/helper/DockerHelper.go b/ci-runner/helper/DockerHelper.go index 853afa54b..84f573ecb 100644 --- a/ci-runner/helper/DockerHelper.go +++ b/ci-runner/helper/DockerHelper.go @@ -31,6 +31,7 @@ import ( "github.com/caarlos0/env" cicxt "github.com/devtron-labs/ci-runner/executor/context" bean2 "github.com/devtron-labs/ci-runner/helper/bean" + "github.com/devtron-labs/common-lib/constants" "github.com/devtron-labs/ci-runner/util" "github.com/devtron-labs/common-lib/utils" @@ -176,6 +177,7 @@ func (impl *DockerHelperImpl) StartDockerDaemonAndDockerLogin(commonWorkflowRequ SecretKey: commonWorkflowRequest.SecretKey, DockerRegistryURL: commonWorkflowRequest.IntermediateDockerRegistryUrl, DockerRegistryType: commonWorkflowRequest.DockerRegistryType, + CredentialsType: commonWorkflowRequest.CredentialsType, }) if err != nil { return err @@ -210,7 +212,7 @@ const CacheModeMax = "max" const CacheModeMin = "min" type DockerCredentials struct { - DockerUsername, DockerPassword, AwsRegion, AccessKey, SecretKey, DockerRegistryURL, DockerRegistryType string + DockerUsername, DockerPassword, AwsRegion, AccessKey, SecretKey, DockerRegistryURL, DockerRegistryType, CredentialsType string } type EnvironmentVariables struct { @@ -224,6 +226,9 @@ func (impl *DockerHelperImpl) GetCommandToExecute(cmd string) *exec.Cmd { } func (impl *DockerHelperImpl) DockerLogin(ciContext cicxt.CiContext, dockerCredentials *DockerCredentials) error { + if dockerCredentials.CredentialsType == string(constants.CredentialsTypeAnonymous) { + return nil + } performDockerLogin := func() error { username := dockerCredentials.DockerUsername pwd := dockerCredentials.DockerPassword diff --git a/ci-runner/helper/EventHelper.go b/ci-runner/helper/EventHelper.go index f892a7b31..f8ebce402 100644 --- a/ci-runner/helper/EventHelper.go +++ b/ci-runner/helper/EventHelper.go @@ -102,6 +102,7 @@ type CommonWorkflowRequest struct { DockerImageTag string `json:"dockerImageTag"` DockerRegistryId string `json:"dockerRegistryId"` DockerRegistryType string `json:"dockerRegistryType"` + CredentialsType string `json:"credentialsType"` DockerRegistryURL string `json:"dockerRegistryURL"` DockerRegistryConnectionConfig *bean.RemoteConnectionConfigBean `json:"dockerRegistryConnectionConfig"` DockerConnection string `json:"dockerConnection"` diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/ci-runner/vendor/modules.txt b/ci-runner/vendor/modules.txt index 65cd46d1d..f0c8d0ac2 100644 --- a/ci-runner/vendor/modules.txt +++ b/ci-runner/vendor/modules.txt @@ -248,7 +248,7 @@ github.com/cespare/xxhash/v2 # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/constants @@ -991,4 +991,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/common-lib/constants/constants.go b/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/common-lib/constants/constants.go +++ b/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/common-lib/helmLib/registry/bean.go b/common-lib/helmLib/registry/bean.go index cd25e430d..0bee1dbc0 100644 --- a/common-lib/helmLib/registry/bean.go +++ b/common-lib/helmLib/registry/bean.go @@ -20,6 +20,7 @@ type Configuration struct { RegistryType string IsPublicRegistry bool RemoteConnectionConfig *bean.RemoteConnectionConfigBean + CredentialsType string } type RegistryConnectionType string @@ -53,3 +54,7 @@ const ( const ( REGISTRY_CREDENTIAL_BASE_PATH = "/home/devtron/registry-credentials" ) + +const ( + HTTP = "http" +) diff --git a/common-lib/helmLib/registry/common.go b/common-lib/helmLib/registry/common.go index ac062c8ea..12ae9806c 100644 --- a/common-lib/helmLib/registry/common.go +++ b/common-lib/helmLib/registry/common.go @@ -14,6 +14,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "os" "strings" ) @@ -176,4 +177,11 @@ func GetTlsConfig(config *Configuration) (*tls.Config, error) { return tlsConfig, nil } -// TODO: add support for certFile, keyFile on UI? +func IsPlainHttp(URL string) bool { + parsedURL, err := url.Parse(URL) + if err != nil { + return false + } + plainHttp := parsedURL.Scheme == HTTP + return plainHttp +} diff --git a/common-lib/helmLib/registry/defaultSettings.go b/common-lib/helmLib/registry/defaultSettings.go index 645af368c..1fa08fded 100644 --- a/common-lib/helmLib/registry/defaultSettings.go +++ b/common-lib/helmLib/registry/defaultSettings.go @@ -1,6 +1,7 @@ package registry import ( + "github.com/devtron-labs/common-lib/constants" "go.uber.org/zap" "helm.sh/helm/v3/pkg/registry" ) @@ -45,7 +46,8 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r } clientOptions := []registry.ClientOption{registry.ClientOptHTTPClient(httpClient)} - if config.RegistryConnectionType == INSECURE_CONNECTION { + + if IsPlainHttp(config.RegistryUrl) { clientOptions = append(clientOptions, registry.ClientOptPlainHTTP()) } @@ -55,7 +57,7 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r return nil, err } - if config != nil && !config.IsPublicRegistry { + if config != nil && !config.IsPublicRegistry && !(config.CredentialsType == string(constants.CredentialsTypeAnonymous)) { registryClient, err = GetLoggedInClient(registryClient, config) if err != nil { return nil, err diff --git a/common-lib/utils/CommonUtils.go b/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/common-lib/utils/CommonUtils.go +++ b/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/common-lib/utils/SqlUtil.go b/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/common-lib/utils/bean/bean.go b/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/common-lib/utils/bean/bean.go +++ b/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/common-lib/utils/sql/UtilStructs.go b/common-lib/utils/sql/AuditLog.go similarity index 100% rename from common-lib/utils/sql/UtilStructs.go rename to common-lib/utils/sql/AuditLog.go diff --git a/git-sensor/go.mod b/git-sensor/go.mod index 2981cea8d..834818f6b 100644 --- a/git-sensor/go.mod +++ b/git-sensor/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.22.4 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c require ( github.com/caarlos0/env v3.5.0+incompatible diff --git a/git-sensor/go.sum b/git-sensor/go.sum index 40a2e1106..d444c9851 100644 --- a/git-sensor/go.sum +++ b/git-sensor/go.sum @@ -27,8 +27,8 @@ github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e/go.mod h1:1TqULGlTey+VNhAu/ag7NJuUvByJemkqodsc9L5PHJk= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/git-sensor/internals/sql/WebhookEventDataMappingRepository.go b/git-sensor/internals/sql/WebhookEventDataMappingRepository.go index 4923541a6..a7b6bf94b 100644 --- a/git-sensor/internals/sql/WebhookEventDataMappingRepository.go +++ b/git-sensor/internals/sql/WebhookEventDataMappingRepository.go @@ -17,7 +17,6 @@ package sql import ( - "github.com/devtron-labs/git-sensor/util" "github.com/go-pg/pg" "github.com/go-pg/pg/orm" "time" @@ -61,15 +60,7 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetCiPipelineMaterialWebhookDa Where("webhook_data_id =? ", webhookParsedDataId). Where("is_active = TRUE "). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return &mapping, nil + return &mapping, err } func (impl WebhookEventDataMappingRepositoryImpl) SaveCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *CiPipelineMaterialWebhookDataMapping) error { @@ -89,15 +80,7 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetMatchedCiPipelineMaterialWe Where("is_active = TRUE "). Where("condition_matched = TRUE "). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return pipelineMaterials, nil + return pipelineMaterials, err } func (impl WebhookEventDataMappingRepositoryImpl) InactivateWebhookDataMappingForPipelineMaterials(ciPipelineMaterialIds []int) error { @@ -127,15 +110,7 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetWebhookPayloadDataForPipeli Offset(offset). Order("updated_on " + sortOrder). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return mappings, nil + return mappings, err } func (impl WebhookEventDataMappingRepositoryImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId int, webhookParsedDataId int) (*CiPipelineMaterialWebhookDataMapping, error) { @@ -149,13 +124,5 @@ func (impl WebhookEventDataMappingRepositoryImpl) GetWebhookPayloadFilterDataFor Where("webhook_data_id =? ", webhookParsedDataId). Where("is_active = TRUE "). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - - return &mapping, nil + return &mapping, err } diff --git a/git-sensor/internals/sql/WebhookEventParsedDataRepository.go b/git-sensor/internals/sql/WebhookEventParsedDataRepository.go index 609b653c3..51890dab6 100644 --- a/git-sensor/internals/sql/WebhookEventParsedDataRepository.go +++ b/git-sensor/internals/sql/WebhookEventParsedDataRepository.go @@ -17,7 +17,6 @@ package sql import ( - "github.com/devtron-labs/git-sensor/util" "github.com/go-pg/pg" "time" ) @@ -54,13 +53,7 @@ func NewWebhookEventParsedDataRepositoryImpl(dbConnection *pg.DB) *WebhookEventP func (impl WebhookEventParsedDataRepositoryImpl) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*WebhookEventParsedData, error) { var webhookEventParsedData WebhookEventParsedData err := impl.dbConnection.Model(&webhookEventParsedData).Where("event_id =? ", eventId).Where("unique_id =? ", uniqueId).Select() - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - return &webhookEventParsedData, nil + return &webhookEventParsedData, err } func (impl WebhookEventParsedDataRepositoryImpl) SaveWebhookParsedEventData(webhookEventParsedData *WebhookEventParsedData) error { @@ -88,12 +81,5 @@ func (impl WebhookEventParsedDataRepositoryImpl) GetWebhookEventParsedDataById(i err := impl.dbConnection.Model(&webhookEventParsedData). Where("id = ? ", id). Select() - - if err != nil { - if util.IsErrNoRows(err) { - return nil, nil - } - return nil, err - } - return &webhookEventParsedData, nil + return &webhookEventParsedData, err } diff --git a/git-sensor/internals/sql/mocks/WebhookEventDataMappingRepository.go b/git-sensor/internals/sql/mocks/WebhookEventDataMappingRepository.go new file mode 100644 index 000000000..79701b03a --- /dev/null +++ b/git-sensor/internals/sql/mocks/WebhookEventDataMappingRepository.go @@ -0,0 +1,201 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + sql "github.com/devtron-labs/git-sensor/internals/sql" + mock "github.com/stretchr/testify/mock" +) + +// WebhookEventDataMappingRepository is an autogenerated mock type for the WebhookEventDataMappingRepository type +type WebhookEventDataMappingRepository struct { + mock.Mock +} + +// GetCiPipelineMaterialWebhookDataMapping provides a mock function with given fields: ciPipelineMaterialId, webhookParsedDataId +func (_m *WebhookEventDataMappingRepository) GetCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId int, webhookParsedDataId int) (*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId, webhookParsedDataId) + + if len(ret) == 0 { + panic("no return value specified for GetCiPipelineMaterialWebhookDataMapping") + } + + var r0 *sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId, webhookParsedDataId) + } + if rf, ok := ret.Get(0).(func(int, int) *sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetMatchedCiPipelineMaterialWebhookDataMappingForPipelineMaterial provides a mock function with given fields: ciPipelineMaterialId +func (_m *WebhookEventDataMappingRepository) GetMatchedCiPipelineMaterialWebhookDataMappingForPipelineMaterial(ciPipelineMaterialId int) ([]*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId) + + if len(ret) == 0 { + panic("no return value specified for GetMatchedCiPipelineMaterialWebhookDataMappingForPipelineMaterial") + } + + var r0 []*sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int) ([]*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId) + } + if rf, ok := ret.Get(0).(func(int) []*sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(ciPipelineMaterialId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookPayloadDataForPipelineMaterialId provides a mock function with given fields: ciPipelineMaterialId, limit, offset, eventTimeSortOrder +func (_m *WebhookEventDataMappingRepository) GetWebhookPayloadDataForPipelineMaterialId(ciPipelineMaterialId int, limit int, offset int, eventTimeSortOrder string) ([]*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookPayloadDataForPipelineMaterialId") + } + + var r0 []*sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int, int, int, string) ([]*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + } + if rf, ok := ret.Get(0).(func(int, int, int, string) []*sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int, int, int, string) error); ok { + r1 = rf(ciPipelineMaterialId, limit, offset, eventTimeSortOrder) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookPayloadFilterDataForPipelineMaterialId provides a mock function with given fields: ciPipelineMaterialId, webhookParsedDataId +func (_m *WebhookEventDataMappingRepository) GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId int, webhookParsedDataId int) (*sql.CiPipelineMaterialWebhookDataMapping, error) { + ret := _m.Called(ciPipelineMaterialId, webhookParsedDataId) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookPayloadFilterDataForPipelineMaterialId") + } + + var r0 *sql.CiPipelineMaterialWebhookDataMapping + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (*sql.CiPipelineMaterialWebhookDataMapping, error)); ok { + return rf(ciPipelineMaterialId, webhookParsedDataId) + } + if rf, ok := ret.Get(0).(func(int, int) *sql.CiPipelineMaterialWebhookDataMapping); ok { + r0 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.CiPipelineMaterialWebhookDataMapping) + } + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(ciPipelineMaterialId, webhookParsedDataId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// InactivateWebhookDataMappingForPipelineMaterials provides a mock function with given fields: ciPipelineMaterialIds +func (_m *WebhookEventDataMappingRepository) InactivateWebhookDataMappingForPipelineMaterials(ciPipelineMaterialIds []int) error { + ret := _m.Called(ciPipelineMaterialIds) + + if len(ret) == 0 { + panic("no return value specified for InactivateWebhookDataMappingForPipelineMaterials") + } + + var r0 error + if rf, ok := ret.Get(0).(func([]int) error); ok { + r0 = rf(ciPipelineMaterialIds) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SaveCiPipelineMaterialWebhookDataMapping provides a mock function with given fields: ciPipelineMaterialWebhookDataMapping +func (_m *WebhookEventDataMappingRepository) SaveCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *sql.CiPipelineMaterialWebhookDataMapping) error { + ret := _m.Called(ciPipelineMaterialWebhookDataMapping) + + if len(ret) == 0 { + panic("no return value specified for SaveCiPipelineMaterialWebhookDataMapping") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.CiPipelineMaterialWebhookDataMapping) error); ok { + r0 = rf(ciPipelineMaterialWebhookDataMapping) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateCiPipelineMaterialWebhookDataMapping provides a mock function with given fields: ciPipelineMaterialWebhookDataMapping +func (_m *WebhookEventDataMappingRepository) UpdateCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *sql.CiPipelineMaterialWebhookDataMapping) error { + ret := _m.Called(ciPipelineMaterialWebhookDataMapping) + + if len(ret) == 0 { + panic("no return value specified for UpdateCiPipelineMaterialWebhookDataMapping") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.CiPipelineMaterialWebhookDataMapping) error); ok { + r0 = rf(ciPipelineMaterialWebhookDataMapping) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewWebhookEventDataMappingRepository creates a new instance of WebhookEventDataMappingRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWebhookEventDataMappingRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *WebhookEventDataMappingRepository { + mock := &WebhookEventDataMappingRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/git-sensor/internals/sql/mocks/WebhookEventParsedDataRepository.go b/git-sensor/internals/sql/mocks/WebhookEventParsedDataRepository.go new file mode 100644 index 000000000..7500db93c --- /dev/null +++ b/git-sensor/internals/sql/mocks/WebhookEventParsedDataRepository.go @@ -0,0 +1,153 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + sql "github.com/devtron-labs/git-sensor/internals/sql" + mock "github.com/stretchr/testify/mock" +) + +// WebhookEventParsedDataRepository is an autogenerated mock type for the WebhookEventParsedDataRepository type +type WebhookEventParsedDataRepository struct { + mock.Mock +} + +// GetWebhookEventParsedDataById provides a mock function with given fields: id +func (_m *WebhookEventParsedDataRepository) GetWebhookEventParsedDataById(id int) (*sql.WebhookEventParsedData, error) { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookEventParsedDataById") + } + + var r0 *sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func(int) (*sql.WebhookEventParsedData, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(int) *sql.WebhookEventParsedData); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookEventParsedDataByIds provides a mock function with given fields: ids, limit +func (_m *WebhookEventParsedDataRepository) GetWebhookEventParsedDataByIds(ids []int, limit int) ([]*sql.WebhookEventParsedData, error) { + ret := _m.Called(ids, limit) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookEventParsedDataByIds") + } + + var r0 []*sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func([]int, int) ([]*sql.WebhookEventParsedData, error)); ok { + return rf(ids, limit) + } + if rf, ok := ret.Get(0).(func([]int, int) []*sql.WebhookEventParsedData); ok { + r0 = rf(ids, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func([]int, int) error); ok { + r1 = rf(ids, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookParsedEventDataByEventIdAndUniqueId provides a mock function with given fields: eventId, uniqueId +func (_m *WebhookEventParsedDataRepository) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) { + ret := _m.Called(eventId, uniqueId) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookParsedEventDataByEventIdAndUniqueId") + } + + var r0 *sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func(int, string) (*sql.WebhookEventParsedData, error)); ok { + return rf(eventId, uniqueId) + } + if rf, ok := ret.Get(0).(func(int, string) *sql.WebhookEventParsedData); ok { + r0 = rf(eventId, uniqueId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(eventId, uniqueId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SaveWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventParsedDataRepository) SaveWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for SaveWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventParsedDataRepository) UpdateWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for UpdateWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewWebhookEventParsedDataRepository creates a new instance of WebhookEventParsedDataRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWebhookEventParsedDataRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *WebhookEventParsedDataRepository { + mock := &WebhookEventParsedDataRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/git-sensor/pkg/RepoManages.go b/git-sensor/pkg/RepoManages.go index 9f73f4674..5aec548f4 100644 --- a/git-sensor/pkg/RepoManages.go +++ b/git-sensor/pkg/RepoManages.go @@ -385,7 +385,7 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq gitCtx = gitCtx.WithCredentials(userName, password). WithTLSData(gitProvider.CaCert, gitProvider.TlsKey, gitProvider.TlsCert, gitProvider.EnableTLSVerification) - checkoutPath, _, _, err := impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) + checkoutPath, err := impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) if err != nil { return material, err } @@ -899,28 +899,28 @@ func (impl RepoManagerImpl) GetWebhookAndCiDataById(id int, ciPipelineMaterialId impl.logger.Debugw("Getting webhook data ", "id", id) webhookDataFromDb, err := impl.webhookEventParsedDataRepository.GetWebhookEventParsedDataById(id) - - if err != nil { - impl.logger.Errorw("error in getting webhook data for Id ", "Id", id, "err", err) - return nil, err - } - - filterData, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId, id) if err != nil { - impl.logger.Errorw("error in getting webhook payload filter data for webhookParsedId ", "Id", id, "err", err) + impl.logger.Errorw("error in getting webhook data for id ", "id", id, "err", err) return nil, err } - webhookData := impl.webhookEventBeanConverter.ConvertFromWebhookParsedDataSqlBean(webhookDataFromDb) webhookAndCiData := &git.WebhookAndCiData{ WebhookData: webhookData, } + filterData, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(ciPipelineMaterialId, id) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("error in getting webhook payload filter data for webhookParsedId ", "ciPipelineMaterialId", ciPipelineMaterialId, "id", id, "err", err) + return nil, err + } else if util2.IsErrNoRows(err) { + impl.logger.Warnw("no webhook payload filter data found for webhookParsedId", "ciPipelineMaterialId", ciPipelineMaterialId, "id", id) + return webhookAndCiData, nil + } + if filterData != nil { webhookAndCiData.ExtraEnvironmentVariables = util.BuildExtraEnvironmentVariablesForCi(filterData.FilterResults, webhookDataFromDb.CiEnvVariableData) } - return webhookAndCiData, nil } @@ -1070,18 +1070,25 @@ func (impl RepoManagerImpl) GetWebhookPayloadDataForPipelineMaterialId(request * } func (impl RepoManagerImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(request *git.WebhookPayloadFilterDataRequest) (*git.WebhookPayloadFilterDataResponse, error) { - impl.logger.Debugw("Getting webhook payload filter data ", "request", request) + impl.logger.Debugw("Getting webhook payload filter data", "request", request) - mapping, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(request.CiPipelineMaterialId, request.ParsedDataId) + parsedData, err := impl.webhookEventParsedDataRepository.GetWebhookEventParsedDataById(request.ParsedDataId) if err != nil { - impl.logger.Errorw("error in getting webhook filter payload data ", "err", err) + impl.logger.Errorw("error in getting parsed webhook data by id", "id", request.ParsedDataId, "err", err) return nil, err } - parsedData, err := impl.webhookEventParsedDataRepository.GetWebhookEventParsedDataById(request.ParsedDataId) - if err != nil { - impl.logger.Errorw("error in getting parsed webhook data ", "err", err) + webhookPayloadFilterDataResponse := &git.WebhookPayloadFilterDataResponse{ + PayloadId: parsedData.PayloadDataId, + } + + mapping, err := impl.webhookEventDataMappingRepository.GetWebhookPayloadFilterDataForPipelineMaterialId(request.CiPipelineMaterialId, request.ParsedDataId) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("error in getting webhook filter payload data", "ciPipelineMaterialId", request.CiPipelineMaterialId, "parsedDataId", request.ParsedDataId, "err", err) return nil, err + } else if util2.IsErrNoRows(err) { + impl.logger.Warnw("no webhook filter payload data found for parsedDataId", "ciPipelineMaterialId", request.CiPipelineMaterialId, "parsedDataId", request.ParsedDataId) + return webhookPayloadFilterDataResponse, nil } filterResults := mapping.FilterResults @@ -1099,11 +1106,7 @@ func (impl RepoManagerImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(req webhookPayloadFilterDataSelectorResponses = append(webhookPayloadFilterDataSelectorResponses, webhookPayloadFilterDataSelectorResponse) } } - - webhookPayloadFilterDataResponse := &git.WebhookPayloadFilterDataResponse{ - PayloadId: parsedData.PayloadDataId, - SelectorsData: webhookPayloadFilterDataSelectorResponses, - } + webhookPayloadFilterDataResponse.SelectorsData = webhookPayloadFilterDataSelectorResponses return webhookPayloadFilterDataResponse, nil } diff --git a/git-sensor/pkg/RepoManages_test.go b/git-sensor/pkg/RepoManages_test.go new file mode 100644 index 000000000..859f0cc2d --- /dev/null +++ b/git-sensor/pkg/RepoManages_test.go @@ -0,0 +1,196 @@ +package pkg + +import ( + "fmt" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/internals/sql/mocks" + "github.com/devtron-labs/git-sensor/pkg/git" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "testing" +) + +func TestRepoManagerImpl_GetWebhookAndCiDataById(t *testing.T) { + type args struct { + id int + ciPipelineMaterialId int + } + tests := []struct { + name string + args args + want *git.WebhookAndCiData + webhookEventParsedDataMockExecution func(*mocks.WebhookEventParsedDataRepository) + webhookEventDataMappingMockExecution func(*mocks.WebhookEventDataMappingRepository) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEvent_ParsedData_And_DataMappings_Found", + args: args{ + id: 1, + ciPipelineMaterialId: 1, + }, + want: &git.WebhookAndCiData{ + ExtraEnvironmentVariables: map[string]string{ + "VAR1": "envValue1", + "VAR2": "envValue2", + "VAR3": "filterValue3", + "VAR4": "filterValue4", + }, + WebhookData: &git.WebhookData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + wantErr: assert.NoError, + webhookEventParsedDataMockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookEventParsedDataById", 1). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "VAR1": "envValue1", + "VAR2": "envValue2", + }, + }, nil).Once() + }, + webhookEventDataMappingMockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetWebhookPayloadFilterDataForPipelineMaterialId", 1, 1). + Return(&sql.CiPipelineMaterialWebhookDataMapping{ + Id: 1, + CiPipelineMaterialId: 1, + FilterResults: []*sql.CiPipelineMaterialWebhookDataMappingFilterResult{ + { + MatchedGroups: map[string]string{ + "VAR1": "filterValue1", + "VAR2": "filterValue2", + }, + }, + { + MatchedGroups: map[string]string{ + "VAR3": "filterValue3", + "VAR4": "filterValue4", + }, + }, + }, + }, nil).Once() + }, + }, + { + name: "WebhookEvent_ParsedData_Found_But_DataMappings_NotFound", + args: args{ + id: 1, + ciPipelineMaterialId: 1, + }, + want: &git.WebhookAndCiData{ + WebhookData: &git.WebhookData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + wantErr: assert.NoError, + webhookEventParsedDataMockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookEventParsedDataById", 1). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "VAR1": "envValue1", + "VAR2": "envValue2", + }, + }, nil).Once() + }, + webhookEventDataMappingMockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetWebhookPayloadFilterDataForPipelineMaterialId", 1, 1). + Return(nil, pg.ErrNoRows).Once() + }, + }, + { + name: "WebhookEvent_ParsedData_And_DataMappings_NotFound", + args: args{ + id: 1, + ciPipelineMaterialId: 1, + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, pg.ErrNoRows.Error(), msgAndArgs...) + }, + webhookEventParsedDataMockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookEventParsedDataById", 1). + Return(nil, pg.ErrNoRows).Once() + }, + webhookEventDataMappingMockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + AssertNotCalled(t, "GetWebhookPayloadFilterDataForPipelineMaterialId", 1, 1) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getRepoManagerImpl(t, tt.webhookEventParsedDataMockExecution, tt.webhookEventDataMappingMockExecution) + got, err := impl.GetWebhookAndCiDataById(tt.args.id, tt.args.ciPipelineMaterialId) + if !tt.wantErr(t, err, fmt.Sprintf("GetWebhookAndCiDataById(%v, %v)", tt.args.id, tt.args.ciPipelineMaterialId)) { + return + } + assert.Equalf(t, tt.want, got, "GetWebhookAndCiDataById(%v, %v)", tt.args.id, tt.args.ciPipelineMaterialId) + }) + } +} + +func getRepoManagerImpl( + t *testing.T, + webhookEventParsedDataMockExecution func(*mocks.WebhookEventParsedDataRepository), + webhookEventDataMappingMockExecution func(*mocks.WebhookEventDataMappingRepository), +) *RepoManagerImpl { + sugaredLogger := getLogger() + webhookEventDataMappingRepository := getWebhookEventDataMappingRepository(t) + if webhookEventDataMappingMockExecution != nil { + webhookEventDataMappingMockExecution(webhookEventDataMappingRepository) + } + webhookEventParsedDataRepositoryMock := getWebhookEventParsedDataRepositoryMock(t) + if webhookEventParsedDataMockExecution != nil { + webhookEventParsedDataMockExecution(webhookEventParsedDataRepositoryMock) + } + webhookEventBeanConverterImpl := git.NewWebhookEventBeanConverterImpl() + return NewRepoManagerImpl( + sugaredLogger, + nil, nil, nil, nil, nil, nil, nil, nil, + webhookEventParsedDataRepositoryMock, + webhookEventDataMappingRepository, + webhookEventBeanConverterImpl, nil, nil, + ) +} + +func getLogger() *zap.SugaredLogger { + return logger.NewSugaredLogger() +} + +func getWebhookEventParsedDataRepositoryMock(t *testing.T) *mocks.WebhookEventParsedDataRepository { + return mocks.NewWebhookEventParsedDataRepository(t) +} + +func getWebhookEventDataMappingRepository(t *testing.T) *mocks.WebhookEventDataMappingRepository { + return mocks.NewWebhookEventDataMappingRepository(t) +} diff --git a/git-sensor/pkg/git/RepositoryManager.go b/git-sensor/pkg/git/RepositoryManager.go index 7aa672f9d..b2a7eb146 100644 --- a/git-sensor/pkg/git/RepositoryManager.go +++ b/git-sensor/pkg/git/RepositoryManager.go @@ -45,7 +45,7 @@ type RepositoryManager interface { InitRepoAndGetSshPrivateKeyPath(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, string, error) FetchRepo(gitCtx GitContext, location string) (errMsg string, err error) BackupGitMaterialBeforeUpdate(detailedExistingMaterial *sql.GitMaterial) error - GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) + GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, err error) GetCheckoutLocation(gitCtx GitContext, material *sql.GitMaterial, url, checkoutPath string) string TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase // Clean cleans a directory @@ -122,24 +122,38 @@ func (impl *RepositoryManagerImpl) getBackupDirForMaterial(material *sql.GitMate return path.Join(BACKUP_BASE_DIR, strconv.Itoa(material.Id), time.Now().Format("02-Jan-2006 15:04:05")) } -func (impl *RepositoryManagerImpl) GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) { +func (impl *RepositoryManagerImpl) GetCheckoutLocationFromGitUrl(material *sql.GitMaterial, cloningMode string) (location string, err error) { //gitRegex := `/(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|\#[-\d\w._]+?)$/` httpsRegex := `^https.*` httpsMatched, err := regexp.MatchString(httpsRegex, material.Url) if httpsMatched { locationWithoutProtocol := strings.ReplaceAll(material.Url, "https://", "") checkoutPath := path.Join(impl.getBaseDirForMaterial(material), locationWithoutProtocol) - return checkoutPath, httpsMatched, false, nil + return checkoutPath, nil + } + httpRegex := `^http.*` + httpMatch, err := regexp.MatchString(httpRegex, material.Url) + if httpMatch { + locationWithoutProtocol := strings.ReplaceAll(material.Url, "http://", "") + checkoutPath := path.Join(impl.getBaseDirForMaterial(material), locationWithoutProtocol) + return checkoutPath, nil } sshRegex := `^git@.*` sshMatched, err := regexp.MatchString(sshRegex, material.Url) if sshMatched { checkoutPath := path.Join(impl.getBaseDirForMaterial(material), material.Url) - return checkoutPath, httpsMatched, sshMatched, nil + return checkoutPath, nil + } + // this regex is used to generic ssh providers like gogs where format is @:/.git + scpLikeSSHRegex := `^[\w-]+@[\w.-]+:[\w./-]+\.git$` + rootMatched, err := regexp.MatchString(scpLikeSSHRegex, material.Url) + if rootMatched { + checkoutPath := path.Join(impl.getBaseDirForMaterial(material), material.Url) + return checkoutPath, nil } - return "", httpsMatched, sshMatched, fmt.Errorf("unsupported format url %s", material.Url) + return "", fmt.Errorf("unsupported format url %s", material.Url) } func (impl *RepositoryManagerImpl) GetCheckoutLocation(gitCtx GitContext, material *sql.GitMaterial, url, checkoutPath string) string { diff --git a/git-sensor/pkg/git/Util.go b/git-sensor/pkg/git/Util.go index 7d4101734..2b96e6eee 100644 --- a/git-sensor/pkg/git/Util.go +++ b/git-sensor/pkg/git/Util.go @@ -60,6 +60,7 @@ func GetProjectName(url string) string { url = url[strings.LastIndex(url, "/")+1:] return strings.TrimSuffix(url, ".git") } + func GetCheckoutPath(url string, cloneLocation string) string { //url= https://github.com/devtron-labs/git-sensor.git cloneLocation= git-base/1/github.com/prakash100198 //then this function returns git-base/1/github.com/prakash100198/SampleGoLangProject/.git @@ -83,11 +84,13 @@ func GetUserNamePassword(gitProvider *sql.GitProvider) (userName, password strin return "", "", fmt.Errorf("unsupported %s", gitProvider.AuthMode) } } + func getSSHPrivateKeyFolderAndFilePath(gitProviderId int) (string, string) { sshPrivateKeyFolderPath := path.Join(SSH_PRIVATE_KEY_DIR, strconv.Itoa(gitProviderId)) sshPrivateKeyFilePath := path.Join(sshPrivateKeyFolderPath, SSH_PRIVATE_KEY_FILE_NAME) return sshPrivateKeyFolderPath, sshPrivateKeyFilePath } + func GetOrCreateSshPrivateKeyOnDisk(gitProviderId int, sshPrivateKeyContent string) (privateKeyPath string, err error) { sshPrivateKeyFolderPath, sshPrivateKeyFilePath := getSSHPrivateKeyFolderAndFilePath(gitProviderId) @@ -232,6 +235,7 @@ func processFileStatOutputNameOnly(commitDiff string) (FileStats, error) { return filestat, nil } + func IsRepoShallowCloned(checkoutPath string) bool { return strings.Contains(checkoutPath, "/.git") } @@ -240,3 +244,7 @@ func getTestBaseDir() string { dir, _ := os.UserHomeDir() return path.Join(dir, "/tmp") } + +var ( + ErrWebhookEventParsedDataNotFound = fmt.Errorf("webhook event parsed data not found") +) diff --git a/git-sensor/pkg/git/Watcher.go b/git-sensor/pkg/git/Watcher.go index 58bd3264d..c0ff1d51a 100644 --- a/git-sensor/pkg/git/Watcher.go +++ b/git-sensor/pkg/git/Watcher.go @@ -195,7 +195,7 @@ func (impl *GitWatcherImpl) pollAndUpdateGitMaterial(materialReq *sql.GitMateria // Helper function to handle SSH key creation and retry fetching material func (impl *GitWatcherImpl) handleSshKeyCreationAndRetry(gitCtx GitContext, material *sql.GitMaterial, location string, gitProvider *sql.GitProvider) (updated bool, repo *GitRepository, errMsg string, err error) { if strings.Contains(material.CheckoutLocation, "/.git") { - location, _, _, err = impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) + location, err = impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode) if err != nil { impl.logger.Errorw("error in getting clone location ", "material", material, "errMsg", errMsg, "err", err) return false, nil, errMsg, err diff --git a/git-sensor/pkg/git/WebhookEventService.go b/git-sensor/pkg/git/WebhookEventService.go index 8d44fcdfb..40a49d0da 100644 --- a/git-sensor/pkg/git/WebhookEventService.go +++ b/git-sensor/pkg/git/WebhookEventService.go @@ -22,6 +22,7 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/internals/util" + util2 "github.com/devtron-labs/git-sensor/util" _ "github.com/robfig/cron/v3" "go.uber.org/zap" "regexp" @@ -94,17 +95,18 @@ func (impl WebhookEventServiceImpl) GetAllGitHostWebhookEventByGitHostName(gitHo func (impl WebhookEventServiceImpl) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) { impl.logger.Debugw("fetching webhook event parsed data for ", "eventId", eventId, "uniqueId", uniqueId) - if len(uniqueId) == 0 { - return nil, nil + impl.logger.Warn("uniqueId is blank. so skipping fetching webhook event parsed data...") + return nil, ErrWebhookEventParsedDataNotFound } - webhookEventParsedData, err := impl.webhookEventParsedDataRepository.GetWebhookParsedEventDataByEventIdAndUniqueId(eventId, uniqueId) - if err != nil { - impl.logger.Errorw("getting error while fetching webhook event parsed data ", "err", err) + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("getting error while fetching webhook event parsed data ", "eventId", eventId, "uniqueId", uniqueId, "err", err) return nil, err + } else if util2.IsErrNoRows(err) { + impl.logger.Warnw("webhook event parsed data not found", "eventId", eventId, "uniqueId", uniqueId) + return nil, ErrWebhookEventParsedDataNotFound } - return webhookEventParsedData, nil } @@ -332,14 +334,20 @@ func (impl WebhookEventServiceImpl) NotifyForAutoCi(material *CiPipelineMaterial return err } -func (impl WebhookEventServiceImpl) HandleMaterialWebhookMappingIntoDb(ciPipelineMaterialId int, webhookParsedDataId int, conditionMatched bool, filterResults []*sql.CiPipelineMaterialWebhookDataMappingFilterResult) error { - impl.logger.Debug("Handling Material webhook mapping into DB") - +func (impl WebhookEventServiceImpl) getCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId int, webhookParsedDataId int) (*sql.CiPipelineMaterialWebhookDataMapping, bool, error) { + var isNewMapping bool mapping, err := impl.webhookEventDataMappingRepository.GetCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId, webhookParsedDataId) - if err != nil { - impl.logger.Errorw("err in getting ci-pipeline vs webhook data mapping", "err", err) - return err + if err != nil && !util2.IsErrNoRows(err) { + impl.logger.Errorw("err in getting ci-pipeline vs webhook data mapping", "ciPipelineMaterialId", ciPipelineMaterialId, "webhookParsedDataId", webhookParsedDataId, "err", err) + return mapping, isNewMapping, err + } else if util2.IsErrNoRows(err) { + isNewMapping = true } + return mapping, isNewMapping, nil +} + +func (impl WebhookEventServiceImpl) HandleMaterialWebhookMappingIntoDb(ciPipelineMaterialId int, webhookParsedDataId int, conditionMatched bool, filterResults []*sql.CiPipelineMaterialWebhookDataMappingFilterResult) error { + impl.logger.Debugw("Handling Material webhook mapping into DB", "ciPipelineMaterialId", ciPipelineMaterialId, "webhookParsedDataId", webhookParsedDataId) ciPipelineMaterialWebhookDataMapping := &sql.CiPipelineMaterialWebhookDataMapping{ CiPipelineMaterialId: ciPipelineMaterialId, @@ -349,8 +357,12 @@ func (impl WebhookEventServiceImpl) HandleMaterialWebhookMappingIntoDb(ciPipelin UpdatedOn: time.Now(), } - isNewMapping := mapping == nil - + // get mapping + mapping, isNewMapping, err := impl.getCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId, webhookParsedDataId) + if err != nil { + impl.logger.Errorw("err in getting ci-pipeline vs webhook data mapping", "ciPipelineMaterialId", ciPipelineMaterialId, "webhookParsedDataId", webhookParsedDataId, "err", err) + return err + } if isNewMapping { // insert into DB impl.logger.Debug("Saving mapping into DB") diff --git a/git-sensor/pkg/git/WebhookEventService_test.go b/git-sensor/pkg/git/WebhookEventService_test.go new file mode 100644 index 000000000..994dc2900 --- /dev/null +++ b/git-sensor/pkg/git/WebhookEventService_test.go @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package git + +import ( + "fmt" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/internals/sql/mocks" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "os" + "testing" + "time" +) + +func TestWebhookEventServiceImpl_GetWebhookParsedEventDataByEventIdAndUniqueId(t *testing.T) { + + type args struct { + eventId int + uniqueId string + } + + tests := []struct { + name string + args args + want *sql.WebhookEventParsedData + mockExecution func(*mocks.WebhookEventParsedDataRepository) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEventParsedData_Found", + args: args{ + eventId: 1, + uniqueId: "uniqueId", + }, + want: &sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, + wantErr: assert.NoError, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + }, + }, + { + name: "WebhookEventParsedData_NotFound", + args: args{ + eventId: 1, + uniqueId: "uniqueId", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, ErrWebhookEventParsedDataNotFound.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, pg.ErrNoRows).Once() + }, + }, + { + name: "Empty_UniqueId", + args: args{ + eventId: 1, + uniqueId: "", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, ErrWebhookEventParsedDataNotFound.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl.AssertNotCalled(t, "GetWebhookParsedEventDataByEventIdAndUniqueId", 2) + }, + }, + { + name: "PG_Timeout_Error", + args: args{ + eventId: 1, + uniqueId: "uniqueId", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventParsedDataRepository) { + mockImpl. + On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, os.ErrDeadlineExceeded).Once() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getWebhookEventServiceImpl(t, tt.mockExecution, nil) + got, err := impl.GetWebhookParsedEventDataByEventIdAndUniqueId(tt.args.eventId, tt.args.uniqueId) + if !tt.wantErr(t, err, fmt.Sprintf("GetWebhookParsedEventDataByEventIdAndUniqueId(%v, %v)", tt.args.eventId, tt.args.uniqueId)) { + return + } + assert.Equalf(t, tt.want, got, "GetWebhookParsedEventDataByEventIdAndUniqueId(%v, %v)", tt.args.eventId, tt.args.uniqueId) + }) + } +} + +func TestWebhookEventServiceImpl_getCiPipelineMaterialWebhookDataMapping(t *testing.T) { + type args struct { + ciPipelineMaterialId int + webhookParsedDataId int + } + tests := []struct { + name string + args args + want *sql.CiPipelineMaterialWebhookDataMapping + isNewEntry bool + mockExecution func(*mocks.WebhookEventDataMappingRepository) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEventDataMapping_Found", + args: args{ + ciPipelineMaterialId: 1, + webhookParsedDataId: 1, + }, + want: &sql.CiPipelineMaterialWebhookDataMapping{ + Id: 1, + CiPipelineMaterialId: 1, + WebhookDataId: 1, + ConditionMatched: true, + IsActive: true, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, + isNewEntry: false, + wantErr: assert.NoError, + mockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetCiPipelineMaterialWebhookDataMapping", 1, 1). + Return(&sql.CiPipelineMaterialWebhookDataMapping{ + Id: 1, + CiPipelineMaterialId: 1, + WebhookDataId: 1, + ConditionMatched: true, + IsActive: true, + CreatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2025, 4, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + }, + }, + { + name: "WebhookEventDataMapping_NotFound", + args: args{ + ciPipelineMaterialId: 1, + webhookParsedDataId: 1, + }, + want: nil, + isNewEntry: true, + wantErr: assert.NoError, + mockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetCiPipelineMaterialWebhookDataMapping", 1, 1). + Return(nil, pg.ErrNoRows).Once() + }, + }, + { + name: "PG_Timeout_Error", + args: args{ + ciPipelineMaterialId: 1, + webhookParsedDataId: 1, + }, + want: nil, + isNewEntry: false, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + mockExecution: func(mockImpl *mocks.WebhookEventDataMappingRepository) { + mockImpl. + On("GetCiPipelineMaterialWebhookDataMapping", 1, 1). + Return(nil, os.ErrDeadlineExceeded).Once() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getWebhookEventServiceImpl(t, nil, tt.mockExecution) + got, isNewMapping, err := impl.getCiPipelineMaterialWebhookDataMapping(tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId) + if !tt.wantErr(t, err, fmt.Sprintf("getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId)) { + return + } + if !tt.wantErr(t, err, fmt.Sprintf("getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId)) { + return + } + assert.Equalf(t, tt.isNewEntry, isNewMapping, "getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId) + assert.Equalf(t, tt.want, got, "getCiPipelineMaterialWebhookDataMapping(%v, %v)", tt.args.ciPipelineMaterialId, tt.args.webhookParsedDataId) + }) + } +} + +func getWebhookEventServiceImpl( + t *testing.T, + webhookEventParsedDataMockExecution func(*mocks.WebhookEventParsedDataRepository), + webhookEventDataMappingMockExecution func(*mocks.WebhookEventDataMappingRepository), +) *WebhookEventServiceImpl { + sugaredLogger := getLogger() + webhookEventDataMappingRepository := getWebhookEventDataMappingRepository(t) + if webhookEventDataMappingMockExecution != nil { + webhookEventDataMappingMockExecution(webhookEventDataMappingRepository) + } + webhookEventParsedDataRepositoryMock := getWebhookEventParsedDataRepositoryMock(t) + if webhookEventParsedDataMockExecution != nil { + webhookEventParsedDataMockExecution(webhookEventParsedDataRepositoryMock) + } + return NewWebhookEventServiceImpl( + sugaredLogger, + nil, + webhookEventParsedDataRepositoryMock, + webhookEventDataMappingRepository, nil, nil, nil, nil, + ) +} + +func getLogger() *zap.SugaredLogger { + return logger.NewSugaredLogger() +} + +func getWebhookEventParsedDataRepositoryMock(t *testing.T) *mocks.WebhookEventParsedDataRepository { + return mocks.NewWebhookEventParsedDataRepository(t) +} + +func getWebhookEventDataMappingRepository(t *testing.T) *mocks.WebhookEventDataMappingRepository { + return mocks.NewWebhookEventDataMappingRepository(t) +} diff --git a/git-sensor/pkg/git/WebhookHandler.go b/git-sensor/pkg/git/WebhookHandler.go index 19af9cf60..ab4bf726e 100644 --- a/git-sensor/pkg/git/WebhookHandler.go +++ b/git-sensor/pkg/git/WebhookHandler.go @@ -17,8 +17,10 @@ package git import ( + "errors" "github.com/devtron-labs/git-sensor/internals/sql" "go.uber.org/zap" + "slices" "strings" "time" ) @@ -82,7 +84,7 @@ func (impl WebhookHandlerImpl) HandleWebhookEvent(webhookEvent *WebhookEvent) er for _, event := range events { if len(event.EventTypesCsv) > 0 { eventTypes := strings.Split(event.EventTypesCsv, ",") - if !contains(eventTypes, eventType) { + if !slices.Contains(eventTypes, eventType) { continue } } @@ -100,42 +102,43 @@ func (impl WebhookHandlerImpl) HandleWebhookEvent(webhookEvent *WebhookEvent) er webhookEventParsedData.EventId = eventId webhookEventParsedData.EventActionType = event.ActionType webhookEventParsedData.PayloadDataId = payloadId - - // fetch webhook parsed data from DB if unique id is not blank - webhookParsedEventGetData, err := impl.webhookEventService.GetWebhookParsedEventDataByEventIdAndUniqueId(eventId, webhookEventParsedData.UniqueId) - if err != nil { - impl.logger.Errorw("error in getting parsed webhook event data", "err", err) - return err + if dbErr := impl.upsertWebhookEventParsedData(eventId, webhookEventParsedData); dbErr != nil { + impl.logger.Errorw("error in upserting webhook event parsed data", "eventId", eventId, "err", dbErr) + // intentionally not returning error here, as we want to continue processing other events } - - // save or update in DB - if webhookParsedEventGetData != nil { - webhookEventParsedData.Id = webhookParsedEventGetData.Id - webhookEventParsedData.CreatedOn = webhookParsedEventGetData.CreatedOn - webhookEventParsedData.UpdatedOn = time.Now() - impl.webhookEventService.UpdateWebhookParsedEventData(webhookEventParsedData) - } else { - webhookEventParsedData.CreatedOn = time.Now() - impl.webhookEventService.SaveWebhookParsedEventData(webhookEventParsedData) - } - // match ci trigger condition and notify err = impl.webhookEventService.MatchCiTriggerConditionAndNotify(event, webhookEventParsedData, fullDataMap) if err != nil { impl.logger.Errorw("error in matching ci trigger condition for webhook after db save", "err", err) return err } - } - return nil } -func contains(s []string, str string) bool { - for _, v := range s { - if v == str { - return true +func (impl WebhookHandlerImpl) upsertWebhookEventParsedData(eventId int, webhookEventParsedData *sql.WebhookEventParsedData) error { + // fetch webhook parsed data from DB if unique id is not blank + webhookParsedEventGetData, err := impl.webhookEventService.GetWebhookParsedEventDataByEventIdAndUniqueId(eventId, webhookEventParsedData.UniqueId) + if err != nil && !errors.Is(err, ErrWebhookEventParsedDataNotFound) { + impl.logger.Errorw("error in getting parsed webhook event data", "eventId", eventId, "uniqueId", webhookEventParsedData.UniqueId, "err", err) + return err + } else if errors.Is(err, ErrWebhookEventParsedDataNotFound) { + impl.logger.Infow("webhook event parsed data not found in db, creating a new one", "eventId", eventId, "uniqueId", webhookEventParsedData.UniqueId) + // save in DB + webhookEventParsedData.CreatedOn = time.Now() + if err = impl.webhookEventService.SaveWebhookParsedEventData(webhookEventParsedData); err != nil { + impl.logger.Errorw("error in saving webhook event parsed data", "err", err) + return err + } + } else { + // update in DB + webhookEventParsedData.Id = webhookParsedEventGetData.Id + webhookEventParsedData.CreatedOn = webhookParsedEventGetData.CreatedOn + webhookEventParsedData.UpdatedOn = time.Now() + if err = impl.webhookEventService.UpdateWebhookParsedEventData(webhookEventParsedData); err != nil { + impl.logger.Errorw("error in updating webhook event parsed data", "err", err) + return err } } - return false + return nil } diff --git a/git-sensor/pkg/git/WebhookHandler_test.go b/git-sensor/pkg/git/WebhookHandler_test.go new file mode 100644 index 000000000..6434b1620 --- /dev/null +++ b/git-sensor/pkg/git/WebhookHandler_test.go @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package git + +import ( + "fmt" + "github.com/devtron-labs/git-sensor/internals/sql" + mocks "github.com/devtron-labs/git-sensor/pkg/git/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "os" + "testing" + "time" +) + +func TestWebhookHandlerImpl_upsertWebhookEventParsedData(t *testing.T) { + type args struct { + eventId int + webhookEventParsedData *sql.WebhookEventParsedData + } + tests := []struct { + name string + args args + mockExecution func(service *mocks.WebhookEventService) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "WebhookEventParsedData_Upsert_Success", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + }, + CreatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + service.On("UpdateWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(nil).Once() + service.AssertNotCalled(t, "SaveWebhookParsedEventData", mock.Anything) + }, + wantErr: assert.NoError, + }, + { + name: "WebhookEventParsedData_Upsert_Error", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(&sql.WebhookEventParsedData{ + Id: 1, + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + }, + CreatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + UpdatedOn: time.Date(2023, time.April, 10, 0, 0, 0, 0, time.UTC), + }, nil).Once() + service.On("UpdateWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(os.ErrDeadlineExceeded).Once() + service.AssertNotCalled(t, "SaveWebhookParsedEventData", mock.Anything) + }, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + }, + { + name: "WebhookEventParsedData_Save_Success", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, ErrWebhookEventParsedDataNotFound).Once() + service.On("SaveWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(nil).Once() + service.AssertNotCalled(t, "UpdateWebhookParsedEventData", mock.Anything) + }, + wantErr: assert.NoError, + }, + { + name: "WebhookEventParsedData_Save_Error", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, ErrWebhookEventParsedDataNotFound).Once() + service.On("SaveWebhookParsedEventData", mock.AnythingOfType("*sql.WebhookEventParsedData")). + Return(os.ErrDeadlineExceeded).Once() + service.AssertNotCalled(t, "UpdateWebhookParsedEventData", mock.Anything) + }, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + }, + { + name: "GetWebhookParsedEventDataByEventIdAndUniqueId_PG_Connection_Error", + args: args{ + eventId: 1, + webhookEventParsedData: &sql.WebhookEventParsedData{ + EventId: 1, + PayloadDataId: 1, + UniqueId: "uniqueId", + EventActionType: "PUSH", + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + CiEnvVariableData: map[string]string{ + "envKey1": "envValue1", + "envKey2": "envValue2", + }, + }, + }, + mockExecution: func(service *mocks.WebhookEventService) { + service.On("GetWebhookParsedEventDataByEventIdAndUniqueId", 1, "uniqueId"). + Return(nil, os.ErrDeadlineExceeded).Once() + service.AssertNotCalled(t, "SaveWebhookParsedEventData", mock.Anything) + service.AssertNotCalled(t, "UpdateWebhookParsedEventData", mock.Anything) + }, + wantErr: func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool { + return assert.EqualError(t, err, os.ErrDeadlineExceeded.Error(), msgAndArgs...) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + impl := getNewWebhookHandlerImplImpl(t, tt.mockExecution) + tt.wantErr(t, impl.upsertWebhookEventParsedData(tt.args.eventId, tt.args.webhookEventParsedData), fmt.Sprintf("upsertWebhookEventParsedData(%v, %v)", tt.args.eventId, tt.args.webhookEventParsedData)) + }) + } +} + +func getNewWebhookHandlerImplImpl( + t *testing.T, + WebhookEventServiceMockExecution func(service *mocks.WebhookEventService), +) *WebhookHandlerImpl { + sugaredLogger := getLogger() + webhookEventService := getWebhookEventServiceMock(t) + if WebhookEventServiceMockExecution != nil { + WebhookEventServiceMockExecution(webhookEventService) + } + return NewWebhookHandlerImpl( + sugaredLogger, + webhookEventService, + nil, + ) +} + +func getWebhookEventServiceMock(t *testing.T) *mocks.WebhookEventService { + return mocks.NewWebhookEventService(t) +} diff --git a/git-sensor/pkg/git/mocks/WebhookEventService.go b/git-sensor/pkg/git/mocks/WebhookEventService.go new file mode 100644 index 000000000..98435275d --- /dev/null +++ b/git-sensor/pkg/git/mocks/WebhookEventService.go @@ -0,0 +1,171 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + sql "github.com/devtron-labs/git-sensor/internals/sql" + mock "github.com/stretchr/testify/mock" +) + +// WebhookEventService is an autogenerated mock type for the WebhookEventService type +type WebhookEventService struct { + mock.Mock +} + +// GetAllGitHostWebhookEventByGitHostId provides a mock function with given fields: gitHostId, gitHostName +func (_m *WebhookEventService) GetAllGitHostWebhookEventByGitHostId(gitHostId int, gitHostName string) ([]*sql.GitHostWebhookEvent, error) { + ret := _m.Called(gitHostId, gitHostName) + + if len(ret) == 0 { + panic("no return value specified for GetAllGitHostWebhookEventByGitHostId") + } + + var r0 []*sql.GitHostWebhookEvent + var r1 error + if rf, ok := ret.Get(0).(func(int, string) ([]*sql.GitHostWebhookEvent, error)); ok { + return rf(gitHostId, gitHostName) + } + if rf, ok := ret.Get(0).(func(int, string) []*sql.GitHostWebhookEvent); ok { + r0 = rf(gitHostId, gitHostName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.GitHostWebhookEvent) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(gitHostId, gitHostName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllGitHostWebhookEventByGitHostName provides a mock function with given fields: gitHostName +func (_m *WebhookEventService) GetAllGitHostWebhookEventByGitHostName(gitHostName string) ([]*sql.GitHostWebhookEvent, error) { + ret := _m.Called(gitHostName) + + if len(ret) == 0 { + panic("no return value specified for GetAllGitHostWebhookEventByGitHostName") + } + + var r0 []*sql.GitHostWebhookEvent + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*sql.GitHostWebhookEvent, error)); ok { + return rf(gitHostName) + } + if rf, ok := ret.Get(0).(func(string) []*sql.GitHostWebhookEvent); ok { + r0 = rf(gitHostName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*sql.GitHostWebhookEvent) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(gitHostName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWebhookParsedEventDataByEventIdAndUniqueId provides a mock function with given fields: eventId, uniqueId +func (_m *WebhookEventService) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) { + ret := _m.Called(eventId, uniqueId) + + if len(ret) == 0 { + panic("no return value specified for GetWebhookParsedEventDataByEventIdAndUniqueId") + } + + var r0 *sql.WebhookEventParsedData + var r1 error + if rf, ok := ret.Get(0).(func(int, string) (*sql.WebhookEventParsedData, error)); ok { + return rf(eventId, uniqueId) + } + if rf, ok := ret.Get(0).(func(int, string) *sql.WebhookEventParsedData); ok { + r0 = rf(eventId, uniqueId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sql.WebhookEventParsedData) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(eventId, uniqueId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MatchCiTriggerConditionAndNotify provides a mock function with given fields: event, webhookEventParsedData, fullDataMap +func (_m *WebhookEventService) MatchCiTriggerConditionAndNotify(event *sql.GitHostWebhookEvent, webhookEventParsedData *sql.WebhookEventParsedData, fullDataMap map[string]string) error { + ret := _m.Called(event, webhookEventParsedData, fullDataMap) + + if len(ret) == 0 { + panic("no return value specified for MatchCiTriggerConditionAndNotify") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.GitHostWebhookEvent, *sql.WebhookEventParsedData, map[string]string) error); ok { + r0 = rf(event, webhookEventParsedData, fullDataMap) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SaveWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventService) SaveWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for SaveWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateWebhookParsedEventData provides a mock function with given fields: webhookEventParsedData +func (_m *WebhookEventService) UpdateWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error { + ret := _m.Called(webhookEventParsedData) + + if len(ret) == 0 { + panic("no return value specified for UpdateWebhookParsedEventData") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*sql.WebhookEventParsedData) error); ok { + r0 = rf(webhookEventParsedData) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewWebhookEventService creates a new instance of WebhookEventService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWebhookEventService(t interface { + mock.TestingT + Cleanup(func()) +}) *WebhookEventService { + mock := &WebhookEventService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/git-sensor/vendor/modules.txt b/git-sensor/vendor/modules.txt index 616be7797..177494985 100644 --- a/git-sensor/vendor/modules.txt +++ b/git-sensor/vendor/modules.txt @@ -66,7 +66,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -488,4 +488,4 @@ gopkg.in/yaml.v3 # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/image-scanner/go.mod b/image-scanner/go.mod index 0c7caee86..f5e9a8cde 100644 --- a/image-scanner/go.mod +++ b/image-scanner/go.mod @@ -74,4 +74,4 @@ require ( mellium.im/sasl v0.3.2 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/image-scanner/go.sum b/image-scanner/go.sum index 0c6997f78..82b0ab807 100644 --- a/image-scanner/go.sum +++ b/image-scanner/go.sum @@ -280,8 +280,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/image-scanner/vendor/modules.txt b/image-scanner/vendor/modules.txt index b924dca8f..823b77b55 100644 --- a/image-scanner/vendor/modules.txt +++ b/image-scanner/vendor/modules.txt @@ -72,7 +72,7 @@ github.com/cespare/xxhash/v2 github.com/coreos/clair/api/v3/clairpb github.com/coreos/clair/database github.com/coreos/clair/ext/versionfmt -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -440,4 +440,4 @@ google.golang.org/protobuf/types/known/wrapperspb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/kubelink/go.mod b/kubelink/go.mod index 0467a8236..0e7b65414 100644 --- a/kubelink/go.mod +++ b/kubelink/go.mod @@ -177,7 +177,7 @@ require ( ) replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 // https://github.com/kubernetes/kubernetes/issues/79384#issuecomment-505627280 k8s.io/api => k8s.io/api v0.29.0 diff --git a/kubelink/go.sum b/kubelink/go.sum index 78324d703..d702e3823 100644 --- a/kubelink/go.sum +++ b/kubelink/go.sum @@ -79,8 +79,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/distribution/v3 v3.0.0-beta.1 h1:X+ELTxPuZ1Xe5MsD3kp2wfGUhc8I+MPfRis8dZ818Ic= diff --git a/kubelink/grpc/applist.pb.go b/kubelink/grpc/applist.pb.go index aab851bcb..c3bf6d220 100644 --- a/kubelink/grpc/applist.pb.go +++ b/kubelink/grpc/applist.pb.go @@ -4263,6 +4263,7 @@ type RegistryCredential struct { Connection string `protobuf:"bytes,11,opt,name=Connection,proto3" json:"Connection,omitempty"` RegistryName string `protobuf:"bytes,12,opt,name=RegistryName,proto3" json:"RegistryName,omitempty"` RegistryCertificate string `protobuf:"bytes,13,opt,name=RegistryCertificate,proto3" json:"RegistryCertificate,omitempty"` + CredentialsType string `protobuf:"bytes,14,opt,name=CredentialsType,proto3" json:"CredentialsType,omitempty"` } func (x *RegistryCredential) Reset() { @@ -4388,6 +4389,13 @@ func (x *RegistryCredential) GetRegistryCertificate() string { return "" } +func (x *RegistryCredential) GetCredentialsType() string { + if x != nil { + return x.CredentialsType + } + return "" +} + type ProxyConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -5333,7 +5341,7 @@ var file_grpc_applist_proto_rawDesc = []byte{ 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x12, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xeb, 0x03, 0x0a, 0x12, 0x52, 0x65, + 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x95, 0x04, 0x0a, 0x12, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x55, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x55, @@ -5364,179 +5372,181 @@ var file_grpc_applist_proto_rawDesc = []byte{ 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x13, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0x29, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, - 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, - 0x72, 0x6c, 0x22, 0xa1, 0x01, 0x0a, 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2a, 0x0a, 0x10, 0x53, 0x53, 0x48, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x10, 0x53, 0x53, 0x48, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, 0x73, 0x73, 0x77, - 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, - 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, - 0x68, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x53, 0x53, 0x48, 0x41, - 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, 0x22, 0xd5, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x4f, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x17, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x52, 0x16, 0x52, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, - 0x6f, 0x64, 0x12, 0x2e, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x3a, 0x0a, 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x53, 0x53, - 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x53, - 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x6f, - 0x0a, 0x13, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, 0x67, 0x65, - 0x64, 0x49, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, - 0x67, 0x65, 0x64, 0x49, 0x6e, 0x12, 0x38, 0x0a, 0x0a, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x4f, 0x43, 0x49, 0x52, - 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x52, 0x0a, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, - 0x4f, 0x0a, 0x17, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x50, 0x75, - 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, - 0x2a, 0x38, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, - 0x4f, 0x58, 0x59, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x48, 0x10, 0x01, 0x12, 0x0a, - 0x0a, 0x06, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x10, 0x02, 0x32, 0xd9, 0x0f, 0x0a, 0x12, 0x41, - 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x39, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x0f, 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, - 0x64, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x14, - 0x4c, 0x69, 0x73, 0x74, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x0f, 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, - 0x2f, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, - 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, - 0x12, 0x2f, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, - 0x00, 0x12, 0x31, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x56, 0x32, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x09, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, - 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x55, 0x6e, - 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, 0x62, 0x65, - 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x48, - 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x46, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, - 0x65, 0x6e, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, - 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, - 0x48, 0x65, 0x6c, 0x6d, 0x41, 0x70, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, - 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x0d, 0x47, 0x65, - 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x11, 0x2e, 0x41, 0x70, - 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, - 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x00, 0x12, 0x40, - 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x6e, 0x69, - 0x66, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, - 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x43, 0x0a, 0x10, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, - 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x19, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, - 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x13, 0x47, 0x65, - 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x12, 0x18, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, - 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, - 0x1b, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, - 0x69, 0x74, 0x68, 0x43, 0x68, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x2e, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x54, 0x79, 0x70, + 0x65, 0x22, 0x29, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x72, 0x6c, 0x22, 0xa1, 0x01, 0x0a, + 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x2a, 0x0a, 0x10, 0x53, 0x53, 0x48, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x53, 0x53, 0x48, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0b, + 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, + 0x0a, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x53, 0x48, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, + 0x12, 0x1e, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, + 0x22, 0xd5, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4f, 0x0a, 0x16, 0x52, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x52, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x2e, 0x0a, 0x0b, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3a, 0x0a, 0x0f, + 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, 0x65, + 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x53, 0x53, 0x48, 0x54, 0x75, 0x6e, 0x6e, + 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x6f, 0x0a, 0x13, 0x4f, 0x43, 0x49, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1e, 0x0a, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x64, 0x49, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x49, 0x73, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x64, 0x49, 0x6e, 0x12, + 0x38, 0x0a, 0x0a, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, + 0x79, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x0a, 0x50, + 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x4f, 0x0a, 0x17, 0x4f, 0x43, 0x49, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, + 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x50, 0x75, 0x73, 0x68, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x2a, 0x38, 0x0a, 0x16, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x10, 0x00, 0x12, + 0x07, 0x0a, 0x03, 0x53, 0x53, 0x48, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x49, 0x52, 0x45, + 0x43, 0x54, 0x10, 0x02, 0x32, 0xd9, 0x0f, 0x0a, 0x12, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x10, 0x4c, + 0x69, 0x73, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x0f, 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x10, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, 0x64, 0x41, 0x70, 0x70, 0x4c, 0x69, + 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x6c, + 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x0f, + 0x2e, 0x41, 0x70, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x14, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x2f, 0x0a, 0x0c, 0x47, 0x65, 0x74, + 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, 0x41, + 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x2f, 0x0a, 0x0c, 0x47, 0x65, + 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, + 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0a, 0x2e, + 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x31, 0x0a, 0x0e, 0x47, + 0x65, 0x74, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x56, 0x32, 0x12, 0x11, 0x2e, + 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0a, 0x2e, 0x41, 0x70, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x34, + 0x0a, 0x09, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, + 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, + 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x55, 0x6e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, + 0x61, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x48, 0x69, 0x62, 0x65, 0x72, 0x6e, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x14, + 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x41, 0x70, + 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x73, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x11, 0x2e, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, + 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x0e, + 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x10, 0x55, 0x6e, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, + 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, + 0x65, 0x72, 0x1a, 0x19, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x3c, 0x0a, 0x12, 0x49, 0x73, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x12, 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, - 0x0f, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x12, 0x17, 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, - 0x0d, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, - 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x4d, 0x0a, 0x11, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, - 0x74, 0x42, 0x75, 0x6c, 0x6b, 0x12, 0x1a, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1a, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x5a, 0x0a, 0x1d, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, - 0x41, 0x6e, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, - 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x57, 0x69, 0x74, 0x68, 0x43, 0x68, 0x61, 0x72, 0x74, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x1d, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, - 0x68, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x19, 0x2e, 0x48, - 0x65, 0x6c, 0x6d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x65, - 0x73, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x43, 0x68, 0x61, 0x72, - 0x74, 0x4e, 0x6f, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x52, 0x0a, 0x1d, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x68, 0x61, 0x72, - 0x74, 0x12, 0x16, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x43, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x12, 0x16, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x13, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x1a, 0x14, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x1a, 0x50, 0x75, 0x73, 0x68, - 0x48, 0x65, 0x6c, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x54, 0x6f, 0x4f, 0x43, 0x49, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x4f, 0x43, - 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x23, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x46, 0x6f, 0x72, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x15, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x46, - 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x3d, - 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x44, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x73, 0x12, 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x12, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, - 0x65, 0x64, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x57, 0x0a, - 0x23, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, - 0x65, 0x65, 0x55, 0x73, 0x69, 0x6e, 0x67, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x12, 0x17, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x65, 0x76, 0x74, 0x72, 0x6f, 0x6e, 0x2d, 0x6c, 0x61, 0x62, - 0x73, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x6c, 0x69, 0x6e, 0x6b, 0x2f, 0x62, 0x65, 0x61, 0x6e, 0x2f, - 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, + 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x18, 0x2e, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x49, + 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x1b, 0x55, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x68, 0x61, + 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x12, 0x49, 0x73, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x12, + 0x12, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, + 0x69, 0x65, 0x72, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0f, 0x52, 0x6f, 0x6c, 0x6c, 0x62, + 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x17, 0x2e, 0x52, 0x6f, 0x6c, + 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0d, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x11, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x42, 0x75, 0x6c, 0x6b, 0x12, + 0x1a, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x42, 0x75, + 0x6c, 0x6b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1d, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x41, 0x6e, 0x64, 0x52, 0x65, 0x74, + 0x72, 0x69, 0x65, 0x76, 0x65, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, 0x2e, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, + 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x68, + 0x61, 0x72, 0x74, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x1d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x75, 0x73, 0x74, 0x6f, + 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x19, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1a, 0x2e, 0x48, 0x65, 0x6c, 0x6d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x43, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x39, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x65, 0x73, 0x12, 0x16, 0x2e, 0x49, 0x6e, + 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x43, 0x68, 0x61, 0x72, 0x74, 0x4e, 0x6f, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x1d, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x57, 0x69, 0x74, 0x68, + 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x68, 0x61, 0x72, 0x74, 0x12, 0x16, 0x2e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, + 0x0a, 0x13, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, + 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x1a, 0x14, 0x2e, 0x4f, 0x43, 0x49, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x49, 0x0a, 0x1a, 0x50, 0x75, 0x73, 0x68, 0x48, 0x65, 0x6c, 0x6d, 0x43, 0x68, + 0x61, 0x72, 0x74, 0x54, 0x6f, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, + 0x12, 0x13, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x4f, 0x43, 0x49, 0x52, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, + 0x23, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x46, 0x6f, 0x72, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x47, + 0x65, 0x74, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, + 0x15, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x46, 0x6c, 0x75, 0x78, 0x41, 0x70, 0x70, + 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x2e, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, + 0x72, 0x1a, 0x12, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, 0x64, 0x41, 0x70, 0x70, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x23, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x55, 0x73, 0x69, 0x6e, + 0x67, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x17, + 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, + 0x65, 0x76, 0x74, 0x72, 0x6f, 0x6e, 0x2d, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x6b, 0x75, 0x62, 0x65, + 0x6c, 0x69, 0x6e, 0x6b, 0x2f, 0x62, 0x65, 0x61, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/kubelink/grpc/applist.proto b/kubelink/grpc/applist.proto index a5df8249e..4e2584ab2 100644 --- a/kubelink/grpc/applist.proto +++ b/kubelink/grpc/applist.proto @@ -474,6 +474,7 @@ message RegistryCredential { string Connection = 11; string RegistryName = 12; string RegistryCertificate = 13; + string CredentialsType = 14; } enum RemoteConnectionMethod { diff --git a/kubelink/pkg/service/helmApplicationService/adapter/adapter.go b/kubelink/pkg/service/helmApplicationService/adapter/adapter.go index bd8190302..35f334d60 100644 --- a/kubelink/pkg/service/helmApplicationService/adapter/adapter.go +++ b/kubelink/pkg/service/helmApplicationService/adapter/adapter.go @@ -24,6 +24,7 @@ func NewRegistryConfig(credential *client.RegistryCredential) (*registry.Configu RegistryCertificateString: credential.RegistryCertificate, RegistryType: credential.RegistryType, IsPublicRegistry: credential.IsPublic, + CredentialsType: credential.CredentialsType, } if credential.Connection == registry.SECURE_WITH_CERT { diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index c92de2cb4..c73a91bf2 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -736,8 +736,7 @@ func (impl *HelmAppServiceImpl) installRelease(ctx context.Context, request *cli chartSpec.CreateNamespace = false impl.logger.Debugw("Installing release", "name", releaseIdentifier.ReleaseName, "namespace", releaseIdentifier.ReleaseNamespace, "dry-run", dryRun) - switch impl.helmReleaseConfig.RunHelmInstallInAsyncMode { - case false: + if runInstallInAsyncMode(request.InstallAppVersionHistoryId, impl.helmReleaseConfig.RunHelmInstallInAsyncMode) { impl.logger.Debugw("Installing release", "name", releaseIdentifier.ReleaseName, "namespace", releaseIdentifier.ReleaseNamespace, "dry-run", dryRun) rel, err := helmClientObj.InstallChart(context.Background(), chartSpec) if err != nil { @@ -748,9 +747,8 @@ func (impl *HelmAppServiceImpl) installRelease(ctx context.Context, request *cli } return nil, err } - return rel, nil - case true: + } else { go func() { helmInstallMessage := commonHelmService.HelmReleaseStatusConfig{ InstallAppVersionHistoryId: int(request.InstallAppVersionHistoryId), @@ -851,8 +849,7 @@ func (impl *HelmAppServiceImpl) UpgradeReleaseWithChartInfo(ctx context.Context, chartSpec.DependencyUpdate = true chartSpec.UpgradeCRDs = true - switch impl.helmReleaseConfig.RunHelmInstallInAsyncMode { - case false: + if runInstallInAsyncMode(request.InstallAppVersionHistoryId, impl.helmReleaseConfig.RunHelmInstallInAsyncMode) { impl.logger.Debug("Upgrading release with chart info") _, err = helmClientObj.UpgradeReleaseWithChartInfo(context.Background(), chartSpec) if UpgradeErr, ok := err.(*driver.StorageDriverError); ok { @@ -871,7 +868,7 @@ func (impl *HelmAppServiceImpl) UpgradeReleaseWithChartInfo(ctx context.Context, } } } - case true: + } else { go func() { impl.logger.Debug("Upgrading release with chart info") _, err = helmClientObj.UpgradeReleaseWithChartInfo(context.Background(), chartSpec) diff --git a/kubelink/pkg/service/helmApplicationService/utils.go b/kubelink/pkg/service/helmApplicationService/utils.go index b2162c1d6..86a02f037 100644 --- a/kubelink/pkg/service/helmApplicationService/utils.go +++ b/kubelink/pkg/service/helmApplicationService/utils.go @@ -33,3 +33,10 @@ const ( func IsReleaseNotFoundInCacheError(err error) bool { return errors.Is(err, k8sInformer.ErrorCacheMissReleaseNotFound) } + +func runInstallInAsyncMode(installedAppVersionHistoryId int32, isAsyncEnabled bool) bool { + if installedAppVersionHistoryId == 0 { + return false + } + return isAsyncEnabled +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go index cd25e430d..0bee1dbc0 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/bean.go @@ -20,6 +20,7 @@ type Configuration struct { RegistryType string IsPublicRegistry bool RemoteConnectionConfig *bean.RemoteConnectionConfigBean + CredentialsType string } type RegistryConnectionType string @@ -53,3 +54,7 @@ const ( const ( REGISTRY_CREDENTIAL_BASE_PATH = "/home/devtron/registry-credentials" ) + +const ( + HTTP = "http" +) diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go index ac062c8ea..12ae9806c 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/common.go @@ -14,6 +14,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "os" "strings" ) @@ -176,4 +177,11 @@ func GetTlsConfig(config *Configuration) (*tls.Config, error) { return tlsConfig, nil } -// TODO: add support for certFile, keyFile on UI? +func IsPlainHttp(URL string) bool { + parsedURL, err := url.Parse(URL) + if err != nil { + return false + } + plainHttp := parsedURL.Scheme == HTTP + return plainHttp +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go index 645af368c..1fa08fded 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/helmLib/registry/defaultSettings.go @@ -1,6 +1,7 @@ package registry import ( + "github.com/devtron-labs/common-lib/constants" "go.uber.org/zap" "helm.sh/helm/v3/pkg/registry" ) @@ -45,7 +46,8 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r } clientOptions := []registry.ClientOption{registry.ClientOptHTTPClient(httpClient)} - if config.RegistryConnectionType == INSECURE_CONNECTION { + + if IsPlainHttp(config.RegistryUrl) { clientOptions = append(clientOptions, registry.ClientOptPlainHTTP()) } @@ -55,7 +57,7 @@ func (s *DefaultSettingsGetterImpl) getRegistryClient(config *Configuration) (*r return nil, err } - if config != nil && !config.IsPublicRegistry { + if config != nil && !config.IsPublicRegistry && !(config.CredentialsType == string(constants.CredentialsTypeAnonymous)) { registryClient, err = GetLoggedInClient(registryClient, config) if err != nil { return nil, err diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/kubelink/vendor/modules.txt b/kubelink/vendor/modules.txt index f602b95a5..e23fc420a 100644 --- a/kubelink/vendor/modules.txt +++ b/kubelink/vendor/modules.txt @@ -127,7 +127,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/constants @@ -1354,7 +1354,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c # go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 # k8s.io/api => k8s.io/api v0.29.0 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.0 diff --git a/kubewatch/go.mod b/kubewatch/go.mod index 145a09333..94969e9ef 100644 --- a/kubewatch/go.mod +++ b/kubewatch/go.mod @@ -245,4 +245,4 @@ replace ( k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/kubewatch/go.sum b/kubewatch/go.sum index b9211517a..dfc0d67b8 100644 --- a/kubewatch/go.sum +++ b/kubewatch/go.sum @@ -719,8 +719,8 @@ github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/kubewatch/vendor/modules.txt b/kubewatch/vendor/modules.txt index 0e03fd429..2b944d7c8 100644 --- a/kubewatch/vendor/modules.txt +++ b/kubewatch/vendor/modules.txt @@ -215,7 +215,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/constants @@ -1760,4 +1760,4 @@ upper.io/db.v3/postgresql # k8s.io/mount-utils => k8s.io/mount-utils v0.29.7 # k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.7 # k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/lens/go.mod b/lens/go.mod index e1694aeb0..c3025fc99 100644 --- a/lens/go.mod +++ b/lens/go.mod @@ -59,4 +59,4 @@ require ( github.com/onsi/gomega v1.18.1 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c diff --git a/lens/go.sum b/lens/go.sum index c576fc93c..72b110c77 100644 --- a/lens/go.sum +++ b/lens/go.sum @@ -22,8 +22,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea h1:76Q2QQCCU/2bwuW0uEEyJpQPlYnm0QqhYmgH7rA8AzU= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c h1:OfO2hSC6BAf7+7SURwCEMHf0NHOekYIK+Dn9NM68A1s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e23..f327d712b 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0..17ccda061 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 000000000..3bdec67be --- /dev/null +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e49..ea16a2f72 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/lens/vendor/modules.txt b/lens/vendor/modules.txt index 3ec236f08..d791a9248 100644 --- a/lens/vendor/modules.txt +++ b/lens/vendor/modules.txt @@ -7,7 +7,7 @@ github.com/caarlos0/env # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -289,4 +289,4 @@ google.golang.org/protobuf/types/known/timestamppb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250411055655-be36c40e11ea +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421050508-c7b75d20165c