Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions client/telemetry/TelemetryEventClient.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,8 @@ func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []be
return clusters, users, k8sServerVersion, hostURL, ssoSetup, HelmAppAccessCount, ChartStoreVisitCount, SkippedOnboarding, HelmAppUpdateCounter, helmChartSuccessfulDeploymentCount, ExternalHelmAppClusterCount
}

// New methods for collecting additional telemetry metrics

func (impl *TelemetryEventClientImpl) SummaryEventForTelemetryEA() {
err := impl.SendSummaryEvent(string(Summary))
if err != nil {
Expand Down Expand Up @@ -310,6 +312,11 @@ func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error {
payload.HelmChartSuccessfulDeploymentCount = helmChartSuccessfulDeploymentCount
payload.ExternalHelmAppClusterCount = ExternalHelmAppClusterCount

// Collect EA-mode compatible telemetry metrics
payload.HelmAppCount = impl.getHelmAppCount()
payload.PhysicalClusterCount, payload.IsolatedClusterCount = impl.getClusterCounts()
payload.ActiveUsersLast30Days = impl.getActiveUsersLast30Days()

payload.ClusterProvider, err = impl.GetCloudProvider()
if err != nil {
impl.logger.Errorw("error while getting cluster provider", "error", err)
Expand Down
41 changes: 39 additions & 2 deletions client/telemetry/TelemetryEventClientExtended.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read"
"github.com/devtron-labs/devtron/pkg/auth/sso"
user2 "github.com/devtron-labs/devtron/pkg/auth/user"
authPolicyRepository "github.com/devtron-labs/devtron/pkg/auth/user/repository"
"github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read"
repository3 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository"
"github.com/devtron-labs/devtron/pkg/build/pipeline/bean"
Expand All @@ -34,6 +35,8 @@ import (
"github.com/devtron-labs/devtron/pkg/cluster/environment"
"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo"
pluginRepository "github.com/devtron-labs/devtron/pkg/plugin/repository"
cvePolicyRepository "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository"
serverDataStore "github.com/devtron-labs/devtron/pkg/server/store"
ucidService "github.com/devtron-labs/devtron/pkg/ucid"
util3 "github.com/devtron-labs/devtron/pkg/util"
Expand Down Expand Up @@ -68,6 +71,11 @@ type TelemetryEventClientImplExtended struct {
chartRepository chartRepoRepository.ChartRepository
ciBuildConfigService pipeline.CiBuildConfigService
gitOpsConfigReadService config.GitOpsConfigReadService
// Additional repositories for FULL-mode telemetry metrics
pluginRepository pluginRepository.GlobalPluginRepository
cvePolicyRepository cvePolicyRepository.CvePolicyRepository
defaultAuthPolicyRepository authPolicyRepository.DefaultAuthPolicyRepository
rbacPolicyRepository authPolicyRepository.RbacPolicyDataRepository
*TelemetryEventClientImpl
}

Expand All @@ -85,7 +93,12 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http
ciBuildConfigService pipeline.CiBuildConfigService, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore,
helmAppClient client.HelmAppClient, installedAppReadService installedAppReader.InstalledAppReadService, userAttributesRepository repository.UserAttributesRepository,
cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl,
gitOpsConfigReadService config.GitOpsConfigReadService, envVariables *util.EnvironmentVariables) (*TelemetryEventClientImplExtended, error) {
gitOpsConfigReadService config.GitOpsConfigReadService, envVariables *util.EnvironmentVariables,
// Optional repositories for additional telemetry metrics
pluginRepository pluginRepository.GlobalPluginRepository,
cvePolicyRepository cvePolicyRepository.CvePolicyRepository,
defaultAuthPolicyRepository authPolicyRepository.DefaultAuthPolicyRepository,
rbacPolicyRepository authPolicyRepository.RbacPolicyDataRepository) (*TelemetryEventClientImplExtended, error) {

cron := cron.New(
cron.WithChain(cron.Recover(cronLogger)))
Expand All @@ -105,6 +118,11 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http
chartRepository: chartRepository,
ciBuildConfigService: ciBuildConfigService,
gitOpsConfigReadService: gitOpsConfigReadService,
// Initialize FULL-mode specific repositories
pluginRepository: pluginRepository,
cvePolicyRepository: cvePolicyRepository,
defaultAuthPolicyRepository: defaultAuthPolicyRepository,
rbacPolicyRepository: rbacPolicyRepository,
TelemetryEventClientImpl: &TelemetryEventClientImpl{
cron: cron,
logger: logger,
Expand Down Expand Up @@ -319,6 +337,21 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
payload.HelmChartSuccessfulDeploymentCount = HelmChartSuccessfulDeploymentCount
payload.ExternalHelmAppClusterCount = ExternalHelmAppClusterCount

// Collect new telemetry metrics
payload.HelmAppCount = impl.getHelmAppCount()
payload.DevtronAppCount = impl.getDevtronAppCount()
payload.JobCount = impl.getJobCount()
payload.JobPipelineCount = impl.getJobPipelineCount()
payload.JobPipelineTriggeredLast24h = impl.getJobPipelineTriggeredLast24h()
payload.JobPipelineSucceededLast24h = impl.getJobPipelineSucceededLast24h()
payload.UserCreatedPluginCount = impl.getUserCreatedPluginCount()
payload.PolicyCount = impl.getPolicyCount()
payload.AppliedPolicyRowCount = impl.getAppliedPolicyRowCount()
payload.PhysicalClusterCount, payload.IsolatedClusterCount = impl.getClusterCounts()
payload.ActiveUsersLast30Days = impl.getActiveUsersLast30Days()
payload.GitOpsPipelineCount = impl.getGitOpsPipelineCount()
payload.HelmPipelineCount = impl.helmPipelineCount()

payload.ClusterProvider, err = impl.GetCloudProvider()
if err != nil {
impl.logger.Errorw("error while getting cluster provider", "error", err)
Expand Down Expand Up @@ -374,7 +407,11 @@ func (impl *TelemetryEventClientImplExtended) getCiBuildTypeData() (int, int, in
func (impl *TelemetryEventClientImplExtended) getCiBuildTypeVsStatusVsCount() (successCount map[bean.CiBuildType]int, failureCount map[bean.CiBuildType]int) {
successCount = make(map[bean.CiBuildType]int)
failureCount = make(map[bean.CiBuildType]int)
buildTypeAndStatusVsCount := impl.ciWorkflowRepository.FindBuildTypeAndStatusDataOfLast1Day()
buildTypeAndStatusVsCount, err := impl.ciWorkflowRepository.FindBuildTypeAndStatusDataOfLast1Day()
if err != nil {
impl.logger.Errorw("error getting build type vs status vs count data", "err", err)
return successCount, failureCount
}
for _, buildTypeCount := range buildTypeAndStatusVsCount {
if buildTypeCount == nil {
continue
Expand Down
20 changes: 20 additions & 0 deletions client/telemetry/bean.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ type TelemetryEventEA struct {
HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"`
ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"`
ClusterProvider string `json:"clusterProvider,omitempty"`
// New telemetry fields
HelmAppCount int `json:"helmAppCount,omitempty"`
PhysicalClusterCount int `json:"physicalClusterCount,omitempty"`
IsolatedClusterCount int `json:"isolatedClusterCount,omitempty"`
ActiveUsersLast30Days int `json:"activeUsersLast30Days,omitempty"`
}

const AppsCount int = 50
Expand Down Expand Up @@ -136,4 +141,19 @@ type TelemetryEventDto struct {
HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"`
ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"`
ClusterProvider string `json:"clusterProvider,omitempty"`
// New telemetry fields
HelmAppCount int `json:"helmAppCount,omitempty"`
DevtronAppCount int `json:"devtronAppCount,omitempty"`
JobCount int `json:"jobCount,omitempty"`
JobPipelineCount int `json:"jobPipelineCount,omitempty"`
JobPipelineTriggeredLast24h int `json:"jobPipelineTriggeredLast24h,omitempty"`
JobPipelineSucceededLast24h int `json:"jobPipelineSucceededLast24h,omitempty"`
UserCreatedPluginCount int `json:"userCreatedPluginCount,omitempty"`
PolicyCount int `json:"policyCount,omitempty"`
AppliedPolicyRowCount int `json:"appliedPolicyRowCount,omitempty"`
PhysicalClusterCount int `json:"physicalClusterCount,omitempty"`
IsolatedClusterCount int `json:"isolatedClusterCount,omitempty"`
ActiveUsersLast30Days int `json:"activeUsersLast30Days,omitempty"`
GitOpsPipelineCount int `json:"gitOpsPipelineCount,omitempty"`
HelmPipelineCount int `json:"helmPipelineCount,omitempty"`
}
58 changes: 58 additions & 0 deletions client/telemetry/telemetryQueries.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package telemetry

// EA-mode compatible telemetry queries - no imports needed for current methods

func (impl *TelemetryEventClientImpl) getHelmAppCount() int {
if impl.installedAppReadService == nil {
impl.logger.Warnw("installedAppReadService not available for helm app count")
return -1
}
count, err := impl.installedAppReadService.GetActiveInstalledAppCount()
if err != nil {
impl.logger.Errorw("error getting helm app count", "err", err)
return -1
}
return count
}

// EA-mode compatible telemetry methods

func (impl *TelemetryEventClientImpl) getClusterCounts() (physicalCount int, isolatedCount int) {
clusters, err := impl.clusterService.FindAllActive()
if err != nil {
impl.logger.Errorw("error getting cluster counts", "err", err)
return -1, -1
}

physicalCount = 0
isolatedCount = 0

for _, cluster := range clusters {
if cluster.IsVirtualCluster {
isolatedCount++
} else {
physicalCount++
}
}

return physicalCount, isolatedCount
}

// Note: FULL-mode specific methods like getDevtronAppCount, getJobCount, etc.
// are now implemented in TelemetryEventClientImplExtended in telemetryQueriesExtended.go

func (impl *TelemetryEventClientImpl) getActiveUsersLast30Days() int {
if impl.userAuditService == nil {
impl.logger.Warnw("userAuditService not available for active users count")
return -1
}

count, err := impl.userAuditService.GetActiveUsersCountInLast30Days()
if err != nil {
impl.logger.Errorw("error getting active users count in last 30 days", "err", err)
return -1
}

impl.logger.Debugw("counted active users in last 30 days", "count", count)
return count
}
137 changes: 137 additions & 0 deletions client/telemetry/telemetryQueriesExtended.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
package telemetry

import (
"github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common"
pluginRepository "github.com/devtron-labs/devtron/pkg/plugin/repository"
)

// FULL-mode specific telemetry methods for TelemetryEventClientImplExtended

func (impl *TelemetryEventClientImplExtended) getDevtronAppCount() int {
devtronAppCount, err := impl.appRepository.FindDevtronAppCount()
if err != nil {
impl.logger.Errorw("error getting all apps for devtron app count", "err", err)
return -1
}
return devtronAppCount
}

func (impl *TelemetryEventClientImplExtended) getJobCount() int {
jobCount, err := impl.appRepository.FindJobCount()
if err != nil {
impl.logger.Errorw("error getting all apps for job count", "err", err)
return -1
}
return jobCount
}

func (impl *TelemetryEventClientImplExtended) getJobPipelineTriggeredLast24h() int {
// Get build type and status data for the last 24 hours
buildTypeStatusData, err := impl.ciWorkflowRepository.FindBuildTypeAndStatusDataOfLast1Day()
if err != nil {
impl.logger.Warnw("no build type status data available for last 24 hours")
return -1
}

// Count job pipeline triggers
// Job pipelines have build type "CI_JOB"
jobTriggeredCount := 0
for _, data := range buildTypeStatusData {
if data.Type == string(common.CI_JOB) {
jobTriggeredCount += data.Count
}
}

return jobTriggeredCount
}

func (impl *TelemetryEventClientImplExtended) getJobPipelineSucceededLast24h() int {
// Get build type and status data for the last 24 hours
buildTypeStatusData, err := impl.ciWorkflowRepository.FindBuildTypeAndStatusDataOfLast1Day()
if err != nil {
impl.logger.Warnw("no build type status data available for last 24 hours")
return -1
}

// Count successful job pipeline runs
// Job pipelines have build type "CI_JOB"
successfulJobCount := 0
for _, data := range buildTypeStatusData {
if data.Type == "CI_JOB" && data.Status == "Succeeded" {
successfulJobCount += data.Count
}
}

impl.logger.Debugw("counted successful job pipeline runs in last 24h", "count", successfulJobCount)
return successfulJobCount
}

func (impl *TelemetryEventClientImplExtended) getUserCreatedPluginCount() int {
// Get all user-created plugins (SHARED type)
plugins, err := impl.pluginRepository.GetAllPluginMinDataByType(string(pluginRepository.PLUGIN_TYPE_SHARED))
if err != nil {
impl.logger.Errorw("error getting user created plugin count", "err", err)
return -1
}

return len(plugins)
}

func (impl *TelemetryEventClientImplExtended) getPolicyCount() int {
// Get global policies
globalPolicies, err := impl.cvePolicyRepository.GetGlobalPolicies()
if err != nil {
impl.logger.Errorw("error getting global CVE policies", "err", err)
return -1
}
return len(globalPolicies)
}

func (impl *TelemetryEventClientImplExtended) getGitOpsPipelineCount() int {
var count int
query := `
SELECT COUNT(DISTINCT p.id)
FROM pipeline p
WHERE p.deleted = false AND p.deployment_app_type = 'argo_cd'
`

dbConnection := impl.cdWorkflowRepository.GetConnection()
_, err := dbConnection.Query(&count, query)
if err != nil {
impl.logger.Errorw("error getting GitOps pipeline count", "err", err)
return -1
}

impl.logger.Debugw("counted GitOps pipelines", "count", count)
return count
}

func (impl *TelemetryEventClientImplExtended) helmPipelineCount() int {
// Get the pipeline repository from cdWorkflowRepository connection
var count int
query := `
SELECT COUNT(DISTINCT p.id)
FROM pipeline p
WHERE p.deleted = false AND p.deployment_app_type = 'helm'
`

dbConnection := impl.cdWorkflowRepository.GetConnection()
_, err := dbConnection.Query(&count, query)
if err != nil {
impl.logger.Errorw("error getting No-GitOps pipeline count", "err", err)
return -1
}

impl.logger.Debugw("counted No-GitOps pipelines", "count", count)
return count
}

// getJobPipelineCount returns 0 for now as implementation is not yet available
func (impl *TelemetryEventClientImplExtended) getJobPipelineCount() int {
Comment thread
prkhrkat marked this conversation as resolved.
return -1
}

// getAppliedPolicyRowCount returns 0 for now as implementation is not yet available
func (impl *TelemetryEventClientImplExtended) getAppliedPolicyRowCount() int {
return -1
}
2 changes: 1 addition & 1 deletion env_gen.json

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions env_gen.md
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,6 @@
| K8s_TCP_KEEPALIVE | int |30 | | | false |
| K8s_TCP_TIMEOUT | int |30 | | | false |
| K8s_TLS_HANDSHAKE_TIMEOUT | int |10 | | | false |
| KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 | | | false |
| KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 | | | false |
| LENS_TIMEOUT | int |0 | Lens microservice timeout. | | false |
| LENS_URL | string |http://lens-milandevtron-service:80 | Lens micro-service URL | | false |
| LIMIT_CI_CPU | string |0.5 | | | false |
Expand Down Expand Up @@ -287,9 +285,13 @@
| DEX_HOST | string |http://localhost | | | false |
| DEX_PORT | string |5556 | | | false |
| GIT_SENSOR_PROTOCOL | string |REST | Protocol to connect with git-sensor micro-service | | false |
| GIT_SENSOR_SERVICE_CONFIG | string |{"loadBalancingPolicy":"pick_first"} | git-sensor grpc service config | | false |
| GIT_SENSOR_TIMEOUT | int |0 | Timeout for getting response from the git-sensor | | false |
| GIT_SENSOR_URL | string |127.0.0.1:7070 | git-sensor micro-service url | | false |
| HELM_CLIENT_URL | string |127.0.0.1:50051 | Kubelink micro-service url | | false |
| KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 | | | false |
| KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 | | | false |
| KUBELINK_GRPC_SERVICE_CONFIG | string |{"loadBalancingPolicy":"round_robin"} | kubelink grpc service config | | false |


## POSTGRES Related Environment Variables
Expand Down
16 changes: 16 additions & 0 deletions internal/sql/repository/app/AppRepository.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ type AppRepository interface {
FindAppAndProjectByIdsIn(ids []int) ([]*App, error)
FetchAppIdsByDisplayNamesForJobs(names []string) (map[int]string, []int, error)
GetActiveCiCdAppsCount() (int, error)
FindDevtronAppCount() (int, error)
FindJobCount() (int, error)

UpdateAppOfferingModeForAppIds(successAppIds []*int, appOfferingMode string, userId int32) error
}
Expand Down Expand Up @@ -520,6 +522,20 @@ func (repo AppRepositoryImpl) GetActiveCiCdAppsCount() (int, error) {
Count()
}

func (repo AppRepositoryImpl) FindDevtronAppCount() (int, error) {
return repo.dbConnection.Model(&App{}).
Where("active=?", true).
Where("app_type=?", helper.CustomApp).
Count()
}

func (repo AppRepositoryImpl) FindJobCount() (int, error) {
return repo.dbConnection.Model(&App{}).
Where("active=?", true).
Where("app_type=?", helper.Job).
Count()
}

func (repo AppRepositoryImpl) UpdateAppOfferingModeForAppIds(successAppIds []*int, appOfferingMode string, userId int32) error {
query := "update app set app_offering_mode = ?,updated_by = ?, updated_on = ? where id in (?);"
var app *App
Expand Down
Loading