Skip to content

Commit 52dd283

Browse files
Ash-expAsh-exp
authored
Merge pull request #249 from devtron-labs/chore-resource-tree-opt
chore: resource tree call memory optimization
2 parents 459b15c + d54bc5c commit 52dd283

8 files changed

Lines changed: 1158 additions & 20 deletions

File tree

‎kubelink/config/GlobalConfig.go‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,30 @@ package config
1818

1919
import "github.com/caarlos0/env"
2020

21+
// CATEGORY=HELM_RELEASE
2122
type HelmReleaseConfig struct {
22-
EnableHelmReleaseCache bool `env:"ENABLE_HELM_RELEASE_CACHE" envDefault:"true"`
23-
MaxCountForHelmRelease int `env:"MAX_COUNT_FOR_HELM_RELEASE" envDefault:"20"`
24-
ManifestFetchBatchSize int `env:"MANIFEST_FETCH_BATCH_SIZE" envDefault:"2"`
25-
RunHelmInstallInAsyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE" envDefault:"false"`
26-
ParentChildGvkMapping string `env:"PARENT_CHILD_GVK_MAPPING" envDefault:""`
27-
ChartWorkingDirectory string `env:"CHART_WORKING_DIRECTORY" envDefault:"/home/devtron/devtroncd/charts/"`
28-
BuildNodesBatchSize int `env:"BUILD_NODES_BATCH_SIZE" envDefault:"2"`
23+
EnableHelmReleaseCache bool `env:"ENABLE_HELM_RELEASE_CACHE" envDefault:"true" description:"Enable helm releases list cache" deprecated:"false" example:"true"`
24+
MaxCountForHelmRelease int `env:"MAX_COUNT_FOR_HELM_RELEASE" envDefault:"20" description:"Max count for helm release history list" deprecated:"false" example:"20"`
25+
ManifestFetchBatchSize int `env:"MANIFEST_FETCH_BATCH_SIZE" envDefault:"2" description:"Manifest fetch parallelism batch size (applied only for parent objects)" deprecated:"false" example:"2"`
26+
RunHelmInstallInAsyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE" envDefault:"false" description:"Run helm install/ upgrade in async mode" deprecated:"false" example:"false"`
27+
ParentChildGvkMapping string `env:"PARENT_CHILD_GVK_MAPPING" envDefault:"" description:"Parent child GVK mapping for resource tree" deprecated:"false" example:""`
28+
ChartWorkingDirectory string `env:"CHART_WORKING_DIRECTORY" envDefault:"/home/devtron/devtroncd/charts/" description:"Helm charts working directory" deprecated:"false" example:"/home/devtron/devtroncd/charts/"`
29+
BuildNodesBatchSize int `env:"BUILD_NODES_BATCH_SIZE" envDefault:"2" description:"Resource tree build nodes parallelism batch size (applied only for depth-1 child objects of a parent object)" deprecated:"false" example:"2"`
30+
FeatChildChildObjectListingPaginationEnable bool `env:"FEAT_CHILD_OBJECT_LISTING_PAGINATION" envDefault:"true" description:"use pagination in listing all the dependent child objects. use 'CHILD_OBJECT_LISTING_PAGE_SIZE' to set the page size." deprecated:"false" example:"true"`
31+
ChildObjectListingPageSize int64 `env:"CHILD_OBJECT_LISTING_PAGE_SIZE" envDefault:"1000" description:"Resource tree child object listing page size" deprecated:"false" example:"100"`
2932
}
3033

3134
func GetHelmReleaseConfig() (*HelmReleaseConfig, error) {
3235
cfg := &HelmReleaseConfig{}
3336
err := env.Parse(cfg)
34-
return cfg, err
37+
if err != nil {
38+
return cfg, err
39+
}
40+
if cfg.ChildObjectListingPageSize <= 10 {
41+
// set the default value for invalid values
42+
cfg.ChildObjectListingPageSize = 1000
43+
}
44+
return cfg, nil
3545
}
3646

3747
func (c *HelmReleaseConfig) IsHelmReleaseCachingEnabled() bool {

‎kubelink/env_gen.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"kubelink","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_NODES_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CHART_WORKING_DIRECTORY","EnvType":"string","EnvValue":"/home/devtron/devtroncd/charts/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_HELM_RELEASE_CACHE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MANIFEST_FETCH_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_COUNT_FOR_HELM_RELEASE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARENT_CHILD_GVK_MAPPING","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]}]
1+
[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"kubelink","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"HELM_RELEASE","Fields":[{"Env":"BUILD_NODES_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"Resource tree build nodes parallelism batch size (applied only for depth-1 child objects of a parent object)","Example":"2","Deprecated":"false"},{"Env":"CHART_WORKING_DIRECTORY","EnvType":"string","EnvValue":"/home/devtron/devtroncd/charts/","EnvDescription":"Helm charts working directory","Example":"/home/devtron/devtroncd/charts/","Deprecated":"false"},{"Env":"CHILD_OBJECT_LISTING_PAGE_SIZE","EnvType":"int64","EnvValue":"1000","EnvDescription":"Resource tree child object listing page size","Example":"100","Deprecated":"false"},{"Env":"ENABLE_HELM_RELEASE_CACHE","EnvType":"bool","EnvValue":"true","EnvDescription":"Enable helm releases list cache","Example":"true","Deprecated":"false"},{"Env":"FEAT_CHILD_OBJECT_LISTING_PAGINATION","EnvType":"bool","EnvValue":"true","EnvDescription":"use pagination in listing all the dependent child objects. use 'CHILD_OBJECT_LISTING_PAGE_SIZE' to set the page size.","Example":"true","Deprecated":"false"},{"Env":"MANIFEST_FETCH_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"Manifest fetch parallelism batch size (applied only for parent objects)","Example":"2","Deprecated":"false"},{"Env":"MAX_COUNT_FOR_HELM_RELEASE","EnvType":"int","EnvValue":"20","EnvDescription":"Max count for helm release history list","Example":"20","Deprecated":"false"},{"Env":"PARENT_CHILD_GVK_MAPPING","EnvType":"string","EnvValue":"","EnvDescription":"Parent child GVK mapping for resource tree","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Run helm install/ upgrade in async mode","Example":"false","Deprecated":"false"}]}]

‎kubelink/env_gen.md‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,8 @@
44
| Key | Type | Default Value | Description | Example | Deprecated |
55
|-------|----------|-------------------|-------------------|-----------------------|------------------|
66
| APP | string |kubelink | | | false |
7-
| BUILD_NODES_BATCH_SIZE | int |2 | | | false |
8-
| CHART_WORKING_DIRECTORY | string |/home/devtron/devtroncd/charts/ | | | false |
97
| CONSUMER_CONFIG_JSON | string | | | | false |
108
| DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false |
11-
| ENABLE_HELM_RELEASE_CACHE | bool |true | | | false |
129
| ENABLE_STATSVIZ | bool |false | | | false |
1310
| K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST | int |25 | | | false |
1411
| K8s_TCP_IDLE_CONN_TIMEOUT | int |300 | | | false |
@@ -18,15 +15,12 @@
1815
| KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 | | | false |
1916
| KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 | | | false |
2017
| LOG_LEVEL | int |-1 | | | false |
21-
| MANIFEST_FETCH_BATCH_SIZE | int |2 | | | false |
22-
| MAX_COUNT_FOR_HELM_RELEASE | int |20 | | | false |
2318
| NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false |
2419
| NATS_MSG_BUFFER_SIZE | int |-1 | | | false |
2520
| NATS_MSG_MAX_AGE | int |86400 | | | false |
2621
| NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false |
2722
| NATS_MSG_REPLICAS | int |0 | | | false |
2823
| NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false |
29-
| PARENT_CHILD_GVK_MAPPING | string | | | | false |
3024
| PG_ADDR | string |127.0.0.1 | | | false |
3125
| PG_DATABASE | string |orchestrator | | | false |
3226
| PG_EXPORT_PROM_METRICS | bool |true | | | false |
@@ -38,7 +32,20 @@
3832
| PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false |
3933
| PG_USER | string | | | | false |
4034
| RUNTIME_CONFIG_LOCAL_DEV | LocalDevMode |false | | | false |
41-
| RUN_HELM_INSTALL_IN_ASYNC_MODE | bool |false | | | false |
4235
| STREAM_CONFIG_JSON | string | | | | false |
4336
| USE_CUSTOM_HTTP_TRANSPORT | bool |false | | | false |
4437

38+
39+
## HELM_RELEASE Related Environment Variables
40+
| Key | Type | Default Value | Description | Example | Deprecated |
41+
|-------|----------|-------------------|-------------------|-----------------------|------------------|
42+
| BUILD_NODES_BATCH_SIZE | int |2 | Resource tree build nodes parallelism batch size (applied only for depth-1 child objects of a parent object) | 2 | false |
43+
| CHART_WORKING_DIRECTORY | string |/home/devtron/devtroncd/charts/ | Helm charts working directory | /home/devtron/devtroncd/charts/ | false |
44+
| CHILD_OBJECT_LISTING_PAGE_SIZE | int64 |1000 | Resource tree child object listing page size | 100 | false |
45+
| ENABLE_HELM_RELEASE_CACHE | bool |true | Enable helm releases list cache | true | false |
46+
| FEAT_CHILD_OBJECT_LISTING_PAGINATION | bool |true | use pagination in listing all the dependent child objects. use 'CHILD_OBJECT_LISTING_PAGE_SIZE' to set the page size. | true | false |
47+
| MANIFEST_FETCH_BATCH_SIZE | int |2 | Manifest fetch parallelism batch size (applied only for parent objects) | 2 | false |
48+
| MAX_COUNT_FOR_HELM_RELEASE | int |20 | Max count for helm release history list | 20 | false |
49+
| PARENT_CHILD_GVK_MAPPING | string | | Parent child GVK mapping for resource tree | | false |
50+
| RUN_HELM_INSTALL_IN_ASYNC_MODE | bool |false | Run helm install/ upgrade in async mode | false | false |
51+

0 commit comments

Comments
 (0)