Skip to content

Commit 5b861f9

Browse files
Ash-expAsh-exp
committed
chore: resource tree call memory optimization
1 parent 439aa2b commit 5b861f9

6 files changed

Lines changed: 238 additions & 65 deletions

File tree

‎kubelink/config/GlobalConfig.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type HelmReleaseConfig struct {
2626
ParentChildGvkMapping string `env:"PARENT_CHILD_GVK_MAPPING" envDefault:""`
2727
ChartWorkingDirectory string `env:"CHART_WORKING_DIRECTORY" envDefault:"/home/devtron/devtroncd/charts/"`
2828
BuildNodesBatchSize int `env:"BUILD_NODES_BATCH_SIZE" envDefault:"2"`
29+
ChildObjectListingLimit int64 `env:"CHILD_OBJECT_LISTING_LIMIT" envDefault:"100"`
2930
}
3031

3132
func GetHelmReleaseConfig() (*HelmReleaseConfig, error) {

‎kubelink/env_gen.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"kubelink","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_NODES_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CHART_WORKING_DIRECTORY","EnvType":"string","EnvValue":"/home/devtron/devtroncd/charts/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_HELM_RELEASE_CACHE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MANIFEST_FETCH_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_COUNT_FOR_HELM_RELEASE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARENT_CHILD_GVK_MAPPING","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]}]
1+
[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"kubelink","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_NODES_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CHART_WORKING_DIRECTORY","EnvType":"string","EnvValue":"/home/devtron/devtroncd/charts/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CHILD_OBJECT_LISTING_LIMIT","EnvType":"int64","EnvValue":"100","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_HELM_RELEASE_CACHE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MANIFEST_FETCH_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_COUNT_FOR_HELM_RELEASE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARENT_CHILD_GVK_MAPPING","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]}]

‎kubelink/env_gen.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
| APP | string |kubelink | | | false |
77
| BUILD_NODES_BATCH_SIZE | int |2 | | | false |
88
| CHART_WORKING_DIRECTORY | string |/home/devtron/devtroncd/charts/ | | | false |
9+
| CHILD_OBJECT_LISTING_LIMIT | int64 |100 | | | false |
910
| CONSUMER_CONFIG_JSON | string | | | | false |
1011
| DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false |
1112
| ENABLE_HELM_RELEASE_CACHE | bool |true | | | false |

‎kubelink/pkg/service/commonHelmService/bean.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,19 @@
1313
* See the License for the specific language governing permissions and
1414
* limitations under the License.
1515
*/
16+
1617
package commonHelmService
1718

1819
import (
1920
"errors"
2021
"github.com/devtron-labs/common-lib/utils/k8s/commonBean"
22+
k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean"
2123
"github.com/devtron-labs/common-lib/workerPool"
2224
"github.com/devtron-labs/kubelink/bean"
2325
"github.com/devtron-labs/kubelink/pkg/asyncProvider"
2426
"go.uber.org/zap"
2527
"k8s.io/apimachinery/pkg/api/meta"
28+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2629
"k8s.io/apimachinery/pkg/runtime/schema"
2730
"k8s.io/client-go/rest"
2831
)
@@ -222,3 +225,114 @@ func (resp *BuildNodeResponse) WithHealthStatusArray(healthStatusArray []*common
222225
var (
223226
ErrorReleaseNotFoundOnCluster = errors.New("release not found")
224227
)
228+
229+
type filterChildrenObjectsResponse struct {
230+
pvcs []unstructured.Unstructured
231+
manifests []*unstructured.Unstructured
232+
}
233+
234+
func newFilterChildrenObjectsResponse() *filterChildrenObjectsResponse {
235+
return &filterChildrenObjectsResponse{}
236+
}
237+
238+
func (resp *filterChildrenObjectsResponse) GetPvcs() []unstructured.Unstructured {
239+
return resp.pvcs
240+
}
241+
242+
func (resp *filterChildrenObjectsResponse) GetManifests() []*unstructured.Unstructured {
243+
return resp.manifests
244+
}
245+
246+
func (resp *filterChildrenObjectsResponse) WithPVCs(pvcs []unstructured.Unstructured) *filterChildrenObjectsResponse {
247+
resp.pvcs = append(resp.pvcs, pvcs...)
248+
return resp
249+
}
250+
251+
func (resp *filterChildrenObjectsResponse) WithManifest(manifest *unstructured.Unstructured) *filterChildrenObjectsResponse {
252+
if manifest == nil {
253+
return resp
254+
}
255+
resp.manifests = append(resp.manifests, manifest)
256+
return resp
257+
}
258+
259+
type filterChildrenObjectsRequest struct {
260+
childGvk schema.GroupVersionResource
261+
pvcs []unstructured.Unstructured
262+
listObjects *unstructured.UnstructuredList
263+
namespace string
264+
parentGvk schema.GroupVersionKind
265+
parentName string
266+
}
267+
268+
func (req *filterChildrenObjectsRequest) IsChildResourceTypePVC() bool {
269+
return req.GetParentGvk().Kind == k8sCommonBean.StatefulSetKind && req.GetChildGvk().Resource == k8sCommonBean.PersistentVolumeClaimsResourceType
270+
}
271+
272+
func (req *filterChildrenObjectsRequest) GetLoggerMetadata(keysAndValues ...any) []any {
273+
metaData := []any{
274+
"namespace", req.namespace,
275+
"childGvk", req.childGvk,
276+
"parentGvk", req.parentGvk,
277+
"parentName", req.parentName,
278+
}
279+
return append(metaData, keysAndValues...)
280+
}
281+
282+
func (req *filterChildrenObjectsRequest) GetChildGvk() schema.GroupVersionResource {
283+
return req.childGvk
284+
}
285+
286+
func (req *filterChildrenObjectsRequest) GetPvcs() []unstructured.Unstructured {
287+
return req.pvcs
288+
}
289+
290+
func (req *filterChildrenObjectsRequest) GetListObjects() *unstructured.UnstructuredList {
291+
return req.listObjects
292+
}
293+
294+
func (req *filterChildrenObjectsRequest) GetNamespace() string {
295+
return req.namespace
296+
}
297+
298+
func (req *filterChildrenObjectsRequest) GetParentGvk() schema.GroupVersionKind {
299+
return req.parentGvk
300+
}
301+
302+
func (req *filterChildrenObjectsRequest) GetParentName() string {
303+
return req.parentName
304+
}
305+
306+
func newFilterChildrenObjectsRequest() *filterChildrenObjectsRequest {
307+
return &filterChildrenObjectsRequest{}
308+
}
309+
310+
func (req *filterChildrenObjectsRequest) WithChildGvk(gvr schema.GroupVersionResource) *filterChildrenObjectsRequest {
311+
req.childGvk = gvr
312+
return req
313+
}
314+
315+
func (req *filterChildrenObjectsRequest) WithPvcs(pvcs []unstructured.Unstructured) *filterChildrenObjectsRequest {
316+
req.pvcs = pvcs
317+
return req
318+
}
319+
320+
func (req *filterChildrenObjectsRequest) WithListObjects(objects *unstructured.UnstructuredList) *filterChildrenObjectsRequest {
321+
req.listObjects = objects
322+
return req
323+
}
324+
325+
func (req *filterChildrenObjectsRequest) WithNamespace(namespace string) *filterChildrenObjectsRequest {
326+
req.namespace = namespace
327+
return req
328+
}
329+
330+
func (req *filterChildrenObjectsRequest) WithParentGvk(parentGvk schema.GroupVersionKind) *filterChildrenObjectsRequest {
331+
req.parentGvk = parentGvk
332+
return req
333+
}
334+
335+
func (req *filterChildrenObjectsRequest) WithParentName(parentName string) *filterChildrenObjectsRequest {
336+
req.parentName = parentName
337+
return req
338+
}

0 commit comments

Comments
 (0)