Skip to content

Commit d94e313

Browse files
authored
Merge pull request #275 from devtron-labs/chore/http-transport-refactoring
chore: http transport service refactoring
2 parents bbd74bc + ba0d761 commit d94e313

64 files changed

Lines changed: 8407 additions & 2043 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

chart-sync/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.22.4
55
toolchain go1.22.6
66

77
replace (
8-
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d
8+
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe
99
helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42
1010
)
1111

chart-sync/go.sum

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
5454
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5555
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5656
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d h1:q0a5FgU3HowrydPq+BjIMHCF0izHjl1oPc71JDLsv+8=
58-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d/go.mod h1:CXQGEo+kZc7JPX5hn4jJf1msal9q/ExSdAYGkHNPnQw=
57+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe h1:jVbCesVCeOSZgSdnR9Rl8Hsx6wZemF+QsHn7XXVOTp0=
58+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe/go.mod h1:HQVUnQI7WHwVq89Bib/18xJqM89S1+xI0O7REctMMrA=
5959
github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY=
6060
github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
6161
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
@@ -292,8 +292,9 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
292292
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
293293
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
294294
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
295-
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
296295
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
296+
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
297+
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
297298
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
298299
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
299300
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

chart-sync/vendor/modules.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ github.com/containerd/platforms
9393
# github.com/davecgh/go-spew v1.1.1
9494
## explicit
9595
github.com/davecgh/go-spew/spew
96-
# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d
96+
# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe
9797
## explicit; go 1.21
9898
github.com/devtron-labs/common-lib/constants
9999
github.com/devtron-labs/common-lib/fetchAllEnv
@@ -791,4 +791,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
791791
# sigs.k8s.io/yaml v1.3.0
792792
## explicit; go 1.12
793793
sigs.k8s.io/yaml
794-
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d
794+
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe

ci-runner/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
toolchain go1.21.8
66

7-
replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d
7+
replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe
88

99
require (
1010
github.com/Knetic/govaluate v3.0.0+incompatible

ci-runner/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
9595
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9696
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
9797
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
98-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d h1:q0a5FgU3HowrydPq+BjIMHCF0izHjl1oPc71JDLsv+8=
99-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d/go.mod h1:CXQGEo+kZc7JPX5hn4jJf1msal9q/ExSdAYGkHNPnQw=
98+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe h1:jVbCesVCeOSZgSdnR9Rl8Hsx6wZemF+QsHn7XXVOTp0=
99+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe/go.mod h1:HQVUnQI7WHwVq89Bib/18xJqM89S1+xI0O7REctMMrA=
100100
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
101101
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
102102
github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=

ci-runner/vendor/modules.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ github.com/cespare/xxhash/v2
248248
# github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
249249
## explicit
250250
github.com/davecgh/go-spew/spew
251-
# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d
251+
# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe
252252
## explicit; go 1.21
253253
github.com/devtron-labs/common-lib/blob-storage
254254
github.com/devtron-labs/common-lib/constants
@@ -991,4 +991,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
991991
# sigs.k8s.io/yaml v1.3.0
992992
## explicit; go 1.12
993993
sigs.k8s.io/yaml
994-
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521130159-a5d26dc2ab8d
994+
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250521214453-c735fc26a0fe

common-lib/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ require (
7373
github.com/leodido/go-urn v1.4.0 // indirect
7474
github.com/moby/docker-image-spec v1.3.1 // indirect
7575
github.com/onsi/ginkgo v1.16.5 // indirect
76+
github.com/stretchr/objx v0.5.2 // indirect
7677
golang.org/x/mod v0.17.0 // indirect
7778
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
7879
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect

common-lib/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
400400
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
401401
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
402402
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
403+
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
404+
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
403405
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
404406
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
405407
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* Copyright (c) 2020-2024. Devtron Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package k8s
18+
19+
import (
20+
"github.com/caarlos0/env"
21+
"go.uber.org/zap"
22+
utilnet "k8s.io/apimachinery/pkg/util/net"
23+
"k8s.io/client-go/rest"
24+
"net"
25+
"net/http"
26+
"time"
27+
)
28+
29+
type DefaultK8sHttpTransportConfig struct{}
30+
31+
type CustomK8sHttpTransportConfig struct {
32+
UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false"`
33+
TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30"`
34+
KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30"`
35+
TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10"`
36+
MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25"`
37+
IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300"`
38+
}
39+
40+
func NewDefaultK8sHttpTransportConfig() *DefaultK8sHttpTransportConfig {
41+
return &DefaultK8sHttpTransportConfig{}
42+
}
43+
44+
func NewCustomK8sHttpTransportConfig(logger *zap.SugaredLogger) *CustomK8sHttpTransportConfig {
45+
customK8sHttpTransportConfig := &CustomK8sHttpTransportConfig{}
46+
err := env.Parse(customK8sHttpTransportConfig)
47+
if err != nil {
48+
logger.Errorw("error in parsing custom k8s http configurations from env", "err", err)
49+
}
50+
return customK8sHttpTransportConfig
51+
}
52+
53+
type TransportType string
54+
55+
const (
56+
TransportTypeDefault TransportType = "default"
57+
TransportTypeOverridden TransportType = "overridden"
58+
)
59+
60+
type HttpTransportConfig struct {
61+
customHttpClientConfig HttpTransportInterface
62+
defaultHttpClientConfig HttpTransportInterface
63+
}
64+
65+
func NewHttpTransportConfig(logger *zap.SugaredLogger) *HttpTransportConfig {
66+
return &HttpTransportConfig{
67+
customHttpClientConfig: NewCustomK8sHttpTransportConfig(logger),
68+
defaultHttpClientConfig: NewDefaultK8sHttpTransportConfig(),
69+
}
70+
}
71+
72+
type HttpTransportInterface interface {
73+
OverrideConfigWithCustomTransport(config *rest.Config) (*rest.Config, error)
74+
}
75+
76+
// OverrideConfigWithCustomTransport
77+
// sets returns the given rest config without any modifications even if UseCustomTransport is enabled.
78+
// This is used when we want to use the default rest.Config provided by the client-go library.
79+
func (impl *DefaultK8sHttpTransportConfig) OverrideConfigWithCustomTransport(config *rest.Config) (*rest.Config, error) {
80+
return config, nil
81+
}
82+
83+
// OverrideConfigWithCustomTransport
84+
// overrides the given rest config with custom transport if UseCustomTransport is enabled.
85+
// if the config already has a defined transport, we don't override it.
86+
func (impl *CustomK8sHttpTransportConfig) OverrideConfigWithCustomTransport(config *rest.Config) (*rest.Config, error) {
87+
if !impl.UseCustomTransport || config.Transport != nil {
88+
return config, nil
89+
}
90+
91+
dial := (&net.Dialer{
92+
Timeout: time.Duration(impl.TimeOut) * time.Second,
93+
KeepAlive: time.Duration(impl.KeepAlive) * time.Second,
94+
}).DialContext
95+
96+
// Get the TLS options for this client config
97+
tlsConfig, err := rest.TLSConfigFor(config)
98+
if err != nil {
99+
return nil, err
100+
}
101+
102+
transport := utilnet.SetTransportDefaults(&http.Transport{
103+
Proxy: config.Proxy,
104+
TLSHandshakeTimeout: time.Duration(impl.TLSHandshakeTimeout) * time.Second,
105+
TLSClientConfig: tlsConfig,
106+
MaxIdleConns: impl.MaxIdleConnsPerHost,
107+
MaxConnsPerHost: impl.MaxIdleConnsPerHost,
108+
MaxIdleConnsPerHost: impl.MaxIdleConnsPerHost,
109+
DialContext: dial,
110+
DisableCompression: config.DisableCompression,
111+
IdleConnTimeout: time.Duration(impl.IdleConnTimeout) * time.Second,
112+
})
113+
114+
rt, err := rest.HTTPWrappersForConfig(config, transport)
115+
if err != nil {
116+
return nil, err
117+
}
118+
119+
config.Transport = rt
120+
config.Timeout = time.Duration(impl.TimeOut) * time.Second
121+
122+
// set default tls config and remove auth/exec provides since we use it in a custom transport.
123+
// we already set tls config in the transport
124+
config.TLSClientConfig = rest.TLSClientConfig{}
125+
config.AuthProvider = nil
126+
config.ExecProvider = nil
127+
128+
return config, nil
129+
}

0 commit comments

Comments
 (0)