Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions controllers/workspace/devworkspace_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,6 @@ func (r *DevWorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request
reqLogger = reqLogger.WithValues(constants.DevWorkspaceIDLoggerKey, workspace.Status.DevWorkspaceId)
reqLogger.Info("Reconciling Workspace", "resolvedConfig", configString)

// Inject ca certificates to the http client, if the certificates configmap is created and defined in the config.
InjectCertificates(r.Client, r.Log)

// Check if the DevWorkspaceRouting instance is marked to be deleted, which is
// indicated by the deletion timestamp being set.
if workspace.GetDeletionTimestamp() != nil {
Expand Down Expand Up @@ -264,7 +261,7 @@ func (r *DevWorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request
WorkspaceNamespace: workspace.Namespace,
Context: ctx,
K8sClient: r.Client,
HttpClient: httpClient,
HttpClient: httpClientsHolder.GetHttpClient(config.Routing),
DefaultResourceRequirements: workspace.Config.Workspace.DefaultContainerResources,
}

Expand Down Expand Up @@ -788,7 +785,7 @@ func (r *DevWorkspaceReconciler) getWorkspaceId(ctx context.Context, workspace *
}

func (r *DevWorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
setupHttpClients(mgr.GetClient(), mgr.GetLogger())
httpClientsHolder = NewDefaultHttpsClientHolder(mgr.GetClient(), mgr.GetLogger())

maxConcurrentReconciles, err := wkspConfig.GetMaxConcurrentReconciles()
if err != nil {
Expand Down
128 changes: 76 additions & 52 deletions controllers/workspace/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"net/url"
"time"

"github.com/devfile/devworkspace-operator/pkg/config"
controller "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"

"k8s.io/apimachinery/pkg/types"

Expand All @@ -32,87 +32,111 @@ import (
"golang.org/x/net/http/httpproxy"
)

var (
httpClient *http.Client
type HttpClientsHolder interface {
GetHttpClient(routingConfig *controller.RoutingConfig) *http.Client
GetHealthCheckHttpClient(routingConfig *controller.RoutingConfig) *http.Client
}

type DefaultHttpsClientHolder struct {
k8s client.Client
logger logr.Logger

client *http.Client
healthCheckHttpClient *http.Client
)
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

var httpClientsHolder HttpClientsHolder

func setupHttpClients(k8s client.Client, logger logr.Logger) {
func NewDefaultHttpsClientHolder(k8s client.Client, logger logr.Logger) *DefaultHttpsClientHolder {
return &DefaultHttpsClientHolder{k8s: k8s, logger: logger}
}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Comment contradicts actual behavior

The comment says "not an issue at all" but this IS an issue - the operator was configured to trust custom certificates. Rebuilding without them on transient failure breaks the trust chain.

Consider updating this comment after fixing the transient failure handling.


Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Critical: Transient ConfigMap failure silently strips custom certificates

When readCertCM fails (transient API error, RBAC issue), newCertsCM becomes nil. The shouldRebuildClients method then detects a "change" and rebuilds the HTTP client with getCaCertPool(nil), stripping all custom CA certificates.

A single transient Kubernetes API error will break TLS trust cluster-wide for all subsequent workspace operations.

Suggested fix: On readCertCM failure, return early without rebuilding clients. Only rebuild when we successfully read a ConfigMap with a different ResourceVersion.

if routingConfig.TLSCertificateConfigmapRef != nil {
    certsCM, err := h.readCertCM(ctx, routingConfig.TLSCertificateConfigmapRef)
    if err != nil {
        h.logger.Error(err, "Failed to read TLS certificate ConfigMap")
        // Preserve existing HTTP clients on transient failure
        return
    }
    newCertsCM = certsCM
}

func (h *DefaultHttpsClientHolder) GetHttpClient(routingConfig *controller.RoutingConfig) *http.Client {
transport := http.DefaultTransport.(*http.Transport).Clone()
healthCheckTransport := http.DefaultTransport.(*http.Transport).Clone()
healthCheckTransport.TLSClientConfig = &tls.Config{
transport.Proxy = h.getProxyFunc(routingConfig)
transport.TLSClientConfig = &tls.Config{
RootCAs: h.getCaCertPool(routingConfig),
}

Comment thread
tolusha marked this conversation as resolved.
return &http.Client{
Transport: transport,
}
}

func (h *DefaultHttpsClientHolder) GetHealthCheckHttpClient(routingConfig *controller.RoutingConfig) *http.Client {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.Proxy = h.getProxyFunc(routingConfig)
transport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
}

globalConfig := config.GetGlobalConfig()
return &http.Client{
Transport: transport,
Timeout: 500 * time.Millisecond,
}
}

if globalConfig.Routing != nil && globalConfig.Routing.ProxyConfig != nil {
func (h *DefaultHttpsClientHolder) getProxyFunc(routingConfig *controller.RoutingConfig) func(*http.Request) (*url.URL, error) {
if routingConfig != nil && routingConfig.ProxyConfig != nil {
proxyConf := httpproxy.Config{}
if globalConfig.Routing.ProxyConfig.HttpProxy != nil {
proxyConf.HTTPProxy = *globalConfig.Routing.ProxyConfig.HttpProxy
if routingConfig.ProxyConfig.HttpProxy != nil {
proxyConf.HTTPProxy = *routingConfig.ProxyConfig.HttpProxy
}
if globalConfig.Routing.ProxyConfig.HttpsProxy != nil {
proxyConf.HTTPSProxy = *globalConfig.Routing.ProxyConfig.HttpsProxy
if routingConfig.ProxyConfig.HttpsProxy != nil {
proxyConf.HTTPSProxy = *routingConfig.ProxyConfig.HttpsProxy
}
if globalConfig.Routing.ProxyConfig.NoProxy != nil {
proxyConf.NoProxy = *globalConfig.Routing.ProxyConfig.NoProxy
if routingConfig.ProxyConfig.NoProxy != nil {
proxyConf.NoProxy = *routingConfig.ProxyConfig.NoProxy
}

proxyFunc := func(req *http.Request) (*url.URL, error) {
return func(req *http.Request) (*url.URL, error) {
return proxyConf.ProxyFunc()(req.URL)
}
transport.Proxy = proxyFunc
healthCheckTransport.Proxy = proxyFunc
}

httpClient = &http.Client{
Transport: transport,
}
healthCheckHttpClient = &http.Client{
Transport: healthCheckTransport,
Timeout: 500 * time.Millisecond,
}
InjectCertificates(k8s, logger)
return nil
}

func InjectCertificates(k8s client.Client, logger logr.Logger) {
if certs, ok := readCertificates(k8s, logger); ok {
func (h *DefaultHttpsClientHolder) getCaCertPool(routingConfig *controller.RoutingConfig) *x509.CertPool {
if certs, ok := h.readCertificates(routingConfig); ok {
var caCertPool *x509.CertPool

systemCertPool, err := x509.SystemCertPool()
if err != nil {
h.logger.Error(err, "Failed to load system cert pool")
caCertPool = x509.NewCertPool()
} else {
caCertPool = systemCertPool
}

for _, certsPem := range certs {
injectCertificates([]byte(certsPem), httpClient.Transport.(*http.Transport), logger)
caCertPool.AppendCertsFromPEM([]byte(certsPem))
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated

return caCertPool
}

return nil
}

func readCertificates(k8s client.Client, logger logr.Logger) (map[string]string, bool) {
configmapRef := config.GetGlobalConfig().Routing.TLSCertificateConfigmapRef
if configmapRef == nil {
func (h *DefaultHttpsClientHolder) readCertificates(routingConfig *controller.RoutingConfig) (map[string]string, bool) {
if routingConfig == nil || routingConfig.TLSCertificateConfigmapRef == nil {
return nil, false
}
configMap := &corev1.ConfigMap{}
namespacedName := &types.NamespacedName{

configmapRef := routingConfig.TLSCertificateConfigmapRef

namespacedName := types.NamespacedName{
Name: configmapRef.Name,
Namespace: configmapRef.Namespace,
}
err := k8s.Get(context.Background(), *namespacedName, configMap)

configMap := &corev1.ConfigMap{}
err := h.k8s.Get(context.Background(), namespacedName, configMap)
if err != nil {
logger.Error(err, "Failed to read configmap with certificates")
h.logger.Error(err, "Failed to read configmap with certificates")
return nil, false
}
return configMap.Data, true
}

func injectCertificates(certsPem []byte, transport *http.Transport, logger logr.Logger) {
caCertPool := transport.TLSClientConfig.RootCAs
if caCertPool == nil {
systemCertPool, err := x509.SystemCertPool()
if err != nil {
logger.Error(err, "Failed to load system cert pool")
caCertPool = x509.NewCertPool()
} else {
caCertPool = systemCertPool
}
}
if ok := caCertPool.AppendCertsFromPEM(certsPem); ok {
transport.TLSClientConfig = &tls.Config{RootCAs: caCertPool}
}
return configMap.Data, true
}
25 changes: 22 additions & 3 deletions controllers/workspace/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,28 @@

package controllers

import "net/http"
import (
"net/http"

controller "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
)

type TestHttpsClientHolder struct {
client *http.Client
healthCheckHttpClient *http.Client
}

func (t *TestHttpsClientHolder) GetHttpClient(_ *controller.RoutingConfig) *http.Client {
return t.client
}

func (t *TestHttpsClientHolder) GetHealthCheckHttpClient(_ *controller.RoutingConfig) *http.Client {
return t.healthCheckHttpClient
}

func SetupHttpClientsForTesting(client *http.Client) {
httpClient = client
healthCheckHttpClient = client
httpClientsHolder = &TestHttpsClientHolder{
client: client,
healthCheckHttpClient: client,
}
}
2 changes: 1 addition & 1 deletion controllers/workspace/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func checkServerStatus(workspace *common.DevWorkspaceWithConfig) (ok bool, respo
}
healthz.Path = path.Join(healthz.Path, "healthz")

resp, err := healthCheckHttpClient.Get(healthz.String())
resp, err := httpClientsHolder.GetHealthCheckHttpClient(workspace.Config.Routing).Get(healthz.String())
if err != nil {
return false, nil, &dwerrors.RetryError{Err: err, Message: "Failed to check server status", RequeueAfter: 1 * time.Second}
}
Expand Down
Loading