Merge pull request #3363 from shajmakh/tls-sched-e2e

openshift-merge-bot[bot] · web-flow · commit cf7b39d31e72 · 2026-05-07T13:04:08.000Z
e2e: add TLS profile modification test
diff --git a/internal/api/features/_topics.json b/internal/api/features/_topics.json
@@ -22,6 +22,7 @@
     "metrics",
     "nrtcrdanns",
     "netpols",
-    "mgselinuxcollect"
+    "mgselinuxcollect",
+    "tlscompliance"
   ]
 }
diff --git a/internal/remoteexec/command.go b/internal/remoteexec/command.go
@@ -20,17 +20,25 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"net"
+	"net/http"
 	"os"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	utilportforward "k8s.io/apimachinery/pkg/util/portforward"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/client-go/transport/spdy"
+	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 )
 
-// ExecCommandOnPod runs command in the pod and returns buffer output
+// CommandOnPod runs command in the pod and returns buffer output
 func CommandOnPod(ctx context.Context, c kubernetes.Interface, pod *corev1.Pod, command ...string) ([]byte, []byte, error) {
 	return CommandOnPodByNames(ctx, c, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, command...)
 }
@@ -76,3 +84,124 @@ func CommandOnPodByNames(ctx context.Context, c kubernetes.Interface, podNamespa
 
 	return outputBuf.Bytes(), errorBuf.Bytes(), nil
 }
+
+// PortForwardToPod opens an SPDY tunnel to the given pod port and calls fn
+// with the resulting net.Conn that speaks directly to that port inside the pod.
+// The data connection is closed after fn returns (before draining the error stream).
+func PortForwardToPod(c kubernetes.Interface, pod *corev1.Pod, podPort string, fn func(conn net.Conn) error) error {
+	return PortForwardToPodByNames(c, pod.Namespace, pod.Name, podPort, fn)
+}
+
+func PortForwardToPodByNames(c kubernetes.Interface, podNamespace, podName, podPort string, fn func(conn net.Conn) error) error {
+	dialer, err := portForwardDialer(c, podNamespace, podName)
+	if err != nil {
+		return fmt.Errorf("port-forward dialer creation failed: %w", err)
+	}
+
+	spdyConn, _, err := dialer.Dial(utilportforward.PortForwardV1Name)
+	if err != nil {
+		return fmt.Errorf("port-forward dial to %s/%s:%s failed: %w", podNamespace, podName, podPort, err)
+	}
+	// by now the connection is established and the port-forward is running
+	defer func() {
+		if err := spdyConn.Close(); err != nil {
+			klog.ErrorS(err, "failed to close SPDY connection")
+		}
+	}()
+
+	// setup the error stream that would be used by kubelet to send back errors
+	headers := http.Header{}
+	headers.Set(corev1.StreamType, corev1.StreamTypeError)
+	headers.Set(corev1.PortHeader, podPort)
+	headers.Set(corev1.PortForwardRequestIDHeader, "0")
+	errorStream, err := spdyConn.CreateStream(headers)
+	if err != nil {
+		return fmt.Errorf("port-forward error stream creation failed: %w", err)
+	}
+	defer func() {
+		if err := errorStream.Close(); err != nil {
+			klog.ErrorS(err, "failed to close error stream")
+		}
+	}()
+
+	errorCh := make(chan error, 1)
+	go func() {
+		buf, err := io.ReadAll(errorStream)
+		if err != nil {
+			errorCh <- err
+			return
+		}
+		if len(buf) > 0 {
+			errorCh <- fmt.Errorf("port-forward to %s/%s:%s: %s", podNamespace, podName, podPort, string(buf))
+			return
+		}
+		errorCh <- nil
+	}()
+
+	// setup the data stream
+	headers.Set(corev1.StreamType, corev1.StreamTypeData)
+	dataStream, err := spdyConn.CreateStream(headers)
+	if err != nil {
+		return fmt.Errorf("port-forward data stream creation failed: %w", err)
+	}
+
+	conn := &streamConn{stream: dataStream}
+	fnErr := fn(conn)
+	// Close the forwarded data stream as soon as fn finishes so the server can
+	// wind down the port-forward and close the error stream. Otherwise io.ReadAll
+	// on errorStream can block forever while we wait on errorCh below, and
+	// deferred closes never run (deadlock).
+	if err := conn.Close(); err != nil {
+		klog.ErrorS(err, "failed to close port-forward data stream")
+	}
+	if fnErr != nil {
+		return fnErr
+	}
+	if err := <-errorCh; err != nil {
+		return err
+	}
+	return nil
+}
+
+func portForwardDialer(c kubernetes.Interface, podNamespace, podName string) (httpstream.Dialer, error) {
+	req := c.CoreV1().RESTClient().
+		Post().
+		Namespace(podNamespace).
+		Resource("pods").
+		Name(podName).
+		SubResource("portforward")
+
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	transport, upgrader, err := spdy.RoundTripperFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, req.URL())
+	return dialer, nil
+}
+
+// streamConn wraps an httpstream.Stream interface as a net.Conn so it can be used with
+// crypto/tls.Client and similar APIs that expect a net.Conn.
+type streamConn struct {
+	stream httpstream.Stream
+}
+
+func (s *streamConn) Read(b []byte) (int, error)  { return s.stream.Read(b) }
+func (s *streamConn) Write(b []byte) (int, error) { return s.stream.Write(b) }
+func (s *streamConn) Close() error                { return s.stream.Close() }
+
+func (s *streamConn) LocalAddr() net.Addr              { return stubAddr{} }
+func (s *streamConn) RemoteAddr() net.Addr             { return stubAddr{} }
+func (s *streamConn) SetDeadline(time.Time) error      { return nil }
+func (s *streamConn) SetReadDeadline(time.Time) error  { return nil }
+func (s *streamConn) SetWriteDeadline(time.Time) error { return nil }
+
+type stubAddr struct{}
+
+func (stubAddr) Network() string { return "spdy" }
+func (stubAddr) String() string  { return "port-forward" }
diff --git a/test/e2e/tls/tls_suite_test.go b/test/e2e/tls/tls_suite_test.go
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2026 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tls
+
+import (
+	"testing"
+
+	e2eclient "github.com/openshift-kni/numaresources-operator/test/internal/clients"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestTLS(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "TLS")
+}
+
+var _ = BeforeSuite(func() {
+	By("Creating all test resources")
+	Expect(e2eclient.ClientsEnabled).To(BeTrue(), "failed to create runtime-controller client")
+})
diff --git a/test/e2e/tls/tls_test.go b/test/e2e/tls/tls_test.go
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2026 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tls
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	ctrltls "github.com/openshift/controller-runtime-common/pkg/tls"
+	libgocrypto "github.com/openshift/library-go/pkg/crypto"
+
+	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
+	"github.com/openshift-kni/numaresources-operator/internal/podlist"
+	e2eclient "github.com/openshift-kni/numaresources-operator/test/internal/clients"
+	"github.com/openshift-kni/numaresources-operator/test/internal/objects"
+	intls "github.com/openshift-kni/numaresources-operator/test/internal/tls"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const schedulerSecurePort = "10259"
+
+var _ = Describe("TLS", func() {
+	It("should reject TLS connections that are not compatible with the profile - negative test", func(ctx context.Context) {
+		By("getting the current OCP TLS profile")
+		tlsProfileSpec, err := ctrltls.FetchAPIServerTLSProfile(ctx, e2eclient.Client)
+		Expect(err).ToNot(HaveOccurred(), "unable to get TLS profile from APIServer")
+
+		tlsConfigFn, _ := ctrltls.NewTLSConfigFromProfile(tlsProfileSpec)
+		tlsCfg := &tls.Config{}
+		tlsConfigFn(tlsCfg)
+		minVersion := tlsCfg.MinVersion
+		klog.InfoS("current TLS minimum version", "version", libgocrypto.TLSVersionToNameOrDie(minVersion))
+
+		belowMinVersion, err := intls.TLSVersionBelow(minVersion)
+		Expect(err).ToNot(HaveOccurred(), "failed to get TLS version below %s", libgocrypto.TLSVersionToNameOrDie(minVersion))
+
+		By("getting the scheduler deployment and pods")
+		nroSchedObj := &nropv1.NUMAResourcesScheduler{}
+		nroSchedKey := objects.NROSchedObjectKey()
+		Expect(e2eclient.Client.Get(ctx, nroSchedKey, nroSchedObj)).To(Succeed(), "failed to get %q in the cluster", nroSchedKey.String())
+
+		deployment, err := podlist.With(e2eclient.Client).DeploymentByOwnerReference(ctx, nroSchedObj.GetUID())
+		Expect(err).ToNot(HaveOccurred(), "failed to get the deployment")
+		Expect(deployment).ToNot(BeNil(), "scheduler deployment not found")
+
+		pods, err := podlist.With(e2eclient.Client).ByDeployment(ctx, *deployment)
+		Expect(err).ToNot(HaveOccurred(), "failed to get the pods")
+		Expect(pods).ToNot(BeEmpty(), "no pods found for the deployment")
+
+		schedulerPod := &pods[0]
+
+		By(fmt.Sprintf("verifying that TLS connections at version %s are rejected by the server", tls.VersionName(belowMinVersion)))
+		err = intls.ProbeMaxTLSVersion(e2eclient.K8sClient, schedulerPod, schedulerSecurePort, belowMinVersion)
+		Expect(err).To(HaveOccurred(), "scheduler server should reject TLS connections capped at %s", tls.VersionName(belowMinVersion))
+		Expect(errors.Is(err, intls.ErrTLSHandshakeRejected)).To(BeTrue(),
+			"expected TLS handshake rejection, got: %v", err)
+
+		By(fmt.Sprintf("verifying that TLS connections with unsupported ciphers are rejected by the server"))
+		if minVersion == tls.VersionTLS13 {
+			klog.InfoS("TLS 1.3 is not configurable, so we cannot test unsupported ciphers")
+			return
+		}
+
+		disallowedCipher := intls.FindDisallowedCipher(tlsProfileSpec.Ciphers)
+		if disallowedCipher == "" {
+			Skip("all known TLS 1.2 ciphers are in the allowed set, nothing to test")
+		}
+		klog.InfoS("testing with disallowed cipher", "cipher", disallowedCipher)
+		err = intls.ProbeTLSCipher(e2eclient.K8sClient, schedulerPod, schedulerSecurePort, disallowedCipher)
+		Expect(err).To(HaveOccurred(), "scheduler server should reject connections with disallowed cipher %s", disallowedCipher)
+		Expect(errors.Is(err, intls.ErrTLSHandshakeRejected)).To(BeTrue(), "expected TLS handshake rejection for cipher %s, got: %v", disallowedCipher, err)
+	})
+
+	It("should adhere to openshift TLS profile - positive test", func(ctx context.Context) {
+		By("getting the current OCP TLS profile")
+		tlsProfileSpec, err := ctrltls.FetchAPIServerTLSProfile(ctx, e2eclient.Client)
+		Expect(err).ToNot(HaveOccurred(), "unable to get TLS profile from APIServer")
+
+		tlsConfigFn, _ := ctrltls.NewTLSConfigFromProfile(tlsProfileSpec)
+		tlsCfg := &tls.Config{}
+		tlsConfigFn(tlsCfg)
+		minVersion := tlsCfg.MinVersion
+		klog.InfoS("current TLS minimum version", "version", libgocrypto.TLSVersionToNameOrDie(minVersion))
+
+		By("getting the scheduler deployment and pods")
+		nroSchedObj := &nropv1.NUMAResourcesScheduler{}
+		nroSchedKey := objects.NROSchedObjectKey()
+		Expect(e2eclient.Client.Get(ctx, nroSchedKey, nroSchedObj)).To(Succeed(), "failed to get %q in the cluster", nroSchedKey.String())
+
+		deployment, err := podlist.With(e2eclient.Client).DeploymentByOwnerReference(ctx, nroSchedObj.GetUID())
+		Expect(err).ToNot(HaveOccurred(), "failed to get the deployment")
+		Expect(deployment).ToNot(BeNil(), "scheduler deployment not found")
+
+		pods, err := podlist.With(e2eclient.Client).ByDeployment(ctx, *deployment)
+		Expect(err).ToNot(HaveOccurred(), "failed to get the pods")
+		Expect(pods).ToNot(BeEmpty(), "no pods found for the deployment")
+
+		schedulerPod := &pods[0]
+
+		By("probing the scheduler HTTPS endpoint to verify TLS connection is accepted")
+		gotVersion, gotCipherID, err := intls.ProbeTLSSettings(e2eclient.K8sClient, schedulerPod, schedulerSecurePort)
+		Expect(err).ToNot(HaveOccurred(), "failed to probe TLS settings on pod %q", schedulerPod.Name)
+
+		gotVersionName := tls.VersionName(gotVersion)
+		gotCipherName := tls.CipherSuiteName(gotCipherID)
+		klog.InfoS("negotiated TLS settings", "version", gotVersionName, "cipher", gotCipherName)
+
+		Expect(gotVersion).To(BeNumerically(">=", minVersion), "negotiated TLS version %s is below the expected minimum %s", gotVersionName, libgocrypto.TLSVersionToNameOrDie(minVersion))
+
+		// TLS 1.3 cipher suites are not configurable and won't appear in
+		// the profile's list; only validate for TLS 1.2 and below.
+		allowedCipherNames := libgocrypto.OpenSSLToIANACipherSuites(tlsProfileSpec.Ciphers)
+		if gotVersion < tls.VersionTLS13 {
+			Expect(gotCipherName).ToNot(BeEmpty(), "could not resolve negotiated cipher suite ID 0x%04x", gotCipherID)
+			Expect(allowedCipherNames).To(ContainElement(gotCipherName), "negotiated cipher %s is not in the allowed set %v", gotCipherName, tlsProfileSpec.Ciphers)
+		}
+	})
+})
diff --git a/test/internal/clients/clients.go b/test/internal/clients/clients.go
@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
+	configv1 "github.com/openshift/api/config/v1"
 	mcov1 "github.com/openshift/api/machineconfiguration/v1"
 	hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 
@@ -48,6 +49,10 @@ var (
 
 func init() {
 	// Setup Scheme for all resources
+	if err := configv1.Install(scheme.Scheme); err != nil {
+		klog.Exit(err.Error())
+	}
+
 	if err := mcov1.AddToScheme(scheme.Scheme); err != nil {
 		klog.Exit(err.Error())
 	}
diff --git a/test/internal/tls/tls.go b/test/internal/tls/tls.go
diff --git a/test/internal/tls/tls_test.go b/test/internal/tls/tls_test.go

Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,7 @@`
`22`	`22`	`"metrics",`
`23`	`23`	`"nrtcrdanns",`
`24`	`24`	`"netpols",`
`25`		`- "mgselinuxcollect"`
	`25`	`+ "mgselinuxcollect",`
	`26`	`+ "tlscompliance"`
`26`	`27`	`]`
`27`	`28`	`}`