diff --git a/e2e-next/clusters/clusters.go b/e2e-next/clusters/clusters.go index c5f6c523a6..1d5429e032 100644 --- a/e2e-next/clusters/clusters.go +++ b/e2e-next/clusters/clusters.go @@ -99,6 +99,18 @@ var ( ) ) +var ( + CertsVClusterName = "certs-test-vcluster" + CertsVCluster = vcluster.Define( + vcluster.WithName(CertsVClusterName), + vcluster.WithVClusterYAML(DefaultVClusterYAML), + vcluster.WithOptions( + DefaultVClusterOptions..., + ), + vcluster.WithDependencies(HostCluster), + ) +) + var ( //go:embed vcluster-servicesync.yaml ServiceSyncVClusterYAMLTemplate string diff --git a/e2e-next/e2e_suite_test.go b/e2e-next/e2e_suite_test.go index a7962eff0e..97d151b800 100644 --- a/e2e-next/e2e_suite_test.go +++ b/e2e-next/e2e_suite_test.go @@ -24,6 +24,7 @@ import ( _ "github.com/loft-sh/vcluster/e2e-next/init" // Import tests + _ "github.com/loft-sh/vcluster/e2e-next/test_certs" _ "github.com/loft-sh/vcluster/e2e-next/test_core/sync" _ "github.com/loft-sh/vcluster/e2e-next/test_core/sync/fromhost" _ "github.com/loft-sh/vcluster/e2e-next/test_deploy" @@ -110,6 +111,7 @@ var _ = SynchronizedBeforeSuite( clusters.InitManifestsVCluster.Setup, clusters.ServiceSyncVCluster.Setup, clusters.FromHostConfigMapsVCluster.Setup, + clusters.CertsVCluster.Setup, )(ctx) Expect(err).NotTo(HaveOccurred()) }) diff --git a/e2e-next/labels/labels.go b/e2e-next/labels/labels.go index bc501e0d25..952e8ed5de 100644 --- a/e2e-next/labels/labels.go +++ b/e2e-next/labels/labels.go @@ -14,4 +14,5 @@ var ( Deploy = Label("deploy") Storage = Label("storage") Security = Label("security") + Certs = Label("certs") ) diff --git a/e2e-next/test_certs/test_rotate.go b/e2e-next/test_certs/test_rotate.go new file mode 100644 index 0000000000..551eca4e56 --- /dev/null +++ b/e2e-next/test_certs/test_rotate.go @@ -0,0 +1,411 @@ +package test_certs + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "os" + "strings" + "time" + + certscmd "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/certs" + "github.com/loft-sh/vcluster/e2e-next/clusters" + "github.com/loft-sh/vcluster/e2e-next/constants" + "github.com/loft-sh/vcluster/e2e-next/labels" + "github.com/loft-sh/vcluster/pkg/certs" + "github.com/loft-sh/vcluster/pkg/cli" + "github.com/loft-sh/vcluster/pkg/cli/flags" + + "github.com/loft-sh/e2e-framework/pkg/setup/cluster" + loftlog "github.com/loft-sh/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// Ordered justification: cert rotation is destructive and cumulative — +// each context modifies the vCluster's certificates, and subsequent +// contexts depend on the state left by prior ones. +var _ = Describe("Certificate Rotation", + Ordered, + labels.PR, + labels.Certs, + cluster.Use(clusters.CertsVCluster), + cluster.Use(clusters.HostCluster), + func() { + var ( + hostClient kubernetes.Interface + vclusterNS = "vcluster-" + clusters.CertsVClusterName + vclusterName = clusters.CertsVClusterName + labelSelector = "app=vcluster,release=" + clusters.CertsVClusterName + + // Fingerprints tracked across contexts + apiserverFingerprintBefore string + apiserverCertBefore *x509.Certificate + caFingerprintBefore string + caCertBefore *x509.Certificate + ) + + BeforeAll(func(ctx context.Context) { + By("Obtaining host client and initial certificate fingerprints", func() { + hostClient = cluster.KubeClientFrom(ctx, constants.GetHostClusterName()) + Expect(hostClient).NotTo(BeNil()) + + secret, err := hostClient.CoreV1().Secrets(vclusterNS).Get(ctx, certs.CertSecretName(vclusterName), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "should obtain the initial cert secret") + + apiserverCertBefore, err = parseCertFromPEM(secret.Data[certs.APIServerCertName]) + Expect(err).NotTo(HaveOccurred(), "should parse apiserver cert") + apiserverFingerprintBefore = certFingerprint(apiserverCertBefore) + + caCertBefore, err = parseCertFromPEM(secret.Data[certs.CACertName]) + Expect(err).NotTo(HaveOccurred(), "should parse CA cert") + caFingerprintBefore = certFingerprint(caCertBefore) + }) + }) + + Context("certs rotate", func() { + It("should preserve CA cert and rotate apiserver cert", func(ctx context.Context) { + By("Executing certs rotate command", func() { + certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: vclusterNS}) + certsCmd.SetArgs([]string{"rotate", vclusterName}) + Expect(certsCmd.Execute()).To(Succeed()) + }) + + By("Waiting for vCluster pods to be ready again", func() { + waitForVClusterReady(ctx, hostClient, vclusterNS, labelSelector) + }) + + By("Verifying CA fingerprint and expiry are unchanged and apiserver fingerprint changed", func() { + secret, err := hostClient.CoreV1().Secrets(vclusterNS).Get(ctx, certs.CertSecretName(vclusterName), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + caCertAfter, err := parseCertFromPEM(secret.Data[certs.CACertName]) + Expect(err).NotTo(HaveOccurred()) + caFingerprintAfter := certFingerprint(caCertAfter) + Expect(caFingerprintAfter).To(Equal(caFingerprintBefore), "CA fingerprint should be unchanged after leaf rotation") + Expect(caCertAfter.NotAfter).To(Equal(caCertBefore.NotAfter), "CA expiry should be unchanged after leaf rotation") + + apiserverCertAfter, err := parseCertFromPEM(secret.Data[certs.APIServerCertName]) + Expect(err).NotTo(HaveOccurred()) + apiserverFingerprintAfter := certFingerprint(apiserverCertAfter) + Expect(apiserverFingerprintAfter).NotTo(Equal(apiserverFingerprintBefore), "apiserver fingerprint should change after rotation") + Expect(apiserverCertAfter.NotAfter.After(apiserverCertBefore.NotAfter)).To(BeTrue(), "new apiserver cert should expire later") + + // Update tracked fingerprints for subsequent contexts + caFingerprintBefore = caFingerprintAfter + apiserverFingerprintBefore = apiserverFingerprintAfter + apiserverCertBefore = apiserverCertAfter + }) + }) + }) + + Context("certs rotate-ca", func() { + It("should rotate both CA and apiserver certs", func(ctx context.Context) { + By("Executing certs rotate-ca command", func() { + certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: vclusterNS}) + certsCmd.SetArgs([]string{"rotate-ca", vclusterName}) + Expect(certsCmd.Execute()).To(Succeed()) + }) + + By("Waiting for vCluster pods to be ready again", func() { + waitForVClusterReady(ctx, hostClient, vclusterNS, labelSelector) + }) + + By("Verifying both CA and apiserver fingerprints changed and expire later", func() { + secret, err := hostClient.CoreV1().Secrets(vclusterNS).Get(ctx, certs.CertSecretName(vclusterName), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + caCertAfter, err := parseCertFromPEM(secret.Data[certs.CACertName]) + Expect(err).NotTo(HaveOccurred()) + caFingerprintAfter := certFingerprint(caCertAfter) + Expect(caFingerprintAfter).NotTo(Equal(caFingerprintBefore), "CA fingerprint should change after CA rotation") + Expect(caCertAfter.NotAfter.After(caCertBefore.NotAfter)).To(BeTrue(), "new CA cert should expire later") + + apiserverCertAfter, err := parseCertFromPEM(secret.Data[certs.APIServerCertName]) + Expect(err).NotTo(HaveOccurred()) + apiserverFingerprintAfter := certFingerprint(apiserverCertAfter) + Expect(apiserverFingerprintAfter).NotTo(Equal(apiserverFingerprintBefore), "apiserver fingerprint should change after CA rotation") + Expect(apiserverCertAfter.NotAfter.After(apiserverCertBefore.NotAfter)).To(BeTrue(), "new apiserver cert should expire later") + + // Update tracked state + caCertBefore = caCertAfter + caFingerprintBefore = caFingerprintAfter + apiserverCertBefore = apiserverCertAfter + apiserverFingerprintBefore = apiserverFingerprintAfter + }) + }) + }) + + Context("expired certificate rotation", func() { + It("should rotate expired certificates", func(ctx context.Context) { + By("Verifying current CA cert is valid", func() { + secret, err := hostClient.CoreV1().Secrets(vclusterNS).Get(ctx, certs.CertSecretName(vclusterName), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + caCert, err := parseCertFromPEM(secret.Data[certs.CACertName]) + Expect(err).NotTo(HaveOccurred()) + Expect(caCert.NotAfter.After(time.Now())).To(BeTrue(), "CA cert should currently be valid") + }) + + By("Setting 1-second validity and rotating CA to create soon-to-expire certs", func() { + Expect(os.Setenv("DEVELOPMENT", "true")).To(Succeed()) + Expect(os.Setenv("VCLUSTER_CERTS_VALIDITYPERIOD", "1s")).To(Succeed()) + defer func() { + os.Unsetenv("DEVELOPMENT") + os.Unsetenv("VCLUSTER_CERTS_VALIDITYPERIOD") + }() + + certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: vclusterNS}) + certsCmd.SetArgs([]string{"rotate-ca", vclusterName}) + Expect(certsCmd.Execute()).To(Succeed()) + }) + + By("Waiting for vCluster pods to be running", func() { + Eventually(func(g Gomega) { + pods, err := hostClient.CoreV1().Pods(vclusterNS).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + g.Expect(err).NotTo(HaveOccurred(), "should list vCluster pods") + g.Expect(pods.Items).NotTo(BeEmpty(), "should have at least one vCluster pod") + for _, pod := range pods.Items { + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning), + "pod %s should be running, got %s", pod.Name, pod.Status.Phase) + } + }).WithPolling(constants.PollingInterval). + WithTimeout(constants.PollingTimeoutLong). + Should(Succeed()) + }) + + By("Waiting for CA cert to expire", func() { + Eventually(func(g Gomega) { + secret, err := hostClient.CoreV1().Secrets(vclusterNS).Get(ctx, certs.CertSecretName(vclusterName), metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + block, _ := pem.Decode(secret.Data[certs.CACertName]) + g.Expect(block).NotTo(BeNil(), "should decode PEM block") + + cert, err := x509.ParseCertificate(block.Bytes) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cert.NotAfter.Before(time.Now())).To(BeTrue(), + "CA cert should be expired, but expires at %s", cert.NotAfter) + }).WithPolling(constants.PollingInterval). + WithTimeout(constants.PollingTimeoutLong). + Should(Succeed()) + }) + + By("Rotating expired CA with normal validity", func() { + certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: vclusterNS}) + certsCmd.SetArgs([]string{"rotate-ca", vclusterName}) + Expect(certsCmd.Execute()).To(Succeed()) + }) + + By("Waiting for vCluster to be fully ready", func() { + waitForVClusterReady(ctx, hostClient, vclusterNS, labelSelector) + }) + + By("Verifying new CA cert is valid", func() { + secret, err := hostClient.CoreV1().Secrets(vclusterNS).Get(ctx, certs.CertSecretName(vclusterName), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + caCert, err := parseCertFromPEM(secret.Data[certs.CACertName]) + Expect(err).NotTo(HaveOccurred()) + Expect(caCert.NotAfter.After(time.Now())).To(BeTrue(), "new CA cert should be valid") + + // Update tracked state + caCertBefore = caCert + caFingerprintBefore = certFingerprint(caCert) + }) + }) + }) + + Context("kube config compatibility", func() { + var restConfigBefore *rest.Config + + BeforeAll(func(ctx context.Context) { + By("Reconnecting to vCluster and saving current TLS config", func() { + cfg, cleanup := connectVCluster(ctx, vclusterName, vclusterNS) + DeferCleanup(cleanup) + + vClient, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + _, err = vClient.CoreV1().Pods(corev1.NamespaceDefault).List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred(), "should be able to list pods with current config") + + restConfigBefore = rest.CopyConfig(cfg) + }) + }) + + Context("after certs rotate", func() { + It("should allow old TLS config after leaf rotation", func(ctx context.Context) { + By("Executing certs rotate command", func() { + certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: vclusterNS}) + certsCmd.SetArgs([]string{"rotate", vclusterName}) + Expect(certsCmd.Execute()).To(Succeed()) + }) + + By("Waiting for vCluster pods to be ready", func() { + waitForVClusterReady(ctx, hostClient, vclusterNS, labelSelector) + }) + + By("Verifying old TLS config still works after leaf rotation", func() { + cfg, cleanup := connectVCluster(ctx, vclusterName, vclusterNS) + DeferCleanup(cleanup) + + // Use new connection's server address but old TLS config + cfg.TLSClientConfig = restConfigBefore.TLSClientConfig + vClient, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + _, err = vClient.CoreV1().Pods(corev1.NamespaceDefault).List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred(), "old TLS config should still work after leaf rotation (CA unchanged)") + }) + }) + }) + + Context("after certs rotate-ca", func() { + It("should reject old TLS config after CA rotation", func(ctx context.Context) { + By("Executing certs rotate-ca command", func() { + certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: vclusterNS}) + certsCmd.SetArgs([]string{"rotate-ca", vclusterName}) + Expect(certsCmd.Execute()).To(Succeed()) + }) + + By("Waiting for vCluster pods to be ready", func() { + waitForVClusterReady(ctx, hostClient, vclusterNS, labelSelector) + }) + + By("Verifying old TLS config fails after CA rotation", func() { + cfg, cleanup := connectVCluster(ctx, vclusterName, vclusterNS) + DeferCleanup(cleanup) + + // Use new connection's server address but old TLS config + cfg.TLSClientConfig = restConfigBefore.TLSClientConfig + vClient, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + _, err = vClient.CoreV1().Pods(corev1.NamespaceDefault).List(ctx, metav1.ListOptions{}) + Expect(err).To(HaveOccurred(), "old TLS config should fail after CA rotation") + + var certErr *tls.CertificateVerificationError + Expect(errors.As(err, &certErr)).To(BeTrue(), + "expected tls.CertificateVerificationError but got: %v", err) + }) + }) + }) + }) + + AfterAll(func(ctx context.Context) { + By("Reconnecting to vCluster to restore suite proxy for subsequent tests", func() { + _, cleanup := connectVCluster(ctx, vclusterName, vclusterNS) + DeferCleanup(cleanup) + }) + }) + }, +) + +// waitForVClusterReady polls until all vCluster pods are running with all containers ready. +func waitForVClusterReady(ctx context.Context, hostClient kubernetes.Interface, namespace, labelSelector string) { + Eventually(func(g Gomega) { + pods, err := hostClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + g.Expect(err).NotTo(HaveOccurred(), "should list vCluster pods") + g.Expect(pods.Items).NotTo(BeEmpty(), "should have at least one vCluster pod") + + for _, pod := range pods.Items { + g.Expect(pod.Status.ContainerStatuses).NotTo(BeEmpty(), + "pod %s should have container statuses", pod.Name) + for i, container := range pod.Status.ContainerStatuses { + g.Expect(container.State.Running).NotTo(BeNil(), + "container %d in pod %s should be running", i, pod.Name) + g.Expect(container.Ready).To(BeTrue(), + "container %d in pod %s should be ready", i, pod.Name) + } + } + }).WithPolling(constants.PollingInterval). + WithTimeout(constants.PollingTimeoutLong). + Should(Succeed()) +} + +// connectVCluster establishes a fresh connection to the vCluster using a background proxy. +// Returns a rest.Config and a cleanup function that removes the temp kubeconfig. +func connectVCluster(ctx context.Context, name, namespace string) (*rest.Config, func()) { + tmpFile, err := os.CreateTemp("", "vcluster-kubeconfig-*.yaml") + Expect(err).NotTo(HaveOccurred()) + tmpPath := tmpFile.Name() + tmpFile.Close() + + cleanupFn := func() { + os.Remove(tmpPath) + } + + options := &cli.ConnectOptions{ + BackgroundProxy: true, + BackgroundProxyImage: constants.GetVClusterImage(), + KubeConfig: tmpPath, + } + globalFlags := &flags.GlobalFlags{ + Namespace: namespace, + } + + err = cli.ConnectHelm(ctx, options, globalFlags, name, nil, loftlog.Discard) + Expect(err).NotTo(HaveOccurred(), "vcluster connect should succeed") + + var cfg *rest.Config + Eventually(func(g Gomega) { + data, err := os.ReadFile(tmpPath) + g.Expect(err).NotTo(HaveOccurred(), "should read kubeconfig file") + g.Expect(data).NotTo(BeEmpty(), "kubeconfig file should not be empty") + + cfg, err = clientcmd.RESTConfigFromKubeConfig(data) + g.Expect(err).NotTo(HaveOccurred(), "should parse kubeconfig") + }).WithPolling(constants.PollingInterval). + WithTimeout(constants.PollingTimeout). + Should(Succeed()) + + // Verify connection works + vClient, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + Eventually(func(g Gomega) { + _, err := vClient.CoreV1().ServiceAccounts(corev1.NamespaceDefault).Get(ctx, "default", metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred(), "should be able to reach vCluster API") + }).WithPolling(constants.PollingInterval). + WithTimeout(constants.PollingTimeout). + Should(Succeed()) + + return cfg, cleanupFn +} + +func parseCertFromPEM(pemData []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(pemData) + if block == nil { + return nil, fmt.Errorf("decoding to PEM block") + } + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("not a certificate") + } + return x509.ParseCertificate(block.Bytes) +} + +func certFingerprint(cert *x509.Certificate) string { + hash := sha256.Sum256(cert.Raw) + fingerprint := hex.EncodeToString(hash[:]) + var formatted strings.Builder + for i := 0; i < len(fingerprint); i += 2 { + if i > 0 { + formatted.WriteString(":") + } + formatted.WriteString(strings.ToUpper(fingerprint[i : i+2])) + } + return formatted.String() +} diff --git a/test/e2e_certs/certs/rotate.go b/test/e2e_certs/certs/rotate.go deleted file mode 100644 index a6a3bbf4d8..0000000000 --- a/test/e2e_certs/certs/rotate.go +++ /dev/null @@ -1,507 +0,0 @@ -package certs - -import ( - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "encoding/pem" - "errors" - "fmt" - "os" - "strings" - "time" - - certscmd "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/certs" - "github.com/loft-sh/vcluster/pkg/certs" - "github.com/loft-sh/vcluster/pkg/cli/flags" - "github.com/loft-sh/vcluster/test/framework" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -var _ = ginkgo.Describe("vCluster cert rotation tests", ginkgo.Ordered, func() { - var ( - f *framework.Framework - secret *corev1.Secret - err error - apiserverCertBefore *x509.Certificate - apiserverFingerprintBefore string - caCertBefore *x509.Certificate - caFingerprintBefore string - ) - - ginkgo.JustBeforeEach(func() { - f = framework.DefaultFramework - }) - - ginkgo.It("should obtain the current cert secret", func() { - secret, err = f.HostClient.CoreV1().Secrets(f.VClusterNamespace).Get(f.Context, certs.CertSecretName(f.VClusterName), metav1.GetOptions{}) - framework.ExpectNoError(err) - }) - - ginkgo.It("should get the fingerprints from the cert secret", func() { - apiserverCertBefore, err = parseCertFromPEM(secret.Data[certs.APIServerCertName]) - framework.ExpectNoError(err) - apiserverFingerprintBefore = certFingerprint(apiserverCertBefore) - - caCertBefore, err = parseCertFromPEM(secret.Data[certs.CACertName]) - framework.ExpectNoError(err) - caFingerprintBefore = certFingerprint(caCertBefore) - }) - - ginkgo.Context("vCluster \"certs rotate\"", ginkgo.Ordered, func() { - ginkgo.It("should execute \"certs rotate\" command", func() { - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"rotate", f.VClusterName}) - err = certsCmd.Execute() - framework.ExpectNoError(err) - }) - - ginkgo.It("should wait until the virtual cluster is ready again", func() { - framework.ExpectNoError(f.WaitForVClusterReady()) - gomega.Eventually(func(g gomega.Gomega) error { - pods, err := f.HostClient.CoreV1().Pods(f.VClusterNamespace).List(f.Context, metav1.ListOptions{ - LabelSelector: "app=vcluster,release=" + f.VClusterName, - }) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(pods.Items).NotTo(gomega.BeEmpty()) - - for _, pod := range pods.Items { - g.Expect(pod.Status.ContainerStatuses).NotTo(gomega.BeEmpty(), - "pod %s should have container statuses", pod.Name) - - for i, container := range pod.Status.ContainerStatuses { - g.Expect(container.State.Running).NotTo(gomega.BeNil(), - "container %d in pod %s should be running", i, pod.Name) - g.Expect(container.Ready).To(gomega.BeTrue(), - "container %d in pod %s should be ready", i, pod.Name) - } - } - - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("should obtain the certs secret with new certificates", func() { - gomega.Eventually(func() error { - secret, err = f.HostClient.CoreV1().Secrets(f.VClusterNamespace).Get(f.Context, certs.CertSecretName(f.VClusterName), metav1.GetOptions{}) - if err != nil { - return err - } - - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeout). - Should(gomega.Succeed()) - }) - - ginkgo.It("should check that the CA certificate fingerprint and its expiry time did not change", func() { - certAfter, err := parseCertFromPEM(secret.Data[certs.CACertName]) - framework.ExpectNoError(err) - - fingerprintAfter := certFingerprint(certAfter) - - // fingerprint should be equal. - gomega.Expect(caFingerprintBefore).To(gomega.Equal(fingerprintAfter)) - - // expiry date should be equal. - gomega.Expect(certAfter.NotAfter).To(gomega.Equal(caCertBefore.NotAfter)) - - // save fingerprint for next round - caFingerprintBefore = fingerprintAfter - }) - - ginkgo.It("should check that the apiservcer certificate fingerprint is different and that it expires later", func() { - apiserverCertAfter, err := parseCertFromPEM(secret.Data[certs.APIServerCertName]) - framework.ExpectNoError(err) - - fingerprintAfter := certFingerprint(apiserverCertAfter) - - // fingerprint should be different. - gomega.Expect(apiserverFingerprintBefore).ToNot(gomega.Equal(fingerprintAfter)) - - // new certificate should expire later than the old one. - gomega.Expect(apiserverCertAfter.NotAfter.After(apiserverCertBefore.NotAfter)).To(gomega.BeTrue()) - - // save fingerprint for next round - apiserverFingerprintBefore = fingerprintAfter - }) - }) - - ginkgo.Context("vCluster \"certs rotate-ca\"", ginkgo.Ordered, func() { - ginkgo.It("should execute \"certs rotate-ca\" command", func() { - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"rotate-ca", f.VClusterName}) - err = certsCmd.Execute() - framework.ExpectNoError(err) - }) - - ginkgo.It("should wait until the virtual cluster is ready again", func() { - framework.ExpectNoError(f.WaitForVClusterReady()) - gomega.Eventually(func(g gomega.Gomega) error { - pods, err := f.HostClient.CoreV1().Pods(f.VClusterNamespace).List(f.Context, metav1.ListOptions{ - LabelSelector: "app=vcluster,release=" + f.VClusterName, - }) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(pods.Items).NotTo(gomega.BeEmpty()) - - for _, pod := range pods.Items { - g.Expect(pod.Status.ContainerStatuses).NotTo(gomega.BeEmpty(), - "pod %s should have container statuses", pod.Name) - - for i, container := range pod.Status.ContainerStatuses { - g.Expect(container.State.Running).NotTo(gomega.BeNil(), - "container %d in pod %s should be running", i, pod.Name) - g.Expect(container.Ready).To(gomega.BeTrue(), - "container %d in pod %s should be ready", i, pod.Name) - } - } - - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("should obtain the current cert secret", func() { - secret, err = f.HostClient.CoreV1().Secrets(f.VClusterNamespace).Get(f.Context, certs.CertSecretName(f.VClusterName), metav1.GetOptions{}) - framework.ExpectNoError(err) - }) - - ginkgo.It("should check that the CA and apiserver certificate fingerprints are different and that they expire later", func() { - apiserverCertAfter, err := parseCertFromPEM(secret.Data[certs.APIServerCertName]) - framework.ExpectNoError(err) - caCertAfter, err := parseCertFromPEM(secret.Data[certs.CACertName]) - framework.ExpectNoError(err) - - apiserverFingerprintAfter := certFingerprint(apiserverCertAfter) - caFingerprintAfter := certFingerprint(caCertAfter) - - // fingerprints should be different. - gomega.Expect(apiserverFingerprintBefore).ToNot(gomega.Equal(apiserverFingerprintAfter)) - gomega.Expect(caFingerprintBefore).ToNot(gomega.Equal(caFingerprintAfter)) - - // new certificates should expire later than the old ones. - gomega.Expect(apiserverCertAfter.NotAfter.After(apiserverCertBefore.NotAfter)).To(gomega.BeTrue()) - gomega.Expect(caCertAfter.NotAfter.After(caCertBefore.NotAfter)).To(gomega.BeTrue()) - }) - }) - - ginkgo.AfterAll(func() { - framework.ExpectNoError(f.RefreshVirtualClient()) - }) -}) - -var _ = ginkgo.Describe("vCluster cert rotation expiration tests", ginkgo.Ordered, func() { - var ( - f *framework.Framework - ) - - ginkgo.JustBeforeEach(func() { - f = framework.DefaultFramework - }) - - ginkgo.It("checking current validity date of CA cert of vCluster", func() { - secret, err := f.HostClient.CoreV1().Secrets(f.VClusterNamespace).Get( - f.Context, certs.CertSecretName(f.VClusterName), metav1.GetOptions{}) - framework.ExpectNoError(err) - - certPEM := secret.Data["ca.crt"] - - block, _ := pem.Decode(certPEM) - gomega.Expect(block).NotTo(gomega.BeNil(), "Failed to decode PEM block") - - cert, err := x509.ParseCertificate(block.Bytes) - framework.ExpectNoError(err) - - gomega.Expect(cert.NotAfter.After(time.Now())).To(gomega.BeTrue(), "CA cert is valid") - - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"check", f.VClusterName}) - - err = certsCmd.Execute() - framework.ExpectNoError(err) - }) - - ginkgo.It("setting validity of ca cert of vCluster to 1 second", func() { - os.Setenv("DEVELOPMENT", "true") - os.Setenv("VCLUSTER_CERTS_VALIDITYPERIOD", "1s") - defer os.Unsetenv("DEVELOPMENT") - defer os.Unsetenv("VCLUSTER_CERTS_VALIDITYPERIOD") - - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"rotate-ca", f.VClusterName}) - - err := certsCmd.Execute() - framework.ExpectNoError(err) - }) - - ginkgo.It("checking the running status of vCluster", func() { - gomega.Eventually(func(g gomega.Gomega) error { - pods, err := f.HostClient.CoreV1().Pods(f.VClusterNamespace).List(f.Context, metav1.ListOptions{ - LabelSelector: "app=vcluster,release=" + f.VClusterName, - }) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(pods.Items).NotTo(gomega.BeEmpty()) - - for _, pod := range pods.Items { - g.Expect(pod.Status.Phase).To(gomega.Equal(corev1.PodRunning)) - } - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("should check if CA cert of vCluster is expired", func() { - gomega.Eventually(func(g gomega.Gomega) error { - secret, err := f.HostClient.CoreV1().Secrets(f.VClusterNamespace).Get( - f.Context, certs.CertSecretName(f.VClusterName), metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - certPEM := secret.Data["ca.crt"] - block, _ := pem.Decode(certPEM) - g.Expect(block).NotTo(gomega.BeNil()) - - cert, err := x509.ParseCertificate(block.Bytes) - g.Expect(err).NotTo(gomega.HaveOccurred()) - if cert.NotAfter.Before(time.Now()) { - return nil - } - return fmt.Errorf("CA cert not expired yet (expires at %s)", cert.NotAfter) - }). - WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("rotating expired CA cert of vCluster", func() { - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"rotate-ca", f.VClusterName}) - - err := certsCmd.Execute() - framework.ExpectNoError(err) - }) - - ginkgo.It("should wait until the vCluster is ready again", func() { - framework.ExpectNoError(f.WaitForVClusterReady()) - gomega.Eventually(func(g gomega.Gomega) error { - pods, err := f.HostClient.CoreV1().Pods(f.VClusterNamespace).List(f.Context, metav1.ListOptions{ - LabelSelector: "app=vcluster,release=" + f.VClusterName, - }) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(pods.Items).NotTo(gomega.BeEmpty()) - - for _, pod := range pods.Items { - g.Expect(pod.Status.ContainerStatuses).NotTo(gomega.BeEmpty(), - "pod %s should have container statuses", pod.Name) - - for i, container := range pod.Status.ContainerStatuses { - g.Expect(container.State.Running).NotTo(gomega.BeNil(), - "container %d in pod %s should be running", i, pod.Name) - g.Expect(container.Ready).To(gomega.BeTrue(), - "container %d in pod %s should be ready", i, pod.Name) - } - } - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("priniting new expiry date and time of vCluster CA cert", func() { - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"check", f.VClusterName}) - - err := certsCmd.Execute() - framework.ExpectNoError(err) - }) - - ginkgo.It("checking new validity date of CA cert of vCluster", func() { - secret, err := f.HostClient.CoreV1().Secrets(f.VClusterNamespace).Get( - f.Context, certs.CertSecretName(f.VClusterName), metav1.GetOptions{}) - framework.ExpectNoError(err) - - certPEM := secret.Data["ca.crt"] - block, _ := pem.Decode(certPEM) - gomega.Expect(block).NotTo(gomega.BeNil(), "Failed to decode PEM block") - - cert, err := x509.ParseCertificate(block.Bytes) - framework.ExpectNoError(err) - - gomega.Expect(cert.NotAfter.After(time.Now())).To(gomega.BeTrue(), "CA cert is valid") - }) - - ginkgo.AfterAll(func() { - framework.ExpectNoError(f.RefreshVirtualClient()) - }) -}) - -var _ = ginkgo.Describe("vCluster cert rotation kube config tests", ginkgo.Ordered, func() { - var ( - f *framework.Framework - restConfigBefore *rest.Config - ) - - ginkgo.JustBeforeEach(func() { - f = framework.DefaultFramework - }) - - ginkgo.It("should be able to use the virtual client", func() { - _, err := f.VClusterClient.CoreV1().Pods(corev1.NamespaceDefault).List(f.Context, metav1.ListOptions{}) - framework.ExpectNoError(err) - - restConfigBefore = f.VClusterConfig - vClusterClient, err := kubernetes.NewForConfig(restConfigBefore) - framework.ExpectNoError(err) - - _, err = vClusterClient.CoreV1().Pods(corev1.NamespaceDefault).List(f.Context, metav1.ListOptions{}) - framework.ExpectNoError(err) - }) - - ginkgo.Context("vCluster \"certs rotate\"", ginkgo.Ordered, func() { - ginkgo.It("should execute \"certs rotate\" command", func() { - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"rotate", f.VClusterName}) - framework.ExpectNoError(certsCmd.Execute()) - }) - - ginkgo.It("should wait until the virtual cluster is ready again", func() { - framework.ExpectNoError(f.WaitForVClusterReady()) - gomega.Eventually(func(g gomega.Gomega) error { - pods, err := f.HostClient.CoreV1().Pods(f.VClusterNamespace).List(f.Context, metav1.ListOptions{ - LabelSelector: "app=vcluster,release=" + f.VClusterName, - }) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(pods.Items).NotTo(gomega.BeEmpty()) - - for _, pod := range pods.Items { - g.Expect(pod.Status.ContainerStatuses).NotTo(gomega.BeEmpty(), - "pod %s should have container statuses", pod.Name) - - for i, container := range pod.Status.ContainerStatuses { - g.Expect(container.State.Running).NotTo(gomega.BeNil(), - "container %d in pod %s should be running", i, pod.Name) - g.Expect(container.Ready).To(gomega.BeTrue(), - "container %d in pod %s should be ready", i, pod.Name) - } - } - - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("should not receive a tls verification error using the old tls config for the virtual client", func() { - framework.ExpectNoError(f.RefreshVirtualClient()) - - cfg := f.VClusterConfig - cfg.TLSClientConfig = restConfigBefore.TLSClientConfig - - vClusterClient, err := kubernetes.NewForConfig(cfg) - framework.ExpectNoError(err) - - _, err = vClusterClient.CoreV1().Pods(corev1.NamespaceDefault).List(f.Context, metav1.ListOptions{}) - framework.ExpectNoError(err) - }) - }) - - ginkgo.Context("vCluster \"certs rotate-ca\"", ginkgo.Ordered, func() { - ginkgo.It("should execute \"certs rotate-ca\" command", func() { - certsCmd := certscmd.NewCertsCmd(&flags.GlobalFlags{Namespace: f.VClusterNamespace}) - certsCmd.SetArgs([]string{"rotate-ca", f.VClusterName}) - framework.ExpectNoError(certsCmd.Execute()) - }) - - ginkgo.It("should wait until the virtual cluster is ready again", func() { - framework.ExpectNoError(f.WaitForVClusterReady()) - gomega.Eventually(func(g gomega.Gomega) error { - pods, err := f.HostClient.CoreV1().Pods(f.VClusterNamespace).List(f.Context, metav1.ListOptions{ - LabelSelector: "app=vcluster,release=" + f.VClusterName, - }) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(pods.Items).NotTo(gomega.BeEmpty()) - - for _, pod := range pods.Items { - g.Expect(pod.Status.ContainerStatuses).NotTo(gomega.BeEmpty(), - "pod %s should have container statuses", pod.Name) - - for i, container := range pod.Status.ContainerStatuses { - g.Expect(container.State.Running).NotTo(gomega.BeNil(), - "container %d in pod %s should be running", i, pod.Name) - g.Expect(container.Ready).To(gomega.BeTrue(), - "container %d in pod %s should be ready", i, pod.Name) - } - } - - return nil - }).WithPolling(time.Second). - WithTimeout(framework.PollTimeoutLong). - Should(gomega.Succeed()) - }) - - ginkgo.It("should receive a tls verification error using the old tls config for the virtual client", func() { - framework.ExpectNoError(f.RefreshVirtualClient()) - - cfg := f.VClusterConfig - cfg.TLSClientConfig = restConfigBefore.TLSClientConfig - - vClusterClient, err := kubernetes.NewForConfig(cfg) - framework.ExpectNoError(err) - - _, err = vClusterClient.CoreV1().Pods(corev1.NamespaceDefault).List(f.Context, metav1.ListOptions{}) - framework.ExpectError(err) - - var certErr *tls.CertificateVerificationError - if !errors.As(err, &certErr) { - framework.Failf("received non-tls verification error: %v", err) - } - }) - }) - - ginkgo.AfterAll(func() { - framework.ExpectNoError(f.RefreshVirtualClient()) - }) -}) - -func parseCertFromPEM(pemData []byte) (*x509.Certificate, error) { - block, _ := pem.Decode(pemData) - if block == nil { - return nil, fmt.Errorf("decoding to PEM block") - } - - if block.Type != "CERTIFICATE" { - return nil, fmt.Errorf("not a certificate") - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("parsing certificate: %w", err) - } - - return cert, nil -} - -func certFingerprint(cert *x509.Certificate) string { - hash := sha256.Sum256(cert.Raw) - fingerprint := hex.EncodeToString(hash[:]) - - // Format as colon-separated hex pairs (like OpenSSL output) - var formatted strings.Builder - for i := 0; i < len(fingerprint); i += 2 { - if i > 0 { - formatted.WriteString(":") - } - formatted.WriteString(strings.ToUpper(fingerprint[i : i+2])) - } - - return formatted.String() -} diff --git a/test/e2e_certs/e2e_suite_test.go b/test/e2e_certs/e2e_suite_test.go deleted file mode 100644 index 325abb7cfc..0000000000 --- a/test/e2e_certs/e2e_suite_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package e2e - -import ( - "context" - "testing" - - "github.com/loft-sh/log" - "github.com/loft-sh/vcluster/test/framework" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - - // Enable cloud provider aut - // Enable cloud provider auth - _ "k8s.io/client-go/plugin/pkg/client/auth" - - // Register tests - _ "github.com/loft-sh/vcluster/test/e2e_certs/certs" -) - -// TestRunE2ETests checks configuration parameters (specified through flags) and then runs -// E2E tests using the Ginkgo runner. -// If a "report directory" is specified, one or more JUnit test reports will be -// generated in this directory, and cluster logs will also be saved. -// This function is called on each Ginkgo node in parallel mode. -func TestRunE2ETests(t *testing.T) { - gomega.RegisterFailHandler(ginkgo.Fail) - err := framework.CreateFramework(context.Background()) - if err != nil { - log.GetInstance().Fatalf("Error setting up framework: %v", err) - } - - var _ = ginkgo.AfterSuite(func() { - err = framework.DefaultFramework.Cleanup() - if err != nil { - log.GetInstance().Warnf("Error executing testsuite cleanup: %v", err) - } - }) - - ginkgo.RunSpecs(t, "VCluster e2ecerts suite") -} diff --git a/test/e2e_certs/values.yaml b/test/e2e_certs/values.yaml deleted file mode 100644 index fd860f5f64..0000000000 --- a/test/e2e_certs/values.yaml +++ /dev/null @@ -1 +0,0 @@ -# empty values file to avoid no file or dir error