feat(CLI): add a check requiring the gateway API (#15409)

When Linkerd is installed with the CLI, we first check to ensure the Gateway API CRDs are installed, and abort if the Gateway API is not present.  However, if Helm is used to the install Linkerd, no such check is performed.  This can lead to an invalid installation because HTTPRoute is a required CRD for Linkerd.

We add a check to `linkerd check` that validates that the Gateway API is installed.

But note the prior art in: #12917

In that PR we _removed_ a check for the Gateway API.  This is because the Gateway API consists of many different CRDs, most of which are optional for Linkerd; only HTTPRoute is required.  Therefore, we don't want to require that all Gateway API CRDs are installed.  For this new check, the HTTPRoute CRD alone is sufficient to make it pass.  This matches the behavior of the condition checked by the `linkerd install` command.

Signed-off-by: Alex Leong <alex@buoyant.io>

Author: adleong

cli/cmd/check.go

@@ -139,12 +139,13 @@ func configureAndRunChecks(cmd *cobra.Command, wout io.Writer, werr io.Writer, o
 	checks := []healthcheck.CategoryID{
 		healthcheck.KubernetesAPIChecks,
 		healthcheck.KubernetesVersionChecks,
+		healthcheck.GatewayAPICRDChecks,
 		healthcheck.LinkerdVersionChecks,
 	}
 
 	crdManifest := bytes.Buffer{}
 	err = renderCRDs(cmd.Context(), nil, &crdManifest, valuespkg.Options{
-		// GatewayAPI CRDs are optional so don't check for them.
+		// Gateway API CRDs are checked separately by GatewayAPICRDChecks.
 		Values: []string{
 			"installGatewayAPI=false",
 		},

cli/cmd/install.go

@@ -28,7 +28,6 @@ import (
 	valuespkg "helm.sh/helm/v3/pkg/cli/values"
 	"helm.sh/helm/v3/pkg/engine"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -206,51 +205,6 @@ func checkNoConfig(ctx context.Context, k8sAPI *k8s.KubernetesAPI) error {
 	return nil
 }
 
-type GatewayAPICRDs int
-
-const (
-	Absent GatewayAPICRDs = iota
-	Linkerd
-	External
-)
-
-// checkGatewayAPICRDs returns true if the Gateway API CRDs are installed in the
-// cluster, and false otherwise.
-func checkGatewayAPICRDs(ctx context.Context, k8sAPI *k8s.KubernetesAPI) (GatewayAPICRDs, error) {
-	crds := k8sAPI.Apiextensions.ApiextensionsV1().CustomResourceDefinitions()
-	result := Absent
-	names := []string{
-		"httproutes.gateway.networking.k8s.io",
-		"grpcroutes.gateway.networking.k8s.io",
-	}
-	for _, name := range names {
-		crd, err := crds.Get(ctx, name, metav1.GetOptions{})
-		if err == nil && crd != nil {
-			if crd.Annotations[k8s.CreatedByAnnotation] != "" {
-				return Linkerd, nil
-			}
-			result = External
-			if !crdIncludesV1(crd) {
-				return result, fmt.Errorf("the %s CRD is missing the v1 version, please upgrade to Gateway API v1.1.1 or later", name)
-			}
-		} else if kerrors.IsNotFound(err) {
-			// No action if CRD is not found.
-		} else {
-			return Absent, err
-		}
-	}
-	return result, nil
-}
-
-func crdIncludesV1(crd *v1.CustomResourceDefinition) bool {
-	for _, version := range crd.Spec.Versions {
-		if version.Name == "v1" {
-			return true
-		}
-	}
-	return false
-}
-
 func installCRDs(ctx context.Context, k8sAPI *k8s.KubernetesAPI, w io.Writer, options valuespkg.Options, format string) error {
 	if err := checkNoConfig(ctx, k8sAPI); err != nil {
 		return err
@@ -390,22 +344,22 @@ func renderChartToBuffer(files []*loader.BufferedFile, values map[string]interfa
 	return &buf, vals, nil
 }
 
-func updateDefaultValues(installed GatewayAPICRDs, defaultValues map[string]interface{}) map[string]interface{} {
-	if installed == Absent {
+func updateDefaultValues(installed healthcheck.GatewayAPICRDs, defaultValues map[string]interface{}) map[string]interface{} {
+	if installed == healthcheck.GatewayAPIAbsent {
 		// if GW API is not installed, default to false
 		defaultValues["installGatewayAPI"] = false
-	} else if installed == Linkerd {
+	} else if installed == healthcheck.GatewayAPILinkerd {
 		// if it is installed by Linkerd, default to true
 		defaultValues["installGatewayAPI"] = true
-	} else if installed == External {
+	} else if installed == healthcheck.GatewayAPIExternal {
 		// if it is external, default to false as we are not managing it
 		defaultValues["installGatewayAPI"] = false
 	}
 
 	return defaultValues
 }
 
-func validateFinalValues(installed GatewayAPICRDs, finalValues map[string]interface{}) error {
+func validateFinalValues(installed healthcheck.GatewayAPICRDs, finalValues map[string]interface{}) error {
 	installing := false
 
 	if installGatewayAPI, ok := finalValues["installGatewayAPI"]; ok {
@@ -416,21 +370,17 @@ func validateFinalValues(installed GatewayAPICRDs, finalValues map[string]interf
 		installing = enableHttpRoutes == true
 	}
 
-	if installed == Absent {
+	if installed == healthcheck.GatewayAPIAbsent {
 		if !installing {
 			// if we are not installing GW API Resources and they are not present, error
-			return errors.New(`The Gateway API CRDs must be installed prior to installing Linkerd. Run:
-
-kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml
-
-or see https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api for more options.`)
+			return healthcheck.GatewayAPICRDsMissingError()
 		}
-	} else if installed == Linkerd {
+	} else if installed == healthcheck.GatewayAPILinkerd {
 		if !installing {
 			// if they are installed and managed by Linkerd, we cannot uninstall them
 			return errors.New("Linkerd is providing GW API, but your current install configuration will remove it")
 		}
-	} else if installed == External {
+	} else if installed == healthcheck.GatewayAPIExternal {
 		if installing {
 			// if they are installed but are external, we cannot be installing as well
 			return errors.New("Linkerd cannot install the Gateway API CRDs because they are already installed by an external source. Please set `installGatewayAPI` to `false`.")
@@ -476,7 +426,7 @@ func renderCRDs(ctx context.Context, k *k8s.KubernetesAPI, w io.Writer, options
 	// If any of the Gateway API CRDs are installed, we default to rendering the
 	// Gateway API CRDs.
 	if k != nil {
-		installed, err := checkGatewayAPICRDs(ctx, k)
+		installed, err := healthcheck.CheckGatewayAPICRDs(ctx, k)
 		if err != nil {
 			return err
 		}

pkg/healthcheck/healthcheck.go

@@ -54,6 +54,11 @@ const (
 	// requirements.
 	KubernetesVersionChecks CategoryID = "kubernetes-version"
 
+	// GatewayAPICRDChecks validates that the Gateway API CRDs exist.
+	// These checks are dependent on the output of KubernetesAPIChecks, so those
+	// checks must be added first.
+	GatewayAPICRDChecks CategoryID = "gateway-api-crd"
+
 	// LinkerdPreInstall* checks enabled by `linkerd check --pre`
 
 	// LinkerdPreInstallChecks adds checks to validate that the control plane
@@ -191,6 +196,14 @@ var ExpectedServiceAccountNames = []string{
 	"linkerd-proxy-injector",
 }
 
+type GatewayAPICRDs int
+
+const (
+	GatewayAPIAbsent GatewayAPICRDs = iota
+	GatewayAPILinkerd
+	GatewayAPIExternal
+)
+
 var (
 	retryWindow = 5 * time.Second
 	// RequestTimeout is the time it takes for a request to timeout
@@ -551,6 +564,20 @@ func (hc *HealthChecker) allCategories() []*Category {
 			},
 			false,
 		),
+		NewCategory(
+			GatewayAPICRDChecks,
+			[]Checker{
+				{
+					description: "Gateway API CRDs are installed",
+					hintAnchor:  "gateway-api-crd",
+					fatal:       true,
+					check: func(ctx context.Context) error {
+						return CheckGatewayAPICRDsInstalled(ctx, hc.kubeAPI)
+					},
+				},
+			},
+			false,
+		),
 		NewCategory(
 			LinkerdPreInstallChecks,
 			[]Checker{
@@ -2201,6 +2228,54 @@ func CheckCustomResourceDefinitions(ctx context.Context, k8sAPI *k8s.KubernetesA
 	return nil
 }
 
+// CheckGatewayAPICRDs returns whether the Gateway API CRDs are installed in the
+// cluster, and whether they were created by Linkerd or an external source.
+func CheckGatewayAPICRDs(ctx context.Context, k8sAPI *k8s.KubernetesAPI) (GatewayAPICRDs, error) {
+	crds := k8sAPI.Apiextensions.ApiextensionsV1().CustomResourceDefinitions()
+	result := GatewayAPIAbsent
+	names := []string{
+		"httproutes.gateway.networking.k8s.io",
+		"grpcroutes.gateway.networking.k8s.io",
+	}
+	for _, name := range names {
+		crd, err := crds.Get(ctx, name, metav1.GetOptions{})
+		if err == nil && crd != nil {
+			if crd.Annotations[k8s.CreatedByAnnotation] != "" {
+				return GatewayAPILinkerd, nil
+			}
+			result = GatewayAPIExternal
+			if !crdIncludesV1(crd) {
+				return result, fmt.Errorf("the %s CRD is missing the v1 version, please upgrade to Gateway API v1.1.1 or later", name)
+			}
+		} else if kerrors.IsNotFound(err) {
+			// No action if CRD is not found.
+		} else {
+			return GatewayAPIAbsent, err
+		}
+	}
+	return result, nil
+}
+
+// CheckGatewayAPICRDsInstalled verifies that the Gateway API CRDs are installed.
+func CheckGatewayAPICRDsInstalled(ctx context.Context, k8sAPI *k8s.KubernetesAPI) error {
+	installed, err := CheckGatewayAPICRDs(ctx, k8sAPI)
+	if err != nil {
+		return err
+	}
+	if installed == GatewayAPIAbsent {
+		return GatewayAPICRDsMissingError()
+	}
+	return nil
+}
+
+func GatewayAPICRDsMissingError() error {
+	return errors.New(`The Gateway API CRDs must be installed prior to installing Linkerd. Run:
+
+kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml
+
+or see https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api for more options.`)
+}
+
 func crdHasVersion(crd *apiextv1.CustomResourceDefinition, version string) bool {
 	for _, crdVersion := range crd.Spec.Versions {
 		if crdVersion.Name == version {
@@ -2210,6 +2285,15 @@ func crdHasVersion(crd *apiextv1.CustomResourceDefinition, version string) bool
 	return false
 }
 
+func crdIncludesV1(crd *apiextv1.CustomResourceDefinition) bool {
+	for _, version := range crd.Spec.Versions {
+		if version.Name == "v1" {
+			return true
+		}
+	}
+	return false
+}
+
 // CheckNodesHaveNonDockerRuntime checks that each node has a non-Docker
 // runtime. This check is only called if proxyInit is not running as root
 // which is a problem for clusters with a Docker container runtime.

pkg/healthcheck/healthcheck_test.go

@@ -75,6 +75,89 @@ func (hc *HealthChecker) addCheckAsCategory(
 	hc.AppendCategories(testCategory)
 }
 
+func TestGatewayAPICRDChecks(t *testing.T) {
+	externalGatewayAPIManifest := `---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: httproutes.gateway.networking.k8s.io
+spec:
+  versions:
+    - name: v1
+`
+	unsupportedGatewayAPIManifest := `---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: httproutes.gateway.networking.k8s.io
+spec:
+  versions:
+    - name: v1alpha1
+`
+
+	testCases := []struct {
+		name        string
+		resources   []string
+		wantSuccess bool
+		wantErr     string
+	}{
+		{
+			name:        "passes when Gateway API CRDs are installed",
+			resources:   []string{externalGatewayAPIManifest},
+			wantSuccess: true,
+		},
+		{
+			name:        "fails when Gateway API CRDs are missing",
+			wantSuccess: false,
+			wantErr:     "The Gateway API CRDs must be installed prior to installing Linkerd",
+		},
+		{
+			name:        "fails when Gateway API CRDs do not include v1",
+			resources:   []string{unsupportedGatewayAPIManifest},
+			wantSuccess: false,
+			wantErr:     "missing the v1 version",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc // pin
+		t.Run(tc.name, func(t *testing.T) {
+			hc := NewHealthChecker(
+				[]CategoryID{GatewayAPICRDChecks},
+				&Options{},
+			)
+
+			var err error
+			hc.kubeAPI, err = k8s.NewFakeAPI(tc.resources...)
+			if err != nil {
+				t.Fatalf("Unexpected error: %s", err)
+			}
+
+			var gotErr error
+			success, _ := hc.RunChecks(func(result *CheckResult) {
+				gotErr = result.Err
+			})
+			if success != tc.wantSuccess {
+				t.Fatalf("expected success=%v, got %v", tc.wantSuccess, success)
+			}
+
+			if tc.wantErr == "" {
+				if gotErr != nil {
+					t.Fatalf("expected no error, got %s", gotErr)
+				}
+				return
+			}
+
+			if gotErr == nil {
+				t.Fatalf("expected error containing %q", tc.wantErr)
+			}
+			if !strings.Contains(gotErr.Error(), tc.wantErr) {
+				t.Fatalf("expected error containing %q, got %q", tc.wantErr, gotErr.Error())
+			}
+		})
+	}
+}
+
 func TestHealthChecker(t *testing.T) {
 	nullObserver := func(*CheckResult) {}
 

test/integration/deep/install_test.go

@@ -95,11 +95,25 @@ func TestInstallCNIPlugin(t *testing.T) {
 		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
 			"'kubectl apply' command failed\n%s", out)
 	}
+}
+
+// TestInstall will install the linkerd control plane to be used in the rest of
+// the deep suite tests.
+func TestInstall(t *testing.T) {
+	err := TestHelper.InstallGatewayAPI()
+	if err != nil {
+		testutil.AnnotatedFatal(t, "failed to install gateway-api", err)
+	}
+
+	// perform a linkerd check
+	checkArgs := []string{"check", "--pre", "--wait=5m"}
+	if TestHelper.CNI() {
+		checkArgs = []string{"check", "--pre", "--linkerd-cni-enabled", "--wait=5m"}
+	}
 
-	// perform a linkerd check with --linkerd-cni-enabled
 	timeout := time.Minute
 	err = testutil.RetryFor(timeout, func() error {
-		out, err = TestHelper.LinkerdRun("check", "--pre", "--linkerd-cni-enabled", "--wait=5m")
+		_, err = TestHelper.LinkerdRun(checkArgs...)
 		if err != nil {
 			return err
 		}
@@ -108,15 +122,6 @@ func TestInstallCNIPlugin(t *testing.T) {
 	if err != nil {
 		testutil.AnnotatedFatal(t, fmt.Sprintf("'linkerd check' command timed-out (%s)", timeout), err)
 	}
-}
-
-// TestInstall will install the linkerd control plane to be used in the rest of
-// the deep suite tests.
-func TestInstall(t *testing.T) {
-	err := TestHelper.InstallGatewayAPI()
-	if err != nil {
-		testutil.AnnotatedFatal(t, "failed to install gateway-api", err)
-	}
 
 	// Install CRDs
 	cmd := []string{