diff --git a/deploy/base1ns-gotemplate/ns_dev.yaml b/deploy/base1ns-gotemplate/ns_dev.yaml index d28644496..cfcef3c3c 100644 --- a/deploy/base1ns-gotemplate/ns_dev.yaml +++ b/deploy/base1ns-gotemplate/ns_dev.yaml @@ -121,6 +121,14 @@ spec: limits.ephemeral-storage: '{{.EPHEMERAL_STORAGE_LIMIT}}' requests.ephemeral-storage: '{{.EPHEMERAL_STORAGE_REQUEST}}' requests.storage: '{{.STORAGE_REQUEST}}' + - apiVersion: v1 + kind: ResourceQuota + metadata: + name: compute-spacerequests + namespace: '{{.SPACE_NAME}}-dev' + spec: + hard: + count/spacerequests.toolchain.dev.openshift.com: "1" - apiVersion: v1 kind: LimitRange metadata: diff --git a/test/e2e/parallel/nstemplatetier_test.go b/test/e2e/parallel/nstemplatetier_test.go index 2a5e16d08..dc82763b5 100644 --- a/test/e2e/parallel/nstemplatetier_test.go +++ b/test/e2e/parallel/nstemplatetier_test.go @@ -301,9 +301,9 @@ func TestTierTemplates(t *testing.T) { allTiers := &toolchainv1alpha1.TierTemplateList{} err = hostAwait.Client.List(context.TODO(), allTiers, client.InNamespace(hostAwait.Namespace), notCreatedByE2e) require.NoError(t, err) - // We have 19 tier templates (base: 3, base1ns: 2, base1nsnoidling: 2, base1ns6didler: 3, appstudio: 3, appstudiolarge: 3, appstudio-env: 3) + // We have 22 tier templates (base: 3, base1ns: 2, base1nsnoidling: 2, base1ns6didler: 3, appstudio: 3, appstudiolarge: 3, appstudio-env: 3, claw: 3) // But we cannot verify the exact number of tiers, because during the operator update it may happen that more TierTemplates are created - assert.GreaterOrEqual(t, len(allTiers.Items), 19) + assert.GreaterOrEqual(t, len(allTiers.Items), 22) } func TestFeatureToggles(t *testing.T) { diff --git a/test/e2e/parallel/spacerequest_test.go b/test/e2e/parallel/spacerequest_test.go index 9d84636e1..4afcae449 100644 --- a/test/e2e/parallel/spacerequest_test.go +++ b/test/e2e/parallel/spacerequest_test.go @@ -11,7 +11,9 @@ import ( . "github.com/codeready-toolchain/toolchain-e2e/testsupport/space" "github.com/codeready-toolchain/toolchain-e2e/testsupport/spaceprovisionerconfig" "github.com/codeready-toolchain/toolchain-e2e/testsupport/wait" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -340,3 +342,73 @@ func TestUpdateSpaceRequest(t *testing.T) { require.NoError(t, err) }) } + +func TestCreateClawSpaceRequest(t *testing.T) { + t.Parallel() + awaitilities := WaitForDeployments(t) + memberAwait := awaitilities.Member1() + + // Create a user with base1ns tier (which includes SpaceRequest RBAC and quota) + user := NewSignupRequest(awaitilities). + ManuallyApprove(). + RequireConditions(wait.ConditionSet(wait.Default(), wait.ApprovedByAdmin())...). + TargetCluster(memberAwait). + SpaceTier("base1ns"). + EnsureMUR(). + Execute(t) + parentSpace := user.Space + + t.Run("provision claw sub-space via SpaceRequest", func(t *testing.T) { + // Create a SpaceRequest for the claw tier in the user's -dev namespace + spaceRequest := NewSpaceRequest(t, + WithSpecTierName("claw"), + WithNamespace(GetDefaultNamespace(parentSpace.Status.ProvisionedNamespaces)), + ) + err := memberAwait.CreateWithCleanup(t, spaceRequest) + require.NoError(t, err) + + // Wait for the sub-space to be created and provisioned + subSpace, err := awaitilities.Host().WaitForSubSpace(t, spaceRequest.Name, spaceRequest.Namespace, parentSpace.GetName(), + wait.UntilSpaceHasTier("claw"), + wait.UntilSpaceHasAnyProvisionedNamespaces(), + ) + require.NoError(t, err) + + // Verify all resources provisioned for the claw sub-space + // (namespace objects, cluster objects, and space roles) + subSpace, _ = VerifyResourcesProvisionedForSpace(t, awaitilities, subSpace.Name, wait.UntilSpaceHasAnyTargetClusterSet()) + + // Verify SpaceRequest status is provisioned + spaceRequest, err = memberAwait.WaitForSpaceRequest(t, types.NamespacedName{Namespace: spaceRequest.GetNamespace(), Name: spaceRequest.GetName()}, + wait.UntilSpaceRequestHasConditions(wait.Provisioned()), + wait.UntilSpaceRequestHasNamespaceAccess(subSpace), + wait.UntilSpaceRequestHasNamespaceAccessWithoutSecretRef(), + ) + require.NoError(t, err) + + t.Run("second SpaceRequest is rejected by quota", func(t *testing.T) { + // The base1ns ns_dev.yaml limits count/spacerequests.toolchain.dev.openshift.com to 1. + // Creating a second SpaceRequest should be rejected by the ResourceQuota. + secondSR := NewSpaceRequest(t, + WithSpecTierName("claw"), + WithNamespace(GetDefaultNamespace(parentSpace.Status.ProvisionedNamespaces)), + ) + err := memberAwait.Client.Create(context.TODO(), secondSR) + require.Error(t, err) + assert.True(t, errors.IsForbidden(err), "expected Forbidden error due to ResourceQuota, got: %v", err) + }) + + t.Run("delete SpaceRequest cleans up sub-space", func(t *testing.T) { + // Delete the SpaceRequest and verify the claw sub-space is removed + err := memberAwait.Client.Delete(context.TODO(), spaceRequest) + require.NoError(t, err) + + err = memberAwait.WaitUntilNamespaceDeleted(t, subSpace.Name, "claw") + require.NoError(t, err) + err = memberAwait.WaitUntilNSTemplateSetDeleted(t, subSpace.Name) + require.NoError(t, err) + err = awaitilities.Host().WaitUntilSpaceAndSpaceBindingsDeleted(t, subSpace.Name) + require.NoError(t, err) + }) + }) +} diff --git a/testsupport/tiers/checks.go b/testsupport/tiers/checks.go index 3b5900251..cb74eef12 100644 --- a/testsupport/tiers/checks.go +++ b/testsupport/tiers/checks.go @@ -32,6 +32,7 @@ const ( base1ns = "base1ns" base1ns6didler = "base1ns6didler" base1nsnoidling = "base1nsnoidling" + claw = "claw" // common CPU limits baseCPULimit = "40000m" @@ -62,6 +63,8 @@ func NewChecksForTier(tier *toolchainv1alpha1.NSTemplateTier) (TierChecks, error return &appstudiolargeTierChecks{appstudioTierChecks{tierName: appstudiolarge}}, nil case appstudioEnv: return &appstudioEnvTierChecks{tierName: appstudioEnv}, nil + case claw: + return &clawTierChecks{tierName: claw}, nil default: return nil, fmt.Errorf("no assertion implementation found for %s", tier.Name) } @@ -113,6 +116,7 @@ func (a *baseTierChecks) GetNamespaceObjectChecks(nsType string) []namespaceObje checks := []namespaceObjectsCheck{ numberOfLimitRanges(1), limitRange("1", "1Gi", "10m", "64Mi"), + resourceQuotaSpaceRequests(), execPodsRole(), crtadminPodsRoleBinding(), crtadminViewRoleBinding(), @@ -197,6 +201,7 @@ func (a *base1nsTierChecks) GetNamespaceObjectChecks(_ string) []namespaceObject corev1.ResourceName("limits.nvidia.com/gpu"): "0", }), resourceQuotaStorage("15Gi", "80Gi", "15Gi", "10"), + resourceQuotaSpaceRequests(), limitRange("1", "1000Mi", "10m", "64Mi"), numberOfLimitRanges(1), execPodsRole(), @@ -525,6 +530,171 @@ func (a *appstudioEnvTierChecks) GetClusterObjectChecks() []clusterObjectsCheck idlers(0, "env")) } +type clawTierChecks struct { + tierName string +} + +func (a *clawTierChecks) GetNamespaceObjectChecks(_ string) []namespaceObjectsCheck { + checks := []namespaceObjectsCheck{ + resourceQuotaComputeDeployNoScope("8", "10Gi", "1", "3Gi"), + resourceQuotaStorage("5Gi", "15Gi", "5Gi", "1"), + limitRange("500m", "512Mi", "10m", "64Mi"), + numberOfLimitRanges(1), + execPodsRole(), + crtadminPodsRoleBinding(), + crtadminViewRoleBinding(), + networkPolicySameNamespace(), + networkPolicyAllowFromIngress(), + networkPolicyAllowFromMonitoring(), + networkPolicyAllowFromOlmNamespaces(), + networkPolicyAllowFromConsoleNamespaces(), + numberOfNetworkPolicies(5), + } + return checks +} + +func (a *clawTierChecks) GetSpaceRoleChecks(spaceRoles map[string][]string) ([]spaceRoleObjectsCheck, error) { + checks := []spaceRoleObjectsCheck{} + roles := 0 + rolebindings := 0 + for role, usernames := range spaceRoles { + switch role { + case "admin": + checks = append(checks, clawUserRole()) + roles++ + for _, userName := range usernames { + checks = append(checks, + clawUserRoleBinding(userName), + clawViewRoleBinding(userName), + ) + rolebindings += 2 + } + default: + return nil, fmt.Errorf("unexpected template name: '%s'", role) + } + } + checks = append(checks, + numberOfToolchainRoles(roles+1), // +1 for `exec-pods` + numberOfToolchainRoleBindings(rolebindings+2), // +2 for `crtadmin-pods` and `crtadmin-view` + ) + return checks, nil +} + +func (a *clawTierChecks) GetExpectedTemplateRefs(t *testing.T, hostAwait *wait.HostAwaitility) TemplateRefs { + templateRefs := GetTemplateRefs(t, hostAwait, a.tierName) + verifyNsTypes(t, a.tierName, templateRefs, "claw") + return templateRefs +} + +func (a *clawTierChecks) GetClusterObjectChecks() []clusterObjectsCheck { + return clusterObjectsChecks( + clusterResourceQuotaClaw(), + numberOfClusterResourceQuotas(1), + idlers(43200, "claw")) +} + +func clusterResourceQuotaClaw() clusterObjectsCheckCreator { + return func() clusterObjectsCheck { + return func(t *testing.T, memberAwait *wait.MemberAwaitility, userName, tierLabel string) { + var err error + hard := make(map[corev1.ResourceName]resource.Quantity) + hard[count("deployments.apps")], err = resource.ParseQuantity("5") + require.NoError(t, err) + hard[count(corev1.ResourcePods)], err = resource.ParseQuantity("10") + require.NoError(t, err) + hard[count("routes.route.openshift.io")], err = resource.ParseQuantity("3") + require.NoError(t, err) + hard[count(corev1.ResourceServices)], err = resource.ParseQuantity("5") + require.NoError(t, err) + hard[count(corev1.ResourceSecrets)], err = resource.ParseQuantity("50") + require.NoError(t, err) + hard[count(corev1.ResourceConfigMaps)], err = resource.ParseQuantity("10") + require.NoError(t, err) + + _, err = memberAwait.WaitForClusterResourceQuota(t, fmt.Sprintf("for-%s-claw", userName), + crqToolchainLabelsWaitCriterion(userName), + clusterResourceQuotaMatches(userName, tierLabel, hard), + ) + require.NoError(t, err) + } + } +} + +func resourceQuotaComputeDeployNoScope(cpuLimit, memoryLimit, cpuRequest, memoryRequest string) namespaceObjectsCheck { + return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, _ string) { + var err error + spec := corev1.ResourceQuotaSpec{ + Hard: make(map[corev1.ResourceName]resource.Quantity), + } + spec.Hard[corev1.ResourceLimitsCPU], err = resource.ParseQuantity(cpuLimit) + require.NoError(t, err) + spec.Hard[corev1.ResourceLimitsMemory], err = resource.ParseQuantity(memoryLimit) + require.NoError(t, err) + spec.Hard[corev1.ResourceRequestsCPU], err = resource.ParseQuantity(cpuRequest) + require.NoError(t, err) + spec.Hard[corev1.ResourceRequestsMemory], err = resource.ParseQuantity(memoryRequest) + require.NoError(t, err) + + criteria := resourceQuotaMatches(ns.Name, "compute-deploy", spec) + _, err = memberAwait.WaitForResourceQuota(t, ns.Name, "compute-deploy", criteria) + require.NoError(t, err) + } +} + +func clawUserRole() spaceRoleObjectsCheck { + return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, owner string) { + role, err := memberAwait.WaitForRole(t, ns, "claw-user", toolchainLabelsWaitCriterion(owner)...) + require.NoError(t, err) + expected := &rbacv1.Role{ + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"claw.sandbox.redhat.com"}, + Resources: []string{"claws"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods/exec"}, + Verbs: []string{"get", "create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + }, + } + assert.Len(t, role.Rules, len(expected.Rules)) + assert.Equal(t, expected.Rules, role.Rules) + } +} + +func clawUserRoleBinding(userName string) spaceRoleObjectsCheck { + return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, owner string) { + rb, err := memberAwait.WaitForRoleBinding(t, ns, userName+"-claw-user", toolchainLabelsWaitCriterion(owner)...) + require.NoError(t, err) + assert.Len(t, rb.Subjects, 1) + assert.Equal(t, "User", rb.Subjects[0].Kind) + assert.Equal(t, userName, rb.Subjects[0].Name) + assert.Equal(t, "claw-user", rb.RoleRef.Name) + assert.Equal(t, "Role", rb.RoleRef.Kind) + assert.Equal(t, "rbac.authorization.k8s.io", rb.RoleRef.APIGroup) + } +} + +func clawViewRoleBinding(userName string) spaceRoleObjectsCheck { + return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, owner string) { + rb, err := memberAwait.WaitForRoleBinding(t, ns, userName+"-view", toolchainLabelsWaitCriterion(owner)...) + require.NoError(t, err) + assert.Len(t, rb.Subjects, 1) + assert.Equal(t, "User", rb.Subjects[0].Kind) + assert.Equal(t, userName, rb.Subjects[0].Name) + assert.Equal(t, "view", rb.RoleRef.Name) + assert.Equal(t, "ClusterRole", rb.RoleRef.Kind) + assert.Equal(t, "rbac.authorization.k8s.io", rb.RoleRef.APIGroup) + } +} + // verifyNsTypes checks that there's a namespace.TemplateRef that begins with `-` for each given templateRef (and no more, no less) func verifyNsTypes(t *testing.T, tier string, templateRefs TemplateRefs, expectedNSTypes ...string) { require.Len(t, templateRefs.Namespaces, len(expectedNSTypes)) @@ -728,6 +898,21 @@ func resourceQuotaStorage(ephemeralLimit, storageRequest, ephemeralRequest, pvcs } } +func resourceQuotaSpaceRequests() namespaceObjectsCheck { + return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, _ string) { + var err error + spec := corev1.ResourceQuotaSpec{ + Hard: make(map[corev1.ResourceName]resource.Quantity), + } + spec.Hard["count/spacerequests.toolchain.dev.openshift.com"], err = resource.ParseQuantity("1") + require.NoError(t, err) + + criteria := resourceQuotaMatches(ns.Name, "compute-spacerequests", spec) + _, err = memberAwait.WaitForResourceQuota(t, ns.Name, "compute-spacerequests", criteria) + require.NoError(t, err) + } +} + func resourceQuotaToolchainCrds(spaceRequestLimit string) namespaceObjectsCheck { return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, _ string) { var err error diff --git a/testsupport/wait/host.go b/testsupport/wait/host.go index 6d51bcf6e..b3bece872 100644 --- a/testsupport/wait/host.go +++ b/testsupport/wait/host.go @@ -50,7 +50,7 @@ import ( ) var ( - BundledNSTemplateTiers []string = []string{"base1ns", "base1nsnoidling", "base1ns6didler", "base"} + BundledNSTemplateTiers []string = []string{"base1ns", "base1nsnoidling", "base1ns6didler", "base", "claw"} CustomNSTemplateTiers []string = []string{"appstudio", "appstudiolarge", "appstudio-env"} AllE2eNSTemplateTiers []string = append(BundledNSTemplateTiers, CustomNSTemplateTiers...) )