Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions deploy/base1ns-gotemplate/ns_dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,14 @@ spec:
limits.ephemeral-storage: '{{.EPHEMERAL_STORAGE_LIMIT}}'
requests.ephemeral-storage: '{{.EPHEMERAL_STORAGE_REQUEST}}'
requests.storage: '{{.STORAGE_REQUEST}}'
- apiVersion: v1
kind: ResourceQuota
metadata:
name: compute-spacerequests
namespace: '{{.SPACE_NAME}}-dev'
spec:
hard:
count/spacerequests.toolchain.dev.openshift.com: "1"
- apiVersion: v1
kind: LimitRange
metadata:
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/parallel/nstemplatetier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -301,9 +301,9 @@ func TestTierTemplates(t *testing.T) {
allTiers := &toolchainv1alpha1.TierTemplateList{}
err = hostAwait.Client.List(context.TODO(), allTiers, client.InNamespace(hostAwait.Namespace), notCreatedByE2e)
require.NoError(t, err)
// We have 19 tier templates (base: 3, base1ns: 2, base1nsnoidling: 2, base1ns6didler: 3, appstudio: 3, appstudiolarge: 3, appstudio-env: 3)
// We have 22 tier templates (base: 3, base1ns: 2, base1nsnoidling: 2, base1ns6didler: 3, appstudio: 3, appstudiolarge: 3, appstudio-env: 3, claw: 3)
// But we cannot verify the exact number of tiers, because during the operator update it may happen that more TierTemplates are created
assert.GreaterOrEqual(t, len(allTiers.Items), 19)
assert.GreaterOrEqual(t, len(allTiers.Items), 22)
}

func TestFeatureToggles(t *testing.T) {
Expand Down
72 changes: 72 additions & 0 deletions test/e2e/parallel/spacerequest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ import (
. "github.com/codeready-toolchain/toolchain-e2e/testsupport/space"
"github.com/codeready-toolchain/toolchain-e2e/testsupport/spaceprovisionerconfig"
"github.com/codeready-toolchain/toolchain-e2e/testsupport/wait"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -340,3 +342,73 @@ func TestUpdateSpaceRequest(t *testing.T) {
require.NoError(t, err)
})
}

func TestCreateClawSpaceRequest(t *testing.T) {
t.Parallel()
awaitilities := WaitForDeployments(t)
memberAwait := awaitilities.Member1()

// Create a user with base1ns tier (which includes SpaceRequest RBAC and quota)
user := NewSignupRequest(awaitilities).
ManuallyApprove().
RequireConditions(wait.ConditionSet(wait.Default(), wait.ApprovedByAdmin())...).
TargetCluster(memberAwait).
SpaceTier("base1ns").
EnsureMUR().
Execute(t)
parentSpace := user.Space

t.Run("provision claw sub-space via SpaceRequest", func(t *testing.T) {
// Create a SpaceRequest for the claw tier in the user's -dev namespace
spaceRequest := NewSpaceRequest(t,
WithSpecTierName("claw"),
WithNamespace(GetDefaultNamespace(parentSpace.Status.ProvisionedNamespaces)),
)
err := memberAwait.CreateWithCleanup(t, spaceRequest)
require.NoError(t, err)

// Wait for the sub-space to be created and provisioned
subSpace, err := awaitilities.Host().WaitForSubSpace(t, spaceRequest.Name, spaceRequest.Namespace, parentSpace.GetName(),
wait.UntilSpaceHasTier("claw"),
wait.UntilSpaceHasAnyProvisionedNamespaces(),
)
require.NoError(t, err)

// Verify all resources provisioned for the claw sub-space
// (namespace objects, cluster objects, and space roles)
subSpace, _ = VerifyResourcesProvisionedForSpace(t, awaitilities, subSpace.Name, wait.UntilSpaceHasAnyTargetClusterSet())

// Verify SpaceRequest status is provisioned
spaceRequest, err = memberAwait.WaitForSpaceRequest(t, types.NamespacedName{Namespace: spaceRequest.GetNamespace(), Name: spaceRequest.GetName()},
wait.UntilSpaceRequestHasConditions(wait.Provisioned()),
wait.UntilSpaceRequestHasNamespaceAccess(subSpace),
wait.UntilSpaceRequestHasNamespaceAccessWithoutSecretRef(),
)
require.NoError(t, err)

t.Run("second SpaceRequest is rejected by quota", func(t *testing.T) {
// The base1ns ns_dev.yaml limits count/spacerequests.toolchain.dev.openshift.com to 1.
// Creating a second SpaceRequest should be rejected by the ResourceQuota.
secondSR := NewSpaceRequest(t,
WithSpecTierName("claw"),
WithNamespace(GetDefaultNamespace(parentSpace.Status.ProvisionedNamespaces)),
)
err := memberAwait.Client.Create(context.TODO(), secondSR)
require.Error(t, err)
assert.True(t, errors.IsForbidden(err), "expected Forbidden error due to ResourceQuota, got: %v", err)
})

t.Run("delete SpaceRequest cleans up sub-space", func(t *testing.T) {
// Delete the SpaceRequest and verify the claw sub-space is removed
err := memberAwait.Client.Delete(context.TODO(), spaceRequest)
require.NoError(t, err)

err = memberAwait.WaitUntilNamespaceDeleted(t, subSpace.Name, "claw")
require.NoError(t, err)
err = memberAwait.WaitUntilNSTemplateSetDeleted(t, subSpace.Name)
require.NoError(t, err)
err = awaitilities.Host().WaitUntilSpaceAndSpaceBindingsDeleted(t, subSpace.Name)
require.NoError(t, err)
})
})
}
185 changes: 185 additions & 0 deletions testsupport/tiers/checks.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
base1ns = "base1ns"
base1ns6didler = "base1ns6didler"
base1nsnoidling = "base1nsnoidling"
claw = "claw"

// common CPU limits
baseCPULimit = "40000m"
Expand Down Expand Up @@ -62,6 +63,8 @@
return &appstudiolargeTierChecks{appstudioTierChecks{tierName: appstudiolarge}}, nil
case appstudioEnv:
return &appstudioEnvTierChecks{tierName: appstudioEnv}, nil
case claw:
return &clawTierChecks{tierName: claw}, nil
default:
return nil, fmt.Errorf("no assertion implementation found for %s", tier.Name)
}
Expand Down Expand Up @@ -113,6 +116,7 @@
checks := []namespaceObjectsCheck{
numberOfLimitRanges(1),
limitRange("1", "1Gi", "10m", "64Mi"),
resourceQuotaSpaceRequests(),
execPodsRole(),
crtadminPodsRoleBinding(),
crtadminViewRoleBinding(),
Expand Down Expand Up @@ -197,6 +201,7 @@
corev1.ResourceName("limits.nvidia.com/gpu"): "0",
}),
resourceQuotaStorage("15Gi", "80Gi", "15Gi", "10"),
resourceQuotaSpaceRequests(),
limitRange("1", "1000Mi", "10m", "64Mi"),
numberOfLimitRanges(1),
execPodsRole(),
Expand Down Expand Up @@ -525,6 +530,171 @@
idlers(0, "env"))
}

type clawTierChecks struct {
tierName string
}

func (a *clawTierChecks) GetNamespaceObjectChecks(_ string) []namespaceObjectsCheck {
checks := []namespaceObjectsCheck{
resourceQuotaComputeDeployNoScope("8", "10Gi", "1", "3Gi"),
resourceQuotaStorage("5Gi", "15Gi", "5Gi", "1"),
limitRange("500m", "512Mi", "10m", "64Mi"),
numberOfLimitRanges(1),
execPodsRole(),
crtadminPodsRoleBinding(),
crtadminViewRoleBinding(),
networkPolicySameNamespace(),
networkPolicyAllowFromIngress(),
networkPolicyAllowFromMonitoring(),
networkPolicyAllowFromOlmNamespaces(),
networkPolicyAllowFromConsoleNamespaces(),
numberOfNetworkPolicies(5),
}
return checks
}

func (a *clawTierChecks) GetSpaceRoleChecks(spaceRoles map[string][]string) ([]spaceRoleObjectsCheck, error) {
checks := []spaceRoleObjectsCheck{}
roles := 0
rolebindings := 0
for role, usernames := range spaceRoles {
switch role {
case "admin":
checks = append(checks, clawUserRole())
roles++
for _, userName := range usernames {
checks = append(checks,
clawUserRoleBinding(userName),
clawViewRoleBinding(userName),
)
rolebindings += 2
}
default:
return nil, fmt.Errorf("unexpected template name: '%s'", role)
}
}
checks = append(checks,
numberOfToolchainRoles(roles+1), // +1 for `exec-pods`
numberOfToolchainRoleBindings(rolebindings+2), // +2 for `crtadmin-pods` and `crtadmin-view`
)
return checks, nil
}

func (a *clawTierChecks) GetExpectedTemplateRefs(t *testing.T, hostAwait *wait.HostAwaitility) TemplateRefs {
templateRefs := GetTemplateRefs(t, hostAwait, a.tierName)
verifyNsTypes(t, a.tierName, templateRefs, "claw")
return templateRefs
}

func (a *clawTierChecks) GetClusterObjectChecks() []clusterObjectsCheck {
return clusterObjectsChecks(
clusterResourceQuotaClaw(),
numberOfClusterResourceQuotas(1),
idlers(43200, "claw"))
}

func clusterResourceQuotaClaw() clusterObjectsCheckCreator {
return func() clusterObjectsCheck {
return func(t *testing.T, memberAwait *wait.MemberAwaitility, userName, tierLabel string) {
var err error
hard := make(map[corev1.ResourceName]resource.Quantity)
hard[count("deployments.apps")], err = resource.ParseQuantity("5")
require.NoError(t, err)
hard[count(corev1.ResourcePods)], err = resource.ParseQuantity("10")
require.NoError(t, err)
hard[count("routes.route.openshift.io")], err = resource.ParseQuantity("3")
require.NoError(t, err)
hard[count(corev1.ResourceServices)], err = resource.ParseQuantity("5")
require.NoError(t, err)
Comment thread
alexeykazakov marked this conversation as resolved.
hard[count(corev1.ResourceSecrets)], err = resource.ParseQuantity("50")
require.NoError(t, err)
hard[count(corev1.ResourceConfigMaps)], err = resource.ParseQuantity("10")
require.NoError(t, err)

_, err = memberAwait.WaitForClusterResourceQuota(t, fmt.Sprintf("for-%s-claw", userName),
crqToolchainLabelsWaitCriterion(userName),
clusterResourceQuotaMatches(userName, tierLabel, hard),
)
require.NoError(t, err)
}
}
}

func resourceQuotaComputeDeployNoScope(cpuLimit, memoryLimit, cpuRequest, memoryRequest string) namespaceObjectsCheck {
return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, _ string) {
var err error
spec := corev1.ResourceQuotaSpec{
Hard: make(map[corev1.ResourceName]resource.Quantity),
}
spec.Hard[corev1.ResourceLimitsCPU], err = resource.ParseQuantity(cpuLimit)
require.NoError(t, err)
spec.Hard[corev1.ResourceLimitsMemory], err = resource.ParseQuantity(memoryLimit)
require.NoError(t, err)
spec.Hard[corev1.ResourceRequestsCPU], err = resource.ParseQuantity(cpuRequest)
require.NoError(t, err)
spec.Hard[corev1.ResourceRequestsMemory], err = resource.ParseQuantity(memoryRequest)
require.NoError(t, err)

criteria := resourceQuotaMatches(ns.Name, "compute-deploy", spec)

Check failure on line 638 in testsupport/tiers/checks.go

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Define a constant instead of duplicating this literal "compute-deploy" 4 times.

See more on https://sonarcloud.io/project/issues?id=codeready-toolchain_toolchain-e2e&issues=AZ5IY1iQIQnTQqJg8rkM&open=AZ5IY1iQIQnTQqJg8rkM&pullRequest=1282
_, err = memberAwait.WaitForResourceQuota(t, ns.Name, "compute-deploy", criteria)
require.NoError(t, err)
}
}

func clawUserRole() spaceRoleObjectsCheck {
return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, owner string) {
role, err := memberAwait.WaitForRole(t, ns, "claw-user", toolchainLabelsWaitCriterion(owner)...)
require.NoError(t, err)
expected := &rbacv1.Role{
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{"claw.sandbox.redhat.com"},
Resources: []string{"claws"},
Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"},
},
{
APIGroups: []string{""},
Resources: []string{"pods/exec"},

Check failure on line 657 in testsupport/tiers/checks.go

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Define a constant instead of duplicating this literal "pods/exec" 3 times.

See more on https://sonarcloud.io/project/issues?id=codeready-toolchain_toolchain-e2e&issues=AZ5L0_dXcdqGO32es5BF&open=AZ5L0_dXcdqGO32es5BF&pullRequest=1282
Verbs: []string{"get", "create"},
},
{
APIGroups: []string{""},
Resources: []string{"secrets"},
Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"},
},
},
}
assert.Len(t, role.Rules, len(expected.Rules))
assert.Equal(t, expected.Rules, role.Rules)
}
}

func clawUserRoleBinding(userName string) spaceRoleObjectsCheck {
return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, owner string) {
rb, err := memberAwait.WaitForRoleBinding(t, ns, userName+"-claw-user", toolchainLabelsWaitCriterion(owner)...)
require.NoError(t, err)
assert.Len(t, rb.Subjects, 1)
assert.Equal(t, "User", rb.Subjects[0].Kind)
assert.Equal(t, userName, rb.Subjects[0].Name)
assert.Equal(t, "claw-user", rb.RoleRef.Name)
assert.Equal(t, "Role", rb.RoleRef.Kind)
assert.Equal(t, "rbac.authorization.k8s.io", rb.RoleRef.APIGroup)
}
}

func clawViewRoleBinding(userName string) spaceRoleObjectsCheck {
return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, owner string) {
rb, err := memberAwait.WaitForRoleBinding(t, ns, userName+"-view", toolchainLabelsWaitCriterion(owner)...)
require.NoError(t, err)
assert.Len(t, rb.Subjects, 1)
assert.Equal(t, "User", rb.Subjects[0].Kind)
assert.Equal(t, userName, rb.Subjects[0].Name)
assert.Equal(t, "view", rb.RoleRef.Name)
assert.Equal(t, "ClusterRole", rb.RoleRef.Kind)
assert.Equal(t, "rbac.authorization.k8s.io", rb.RoleRef.APIGroup)
}
}

// verifyNsTypes checks that there's a namespace.TemplateRef that begins with `<tier>-<type>` for each given templateRef (and no more, no less)
func verifyNsTypes(t *testing.T, tier string, templateRefs TemplateRefs, expectedNSTypes ...string) {
require.Len(t, templateRefs.Namespaces, len(expectedNSTypes))
Expand Down Expand Up @@ -728,6 +898,21 @@
}
}

func resourceQuotaSpaceRequests() namespaceObjectsCheck {
return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, _ string) {
var err error
spec := corev1.ResourceQuotaSpec{
Hard: make(map[corev1.ResourceName]resource.Quantity),
}
spec.Hard["count/spacerequests.toolchain.dev.openshift.com"], err = resource.ParseQuantity("1")
require.NoError(t, err)

criteria := resourceQuotaMatches(ns.Name, "compute-spacerequests", spec)
_, err = memberAwait.WaitForResourceQuota(t, ns.Name, "compute-spacerequests", criteria)
require.NoError(t, err)
}
}

func resourceQuotaToolchainCrds(spaceRequestLimit string) namespaceObjectsCheck {
return func(t *testing.T, ns *corev1.Namespace, memberAwait *wait.MemberAwaitility, _ string) {
var err error
Expand Down
2 changes: 1 addition & 1 deletion testsupport/wait/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ import (
)

var (
BundledNSTemplateTiers []string = []string{"base1ns", "base1nsnoidling", "base1ns6didler", "base"}
BundledNSTemplateTiers []string = []string{"base1ns", "base1nsnoidling", "base1ns6didler", "base", "claw"}
CustomNSTemplateTiers []string = []string{"appstudio", "appstudiolarge", "appstudio-env"}
AllE2eNSTemplateTiers []string = append(BundledNSTemplateTiers, CustomNSTemplateTiers...)
)
Expand Down
Loading