diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index eed314ad..ef31ad2e 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -32,7 +32,7 @@ jobs: - name: "setup go" uses: actions/setup-go@v4 with: - go-version: '^1.18.1' + go-version: '1.20' - name: "shardingsphere operator test" run: | cd shardingsphere-operator diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 955e5fe2..5b65bead 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -33,7 +33,7 @@ jobs: - name: "setup go" uses: actions/setup-go@v4 with: - go-version: '^1.18.1' + go-version: '1.20' - name: "shardingsphere operator test" run: | cd shardingsphere-operator diff --git a/examples/operator/shardingsphere-storageprovider-aws-aurora-cluster.yaml b/examples/operator/shardingsphere-storageprovider-aws-aurora-cluster.yaml index 903a8fcb..c3a54ef7 100644 --- a/examples/operator/shardingsphere-storageprovider-aws-aurora-cluster.yaml +++ b/examples/operator/shardingsphere-storageprovider-aws-aurora-cluster.yaml @@ -20,6 +20,8 @@ apiVersion: shardingsphere.apache.org/v1alpha1 kind: StorageProvider metadata: name: aws-aurora-cluster-mysql-5.7 + annotations: + storageproviders.shardingsphere.apache.org/allowed-namespaces: default spec: provisioner: storageproviders.shardingsphere.apache.org/aws-aurora reclaimPolicy: Delete diff --git a/examples/operator/shardingsphere-storageprovider-aws-rds-cluster.yaml b/examples/operator/shardingsphere-storageprovider-aws-rds-cluster.yaml index 2eb0e849..2ec8ff2e 100644 --- a/examples/operator/shardingsphere-storageprovider-aws-rds-cluster.yaml +++ b/examples/operator/shardingsphere-storageprovider-aws-rds-cluster.yaml @@ -19,6 +19,8 @@ apiVersion: shardingsphere.apache.org/v1alpha1 kind: StorageProvider metadata: name: aws-rds-cluster-mysql-8.0.32 + annotations: + storageproviders.shardingsphere.apache.org/allowed-namespaces: default spec: provisioner: storageproviders.shardingsphere.apache.org/aws-rds-cluster reclaimPolicy: Delete diff --git a/examples/operator/shardingsphere-storageprovider-aws-rds-instance.yaml b/examples/operator/shardingsphere-storageprovider-aws-rds-instance.yaml index 135d35e7..4ae4297e 100644 --- a/examples/operator/shardingsphere-storageprovider-aws-rds-instance.yaml +++ b/examples/operator/shardingsphere-storageprovider-aws-rds-instance.yaml @@ -20,6 +20,8 @@ apiVersion: shardingsphere.apache.org/v1alpha1 kind: StorageProvider metadata: name: aws-rds-instance-mysql-5.7 + annotations: + storageproviders.shardingsphere.apache.org/allowed-namespaces: default spec: provisioner: storageproviders.shardingsphere.apache.org/aws-rds-instance reclaimPolicy: Delete diff --git a/shardingsphere-operator/.cache/go-build/README b/shardingsphere-operator/.cache/go-build/README new file mode 100644 index 00000000..a59d0c92 --- /dev/null +++ b/shardingsphere-operator/.cache/go-build/README @@ -0,0 +1,4 @@ +This directory holds cached build artifacts from the Go build system. +Run "go clean -cache" if the directory is getting too large. +Run "go clean -fuzzcache" to delete the fuzz cache. +See golang.org to learn more about Go. diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/bou.ke/monkey/@v/v1.0.2.lock b/shardingsphere-operator/.cache/go-mod/cache/download/bou.ke/monkey/@v/v1.0.2.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/!d!a!t!a-!d!o!g/go-sqlmock/@v/v1.5.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/!d!a!t!a-!d!o!g/go-sqlmock/@v/v1.5.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/antlr/antlr4/@v/v0.0.0-20181218183524-be58ebffde8e.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/antlr/antlr4/@v/v0.0.0-20181218183524-be58ebffde8e.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/chaos-mesh/chaos-mesh/api/@v/v0.0.0-20230517110555-afab5b4a7813.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/chaos-mesh/chaos-mesh/api/@v/v0.0.0-20230517110555-afab5b4a7813.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/cloudnative-pg/cloudnative-pg/@v/v1.20.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/cloudnative-pg/cloudnative-pg/@v/v1.20.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/database-mesh/golang-sdk/@v/v0.0.0-20230608051131-717115b848ac.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/database-mesh/golang-sdk/@v/v0.0.0-20230608051131-717115b848ac.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/go-logr/logr/@v/v1.2.4.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/go-logr/logr/@v/v1.2.4.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/go-sql-driver/mysql/@v/v1.7.1.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/go-sql-driver/mysql/@v/v1.7.1.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/golang/mock/@v/v1.6.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/golang/mock/@v/v1.6.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/onsi/ginkgo/v2/@v/v2.9.2.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/onsi/ginkgo/v2/@v/v2.9.2.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/onsi/gomega/@v/v1.27.6.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/onsi/gomega/@v/v1.27.6.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/prometheus/client_golang/@v/v1.14.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/prometheus/client_golang/@v/v1.14.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/github.com/stretchr/testify/@v/v1.8.1.lock b/shardingsphere-operator/.cache/go-mod/cache/download/github.com/stretchr/testify/@v/v1.8.1.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/go.uber.org/zap/@v/v1.24.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/go.uber.org/zap/@v/v1.24.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/golang.org/x/mod/@v/v0.9.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/golang.org/x/mod/@v/v0.9.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/gopkg.in/yaml.v2/@v/v2.4.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/gopkg.in/yaml.v2/@v/v2.4.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/api/@v/v0.26.4.lock b/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/api/@v/v0.26.4.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/apimachinery/@v/v0.26.4.lock b/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/apimachinery/@v/v0.26.4.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/autoscaler/vertical-pod-autoscaler/@v/v0.14.0.lock b/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/autoscaler/vertical-pod-autoscaler/@v/v0.14.0.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/client-go/@v/v0.26.3.lock b/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/client-go/@v/v0.26.3.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/utils/@v/v0.0.0-20230505201702-9f6742963106.lock b/shardingsphere-operator/.cache/go-mod/cache/download/k8s.io/utils/@v/v0.0.0-20230505201702-9f6742963106.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/.cache/go-mod/cache/download/sigs.k8s.io/controller-runtime/@v/v0.14.6.lock b/shardingsphere-operator/.cache/go-mod/cache/download/sigs.k8s.io/controller-runtime/@v/v0.14.6.lock new file mode 100644 index 00000000..e69de29b diff --git a/shardingsphere-operator/Makefile b/shardingsphere-operator/Makefile index ce6835da..42ff7abc 100644 --- a/shardingsphere-operator/Makefile +++ b/shardingsphere-operator/Makefile @@ -121,6 +121,7 @@ CHECK_MOCKGEN ?= $(LOCALBIN)/mockgen ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.7 CONTROLLER_TOOLS_VERSION ?= v0.9.0 +ENVTEST_SETUP_VERSION ?= release-0.14 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize @@ -136,7 +137,7 @@ $(CONTROLLER_GEN): $(LOCALBIN) .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_SETUP_VERSION) .PHONY: lint lint: check-lint diff --git a/shardingsphere-operator/api/v1alpha1/storageprovider_types.go b/shardingsphere-operator/api/v1alpha1/storageprovider_types.go index 02cb15fd..9bfa70a1 100644 --- a/shardingsphere-operator/api/v1alpha1/storageprovider_types.go +++ b/shardingsphere-operator/api/v1alpha1/storageprovider_types.go @@ -35,6 +35,9 @@ const ( AnnotationsMasterUsername = "storageproviders.shardingsphere.apache.org/master-username" AnnotationsMasterUserPassword = "storageproviders.shardingsphere.apache.org/master-user-password" AnnotationsFinalSnapshotIdentifier = "storageproviders.shardingsphere.apache.org/final-snapshot-identifier" + // AnnotationsAllowedNamespaces declares which namespaces are allowed to reference this cluster-scoped StorageProvider. + // Value is a comma-separated list, e.g. "default,tenant-a", or "*" to allow all namespaces. + AnnotationsAllowedNamespaces = "storageproviders.shardingsphere.apache.org/allowed-namespaces" ProvisionerAWSRDSInstance = "storageproviders.shardingsphere.apache.org/aws-rds-instance" ProvisionerAWSRDSCluster = "storageproviders.shardingsphere.apache.org/aws-rds-cluster" diff --git a/shardingsphere-operator/pkg/controllers/storage_ndoe_controller_test.go b/shardingsphere-operator/pkg/controllers/storage_ndoe_controller_test.go index ce2e4e22..ccc375fb 100644 --- a/shardingsphere-operator/pkg/controllers/storage_ndoe_controller_test.go +++ b/shardingsphere-operator/pkg/controllers/storage_ndoe_controller_test.go @@ -98,7 +98,11 @@ var _ = Describe("StorageNode Controller Mock Test For AWS Rds Instance", func() // create default resource dbClass := &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultTestStorageProvider, + Name: defaultTestStorageProvider, + Namespace: defaultTestNamespace, + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: defaultTestNamespace, + }, }, Spec: v1alpha1.StorageProviderSpec{ Provisioner: v1alpha1.ProvisionerAWSRDSInstance, @@ -131,7 +135,8 @@ var _ = Describe("StorageNode Controller Mock Test For AWS Rds Instance", func() })).Should(Succeed()) Expect(fakeClient.Delete(ctx, &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultTestStorageProvider, + Name: defaultTestStorageProvider, + Namespace: defaultTestNamespace, }, })).Should(Succeed()) @@ -640,7 +645,11 @@ var _ = Describe("StorageNode Controller Mock Test For AWS Rds Instance", func() storageProvider := &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultTestStorageProvider, + Name: defaultTestStorageProvider, + Namespace: defaultTestNamespace, + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: defaultTestNamespace, + }, }, Spec: v1alpha1.StorageProviderSpec{ Provisioner: v1alpha1.ProvisionerAWSRDSInstance, @@ -741,7 +750,11 @@ var _ = Describe("StorageNode Controller Mock Test For AWS Aurora", func() { BeforeEach(func() { provider = &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ - Name: "aws-aurora", + Name: "aws-aurora", + Namespace: defaultTestNamespace, + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: defaultTestNamespace, + }, }, Spec: v1alpha1.StorageProviderSpec{ Provisioner: v1alpha1.ProvisionerAWSAurora, @@ -1191,7 +1204,11 @@ var _ = Describe("StorageNode Controller Mock Test For AWS RDS Cluster", func() BeforeEach(func() { provider = &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ - Name: providerName, + Name: providerName, + Namespace: defaultTestNamespace, + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: defaultTestNamespace, + }, }, Spec: v1alpha1.StorageProviderSpec{ Provisioner: v1alpha1.ProvisionerAWSRDSCluster, @@ -1635,3 +1652,56 @@ var _ = Describe("StorageNode Controller Mock Test For AWS RDS Cluster", func() }) }) }) + +var _ = Describe("validateStorageProviderNamespace", func() { + It("should allow when namespace is in allow list", func() { + node := &v1alpha1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant-a", + }, + } + provider := &v1alpha1.StorageProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sp", + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: "tenant-a,tenant-b", + }, + }, + } + Expect(validateStorageProviderNamespace(node, provider)).To(Succeed()) + }) + + It("should deny when namespace is not in allow list", func() { + node := &v1alpha1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant-c", + }, + } + provider := &v1alpha1.StorageProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sp", + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: "tenant-a,tenant-b", + }, + }, + } + Expect(validateStorageProviderNamespace(node, provider)).To(HaveOccurred()) + }) +}) + +var _ = Describe("storageProviderObjectKey", func() { + It("should include storageProvider name and storagenode namespace", func() { + node := &v1alpha1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant-a", + }, + Spec: v1alpha1.StorageNodeSpec{ + StorageProviderName: "sp-a", + }, + } + + key := storageProviderObjectKey(node) + Expect(key.Name).To(Equal("sp-a")) + Expect(key.Namespace).To(Equal("tenant-a")) + }) +}) diff --git a/shardingsphere-operator/pkg/controllers/storage_node_controller.go b/shardingsphere-operator/pkg/controllers/storage_node_controller.go index c221e87a..db079c7c 100644 --- a/shardingsphere-operator/pkg/controllers/storage_node_controller.go +++ b/shardingsphere-operator/pkg/controllers/storage_node_controller.go @@ -249,11 +249,16 @@ func (r *StorageNodeReconciler) getStorageProvider(ctx context.Context, node *v1 storageProvider = &v1alpha1.StorageProvider{} - if err := r.Get(ctx, client.ObjectKey{Name: node.Spec.StorageProviderName}, storageProvider); err != nil { + if err := r.Get(ctx, storageProviderObjectKey(node), storageProvider); err != nil { r.Log.Error(err, fmt.Sprintf("unable to fetch storageProvider %s", node.Spec.StorageProviderName)) r.Recorder.Event(node, corev1.EventTypeWarning, "storageProviderNotFound", fmt.Sprintf("storageProvider %s not found", node.Spec.StorageProviderName)) return nil, err } + if err := validateStorageProviderNamespace(node, storageProvider); err != nil { + r.Log.Error(err, fmt.Sprintf("storageProvider %s is not accessible from namespace %s", node.Spec.StorageProviderName, node.Namespace)) + r.Recorder.Event(node, corev1.EventTypeWarning, "storageProviderNamespaceDenied", err.Error()) + return nil, err + } // check provisioner // aws-like provisioner need aws rds client @@ -269,6 +274,30 @@ func (r *StorageNodeReconciler) getStorageProvider(ctx context.Context, node *v1 return storageProvider, nil } +func storageProviderObjectKey(node *v1alpha1.StorageNode) client.ObjectKey { + return client.ObjectKey{ + Name: node.Spec.StorageProviderName, + Namespace: node.Namespace, + } +} + +func validateStorageProviderNamespace(node *v1alpha1.StorageNode, storageProvider *v1alpha1.StorageProvider) error { + if storageProvider == nil { + return fmt.Errorf("storageProvider is nil") + } + allowedNamespaces, ok := storageProvider.Annotations[v1alpha1.AnnotationsAllowedNamespaces] + if !ok || strings.TrimSpace(allowedNamespaces) == "" { + return fmt.Errorf("storageProvider %s missing annotation %s", storageProvider.Name, v1alpha1.AnnotationsAllowedNamespaces) + } + for _, namespace := range strings.Split(allowedNamespaces, ",") { + namespace = strings.TrimSpace(namespace) + if namespace == "*" || namespace == node.Namespace { + return nil + } + } + return fmt.Errorf("namespace %s is not allowed to use storageProvider %s", node.Namespace, storageProvider.Name) +} + // nolint:gocritic func computeDesiredState(status v1alpha1.StorageNodeStatus) v1alpha1.StorageNodeStatus { // Initialize a new status object based on the current state diff --git a/shardingsphere-operator/test/e2e/storage_node_controller_test.go b/shardingsphere-operator/test/e2e/storage_node_controller_test.go index 1fac6f8d..3c5f901f 100644 --- a/shardingsphere-operator/test/e2e/storage_node_controller_test.go +++ b/shardingsphere-operator/test/e2e/storage_node_controller_test.go @@ -47,6 +47,9 @@ var _ = Describe("StorageNode Controller Suite Test For AWS RDS Instance", func( StorageProvider := &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ Name: storageProviderName, + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: "default", + }, }, Spec: v1alpha1.StorageProviderSpec{ Provisioner: v1alpha1.ProvisionerAWSRDSInstance, @@ -294,6 +297,9 @@ var _ = Describe("StorageNode Controller Suite Test For AWS Aurora Cluster", fun provider := &v1alpha1.StorageProvider{ ObjectMeta: metav1.ObjectMeta{ Name: storageProviderName, + Annotations: map[string]string{ + v1alpha1.AnnotationsAllowedNamespaces: "default", + }, }, Spec: v1alpha1.StorageProviderSpec{ Provisioner: v1alpha1.ProvisionerAWSAurora,