Add default permission for replica balancing (#770)

"collection-admin-edit" for Solr 9.9+ and "/____v2/cluster/replicas/balance" for previous versions

(cherry picked from commit 494471b)

Author: HoustonPutman

controllers/util/solr_security_util.go

@@ -117,6 +117,9 @@ func reconcileForBasicAuthWithBootstrappedSecurityJson(ctx context.Context, clie
 
 		// supply the bootstrap security.json to the initContainer via a simple BASE64 encoding env var
 		security.SecurityJson = string(bootstrapSecret.Data[SecurityJsonFile])
+		security.SecurityJsonSrc = &corev1.EnvVarSource{
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{Name: bootstrapSecret.Name}, Key: SecurityJsonFile}}
 		basicAuthSecret = authSecret
 	}
 
@@ -393,7 +396,9 @@ func generateSecurityJson(solrCloud *solr.SolrCloud) map[string][]byte {
           { "name": "k8s-metrics", "role":"k8s", "collection": null, "path":"/admin/metrics" },
           { "name": "k8s-zk", "role":"k8s", "collection": null, "path":"/admin/zookeeper/status" },
           { "name": "k8s-ping", "role":"k8s", "collection": "*", "path":"/admin/ping" },
-          { "name": "read", "role":["admin","users"] },
+          { "name": "k8s-replica-balancing", "role":"k8s", "collection": null, "path":"/____v2/cluster/replicas/balance" },
+          { "name": "collection-admin-edit", "role":"k8s" },
+          { "name": "read", "role":["admin","users","k8s"] },
           { "name": "update", "role":["admin"] },
           { "name": "security-read", "role": ["admin"] },
           { "name": "security-edit", "role": ["admin"] },

docs/solr-cloud/solr-cloud-crd.md

@@ -1032,6 +1032,16 @@ Take a moment to review these authorization rules so that you're aware of the ro
         "collection": "*",
         "path": "/admin/ping"
       },
+      {
+        "name": "k8s-replica-balancing",
+        "role": "k8s",
+        "collection": null,
+        "path": "/____v2/cluster/replicas/balance"
+      },
+      {
+        "name": "collection-admin-edit",
+        "role": "k8s"
+      },
       {
         "name": "read",
         "role": [ "admin", "users" ]
@@ -1165,6 +1175,12 @@ Users need to ensure their `security.json` contains the user supplied in the `ba
 /admin/metrics
 /admin/ping (for collection="*")
 /admin/zookeeper/status
+/____v2/cluster/replicas/balance
+```
+
+And the following named permissions:
+```aiignore
+collection-admin-edit
 ```
 _Tip: see the authorization rules defined by the default `security.json` as a guide for configuring access for the operator user_
 

helm/solr-operator/Chart.yaml

@@ -78,6 +78,13 @@ annotations:
           url: https://github.com/apache/solr-operator/issues/759
         - name: Github PR
           url: https://github.com/apache/solr-operator/pull/769
+    - kind: fixed
+      description: "Fix default bootstrapped permissions for replica balancing"
+      links:
+        - name: Github Issue
+          url: https://github.com/apache/solr-operator/issues/653
+        - name: Github PR
+          url: https://github.com/apache/solr-operator/pull/770
   artifacthub.io/images: |
     - name: solr-operator
       image: apache/solr-operator:v0.9.1-prerelease

tests/e2e/solrcloud_security_json_test.go

@@ -20,9 +20,14 @@ package e2e
 import (
 	"context"
 	solrv1beta1 "github.com/apache/solr-operator/api/v1beta1"
+	"github.com/apache/solr-operator/controllers"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"time"
 )
 
 var _ = FDescribe("E2E - SolrCloud - Security JSON", func() {
@@ -35,10 +40,6 @@ var _ = FDescribe("E2E - SolrCloud - Security JSON", func() {
 	})
 
 	JustBeforeEach(func(ctx context.Context) {
-		By("generating the security.json secret and basic auth secret")
-		generateSolrSecuritySecret(ctx, solrCloud)
-		generateSolrBasicAuthSecret(ctx, solrCloud)
-
 		By("creating the SolrCloud")
 		Expect(k8sClient.Create(ctx, solrCloud)).To(Succeed())
 
@@ -50,20 +51,64 @@ var _ = FDescribe("E2E - SolrCloud - Security JSON", func() {
 		solrCloud = expectSolrCloudToBeReady(ctx, solrCloud)
 
 		By("creating a first Solr Collection")
-		createAndQueryCollection(ctx, solrCloud, "basic", 1, 1)
+		createAndQueryCollection(ctx, solrCloud, "basic", 2, 1)
 	})
 
-	FContext("Provided Zookeeper", func() {
-		BeforeEach(func() {
+	FContext("Provided Security JSON", func() {
+		BeforeEach(func(ctx context.Context) {
 			solrCloud.Spec.ZookeeperRef = &solrv1beta1.ZookeeperRef{
 				ProvidedZookeeper: &solrv1beta1.ZookeeperSpec{
 					Replicas:  pointer.Int32(1),
 					Ephemeral: &solrv1beta1.ZKEphemeral{},
 				},
 			}
+
+			solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{
+				AuthenticationType: "Basic",
+				BasicAuthSecret:    solrCloud.Name + "-basic-auth-secret",
+				BootstrapSecurityJson: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: solrCloud.Name + "-security-secret",
+					},
+					Key: "security.json",
+				},
+			}
+
+			By("generating the security.json secret and basic auth secret")
+			generateSolrSecuritySecret(ctx, solrCloud)
+			generateSolrBasicAuthSecret(ctx, solrCloud)
 		})
 
 		// All testing will be done in the "JustBeforeEach" logic, no additional tests required here
 		FIt("Starts correctly", func(ctx context.Context) {})
 	})
+
+	FContext("Bootstrapped Security", func() {
+
+		BeforeEach(func() {
+			solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{
+				AuthenticationType: "Basic",
+			}
+		})
+
+		FIt("Scales up with replica migration", func(ctx context.Context) {
+			originalSolrCloud := solrCloud.DeepCopy()
+			solrCloud.Spec.Replicas = pointer.Int32(int32(2))
+			By("triggering a scale up via solrCloud replicas")
+			Expect(k8sClient.Patch(ctx, solrCloud, client.MergeFrom(originalSolrCloud))).To(Succeed(), "Could not patch SolrCloud replicas to initiate scale down")
+
+			By("make sure scaleDown happens without a clusterLock and eventually the replicas are removed")
+			// Once the scale down actually occurs, the statefulSet annotations should be removed very soon
+			expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Minute*2, time.Millisecond*500, func(g Gomega, found *appsv1.StatefulSet) {
+				g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(2)), "StatefulSet should eventually have 2 pods.")
+				clusterOp, err := controllers.GetCurrentClusterOp(found)
+				g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+				g.Expect(clusterOp).To(BeNil(), "StatefulSet should have a ScaleDown lock after scaling is complete.")
+			})
+
+			queryCollection(ctx, solrCloud, "basic", 0)
+
+			// TODO: When balancing is in all Operator supported Solr versions, add a test to make sure balancing occurred
+		})
+	})
 })

tests/e2e/suite_test.go

@@ -106,6 +106,7 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) {
 	var err error
 	k8sConfig, err = config.GetConfig()
 	Expect(err).NotTo(HaveOccurred(), "Could not load in default kubernetes config")
+	k8sConfig.Timeout = time.Minute
 	Expect(zkApi.AddToScheme(scheme.Scheme)).To(Succeed())
 	Expect(certManagerApi.AddToScheme(scheme.Scheme)).To(Succeed())
 	k8sClient, err = client.New(k8sConfig, client.Options{Scheme: scheme.Scheme})
@@ -338,14 +339,33 @@ func writeAllSolrInfoToFiles(ctx context.Context, directory string, namespace st
 	}
 
 	foundServices := &corev1.ServiceList{}
-	Expect(k8sClient.List(ctx, foundServices, listOps)).To(Succeed(), "Could not fetch Solr pods")
+	Expect(k8sClient.List(ctx, foundServices, listOps)).To(Succeed(), "Could not fetch Solr services")
 	Expect(foundServices).ToNot(BeNil(), "No Solr services could be found")
 	for _, service := range foundServices.Items {
 		writeAllServiceInfoToFiles(
 			directory+service.Name+".service",
 			&service,
 		)
 	}
+
+	// Unfortunately the secrets don't have a technology label
+	req, err = labels.NewRequirement("solr-cloud", selection.Exists, make([]string, 0))
+	Expect(err).ToNot(HaveOccurred())
+
+	labelSelector = labels.Everything().Add(*req)
+	listOps = &client.ListOptions{
+		Namespace:     namespace,
+		LabelSelector: labelSelector,
+	}
+
+	foundSecrets := &corev1.SecretList{}
+	Expect(k8sClient.List(ctx, foundSecrets, listOps)).To(Succeed(), "Could not fetch Solr secrets")
+	for _, secret := range foundSecrets.Items {
+		writeAllSecretInfoToFiles(
+			directory+secret.Name+".secret",
+			&secret,
+		)
+	}
 }
 
 // writeSolrClusterStatusInfoToFile writes the following each to a separate file with the given base name & directory.
@@ -401,6 +421,19 @@ func writeAllServiceInfoToFiles(baseFilename string, service *corev1.Service) {
 	Expect(writeErr).ToNot(HaveOccurred(), "Could not write service json to file")
 }
 
+// writeAllSecretInfoToFiles writes the following each to a separate file with the given base name & directory.
+//   - Service
+func writeAllSecretInfoToFiles(baseFilename string, secret *corev1.Secret) {
+	// Write service to a file
+	statusFile, err := os.Create(baseFilename + ".json")
+	defer statusFile.Close()
+	Expect(err).ToNot(HaveOccurred(), "Could not open file to save secret status: %s", baseFilename+".json")
+	jsonBytes, marshErr := json.MarshalIndent(secret, "", "\t")
+	Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize secret json")
+	_, writeErr := statusFile.Write(jsonBytes)
+	Expect(writeErr).ToNot(HaveOccurred(), "Could not write secret json to file")
+}
+
 // writeAllPodInfoToFile writes the following each to a separate file with the given base name & directory.
 //   - Pod Spec/Status
 //   - Pod Events

tests/e2e/test_utils_test.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -233,7 +234,7 @@ func createAndQueryCollectionWithGomega(ctx context.Context, solrCloud *solrv1be
 	}
 
 	additionalOffset += 1
-	g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) {
+	g.EventuallyWithOffset(additionalOffset, func(innerG Gomega, ctx context.Context) {
 		response, err := callSolrApiInPod(
 			ctx,
 			solrCloud,
@@ -246,7 +247,7 @@ func createAndQueryCollectionWithGomega(ctx context.Context, solrCloud *solrv1be
 	}).Within(time.Second*10).WithContext(ctx).Should(Succeed(), "Collection creation command start was not successful")
 	// Only wait 5 seconds when trying to create the asyncCommand
 
-	g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) {
+	g.EventuallyWithOffset(additionalOffset, func(innerG Gomega, ctx context.Context) {
 		response, err := callSolrApiInPod(
 			ctx,
 			solrCloud,
@@ -271,7 +272,7 @@ func createAndQueryCollectionWithGomega(ctx context.Context, solrCloud *solrv1be
 		innerG.Expect(response).To(ContainSubstring("\"state\":\"completed\""), "Did not finish creating Solr Collection in time")
 	}).Within(time.Second*40).WithContext(ctx).Should(Succeed(), "Collection creation was not successful")
 
-	g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) {
+	g.EventuallyWithOffset(additionalOffset, func(innerG Gomega, ctx context.Context) {
 		response, err := callSolrApiInPod(
 			ctx,
 			solrCloud,
@@ -296,7 +297,7 @@ func queryCollection(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, coll
 }
 
 func queryCollectionWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, docCount int, g Gomega, additionalOffset ...int) {
-	g.EventuallyWithOffset(resolveOffset(additionalOffset), func(innerG Gomega) {
+	g.EventuallyWithOffset(resolveOffset(additionalOffset), func(innerG Gomega, ctx context.Context) {
 		response, err := callSolrApiInPod(
 			ctx,
 			solrCloud,
@@ -476,6 +477,20 @@ func callSolrApiInPod(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, htt
 		queryParamsString = "?" + queryParamsString
 	}
 
+	toolOpts := ""
+	if solrCloud.Spec.SolrSecurity != nil && solrCloud.Spec.SolrSecurity.AuthenticationType == solrv1beta1.Basic {
+		basicAuthSecretName := solrCloud.BasicAuthSecretName()
+		basicAuthSecret := &corev1.Secret{}
+		if err = k8sClient.Get(ctx, resourceKey(solrCloud, basicAuthSecretName), basicAuthSecret); err != nil {
+			return "", err
+		}
+		toolOpts =
+			"JAVA_TOOL_OPTIONS=\"-Dbasicauth=" +
+				string(basicAuthSecret.Data[corev1.BasicAuthUsernameKey]) + ":" + string(basicAuthSecret.Data[corev1.BasicAuthPasswordKey]) +
+				" -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\""
+	}
+	GinkgoLogr.Info(toolOpts)
+
 	command := []string{
 		"solr",
 		"api",
@@ -489,6 +504,12 @@ func callSolrApiInPod(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, htt
 			apiPath,
 			queryParamsString),
 	}
+	if toolOpts != "" {
+		commandString := fmt.Sprintf("%s %s", toolOpts, strings.Join(command, " "))
+		commandString = regexp.MustCompile(`\s+`).ReplaceAllString(strings.TrimSpace(commandString), " ")
+
+		command = []string{"sh", "-c", fmt.Sprintf("%q", commandString)}
+	}
 	return runExecForContainer(ctx, util.SolrNodeContainer, solrCloud.GetRandomSolrPodName(), solrCloud.Namespace, command)
 }
 
@@ -655,17 +676,6 @@ func generateBaseSolrCloudWithSecurityJSON(replicas int) *solrv1beta1.SolrCloud
 		solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{}
 	}
 
-	solrCloud.Spec.SolrSecurity.BootstrapSecurityJson = &corev1.SecretKeySelector{
-		LocalObjectReference: corev1.LocalObjectReference{
-			Name: solrCloud.Name + "-security-secret",
-		},
-		Key: "security.json",
-	}
-
-	solrCloud.Spec.SolrSecurity.AuthenticationType = "Basic"
-
-	solrCloud.Spec.SolrSecurity.BasicAuthSecret = solrCloud.Name + "-basic-auth-secret"
-
 	return solrCloud
 }
 

tests/scripts/manage_e2e_tests.sh

@@ -76,7 +76,7 @@ if [[ -z "${KUBERNETES_VERSION:-}" ]]; then
   KUBERNETES_VERSION="v1.26.6"
 fi
 if [[ -z "${SOLR_IMAGE:-}" ]]; then
-  SOLR_IMAGE="${SOLR_VERSION:-9.4.0}"
+  SOLR_IMAGE="${SOLR_VERSION:-9.8.1}"
 fi
 if [[ "${SOLR_IMAGE}" != *":"* ]]; then
   SOLR_IMAGE="solr:${SOLR_IMAGE}"