Merge pull request #1319 from jcechace/PBM-1668-ixq-member-bound

PBM-1668 add index quorum limit handling

Author: jcechace

cmd/pbm/main.go

@@ -835,7 +835,7 @@ func (app *pbmApp) buildRestoreCmd() *cobra.Command {
 	)
 	restoreCmd.Flags().StringVar(
 		&restoreOptions.indexCommitQuorum, "index-commit-quorum", "",
-		"Index commit quorum for logical restore index builds: majority, votingMembers, or a positive integer.",
+		"Index commit quorum for logical restore index builds: majority, votingMembers, or an integer from 1 to 50.",
 	)
 	restoreCmd.Flags().StringVar(
 		&restoreOptions.ns, "ns", "",

packaging/conf/pbm-conf-reference.yml

@@ -253,7 +253,7 @@
 #  numParallelCollections:
 
 ## Set the commit quorum for index creation during logical restore.
-## Supported values are majority, votingMembers, or a positive integer.
+## Supported values are majority, votingMembers, or an integer from 1 to 50.
 ## Default value is votingMembers.
 #  indexCommitQuorum: votingMembers
 

pbm/config/ixquorum.go

@@ -14,7 +14,10 @@ const (
 	IndexCommitQuorumVotingMembers IndexCommitQuorum = "votingMembers"
 )
 
-const DefaultRestoreIndexCommitQuorum = IndexCommitQuorumVotingMembers
+const (
+	DefaultRestoreIndexCommitQuorum = IndexCommitQuorumVotingMembers
+	MaxIndexCommitQuorum            = 50
+)
 
 // CommandValue returns the value shape expected by MongoDB's createIndexes
 // commitQuorum field. MongoDB accepts symbolic string values or an integer
@@ -40,7 +43,7 @@ func ValidateIndexCommitQuorum(q IndexCommitQuorum) error {
 	}
 
 	n, err := strconv.ParseInt(val, 10, 32)
-	if err != nil || n <= 0 {
+	if err != nil || n <= 0 || n > MaxIndexCommitQuorum {
 		return errors.Errorf("invalid restore.indexCommitQuorum %q", val)
 	}
 

pbm/config/ixquorum_test.go

@@ -35,7 +35,9 @@ func TestValidateIndexCommitQuorum(t *testing.T) {
 		{name: "majority", value: IndexCommitQuorumMajority},
 		{name: "votingMembers", value: IndexCommitQuorumVotingMembers},
 		{name: "positive integer", value: "3"},
+		{name: "max integer", value: "50"},
 		{name: "zero", value: "0", wantErr: true},
+		{name: "above max integer", value: "51", wantErr: true},
 		{name: "negative integer", value: "-1", wantErr: true},
 		{name: "unknown string", value: "whatever", wantErr: true},
 		{name: "leading whitespace", value: " majority", wantErr: true},

pbm/restore/logical.go

@@ -36,6 +36,8 @@ import (
 	"github.com/percona/percona-backup-mongodb/pbm/version"
 )
 
+const mongoErrUnsatisfiableCommitQuorum = 278
+
 // mDBCl represents mDB client iterface for the DB related ops.
 type mDBCl interface {
 	runCmdShardsvrDropDatabase(ctx context.Context, db string, configDBDoc *configDatabasesDoc) error
@@ -1316,16 +1318,22 @@ func (r *Restore) restoreIndexes(ctx context.Context, nss []string) error {
 			delete(index.Options, "v")
 		}
 
-		rawCommand := bson.D{
-			{"createIndexes", ns.Collection},
-			{"indexes", indexes},
-			{"ignoreUnknownIndexOptions", true},
-			{"commitQuorum", r.indexCommitQuorum.CommandValue()},
-		}
+		commitQuorum := r.indexCommitQuorum.CommandValue()
+		rawCommand := createIndexesCommand(ns.Collection, indexes, commitQuorum)
 
 		r.log.Info("restoring indexes for %s.%s: %s",
 			ns.DB, ns.Collection, strings.Join(indexNames, ", "))
 		err := r.nodeConn.Database(ns.DB).RunCommand(ctx, rawCommand).Err()
+		if shouldRetryWithDefaultIndexCommitQuorum(err, commitQuorum) {
+			commitQuorum = config.DefaultRestoreIndexCommitQuorum.CommandValue()
+			r.log.Debug("createIndexes for %s.%s failed with MongoDB error: %v", ns.DB, ns.Collection, err)
+			r.log.Warning(
+				"index commit quorum cannot be satisfied for %s.%s, retrying with %s",
+				ns.DB, ns.Collection, commitQuorum,
+			)
+			rawCommand = createIndexesCommand(ns.Collection, indexes, commitQuorum)
+			err = r.nodeConn.Database(ns.DB).RunCommand(ctx, rawCommand).Err()
+		}
 		if err != nil {
 			return errors.Wrapf(err, "createIndexes for %s.%s", ns.DB, ns.Collection)
 		}
@@ -1334,6 +1342,27 @@ func (r *Restore) restoreIndexes(ctx context.Context, nss []string) error {
 	return nil
 }
 
+func createIndexesCommand(collection string, indexes []*idx.IndexDocument, commitQuorum any) bson.D {
+	return bson.D{
+		{"createIndexes", collection},
+		{"indexes", indexes},
+		{"ignoreUnknownIndexOptions", true},
+		{"commitQuorum", commitQuorum},
+	}
+}
+
+func shouldRetryWithDefaultIndexCommitQuorum(err error, commitQuorum any) bool {
+	if err == nil {
+		return false
+	}
+	if _, ok := commitQuorum.(int32); !ok {
+		return false
+	}
+
+	var cmdErr mongo.CommandError
+	return errors.As(err, &cmdErr) && cmdErr.HasErrorCode(mongoErrUnsatisfiableCommitQuorum)
+}
+
 func (r *Restore) updateRouterConfig(ctx context.Context) error {
 	if len(r.sMap) == 0 || !r.nodeInfo.IsSharded() {
 		return nil

pbm/restore/logical_test.go

@@ -8,7 +8,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
 
 	"github.com/percona/percona-backup-mongodb/pbm/backup"
 	"github.com/percona/percona-backup-mongodb/pbm/config"
@@ -216,6 +218,42 @@ func TestResolveNamespace(t *testing.T) {
 	})
 }
 
+func TestShouldRetryWithDefaultIndexCommitQuorum(t *testing.T) {
+	tests := []struct {
+		name         string
+		err          error
+		commitQuorum any
+		want         bool
+	}{
+		{name: "nil error", err: nil, commitQuorum: int32(3), want: false},
+		{
+			name:         "numeric unsatisfiable quorum",
+			err:          mongo.CommandError{Code: mongoErrUnsatisfiableCommitQuorum},
+			commitQuorum: int32(3),
+			want:         true,
+		},
+		{
+			name:         "string unsatisfiable quorum",
+			err:          mongo.CommandError{Code: mongoErrUnsatisfiableCommitQuorum},
+			commitQuorum: string(config.IndexCommitQuorumVotingMembers),
+			want:         false,
+		},
+		{
+			name:         "numeric other command error",
+			err:          mongo.CommandError{Code: 9},
+			commitQuorum: int32(3),
+			want:         false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := shouldRetryWithDefaultIndexCommitQuorum(tt.err, tt.commitQuorum)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}
+
 func TestShouldRestoreUsersAndRoles(t *testing.T) {
 	checkOpt := func(t testing.TB, got, want restoreUsersAndRolesOption) {
 		t.Helper()