[4.1.0 Backport] CBG-5484: Support resync invalidate principals on both metadata collections (#8383)

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

Author: bbrks

base/constants.go

@@ -36,6 +36,10 @@ const (
 	TestEnvTLSSkipVerify     = "SG_TEST_TLS_SKIP_VERIFY"
 	DefaultTestTLSSkipVerify = true
 
+	// Env variable to enable _system._mobile collection for SG metadata
+	TestEnvUseSystemMetadataCollection     = "SG_TEST_USE_SYSTEM_METADATA_COLLECTION"
+	DefaultTestUseSystemMetadataCollection = false
+
 	// Walrus by default, but can set to "Couchbase" to have it use http://127.0.0.1:8091
 	TestEnvSyncGatewayBackingStore = "SG_TEST_BACKING_STORE"
 	TestEnvBackingStoreCouchbase   = "Couchbase"

base/util_testing.go

@@ -227,6 +227,21 @@ func TestTLSSkipVerify() bool {
 	return val
 }
 
+// TestUseSystemMetadataCollection returns true if Sync Gateway should use the system metadata collection in tests. Default: DefaultTestUseSystemMetadataCollection
+func TestUseSystemMetadataCollection() bool {
+	useSystemMetadataCollection, isSet := os.LookupEnv(TestEnvUseSystemMetadataCollection)
+	if !isSet {
+		return DefaultTestUseSystemMetadataCollection
+	}
+
+	val, err := strconv.ParseBool(useSystemMetadataCollection)
+	if err != nil {
+		panic(fmt.Sprintf("unable to parse %q value %q: %v", TestEnvUseSystemMetadataCollection, useSystemMetadataCollection, err))
+	}
+
+	return val
+}
+
 func TestUseCouchbaseServerDockerName() (bool, string) {
 	testX509CouchbaseServerDockerName, isSet := os.LookupEnv(TestEnvCouchbaseServerDockerName)
 	if !isSet {

db/background_mgr_resync_dcp.go

@@ -834,10 +834,18 @@ type ResyncManagerStatusDocDCP struct {
 
 // initializePrincipalDocsIndex creates the metadata indexes required for resync
 func initializePrincipalDocsIndex(ctx context.Context, db *DatabaseContext) error {
-	n1qlStore, ok := base.AsN1QLStore(db.MetadataStore)
-	if !ok {
-		return errors.New("Cannot create indexes on non-Couchbase data store.")
+	var dataStores []base.DataStore
+
+	if metadataStore, ok := db.MetadataStore.(*base.MetadataStore); ok {
+		// need to ensure both primary and fallback have index
+		dataStores = append(dataStores, metadataStore.Primary())
+		if !metadataStore.MigrationComplete() {
+			dataStores = append(dataStores, metadataStore.Fallback())
+		}
+	} else {
+		dataStores = append(dataStores, db.MetadataStore)
 	}
+
 	options := InitializeIndexOptions{
 		WaitForIndexesOnlineOption: base.WaitForIndexesDefault,
 		NumReplicas:                db.Options.NumIndexReplicas,
@@ -846,7 +854,17 @@ func initializePrincipalDocsIndex(ctx context.Context, db *DatabaseContext) erro
 		NumPartitions:              db.numIndexPartitions(),
 	}
 
-	return InitializeIndexes(ctx, n1qlStore, options)
+	var me *base.MultiError
+	for _, ds := range dataStores {
+		n1qlStore, ok := base.AsN1QLStore(ds)
+		if !ok {
+			return fmt.Errorf("Cannot create indexes on non-Couchbase data store.")
+		}
+		if err := InitializeIndexes(ctx, n1qlStore, options); err != nil {
+			me = me.Append(err)
+		}
+	}
+	return me.ErrorOrNil()
 }
 
 // getResyncDCPClientOptions returns the default set of DCPClientOptions suitable for resync. collectionIDs

db/indexes.go

@@ -413,7 +413,16 @@ func InitializeIndexes(ctx context.Context, n1QLStore base.N1QLStore, options In
 		return nil
 	}
 
-	indexesMeta, err := base.GetIndexesMeta(ctx, n1QLStore, requiredIndexes.FullIndexNames())
+	var (
+		indexesMeta map[string]base.IndexMeta
+		err         error
+	)
+	// TODO: Could probably always route through `GetSystemCollectionIndexesMeta` but will defer for post-4.1 cleanup?
+	if base.IsMobileSystemCollection(n1QLStore) {
+		indexesMeta, err = base.GetSystemCollectionIndexesMeta(ctx, n1QLStore, n1QLStore.ScopeName(), n1QLStore.CollectionName(), requiredIndexes.FullIndexNames())
+	} else {
+		indexesMeta, err = base.GetIndexesMeta(ctx, n1QLStore, requiredIndexes.FullIndexNames())
+	}
 	if err != nil {
 		base.WarnfCtx(ctx, "Error getting indexes meta: %v - continuing to create", err)
 	}
@@ -432,7 +441,7 @@ func InitializeIndexes(ctx context.Context, n1QLStore base.N1QLStore, options In
 
 	if len(requiredIndexes) == 0 {
 		// all indexes are already online so we can return early without any per-index queries
-		base.InfofCtx(ctx, base.KeyAll, "Indexes ready - already online when checked")
+		base.InfofCtx(ctx, base.KeyAll, "Indexes ready for %q - already online when checked", base.MD(n1QLStore.GetName()))
 		return nil
 	}
 

rest/adminapitest/resync_test.go

@@ -222,6 +222,20 @@ func TestResyncRegenerateSequencesPrincipals(t *testing.T) {
 }
 
 func TestResyncInvalidatePrincipals(t *testing.T) {
+	var tests = []struct {
+		name                             string
+		useSystemScopeMetadataCollection bool
+	}{
+		{
+			name:                             "useSystemScopeMetadataCollection=false",
+			useSystemScopeMetadataCollection: false,
+		},
+		{
+			name:                             "useSystemScopeMetadataCollection=true",
+			useSystemScopeMetadataCollection: true,
+		},
+	}
+
 	initialSyncFn := `
 	function(doc) {
 		access(doc.userName, "channelABC");
@@ -236,71 +250,82 @@ func TestResyncInvalidatePrincipals(t *testing.T) {
 		role(doc.userName, "role:roleDEF");
 	}`
 
-	rt := rest.NewRestTester(t, &rest.RestTesterConfig{
-		PersistentConfig: true,
-		SyncFn:           initialSyncFn,
-	})
-	defer rt.Close()
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// this condition is not allowed via regular db config-level validation
+			// but RestTester code doesn't perform full dbconfig validation to block this
+			if test.useSystemScopeMetadataCollection && base.TestsDisableGSI() {
+				t.Skip("use_views not supported with system scoped metadata collection")
+			}
 
-	dbConfig := rt.NewDbConfig()
-	ds := rt.TestBucket.GetSingleDataStore()
-	scopeName := ds.ScopeName()
-	collectionName := ds.CollectionName()
+			rt := rest.NewRestTester(t, &rest.RestTesterConfig{
+				PersistentConfig:                 true,
+				SyncFn:                           initialSyncFn,
+				UseSystemScopeMetadataCollection: base.Ptr(test.useSystemScopeMetadataCollection),
+			})
+			defer rt.Close()
+
+			dbConfig := rt.NewDbConfig()
+			ds := rt.TestBucket.GetSingleDataStore()
+			scopeName := ds.ScopeName()
+			collectionName := ds.CollectionName()
 
-	rest.RequireStatus(t, rt.CreateDatabase("db1", dbConfig), http.StatusCreated)
+			rest.RequireStatus(t, rt.CreateDatabase("db1", dbConfig), http.StatusCreated)
 
-	// Set up user and role
-	username := "alice"
-	rolename := "foo"
-	grantingDocID := "grantDoc"
-	grantingDocBody := `{
+			// Set up user and role
+			username := "alice"
+			rolename := "foo"
+			grantingDocID := "grantDoc"
+			grantingDocBody := `{
 		"userName":"alice",
 		"roleName":"foo"
 	}`
-	rt.CreateUser(username, nil)
+			rt.CreateUser(username, nil)
 
-	response := rt.SendAdminRequest(http.MethodPut, "/{{.db}}/_role/"+rolename, rest.GetRolePayload(t, rolename, rt.GetSingleDataStore(), nil))
-	rest.RequireStatus(t, response, http.StatusCreated)
+			response := rt.SendAdminRequest(http.MethodPut, "/{{.db}}/_role/"+rolename, rest.GetRolePayload(t, rolename, rt.GetSingleDataStore(), nil))
+			rest.RequireStatus(t, response, http.StatusCreated)
 
-	// Write doc to perform dynamic grants
-	response = rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+grantingDocID, grantingDocBody)
-	rest.RequireStatus(t, response, http.StatusCreated)
+			// Write doc to perform dynamic grants
+			response = rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/"+grantingDocID, grantingDocBody)
+			rest.RequireStatus(t, response, http.StatusCreated)
 
-	ctx := rt.Context()
-	user, err := rt.GetDatabase().Authenticator(ctx).GetUser(username)
-	require.NoError(t, err)
-	channels := user.CollectionChannels(scopeName, collectionName)
-	_, ok := channels["channelABC"]
-	require.True(t, ok, "user should have channel channelABC")
-	roles := user.RoleNames()
-	_, ok = roles["roleABC"]
-	require.True(t, ok, "user should have role roleABC")
+			ctx := rt.Context()
+			user, err := rt.GetDatabase().Authenticator(ctx).GetUser(username)
+			require.NoError(t, err)
+			channels := user.CollectionChannels(scopeName, collectionName)
+			_, ok := channels["channelABC"]
+			require.True(t, ok, "user should have channel channelABC")
+			roles := user.RoleNames()
+			_, ok = roles["roleABC"]
+			require.True(t, ok, "user should have role roleABC")
 
-	rt.TakeDbOffline()
+			rt.TakeDbOffline()
 
-	// Update the sync function
-	rt.SyncFn = updatedSyncFn
-	updatedDbConfig := rt.NewDbConfig()
-	rt.UpsertDbConfig("db1", updatedDbConfig)
-	rt.TakeDbOffline()
+			// Update the sync function
+			rt.SyncFn = updatedSyncFn
+			updatedDbConfig := rt.NewDbConfig()
+			rt.UpsertDbConfig("db1", updatedDbConfig)
+			rt.TakeDbOffline()
 
-	// Run resync
-	rest.RequireStatus(t, rt.SendAdminRequest(http.MethodPost, "/{{.db}}/_resync?action=start", ""), http.StatusOK)
-	_ = rt.WaitForResyncDCPStatus(db.BackgroundProcessStateCompleted)
+			// Run resync
+			rest.RequireStatus(t, rt.SendAdminRequest(http.MethodPost, "/{{.db}}/_resync?action=start", ""), http.StatusOK)
+			_ = rt.WaitForResyncDCPStatus(db.BackgroundProcessStateCompleted)
 
-	// validate user channels and roles have been updated
-	user, err = rt.GetDatabase().Authenticator(ctx).GetUser(username)
-	require.NoError(t, err)
-	channels = user.CollectionChannels(scopeName, collectionName)
-	_, ok = channels["channelABC"]
-	require.False(t, ok, "user should not have channel channelABC")
-	_, ok = channels["channelDEF"]
-	require.True(t, ok, "user should have channel channelDEF")
-	roles = user.RoleNames()
-	_, ok = roles["roleABC"]
-	require.False(t, ok, "user should not have role roleABC")
-	_, ok = roles["roleDEF"]
-	require.True(t, ok, "user should have role roleDEF")
+			// validate user channels and roles have been updated
+			user, err = rt.GetDatabase().Authenticator(ctx).GetUser(username)
+			require.NoError(t, err)
+			channels = user.CollectionChannels(scopeName, collectionName)
+			_, ok = channels["channelABC"]
+			require.False(t, ok, "user should not have channel channelABC")
+			_, ok = channels["channelDEF"]
+			require.True(t, ok, "user should have channel channelDEF")
+			roles = user.RoleNames()
+			_, ok = roles["roleABC"]
+			require.False(t, ok, "user should not have role roleABC")
+			_, ok = roles["roleDEF"]
+			require.True(t, ok, "user should have role roleDEF")
+		})
+	}
 }
 
 func TestResyncDoesNotWriteDocBody(t *testing.T) {

rest/utilities_testing.go

@@ -51,33 +51,34 @@ import (
 
 // RestTesterConfig represents configuration for sync gateway
 type RestTesterConfig struct {
-	GuestEnabled                    bool                        // If this is true, Admin Party is in full effect
-	SyncFn                          string                      // put the sync() function source in here (optional)
-	ImportFilter                    string                      // put the import filter function source in here (optional)
-	DatabaseConfig                  *DatabaseConfig             // Supports additional config options.  BucketConfig, Name, Sync, Unsupported will be ignored (overridden)
-	MutateStartupConfig             func(config *StartupConfig) // Function to mutate the startup configuration before the server context gets created. This overrides options the RT sets.
-	InitSyncSeq                     uint64                      // If specified, initializes _sync:seq on bucket creation.  Not supported when running against walrus
-	AllowConflicts                  bool                        // Enable conflicts mode.  By default, conflicts will not allowed
-	EnableUserQueries               bool                        // Enable the feature-flag for user N1QL/etc queries
-	CustomTestBucket                *base.TestBucket            // If set, use this bucket instead of requesting a new one.
-	LeakyBucketConfig               *base.LeakyBucketConfig     // Set to create and use a leaky bucket on the RT and DB. A test bucket cannot be passed in if using this option.
-	adminInterface                  string                      // adminInterface overrides the default admin interface.
-	SgReplicateEnabled              bool                        // SgReplicateManager disabled by default for RestTester
-	AutoImport                      *bool
-	HideProductInfo                 bool
-	AdminInterfaceAuthentication    bool
-	metricsInterfaceAuthentication  bool
-	enableAdminAuthPermissionsCheck bool
-	useTLSServer                    bool // If true, TLS will be required for communications with CBS. Default: false
-	PersistentConfig                bool
-	GroupID                         *string
-	serverless                      bool // Runs SG in serverless mode. Must be used in conjunction with persistent config
-	collectionConfig                collectionConfiguration
-	numCollections                  int
-	nodeClusterCompatVersion        *base.ClusterCompatVersion // alternate cluster compat version this node identifies as. Defaults to base.NodeClusterCompatVersion.
-	allowDbConfigEnvVars            *bool
-	maxConcurrentRevs               *int
-	UseXattrConfig                  bool
+	GuestEnabled                     bool                        // If this is true, Admin Party is in full effect
+	SyncFn                           string                      // put the sync() function source in here (optional)
+	ImportFilter                     string                      // put the import filter function source in here (optional)
+	DatabaseConfig                   *DatabaseConfig             // Supports additional config options.  BucketConfig, Name, Sync, Unsupported will be ignored (overridden)
+	MutateStartupConfig              func(config *StartupConfig) // Function to mutate the startup configuration before the server context gets created. This overrides options the RT sets.
+	InitSyncSeq                      uint64                      // If specified, initializes _sync:seq on bucket creation.  Not supported when running against walrus
+	AllowConflicts                   bool                        // Enable conflicts mode.  By default, conflicts will not allowed
+	EnableUserQueries                bool                        // Enable the feature-flag for user N1QL/etc queries
+	CustomTestBucket                 *base.TestBucket            // If set, use this bucket instead of requesting a new one.
+	LeakyBucketConfig                *base.LeakyBucketConfig     // Set to create and use a leaky bucket on the RT and DB. A test bucket cannot be passed in if using this option.
+	adminInterface                   string                      // adminInterface overrides the default admin interface.
+	SgReplicateEnabled               bool                        // SgReplicateManager disabled by default for RestTester
+	AutoImport                       *bool
+	HideProductInfo                  bool
+	AdminInterfaceAuthentication     bool
+	metricsInterfaceAuthentication   bool
+	enableAdminAuthPermissionsCheck  bool
+	useTLSServer                     bool // If true, TLS will be required for communications with CBS. Default: false
+	PersistentConfig                 bool
+	GroupID                          *string
+	serverless                       bool // Runs SG in serverless mode. Must be used in conjunction with persistent config
+	collectionConfig                 collectionConfiguration
+	numCollections                   int
+	nodeClusterCompatVersion         *base.ClusterCompatVersion // alternate cluster compat version this node identifies as. Defaults to base.NodeClusterCompatVersion.
+	allowDbConfigEnvVars             *bool
+	maxConcurrentRevs                *int
+	UseXattrConfig                   bool
+	UseSystemScopeMetadataCollection *bool
 }
 
 type collectionConfiguration uint8
@@ -274,6 +275,12 @@ func (rt *RestTester) Bucket() base.Bucket {
 		}
 	}
 
+	if rt.RestTesterConfig.UseSystemScopeMetadataCollection != nil {
+		sc.Bootstrap.UseSystemMetadataCollection = rt.RestTesterConfig.UseSystemScopeMetadataCollection
+	} else {
+		sc.Bootstrap.UseSystemMetadataCollection = base.Ptr(base.TestUseSystemMetadataCollection())
+	}
+
 	if rt.RestTesterConfig.GroupID != nil {
 		sc.Bootstrap.ConfigGroupID = *rt.RestTesterConfig.GroupID
 	} else if rt.RestTesterConfig.PersistentConfig {