CBG-5495 add RestTestCluster (#8407)

Author: torcolvin

rest/adminapitest/resync_test.go

@@ -534,22 +534,20 @@ func TestDistributedResync(t *testing.T) {
 
 	t.Skip("Long-running dev-time test used to test multi-node cbgt rebalance during resync")
 
-	// Create first rest tester with persistent config
-	rt1 := rest.NewRestTesterPersistentConfig(t)
-	defer rt1.Close()
-
-	// Create second RT targeting the same bucket, with matching groupID and config polling enabled
-	rt2Config := &rest.RestTesterConfig{
-		GroupID: &rt1.ServerContext().Config.Bootstrap.ConfigGroupID,
+	ctx := base.TestCtx(t)
+	rtc := rest.NewRestTesterCluster(t, &rest.RestTesterClusterConfig{
+		NumNodes: 2,
 		MutateStartupConfig: func(config *rest.StartupConfig) {
 			// enable config polling
 			config.Bootstrap.ConfigUpdateFrequency = base.NewConfigDuration(50 * time.Millisecond)
 		},
-		CustomTestBucket: rt1.TestBucket.NoCloseClone(),
-		PersistentConfig: true,
-	}
-	rt2 := rest.NewRestTester(t, rt2Config)
-	defer rt2.Close()
+	})
+	defer rtc.Close(ctx)
+
+	rt1 := rtc.Node(0)
+	rt2 := rtc.Node(1)
+
+	rest.RequireStatus(t, rt1.CreateDatabase("db", rt1.NewDbConfig()), http.StatusCreated)
 
 	// wait for db to be online on rt1
 	rt1.WaitForDBState(db.RunStateString[db.DBOnline])

rest/cluster_compat_test.go

@@ -752,22 +752,14 @@ func refreshAndGetRegistry(t *testing.T, rt *RestTester, bucketName string) *Gat
 // IsConfigFullyApplied returns true once both nodes have applied the config.
 func TestIsConfigFullyAppliedTwoNodeConvergence(t *testing.T) {
 	ctx := base.TestCtx(t)
-	tb := base.GetTestBucket(t)
-	defer tb.Close(ctx)
-
-	groupID := t.Name()
-	rtConfig := &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	}
+	rtc := NewRestTesterCluster(t, &RestTesterClusterConfig{
+		NumNodes: 2,
+	})
+	defer rtc.Close(ctx)
 
-	rtA := NewRestTester(t, rtConfig)
-	defer rtA.Close()
-	rtB := NewRestTester(t, rtConfig)
-	defer rtB.Close()
-
-	bucketName := tb.GetName()
+	rtA := rtc.Node(0)
+	rtB := rtc.Node(1)
+	bucketName := rtc.testBucket.GetName()
 
 	dbConfig := rtA.NewDbConfig()
 	resp := rtA.CreateDatabase("db", dbConfig)
@@ -782,7 +774,7 @@ func TestIsConfigFullyAppliedTwoNodeConvergence(t *testing.T) {
 	require.NotEmpty(t, dbVersion)
 
 	registry := refreshAndGetRegistry(t, rtA, bucketName)
-	acked, missing, err := registry.IsConfigFullyApplied(ctx, groupID, "db", dbVersion)
+	acked, missing, err := registry.IsConfigFullyApplied(ctx, rtc.groupID, "db", dbVersion)
 	require.NoError(t, err)
 	assert.True(t, acked, "both nodes should have acked; missing: %v (registry nodes: %v)", missing, registry.Nodes)
 	assert.Empty(t, missing)
@@ -793,22 +785,14 @@ func TestIsConfigFullyAppliedTwoNodeConvergence(t *testing.T) {
 // node A's view even though B never acked the latest version.
 func TestIsConfigFullyAppliedStaleNodePruned(t *testing.T) {
 	ctx := base.TestCtx(t)
-	tb := base.GetTestBucket(t)
-	defer tb.Close(ctx)
-
-	groupID := t.Name()
-	rtConfig := &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	}
-
-	rtA := NewRestTester(t, rtConfig)
-	defer rtA.Close()
-	rtB := NewRestTester(t, rtConfig)
-	defer rtB.Close()
+	rtc := NewRestTesterCluster(t, &RestTesterClusterConfig{
+		NumNodes: 2,
+	})
+	defer rtc.Close(ctx)
 
-	bucketName := tb.GetName()
+	rtA := rtc.Node(0)
+	rtB := rtc.Node(1)
+	bucketName := rtc.testBucket.GetName()
 
 	dbConfig := rtA.NewDbConfig()
 	resp := rtA.CreateDatabase("db", dbConfig)
@@ -825,7 +809,7 @@ func TestIsConfigFullyAppliedStaleNodePruned(t *testing.T) {
 	assert.NotContains(t, maps.Keys(registry.Nodes), rtB.ServerContext().NodeUID, "stale node B should be pruned")
 
 	dbVersion := rtA.ServerContext().ClusterCompat.getAppliedDBVersionsForBucket(bucketName)["db"]
-	acked, missing, err := registry.IsConfigFullyApplied(ctx, groupID, "db", dbVersion)
+	acked, missing, err := registry.IsConfigFullyApplied(ctx, rtc.groupID, "db", dbVersion)
 	require.NoError(t, err)
 	assert.True(t, acked, "gate should pass after stale node is pruned; missing: %v", missing)
 	assert.Empty(t, missing)
@@ -835,20 +819,14 @@ func TestIsConfigFullyAppliedStaleNodePruned(t *testing.T) {
 // picks up the config via polling, it re-acks and IsConfigFullyApplied remains true.
 func TestIsConfigFullyAppliedNodeRestart(t *testing.T) {
 	ctx := base.TestCtx(t)
-	tb := base.GetTestBucket(t)
-	defer tb.Close(ctx)
-
-	groupID := t.Name()
-	rtConfig := &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	}
+	rtc := NewRestTesterCluster(t, &RestTesterClusterConfig{
+		NumNodes: 2,
+	})
+	defer rtc.Close(ctx)
 
-	rtA := NewRestTester(t, rtConfig)
-	defer rtA.Close()
-
-	bucketName := tb.GetName()
+	rtA := rtc.Node(0)
+	rtB := rtc.Node(1)
+	bucketName := rtc.testBucket.GetName()
 
 	dbConfig := rtA.NewDbConfig()
 	resp := rtA.CreateDatabase("db", dbConfig)
@@ -857,15 +835,12 @@ func TestIsConfigFullyAppliedNodeRestart(t *testing.T) {
 	dbVersion := rtA.ServerContext().ClusterCompat.getAppliedDBVersionsForBucket(bucketName)["db"]
 	require.NotEmpty(t, dbVersion)
 
-	rtB := NewRestTester(t, rtConfig)
-	defer rtB.Close()
-
 	rtB.ServerContext().ForceDbConfigsReload(t, ctx)
 
 	refreshAndGetRegistry(t, rtB, bucketName)
 
 	registry := refreshAndGetRegistry(t, rtA, bucketName)
-	acked, missing, err := registry.IsConfigFullyApplied(ctx, groupID, "db", dbVersion)
+	acked, missing, err := registry.IsConfigFullyApplied(ctx, rtc.groupID, "db", dbVersion)
 	require.NoError(t, err)
 	assert.True(t, acked, "restarted node B should have re-acked; missing: %v", missing)
 	assert.Empty(t, missing)
@@ -880,8 +855,6 @@ func TestIsConfigFullyAppliedCrossGroupIgnored(t *testing.T) {
 	tb := base.GetTestBucket(t)
 	defer tb.Close(ctx)
 
-	groupA := t.Name() + "-A"
-	groupB := t.Name() + "-B"
 	bucketName := tb.GetName()
 
 	twoCollectionScopesConfig := GetCollectionsConfig(t, tb, 2)
@@ -895,9 +868,9 @@ func TestIsConfigFullyAppliedCrossGroupIgnored(t *testing.T) {
 	rtA := NewRestTester(t, &RestTesterConfig{
 		CustomTestBucket: tb.NoCloseClone(),
 		PersistentConfig: true,
-		GroupID:          &groupA,
 	})
 	defer rtA.Close()
+	groupA := rtA.ServerContext().Config.Bootstrap.ConfigGroupID
 
 	dbConfigA := rtA.NewDbConfig()
 	dbConfigA.Scopes = collection1ScopesConfig
@@ -911,7 +884,6 @@ func TestIsConfigFullyAppliedCrossGroupIgnored(t *testing.T) {
 	rtB := NewRestTester(t, &RestTesterConfig{
 		CustomTestBucket: tb.NoCloseClone(),
 		PersistentConfig: true,
-		GroupID:          &groupB,
 	})
 	defer rtB.Close()
 
@@ -947,22 +919,14 @@ func TestIsConfigFullyAppliedCrossGroupIgnored(t *testing.T) {
 // and IsConfigFullyApplied for V1 becomes true.
 func TestIsConfigFullyAppliedRollback(t *testing.T) {
 	ctx := base.TestCtx(t)
-	tb := base.GetTestBucket(t)
-	defer tb.Close(ctx)
-
-	groupID := t.Name()
-	rtConfig := &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	}
-
-	rtA := NewRestTester(t, rtConfig)
-	defer rtA.Close()
-	rtB := NewRestTester(t, rtConfig)
-	defer rtB.Close()
+	rtc := NewRestTesterCluster(t, &RestTesterClusterConfig{
+		NumNodes: 2,
+	})
+	defer rtc.Close(ctx)
 
-	bucketName := tb.GetName()
+	rtA := rtc.Node(0)
+	rtB := rtc.Node(1)
+	bucketName := rtc.testBucket.GetName()
 
 	dbConfig := rtA.NewDbConfig()
 	resp := rtA.CreateDatabase("db", dbConfig)
@@ -989,11 +953,11 @@ func TestIsConfigFullyAppliedRollback(t *testing.T) {
 
 	registry := refreshAndGetRegistry(t, rtA, bucketName)
 
-	acked, _, err := registry.IsConfigFullyApplied(ctx, groupID, "db", versionV)
+	acked, _, err := registry.IsConfigFullyApplied(ctx, rtc.groupID, "db", versionV)
 	require.NoError(t, err)
 	assert.False(t, acked, "old version V should no longer be acked after rollback to V1")
 
-	acked, missing, err := registry.IsConfigFullyApplied(ctx, groupID, "db", versionV1)
+	acked, missing, err := registry.IsConfigFullyApplied(ctx, rtc.groupID, "db", versionV1)
 	require.NoError(t, err)
 	assert.True(t, acked, "new version V1 should be acked by both nodes; missing: %v", missing)
 	assert.Empty(t, missing)
@@ -1005,22 +969,14 @@ func TestIsConfigFullyAppliedRollback(t *testing.T) {
 // tracking should be removed from the second node's clusterCompatManager.
 func TestIsConfigFullyAppliedDeleteDBRemovesTracking(t *testing.T) {
 	ctx := base.TestCtx(t)
-	tb := base.GetTestBucket(t)
-	defer tb.Close(ctx)
-
-	groupID := t.Name()
-	rtConfig := &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	}
-
-	rtA := NewRestTester(t, rtConfig)
-	defer rtA.Close()
-	rtB := NewRestTester(t, rtConfig)
-	defer rtB.Close()
+	rtc := NewRestTesterCluster(t, &RestTesterClusterConfig{
+		NumNodes: 2,
+	})
+	defer rtc.Close(ctx)
 
-	bucketName := tb.GetName()
+	rtA := rtc.Node(0)
+	rtB := rtc.Node(1)
+	bucketName := rtc.testBucket.GetName()
 
 	dbConfig := rtA.NewDbConfig()
 	resp := rtA.CreateDatabase("db", dbConfig)
@@ -1110,7 +1066,6 @@ func TestClusterCompatMultipleDBsSameBucket(t *testing.T) {
 	tb := base.GetTestBucket(t)
 	defer tb.Close(ctx)
 
-	groupID := t.Name()
 	bucketName := tb.GetName()
 
 	twoCollectionScopesConfig := GetCollectionsConfig(t, tb, 2)
@@ -1124,7 +1079,6 @@ func TestClusterCompatMultipleDBsSameBucket(t *testing.T) {
 	rt := NewRestTester(t, &RestTesterConfig{
 		CustomTestBucket: tb.NoCloseClone(),
 		PersistentConfig: true,
-		GroupID:          &groupID,
 	})
 	defer rt.Close()
 
@@ -1162,27 +1116,23 @@ func TestClusterCompatMultipleDBsSameBucket(t *testing.T) {
 func TestClusterCompatApplyConfigsBatchVersions(t *testing.T) {
 	base.TestRequiresCollections(t)
 	ctx := base.TestCtx(t)
-	tb := base.GetTestBucket(t)
-	defer tb.Close(ctx)
+	rtc := NewRestTesterCluster(t, &RestTesterClusterConfig{
+		NumNodes: 2,
+	})
+	defer rtc.Close(ctx)
 
-	groupID := t.Name()
-	bucketName := tb.GetName()
+	rtA := rtc.Node(0)
+	rtB := rtc.Node(1)
+	bucketName := rtc.testBucket.GetName()
 
-	twoCollectionScopesConfig := GetCollectionsConfig(t, tb, 2)
+	twoCollectionScopesConfig := GetCollectionsConfig(t, rtc.testBucket, 2)
 	dataStoreNames := GetDataStoreNamesFromScopesConfig(twoCollectionScopesConfig)
 	scopeName := dataStoreNames[0].ScopeName()
 	collection1Name := dataStoreNames[0].CollectionName()
 	collection2Name := dataStoreNames[1].CollectionName()
 	collection1ScopesConfig := ScopesConfig{scopeName: ScopeConfig{Collections: map[string]*CollectionConfig{collection1Name: {}}}}
 	collection2ScopesConfig := ScopesConfig{scopeName: ScopeConfig{Collections: map[string]*CollectionConfig{collection2Name: {}}}}
 
-	rtA := NewRestTester(t, &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	})
-	defer rtA.Close()
-
 	dbConfig1 := rtA.NewDbConfig()
 	dbConfig1.Scopes = collection1ScopesConfig
 	resp := rtA.CreateDatabase("db1", dbConfig1)
@@ -1193,13 +1143,6 @@ func TestClusterCompatApplyConfigsBatchVersions(t *testing.T) {
 	resp = rtA.CreateDatabase("db2", dbConfig2)
 	RequireStatus(t, resp, http.StatusCreated)
 
-	rtB := NewRestTester(t, &RestTesterConfig{
-		CustomTestBucket: tb.NoCloseClone(),
-		PersistentConfig: true,
-		GroupID:          &groupID,
-	})
-	defer rtB.Close()
-
 	rtB.ServerContext().ForceDbConfigsReload(t, ctx)
 
 	ccmB := rtB.ServerContext().ClusterCompat
@@ -1228,16 +1171,14 @@ func TestIsConfigFullyAppliedNoEligibleAckersIntegration(t *testing.T) {
 	tb := base.GetTestBucket(t)
 	defer tb.Close(ctx)
 
-	groupA := t.Name() + "-A"
-	groupB := t.Name() + "-B"
 	bucketName := tb.GetName()
 
 	rt := NewRestTester(t, &RestTesterConfig{
 		CustomTestBucket: tb.NoCloseClone(),
 		PersistentConfig: true,
-		GroupID:          &groupA,
 	})
 	defer rt.Close()
+	groupA := rt.ServerContext().Config.Bootstrap.ConfigGroupID
 
 	dbConfig := rt.NewDbConfig()
 	resp := rt.CreateDatabase("db", dbConfig)
@@ -1252,8 +1193,9 @@ func TestIsConfigFullyAppliedNoEligibleAckersIntegration(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, acked, "group A node should have acked; missing: %v", missing)
 
-	acked, missing, err = registry.IsConfigFullyApplied(ctx, groupB, "db", dbVersion)
-	require.ErrorIs(t, err, ErrNoEligibleAckers, "group B has no nodes, should return ErrNoEligibleAckers")
+	// Any group with no registered nodes returns ErrNoEligibleAckers.
+	acked, missing, err = registry.IsConfigFullyApplied(ctx, t.Name(), "db", dbVersion)
+	require.ErrorIs(t, err, ErrNoEligibleAckers, "a group with no nodes should return ErrNoEligibleAckers")
 	assert.False(t, acked)
 	assert.Empty(t, missing)
 }