Fix DD hang during multi-region failover by decoupling health reporting from remote TC startup

Author: MarkSh1

fdbserver/datadistributor/DDTeamCollection.actor.cpp

@@ -4470,8 +4470,8 @@ DDTeamCollection::DDTeamCollection(DDTeamCollectionInitParams const& params)
     output(params.output), unhealthyServers(0), storageWiggler(makeReference<StorageWiggler>(this)),
     processingWiggle(params.processingWiggle), shardsAffectedByTeamFailure(params.shardsAffectedByTeamFailure),
     initialFailureReactionDelay(
-        delayed(params.readyToStart, SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskPriority::DataDistribution)),
-    initializationDoneActor(logOnCompletion(params.readyToStart && initialFailureReactionDelay)), recruitingStream(0),
+        delayed(params.onHealthy, SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskPriority::DataDistribution)),
+    initializationDoneActor(logOnCompletion(params.onHealthy && initialFailureReactionDelay)), recruitingStream(0),
     restartRecruiting(SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY), healthyTeamCount(0),
     zeroHealthyTeams(params.zeroHealthyTeams), optimalTeamCount(0), zeroOptimalTeams(true), isTssRecruiting(false),
     includedDCs(params.includedDCs), otherTrackedDCs(params.otherTrackedDCs),
@@ -6266,6 +6266,7 @@ class DDTeamCollectionUnitTest {
 		                                                     {},
 		                                                     {},
 		                                                     Future<Void>(Void()),
+		                                                     Future<Void>(Void()),
 		                                                     makeReference<AsyncVar<bool>>(true),
 		                                                     IsPrimary::True,
 		                                                     makeReference<AsyncVar<bool>>(false),
@@ -6321,6 +6322,7 @@ class DDTeamCollectionUnitTest {
 		                                                     {},
 		                                                     {},
 		                                                     Future<Void>(Void()),
+		                                                     Future<Void>(Void()),
 		                                                     makeReference<AsyncVar<bool>>(true),
 		                                                     IsPrimary::True,
 		                                                     makeReference<AsyncVar<bool>>(false),

fdbserver/datadistributor/DataDistribution.cpp

@@ -2882,6 +2882,7 @@ Future<Void> dataDistribution(Reference<DataDistributor> self,
 			    self->primaryDcId,
 			    self->configuration.usableRegions > 1 ? self->remoteDcIds : std::vector<Optional<Key>>(),
 			    self->initialized.getFuture(),
+			    self->initialized.getFuture(),
 			    zeroHealthyTeams[0],
 			    IsPrimary::True,
 			    processingUnhealthy,
@@ -2908,10 +2909,10 @@ Future<Void> dataDistribution(Reference<DataDistributor> self,
 				                                self->configuration,
 				                                self->remoteDcIds,
 				                                Optional<std::vector<Optional<Key>>>(),
-				                                // In multi-region configurations DD only needs the cluster to
-				                                // reach ACCEPTING_COMMITS before continuing startup. Waiting for
-				                                // ALL_LOGS_RECRUITED can hang during degraded failover scenarios
-				                                // such as when the primary DC is down.
+				                                // Remote TC must wait for ALL_LOGS_RECRUITED before actually
+				                                // starting (building teams, moving data), but reports Healthy
+				                                // status at ACCEPTING_COMMITS for faster visibility.
+				                                self->initialized.getFuture() && remoteRecovered(self->dbInfo),
 				                                self->initialized.getFuture() && waitForAcceptingCommits(self->dbInfo),
 				                                zeroHealthyTeams[1],
 				                                IsPrimary::False,

fdbserver/datadistributor/include/fdbserver/datadistributor/DDTeamCollection.h

@@ -193,6 +193,7 @@ struct DDTeamCollectionInitParams {
 	std::vector<Optional<Key>> includedDCs;
 	Optional<std::vector<Optional<Key>>> otherTrackedDCs;
 	Future<Void> readyToStart;
+	Future<Void> onHealthy;
 	Reference<AsyncVar<bool>> zeroHealthyTeams;
 	IsPrimary primary;
 	Reference<AsyncVar<bool>> processingUnhealthy;

fdbserver/workloads/PerpetualWiggleStatsWorkload.cpp

@@ -199,6 +199,7 @@ struct PerpetualWiggleStatsWorkload : public TestWorkload {
 		                                {},
 		                                {},
 		                                Future<Void>(Void()),
+										Future<Void>(Void()),
 		                                makeReference<AsyncVar<bool>>(true),
 		                                IsPrimary::True,
 		                                makeReference<AsyncVar<bool>>(false),