Add configurable DNS removal delay in simulator

fdbrpc/HTTPServer.cpp

@@ -19,7 +19,9 @@
  */
 
 #include "fdbrpc/HTTP.h"
+#include "flow/CodeProbe.h"
 #include "flow/IRandom.h"
+#include "flow/Knobs.h"
 #include "flow/Trace.h"
 #include "fdbrpc/simulator.h"
 Future<Void> callbackHandler(Reference<IConnection> conn,
@@ -159,6 +161,21 @@ void HTTP::SimRegisteredHandlerContext::removeIp(IPAddress ip) {
 	updateDNS();
 }
 
+Future<Void> HTTP::SimRegisteredHandlerContext::delayedRemoveIp(IPAddress ip, double delaySeconds) {
+	CODE_PROBE(true,
+	           "Simulated delayed DNS removal for dead process",
+	           probe::context::sim2,
+	           probe::assert::simOnly);
+	TraceEvent("SimDNSDelayedRemoval")
+	    .detail("Hostname", hostname)
+	    .detail("Service", service)
+	    .detail("IP", ip.toString())
+	    .detail("DelaySecs", delaySeconds);
+	co_await ::delay(delaySeconds);
+	removeIp(ip);
+	co_return;
+}
+
 struct AlwaysFailRequestHandler final : HTTP::IRequestHandler, ReferenceCounted<AlwaysFailRequestHandler> {
 	Future<Void> handleRequest(Reference<HTTP::IncomingRequest> req,
 	                           Reference<HTTP::OutgoingResponse> response) override {

fdbrpc/include/fdbrpc/HTTP.h

@@ -177,6 +177,7 @@ struct SimRegisteredHandlerContext : ReferenceCounted<SimRegisteredHandlerContex
 
 	void addAddress(NetworkAddress addr);
 	void removeIp(IPAddress addr);
+	Future<Void> delayedRemoveIp(IPAddress addr, double delay);
 
 private:
 	std::vector<NetworkAddress> addresses;

fdbrpc/sim2.cpp

@@ -2223,9 +2223,18 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		}
 		ASSERT(found);
 
-		// FIXME: potentially instead delay removing from DNS for a bit so we still briefly try to talk to dead server
-		for (auto& it : httpHandlers) {
-			it.second->removeIp(p->address.ip);
+		double dnsRemovalDelay = FLOW_KNOBS->SIM_DNS_REMOVAL_MAX_DELAY;
+		if (dnsRemovalDelay > 0) {
+			// Simulate real-world DNS caching: keep stale DNS entries briefly so
+			// clients may still attempt to talk to the dead server's address.
+			double actualDelay = deterministicRandom()->random01() * dnsRemovalDelay;
+			for (auto& it : httpHandlers) {
+				uncancellable(it.second->delayedRemoveIp(p->address.ip, actualDelay));
+			}
+		} else {
+			for (auto& it : httpHandlers) {
+				it.second->removeIp(p->address.ip);
+			}
 		}
 	}
 

flow/Knobs.cpp

@@ -242,6 +242,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( MAX_BUGGIFIED_DELAY,                                   0 ); if( randomize && buggify() ) MAX_BUGGIFIED_DELAY =  0.2 * deterministicRandom()->random01();
 	init( MAX_RUNLOOP_SLEEP_DELAY,                               0 );
 	init( SIM_CONNECT_ERROR_MODE,                                0 ); if( randomize && buggify() ) SIM_CONNECT_ERROR_MODE = deterministicRandom()->randomInt(0,3);
+	init( SIM_DNS_REMOVAL_MAX_DELAY,                             0 ); if( randomize && buggify() ) SIM_DNS_REMOVAL_MAX_DELAY = 2.0 * deterministicRandom()->random01();
 
 	//Tracefiles
 	init( ZERO_LENGTH_FILE_PAD,                                  1 );

flow/include/flow/Knobs.h

@@ -296,6 +296,7 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 	double MAX_BUGGIFIED_DELAY;
 	double MAX_RUNLOOP_SLEEP_DELAY;
 	int SIM_CONNECT_ERROR_MODE;
+	double SIM_DNS_REMOVAL_MAX_DELAY;
 	double SIM_SPEEDUP_AFTER_SECONDS;
 	int MAX_TRACE_LINES;