diff --git a/fdbrpc/HTTPServer.cpp b/fdbrpc/HTTPServer.cpp index 8ab4f4bddcb..597bb1d79e9 100644 --- a/fdbrpc/HTTPServer.cpp +++ b/fdbrpc/HTTPServer.cpp @@ -19,7 +19,9 @@ */ #include "fdbrpc/HTTP.h" +#include "flow/CodeProbe.h" #include "flow/IRandom.h" +#include "flow/Knobs.h" #include "flow/Trace.h" #include "fdbrpc/simulator.h" Future callbackHandler(Reference conn, @@ -159,6 +161,21 @@ void HTTP::SimRegisteredHandlerContext::removeIp(IPAddress ip) { updateDNS(); } +Future HTTP::SimRegisteredHandlerContext::delayedRemoveIp(IPAddress ip, double delaySeconds) { + CODE_PROBE(true, + "Simulated delayed DNS removal for dead process", + probe::context::sim2, + probe::assert::simOnly); + TraceEvent("SimDNSDelayedRemoval") + .detail("Hostname", hostname) + .detail("Service", service) + .detail("IP", ip.toString()) + .detail("DelaySecs", delaySeconds); + co_await ::delay(delaySeconds); + removeIp(ip); + co_return; +} + struct AlwaysFailRequestHandler final : HTTP::IRequestHandler, ReferenceCounted { Future handleRequest(Reference req, Reference response) override { diff --git a/fdbrpc/include/fdbrpc/HTTP.h b/fdbrpc/include/fdbrpc/HTTP.h index f4ff40826ec..5b0c991308c 100644 --- a/fdbrpc/include/fdbrpc/HTTP.h +++ b/fdbrpc/include/fdbrpc/HTTP.h @@ -177,6 +177,7 @@ struct SimRegisteredHandlerContext : ReferenceCounted delayedRemoveIp(IPAddress addr, double delay); private: std::vector addresses; diff --git a/fdbrpc/sim2.cpp b/fdbrpc/sim2.cpp index 594d4fa7516..30e71c62560 100644 --- a/fdbrpc/sim2.cpp +++ b/fdbrpc/sim2.cpp @@ -2223,9 +2223,18 @@ class Sim2 final : public ISimulator, public INetworkConnections { } ASSERT(found); - // FIXME: potentially instead delay removing from DNS for a bit so we still briefly try to talk to dead server - for (auto& it : httpHandlers) { - it.second->removeIp(p->address.ip); + double dnsRemovalDelay = FLOW_KNOBS->SIM_DNS_REMOVAL_MAX_DELAY; + if (dnsRemovalDelay > 0) { + // Simulate real-world DNS caching: keep stale DNS entries briefly so + // clients may still attempt to talk to the dead server's address. + double actualDelay = deterministicRandom()->random01() * dnsRemovalDelay; + for (auto& it : httpHandlers) { + uncancellable(it.second->delayedRemoveIp(p->address.ip, actualDelay)); + } + } else { + for (auto& it : httpHandlers) { + it.second->removeIp(p->address.ip); + } } } diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index fd9d3700d0c..60d93dcfa20 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -242,6 +242,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { init( MAX_BUGGIFIED_DELAY, 0 ); if( randomize && buggify() ) MAX_BUGGIFIED_DELAY = 0.2 * deterministicRandom()->random01(); init( MAX_RUNLOOP_SLEEP_DELAY, 0 ); init( SIM_CONNECT_ERROR_MODE, 0 ); if( randomize && buggify() ) SIM_CONNECT_ERROR_MODE = deterministicRandom()->randomInt(0,3); + init( SIM_DNS_REMOVAL_MAX_DELAY, 0 ); if( randomize && buggify() ) SIM_DNS_REMOVAL_MAX_DELAY = 2.0 * deterministicRandom()->random01(); //Tracefiles init( ZERO_LENGTH_FILE_PAD, 1 ); diff --git a/flow/include/flow/Knobs.h b/flow/include/flow/Knobs.h index 52165d133f0..ddc457f5313 100644 --- a/flow/include/flow/Knobs.h +++ b/flow/include/flow/Knobs.h @@ -296,6 +296,7 @@ class FlowKnobs : public KnobsImpl { double MAX_BUGGIFIED_DELAY; double MAX_RUNLOOP_SLEEP_DELAY; int SIM_CONNECT_ERROR_MODE; + double SIM_DNS_REMOVAL_MAX_DELAY; double SIM_SPEEDUP_AFTER_SECONDS; int MAX_TRACE_LINES;