Prune runtime-deleted object dictionary every 15 Minutes

Author: jschmidt-icinga

lib/remote/apilistener-configsync.cpp

@@ -552,6 +552,38 @@ void ApiListener::SendRuntimeConfigObjects(const JsonRpcConnection::Ptr& aclient
 		<< "Finished syncing runtime objects to endpoint '" << endpoint->GetName() << "'.";
 }
 
+/**
+ * Prunes the list of deleted runtime objects.
+ *
+ * This takes into account the longest log duration of all the endpoints in the cluster and
+ * then deletes the entries for objects that are older than that.
+ *
+ * The reasoning is that once a deletion is older than log duration of an endpoint it is
+ * unlikely that we could still get any delayed updates to the object versions that were deleted.
+ * At that point, either all endpoints are up-to-date, or the problematic messages have been
+ * dropped from the replay log anyway.
+ */
+void ApiListener::PruneDeletedRuntimeObjects()
+{
+	auto deletedRuntimeObjects = GetDeletedRuntimeObjects();
+
+	double maxLogDuration = 0;
+	for (const auto &endpoint : ConfigType::GetObjectsByType<Endpoint>()) {
+		maxLogDuration = std::max(maxLogDuration, endpoint->GetLogDuration());
+	}
+	double cutoff = Utility::GetTime() - maxLogDuration;
+
+	ObjectLock lock(deletedRuntimeObjects);
+
+	for (auto it = deletedRuntimeObjects->Begin(); it != deletedRuntimeObjects->End();) {
+		if (it->second < cutoff) {
+			it = deletedRuntimeObjects->Remove(it);
+		} else {
+			it++;
+		};
+	}
+}
+
 static String MakeDeletionTimestampKey(const String& typeName, const String& objName)
 {
 	return typeName + ":" + objName;

lib/remote/apilistener.cpp

@@ -297,6 +297,12 @@ void ApiListener::Start(bool runtimeCreated)
 	m_Timer->Start();
 	m_Timer->Reschedule(0);
 
+	m_DeletedRuntimeObjectsTimer = Timer::Create();
+	m_DeletedRuntimeObjectsTimer->OnTimerExpired.connect([this](const Timer* const&) { PruneDeletedRuntimeObjects(); });
+	m_DeletedRuntimeObjectsTimer->SetInterval(15 * 60);
+	m_DeletedRuntimeObjectsTimer->Start();
+	m_DeletedRuntimeObjectsTimer->Reschedule(0);
+
 	m_ReconnectTimer = Timer::Create();
 	m_ReconnectTimer->OnTimerExpired.connect([this](const Timer * const&) { ApiReconnectTimerHandler(); });
 	m_ReconnectTimer->SetInterval(10);
@@ -379,6 +385,7 @@ void ApiListener::Stop(bool runtimeDeleted)
 	m_CleanupCertificateRequestsTimer->Stop(true);
 	m_AuthorityTimer->Stop(true);
 	m_ReconnectTimer->Stop(true);
+	m_DeletedRuntimeObjectsTimer->Stop(true);
 	m_Timer->Stop(true);
 	m_RenewOwnCertTimer->Stop(true);
 

lib/remote/apilistener.hpp

@@ -183,6 +183,7 @@ class ApiListener final : public ObjectImpl<ApiListener>
 	std::set<HttpServerConnection::Ptr> m_HttpClients;
 
 	Timer::Ptr m_Timer;
+	Timer::Ptr m_DeletedRuntimeObjectsTimer;
 	Timer::Ptr m_ReconnectTimer;
 	Timer::Ptr m_AuthorityTimer;
 	Timer::Ptr m_CleanupCertificateRequestsTimer;
@@ -204,6 +205,7 @@ class ApiListener final : public ObjectImpl<ApiListener>
 	void CleanupCertificateRequestsTimerHandler();
 	void CheckApiPackageIntegrity();
 
+	void PruneDeletedRuntimeObjects();
 	double GetRuntimeObjectDeletionTs(const String& typeName, const String& objName);
 	bool UpdateRuntimeObjectDeletionTs(const String& typeName, const String& objName, double ts);