ApiListener: Sync runtime configs in order

lib/remote/apilistener-configsync.cpp

@@ -5,11 +5,14 @@
 #include "remote/configobjectutility.hpp"
 #include "remote/jsonrpc.hpp"
 #include "base/configtype.hpp"
-#include "base/json.hpp"
 #include "base/convert.hpp"
+#include "base/dependencygraph.hpp"
+#include "base/json.hpp"
 #include "config/vmops.hpp"
 #include "remote/configobjectslock.hpp"
 #include <fstream>
+#include <set>
+#include <functional>
 
 using namespace icinga;
 
@@ -393,6 +396,53 @@ void ApiListener::UpdateConfigObject(const ConfigObject::Ptr& object, const Mess
 	}
 }
 
+/**
+ * Syncs the specified object and its direct and intermediate parents to the provided client
+ * in topological order of their dependency graph recursively.
+ *
+ * Objects that the client does not have access to are skipped without going through their dependency graph.
+ * Likewise, if the origin parameter is set, the object parent graph will not be traversed; instead, the object
+ * will be directly forwarded to the ApiListener::UpdateConfigObject method.
+ *
+ * @param object ConfigObject::Ptr The config object you want to sync.
+ * @param origin MessageOrigin::Ptr The source of that config update event; pass nullptr if it's from local endpoint.
+ * @param client JsonRpcConnection::Ptr The JsonRpc client you send the update to.
+ * @param syncedObjects std::unordered_set<ConfigObject*> Used to cache the already synced objects.
+ *                      It won't be used if the origin parameter is set.
+ */
+void ApiListener::UpdateConfigObjectWithParents(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
+	const JsonRpcConnection::Ptr& client, std::unordered_set<ConfigObject*>& syncedObjects)
+{
+	if (!origin) {
+		// Don't sync already synced objects.
+		if (syncedObjects.find(object.get()) != syncedObjects.end()) {
+			return;
+		}
+
+		Endpoint::Ptr endpoint = client->GetEndpoint();
+		ASSERT(endpoint);
+
+		Zone::Ptr azone = endpoint->GetZone();
+
+		/* don't sync objects for non-matching parent-child zones */
+		if (!azone->CanAccessObject(object)) {
+			return;
+		}
+		syncedObjects.emplace(object.get());
+
+		for (const Object::Ptr& parent : DependencyGraph::GetParents(object)) {
+			// Actually, the following dynamic cast should never fail, since the DependencyGraph class
+			// expects the types to always be of type Object::Ptr and such an object is supposed to always
+			// point to an instance of the specific derived ConfigObject class. See TypeHelper<>::GetFactory().
+			if (ConfigObject::Ptr parentObj = dynamic_pointer_cast<ConfigObject>(parent)) {
+				UpdateConfigObjectWithParents(parentObj, nullptr, client, syncedObjects);
+			}
+		}
+	}
+
+	/* send the config object to the connected client */
+	UpdateConfigObject(object, origin, client);
+}
 
 void ApiListener::DeleteConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
 	const JsonRpcConnection::Ptr& client)
@@ -454,19 +504,17 @@ void ApiListener::SendRuntimeConfigObjects(const JsonRpcConnection::Ptr& aclient
 	Log(LogInformation, "ApiListener")
 		<< "Syncing runtime objects to endpoint '" << endpoint->GetName() << "'.";
 
-	for (const Type::Ptr& type : Type::GetAllTypes()) {
-		auto *dtype = dynamic_cast<ConfigType *>(type.get());
-
-		if (!dtype)
-			continue;
-
-		for (const ConfigObject::Ptr& object : dtype->GetObjects()) {
-			/* don't sync objects for non-matching parent-child zones */
-			if (!azone->CanAccessObject(object))
-				continue;
-
-			/* send the config object to the connected client */
-			UpdateConfigObject(object, nullptr, aclient);
+	std::unordered_set<ConfigObject*> syncedObjects;
+	for (const Type::Ptr& type : Type::GetConfigTypesSortedByLoadDependencies()) {
+		if (auto *ctype = dynamic_cast<ConfigType *>(type.get()); ctype) {
+			for (const ConfigObject::Ptr& object : ctype->GetObjects()) {
+				// All objects must be synced sorted by their dependency graph.
+				// Otherwise, downtimes/comments etc. might get synced before their respective Checkables, which will
+				// result in comments and downtimes being ignored by the other endpoint since it does not yet know
+				// about their checkables. Given that the runtime config updates event does not trigger a reload on the
+				// remote endpoint, these objects won't be synced again until the next reload.
+				UpdateConfigObjectWithParents(object, nullptr, aclient, syncedObjects);
+			}
 		}
 	}
 

lib/remote/apilistener.hpp

@@ -247,6 +247,8 @@ class ApiListener final : public ObjectImpl<ApiListener>
 	/* configsync */
 	void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
 		const JsonRpcConnection::Ptr& client = nullptr);
+	void UpdateConfigObjectWithParents(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
+		const JsonRpcConnection::Ptr& client = nullptr, std::unordered_set<ConfigObject*>& syncedObjects);
 	void DeleteConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
 		const JsonRpcConnection::Ptr& client = nullptr);
 	void SendRuntimeConfigObjects(const JsonRpcConnection::Ptr& aclient);