From 5b4250866b08aee9a0e7bb90d8197101b1a41cbe Mon Sep 17 00:00:00 2001 From: Daniel Edwards Date: Thu, 1 Aug 2024 11:51:47 +0200 Subject: [PATCH] metrics: add experimental internal metrics (#85) Issue: SILKIT-1379 Subject: Creation, collection, sending, receiving, and storing various metrics as JSON. The metrics can be used to get insights into internal behavior of individual SIL Kit participants or simulations. Currently available metrics: COUNTER: Collects single unsigned integer value. STATISTICS: Collects single double values and accumulates minimum, maximum, exponential incremental mean, and standard deviation. STRING_LIST: Collects a list of strings. If metrics are sent to remote participants, the participant that collects these, should be started first. The registry also supports collection of remote metrics, which is preferable, since it must be started first anyway. Collection and storage of the metrics is configured via the participant configuration file (or the registry configuration file). Metrics are only sent if they have changed. --- SilKit/source/CMakeLists.txt | 2 + SilKit/source/config/Configuration.hpp | 1 + .../config/ParticipantConfiguration.hpp | 30 ++ .../ParticipantConfigurationFromXImpl.cpp | 105 +++++- .../config/ParticipantConfiguration_Full.json | 13 + .../config/ParticipantConfiguration_Full.yaml | 10 + .../config/Test_ParticipantConfiguration.cpp | 13 + SilKit/source/config/YamlConversion.cpp | 90 +++++ SilKit/source/config/YamlConversion.hpp | 4 + SilKit/source/config/YamlSchema.cpp | 88 +++-- SilKit/source/core/internal/CMakeLists.txt | 1 + .../core/internal/IParticipantInternal.hpp | 21 +- .../core/internal/ServiceConfigKeys.hpp | 2 + .../internal/traits/SilKitMsgSerdesName.hpp | 1 + .../core/internal/traits/SilKitMsgTraits.hpp | 1 + .../core/internal/traits/SilKitMsgVersion.hpp | 1 + .../traits/SilKitServiceConfigTraits.hpp | 4 + .../internal/traits/SilKitServices_fwd.hpp | 5 + .../NullConnectionParticipant.cpp | 3 +- .../core/mock/participant/MockParticipant.hpp | 85 +++++ SilKit/source/core/participant/CMakeLists.txt | 1 + .../source/core/participant/Participant.hpp | 33 +- .../core/participant/Participant_impl.hpp | 160 +++++++- SilKit/source/core/vasio/CMakeLists.txt | 1 + .../source/core/vasio/SerializedMessage.hpp | 1 + .../core/vasio/Test_VAsioConnection.cpp | 3 +- SilKit/source/core/vasio/VAsioConnection.cpp | 33 +- SilKit/source/core/vasio/VAsioConnection.hpp | 9 +- SilKit/source/core/vasio/VAsioRegistry.cpp | 71 +++- SilKit/source/core/vasio/VAsioRegistry.hpp | 13 + SilKit/source/services/CMakeLists.txt | 1 + SilKit/source/services/logging/Logger.cpp | 14 +- .../services/logging/SilKitFmtFormatters.hpp | 4 + SilKit/source/services/metrics/CMakeLists.txt | 64 ++++ ...tricsSinksFromParticipantConfiguration.cpp | 62 ++++ ...tricsSinksFromParticipantConfiguration.hpp | 23 ++ .../services/metrics/ICounterMetric.hpp | 18 + .../services/metrics/IMetricsManager.hpp | 24 ++ .../services/metrics/IMetricsProcessor.hpp | 19 + .../services/metrics/IMetricsSender.hpp | 17 + .../source/services/metrics/IMetricsSink.hpp | 22 ++ .../services/metrics/IMetricsTimerThread.hpp | 21 ++ .../metrics/IMsgForMetricsReceiver.hpp | 30 ++ .../services/metrics/IMsgForMetricsSender.hpp | 30 ++ .../services/metrics/IStatisticMetric.hpp | 15 + .../services/metrics/IStringListMetric.hpp | 18 + SilKit/source/services/metrics/Metrics.hpp | 27 ++ .../services/metrics/MetricsDatatypes.cpp | 62 ++++ .../services/metrics/MetricsDatatypes.hpp | 56 +++ .../services/metrics/MetricsJsonSink.cpp | 56 +++ .../services/metrics/MetricsJsonSink.hpp | 27 ++ .../services/metrics/MetricsManager.cpp | 338 +++++++++++++++++ .../services/metrics/MetricsManager.hpp | 81 ++++ .../services/metrics/MetricsProcessor.cpp | 84 +++++ .../services/metrics/MetricsProcessor.hpp | 47 +++ .../services/metrics/MetricsReceiver.cpp | 51 +++ .../services/metrics/MetricsReceiver.hpp | 50 +++ .../services/metrics/MetricsRemoteSink.cpp | 29 ++ .../services/metrics/MetricsRemoteSink.hpp | 26 ++ .../source/services/metrics/MetricsSender.cpp | 45 +++ .../source/services/metrics/MetricsSender.hpp | 43 +++ .../source/services/metrics/MetricsSerdes.cpp | 33 ++ .../source/services/metrics/MetricsSerdes.hpp | 19 + .../services/metrics/MetricsTimerThread.cpp | 96 +++++ .../services/metrics/MetricsTimerThread.hpp | 36 ++ .../services/metrics/Test_MetricsJsonSink.cpp | 105 ++++++ .../services/metrics/Test_MetricsManager.cpp | 122 ++++++ .../metrics/Test_MetricsProcessor.cpp | 159 ++++++++ .../metrics/Test_MetricsRemoteSink.cpp | 51 +++ .../services/metrics/Test_MetricsSerdes.cpp | 38 ++ .../orchestration/TimeSyncService.cpp | 26 +- .../orchestration/TimeSyncService.hpp | 5 + .../source/services/rpc/RpcTestUtilities.hpp | 3 +- SilKit/source/util/CMakeLists.txt | 16 +- SilKit/source/util/ExecutionEnvironment.cpp | 346 ++++++++++++++++++ SilKit/source/util/ExecutionEnvironment.hpp | 25 ++ SilKit/source/util/StringHelpers.cpp | 135 ++++--- SilKit/source/util/StringHelpers.hpp | 9 + SilKit/source/util/tests/Test_Filesystem.cpp | 35 +- Utilities/SilKitRegistry/Registry.cpp | 2 + .../config/RegistryConfiguration.cpp | 12 + .../config/RegistryConfiguration.hpp | 9 + .../config/RegistryYamlConversion.cpp | 30 ++ .../config/RegistryYamlConversion.hpp | 1 + 84 files changed, 3382 insertions(+), 140 deletions(-) create mode 100644 SilKit/source/services/metrics/CMakeLists.txt create mode 100644 SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.cpp create mode 100644 SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.hpp create mode 100644 SilKit/source/services/metrics/ICounterMetric.hpp create mode 100644 SilKit/source/services/metrics/IMetricsManager.hpp create mode 100644 SilKit/source/services/metrics/IMetricsProcessor.hpp create mode 100644 SilKit/source/services/metrics/IMetricsSender.hpp create mode 100644 SilKit/source/services/metrics/IMetricsSink.hpp create mode 100644 SilKit/source/services/metrics/IMetricsTimerThread.hpp create mode 100644 SilKit/source/services/metrics/IMsgForMetricsReceiver.hpp create mode 100644 SilKit/source/services/metrics/IMsgForMetricsSender.hpp create mode 100644 SilKit/source/services/metrics/IStatisticMetric.hpp create mode 100644 SilKit/source/services/metrics/IStringListMetric.hpp create mode 100644 SilKit/source/services/metrics/Metrics.hpp create mode 100644 SilKit/source/services/metrics/MetricsDatatypes.cpp create mode 100644 SilKit/source/services/metrics/MetricsDatatypes.hpp create mode 100644 SilKit/source/services/metrics/MetricsJsonSink.cpp create mode 100644 SilKit/source/services/metrics/MetricsJsonSink.hpp create mode 100644 SilKit/source/services/metrics/MetricsManager.cpp create mode 100644 SilKit/source/services/metrics/MetricsManager.hpp create mode 100644 SilKit/source/services/metrics/MetricsProcessor.cpp create mode 100644 SilKit/source/services/metrics/MetricsProcessor.hpp create mode 100644 SilKit/source/services/metrics/MetricsReceiver.cpp create mode 100644 SilKit/source/services/metrics/MetricsReceiver.hpp create mode 100644 SilKit/source/services/metrics/MetricsRemoteSink.cpp create mode 100644 SilKit/source/services/metrics/MetricsRemoteSink.hpp create mode 100644 SilKit/source/services/metrics/MetricsSender.cpp create mode 100644 SilKit/source/services/metrics/MetricsSender.hpp create mode 100644 SilKit/source/services/metrics/MetricsSerdes.cpp create mode 100644 SilKit/source/services/metrics/MetricsSerdes.hpp create mode 100644 SilKit/source/services/metrics/MetricsTimerThread.cpp create mode 100644 SilKit/source/services/metrics/MetricsTimerThread.hpp create mode 100644 SilKit/source/services/metrics/Test_MetricsJsonSink.cpp create mode 100644 SilKit/source/services/metrics/Test_MetricsManager.cpp create mode 100644 SilKit/source/services/metrics/Test_MetricsProcessor.cpp create mode 100644 SilKit/source/services/metrics/Test_MetricsRemoteSink.cpp create mode 100644 SilKit/source/services/metrics/Test_MetricsSerdes.cpp create mode 100644 SilKit/source/util/ExecutionEnvironment.cpp create mode 100644 SilKit/source/util/ExecutionEnvironment.hpp diff --git a/SilKit/source/CMakeLists.txt b/SilKit/source/CMakeLists.txt index cdd558eaf..322639c43 100644 --- a/SilKit/source/CMakeLists.txt +++ b/SilKit/source/CMakeLists.txt @@ -183,7 +183,9 @@ list(APPEND SilKitImplObjectLibraries O_SilKit_Services_Orchestration O_SilKit_Services_PubSub O_SilKit_Services_Rpc + O_SilKit_Services_Metrics O_SilKit_Tracing + O_SilKit_Util O_SilKit_Util_FileHelpers O_SilKit_Util_StringHelpers O_SilKit_Util_Filesystem diff --git a/SilKit/source/config/Configuration.hpp b/SilKit/source/config/Configuration.hpp index c6f7ec6f3..a4e9fe826 100644 --- a/SilKit/source/config/Configuration.hpp +++ b/SilKit/source/config/Configuration.hpp @@ -168,6 +168,7 @@ struct SimulatedNetwork inline bool operator==(const Sink& lhs, const Sink& rhs); inline bool operator<(const Sink& lhs, const Sink& rhs); inline bool operator>(const Sink& lhs, const Sink& rhs); + inline bool operator==(const Logging& lhs, const Logging& rhs); inline bool operator==(const TraceSink& lhs, const TraceSink& rhs); inline bool operator==(const TraceSource& lhs, const TraceSource& rhs); diff --git a/SilKit/source/config/ParticipantConfiguration.hpp b/SilKit/source/config/ParticipantConfiguration.hpp index c521d64f4..048572973 100644 --- a/SilKit/source/config/ParticipantConfiguration.hpp +++ b/SilKit/source/config/ParticipantConfiguration.hpp @@ -229,6 +229,30 @@ struct Tracing std::vector traceSources; }; +// ================================================================================ +// Metrics service +// ================================================================================ + +struct MetricsSink +{ + enum class Type + { + Undefined, + JsonFile, + Remote, + }; + + Type type{Type::Undefined}; + std::string name; +}; + +//! \brief Metrics configuration +struct Metrics +{ + std::vector sinks; + bool collectFromRemote{false}; +}; + // ================================================================================ // Extensions // ================================================================================ @@ -283,6 +307,7 @@ struct TimeSynchronization struct Experimental { TimeSynchronization timeSynchronization; + Metrics metrics; }; // ================================================================================ @@ -336,12 +361,17 @@ bool operator==(const RpcServer& lhs, const RpcServer& rhs); bool operator==(const RpcClient& lhs, const RpcClient& rhs); bool operator==(const HealthCheck& lhs, const HealthCheck& rhs); bool operator==(const Tracing& lhs, const Tracing& rhs); +bool operator==(const MetricsSink& lhs, const MetricsSink& rhs); +bool operator==(const Metrics& lhs, const Metrics& rhs); bool operator==(const Extensions& lhs, const Extensions& rhs); bool operator==(const Middleware& lhs, const Middleware& rhs); bool operator==(const ParticipantConfiguration& lhs, const ParticipantConfiguration& rhs); bool operator==(const TimeSynchronization& lhs, const TimeSynchronization& rhs); bool operator==(const Experimental& lhs, const Experimental& rhs); +bool operator<(const MetricsSink& lhs, const MetricsSink& rhs); +bool operator>(const MetricsSink& lhs, const MetricsSink& rhs); + } // namespace v1 } // namespace Config } // namespace SilKit diff --git a/SilKit/source/config/ParticipantConfigurationFromXImpl.cpp b/SilKit/source/config/ParticipantConfigurationFromXImpl.cpp index 305993e24..6edc0230a 100644 --- a/SilKit/source/config/ParticipantConfigurationFromXImpl.cpp +++ b/SilKit/source/config/ParticipantConfigurationFromXImpl.cpp @@ -76,9 +76,18 @@ struct TimeSynchronizationCache SilKit::Util::Optional animationFactor; }; +struct MetricsCache +{ + SilKit::Util::Optional collectFromRemote; + std::set jsonFileSinks; + std::set fileNames; + SilKit::Util::Optional remoteSink; +}; + struct ExperimentalCache { TimeSynchronizationCache timeSynchronizationCache; + MetricsCache metricsCache; }; struct ConfigIncludeData @@ -368,12 +377,66 @@ void CacheTimeSynchronization(const YAML::Node& root, TimeSynchronizationCache& PopulateCacheField(root, "TimeSynchronization", "AnimationFactor", cache.animationFactor); } +void CacheMetrics(const YAML::Node& root, MetricsCache& cache) +{ + PopulateCacheField(root, "Metrics", "CollectFromRemote", cache.collectFromRemote); + + if (root["Sinks"]) + { + for (const auto& sinkNode : root["Sinks"]) + { + auto sink = parse_as(sinkNode); + if (sink.type == MetricsSink::Type::JsonFile) + { + if (cache.fileNames.count(sink.name) == 0) + { + cache.jsonFileSinks.insert(sink); + cache.fileNames.insert(sink.name); + } + else + { + std::stringstream error_msg; + error_msg << "JSON file metrics sink " << sink.name << " already exists!"; + throw SilKit::ConfigurationError(error_msg.str()); + } + } + else if (sink.type == MetricsSink::Type::Remote) + { + if (!cache.remoteSink.has_value()) + { + // Replace the already included sink with this one + // since we have not set it yet + cache.remoteSink = sink; + } + else + { + std::stringstream error_msg; + error_msg << "Remote metrics sink already exists!"; + throw SilKit::ConfigurationError(error_msg.str()); + } + } + else + { + std::stringstream error_msg; + error_msg << "Invalid MetricsSink::Type(" + << static_cast>(sink.type) << ")"; + throw SilKit::ConfigurationError(error_msg.str()); + } + } + } +} + void CacheExperimental(const YAML::Node& root, ExperimentalCache& cache) { if (root["TimeSynchronization"]) { CacheTimeSynchronization(root["TimeSynchronization"], cache.timeSynchronizationCache); } + + if (root["Metrics"]) + { + CacheMetrics(root["Metrics"], cache.metricsCache); + } } void PopulateCaches(const YAML::Node& config, ConfigIncludeData& configIncludeData) @@ -491,9 +554,27 @@ void MergeTimeSynchronizationCache(const TimeSynchronizationCache& cache, TimeSy MergeCacheField(cache.animationFactor, timeSynchronization.animationFactor); } +void MergeMetricsCache(const MetricsCache& cache, Metrics& metrics) +{ + MergeCacheField(cache.collectFromRemote, metrics.collectFromRemote); + MergeCacheSet(cache.jsonFileSinks, metrics.sinks); + + if (cache.remoteSink.has_value() && metrics.collectFromRemote) + { + throw SilKit::ConfigurationError{ + "Cannot have 'Remote' metrics sink together with 'CollectFromRemote' being true"}; + } + + if (cache.remoteSink.has_value()) + { + metrics.sinks.push_back(cache.remoteSink.value()); + } +} + void MergeExperimentalCache(const ExperimentalCache& cache, Experimental& experimental) { MergeTimeSynchronizationCache(cache.timeSynchronizationCache, experimental.timeSynchronization); + MergeMetricsCache(cache.metricsCache, experimental.metrics); } @@ -675,6 +756,16 @@ bool operator==(const Tracing& lhs, const Tracing& rhs) return lhs.traceSinks == rhs.traceSinks && lhs.traceSources == rhs.traceSources; } +bool operator==(const MetricsSink& lhs, const MetricsSink& rhs) +{ + return lhs.type == rhs.type && lhs.name == rhs.name; +} + +bool operator==(const Metrics& lhs, const Metrics& rhs) +{ + return lhs.sinks == rhs.sinks && lhs.collectFromRemote == rhs.collectFromRemote; +} + bool operator==(const Extensions& lhs, const Extensions& rhs) { return lhs.searchPathHints == rhs.searchPathHints; @@ -696,7 +787,7 @@ bool operator==(const ParticipantConfiguration& lhs, const ParticipantConfigurat && lhs.flexrayControllers == rhs.flexrayControllers && lhs.dataPublishers == rhs.dataPublishers && lhs.dataSubscribers == rhs.dataSubscribers && lhs.rpcClients == rhs.rpcClients && lhs.rpcServers == rhs.rpcServers && lhs.logging == rhs.logging && lhs.healthCheck == rhs.healthCheck - && lhs.tracing == rhs.tracing && lhs.extensions == rhs.extensions; + && lhs.tracing == rhs.tracing && lhs.extensions == rhs.extensions && lhs.experimental == rhs.experimental; } bool operator==(const TimeSynchronization& lhs, const TimeSynchronization& rhs) @@ -706,7 +797,17 @@ bool operator==(const TimeSynchronization& lhs, const TimeSynchronization& rhs) bool operator==(const Experimental& lhs, const Experimental& rhs) { - return lhs.timeSynchronization == rhs.timeSynchronization; + return lhs.timeSynchronization == rhs.timeSynchronization && lhs.metrics == rhs.metrics; +} + +bool operator<(const MetricsSink& lhs, const MetricsSink& rhs) +{ + return std::make_tuple(lhs.type, lhs.name) < std::make_tuple(rhs.type, rhs.name); +} + +bool operator>(const MetricsSink& lhs, const MetricsSink& rhs) +{ + return !(lhs < rhs); } } // namespace v1 diff --git a/SilKit/source/config/ParticipantConfiguration_Full.json b/SilKit/source/config/ParticipantConfiguration_Full.json index bf5cb003f..6009b8af6 100644 --- a/SilKit/source/config/ParticipantConfiguration_Full.json +++ b/SilKit/source/config/ParticipantConfiguration_Full.json @@ -233,6 +233,19 @@ "Experimental": { "TimeSynchronization": { "AnimationFactor": 1.5 + }, + "Metrics": { + "CollectFromRemote": false, + "Sinks": [ + { + "Type": "JsonFile", + "Name": "MyJsonMetrics" + }, + { + "Type": "Remote", + "Name": "MyRemoteMetricsSink" + } + ] } } } \ No newline at end of file diff --git a/SilKit/source/config/ParticipantConfiguration_Full.yaml b/SilKit/source/config/ParticipantConfiguration_Full.yaml index f5408b313..484d5d9bd 100644 --- a/SilKit/source/config/ParticipantConfiguration_Full.yaml +++ b/SilKit/source/config/ParticipantConfiguration_Full.yaml @@ -167,3 +167,13 @@ Middleware: TcpReceiveBufferSize: 3456 RegistryAsFallbackProxy: false ConnectTimeoutSeconds: 1.234 +Experimental: + TimeSynchronization: + AnimationFactor: 1.5 + Metrics: + CollectFromRemote: false + Sinks: + - Type: JsonFile + Name: MyJsonMetrics + - Type: Remote + Name: MyRemoteMetricsSink \ No newline at end of file diff --git a/SilKit/source/config/Test_ParticipantConfiguration.cpp b/SilKit/source/config/Test_ParticipantConfiguration.cpp index 43b7569ea..349e27ff2 100644 --- a/SilKit/source/config/Test_ParticipantConfiguration.cpp +++ b/SilKit/source/config/Test_ParticipantConfiguration.cpp @@ -255,4 +255,17 @@ TEST_F(Test_ParticipantConfiguration, full_configuration_file_json_yaml_equal) ASSERT_TRUE(participantConfigJson == participantConfigYaml); } +TEST_F(Test_ParticipantConfiguration, remote_metric_sink_collect_from_remote_fails) +{ + constexpr auto configurationString = R"( +Metrics: + CollectFromRemote: true + Sinks: + - Type: Remote +)"; + + ASSERT_THROW(SilKit::Config::ParticipantConfigurationFromStringImpl(configurationString), + SilKit::ConfigurationError); +} + } // anonymous namespace diff --git a/SilKit/source/config/YamlConversion.cpp b/SilKit/source/config/YamlConversion.cpp index e06522c5f..58b59b094 100644 --- a/SilKit/source/config/YamlConversion.cpp +++ b/SilKit/source/config/YamlConversion.cpp @@ -964,6 +964,94 @@ bool Converter::decode(const Node& node, TraceSource::Type& obj) return true; } +// Metrics + +template <> +Node Converter::encode(const MetricsSink::Type& obj) +{ + Node node; + switch (obj) + { + case MetricsSink::Type::Undefined: + node = "Undefined"; + break; + case MetricsSink::Type::JsonFile: + node = "JsonFile"; + break; + case MetricsSink::Type::Remote: + node = "Remote"; + break; + default: + throw ConfigurationError{"Unknown MetricsSink Type"}; + } + return node; +} + +template <> +bool Converter::decode(const Node& node, MetricsSink::Type& obj) +{ + auto&& str = parse_as(node); + if (str == "Undefined" || str == "") + { + obj = MetricsSink::Type::Undefined; + } + else if (str == "JsonFile") + { + obj = MetricsSink::Type::JsonFile; + } + else if (str == "Remote") + { + obj = MetricsSink::Type::Remote; + } + else + { + throw ConversionError{node, "Unknown MetricsSink::Type: " + str + "."}; + } + return true; +} + +template <> +Node Converter::encode(const MetricsSink& obj) +{ + Node node; + node["Type"] = obj.type; + if (!obj.name.empty()) + { + node["Name"] = obj.name; + } + return node; +} + +template <> +bool Converter::decode(const Node& node, MetricsSink& obj) +{ + obj.type = parse_as(node["Type"]); + optional_decode(obj.name, node, "Name"); + return true; +} + +template <> +Node Converter::encode(const Metrics& obj) +{ + Node node; + optional_encode(obj.sinks, node, "Sinks"); + if (obj.collectFromRemote) + { + node["CollectFromRemote"] = obj.collectFromRemote; + } + return node; +} + +template <> +bool Converter::decode(const Node& node, Metrics& obj) +{ + optional_decode(obj.sinks, node, "Sinks"); + optional_decode(obj.collectFromRemote, node, "CollectFromRemote"); + return true; +} + +// Extensions + template <> Node Converter::encode(const Extensions& obj) { @@ -1038,12 +1126,14 @@ Node Converter::encode(const Experimental& obj) Node node; non_default_encode(obj.timeSynchronization, node, "TimeSynchronization", defaultObj.timeSynchronization); + non_default_encode(obj.metrics, node, "Metrics", defaultObj.metrics); return node; } template <> bool Converter::decode(const Node& node, Experimental& obj) { optional_decode(obj.timeSynchronization, node, "TimeSynchronization"); + optional_decode(obj.metrics, node, "Metrics"); return true; } diff --git a/SilKit/source/config/YamlConversion.hpp b/SilKit/source/config/YamlConversion.hpp index 654f8b5ab..48a98b627 100644 --- a/SilKit/source/config/YamlConversion.hpp +++ b/SilKit/source/config/YamlConversion.hpp @@ -77,6 +77,10 @@ DEFINE_SILKIT_CONVERT(TraceSink::Type); DEFINE_SILKIT_CONVERT(TraceSource); DEFINE_SILKIT_CONVERT(TraceSource::Type); +DEFINE_SILKIT_CONVERT(MetricsSink); +DEFINE_SILKIT_CONVERT(MetricsSink::Type); +DEFINE_SILKIT_CONVERT(Metrics); + DEFINE_SILKIT_CONVERT(Middleware); DEFINE_SILKIT_CONVERT(Extensions); diff --git a/SilKit/source/config/YamlSchema.cpp b/SilKit/source/config/YamlSchema.cpp index d570e69d7..c72ff7e4b 100644 --- a/SilKit/source/config/YamlSchema.cpp +++ b/SilKit/source/config/YamlSchema.cpp @@ -30,42 +30,46 @@ auto MakeYamlSchema() -> YamlSchemaElem { // Note: Keep these definitions in sync with ParticipantConfiguration.schema.json, // which is currently the main reference for valid configuration files. - YamlSchemaElem replay("Replay", - { - {"UseTraceSource"}, - {"Direction"}, - {"MdfChannel", { - {"ChannelName"}, {"ChannelSource"}, {"ChannelPath"}, - {"GroupName"}, {"GroupSource"}, {"GroupPath"}, - } - } - } - ); - YamlSchemaElem traceSinks("TraceSinks", - { - {"Name"}, - {"OutputPath"}, - {"Type"}, - } - ); - YamlSchemaElem traceSources("TraceSources", - { - {"Name"}, - {"InputPath"}, - {"Type"}, - } - ); - YamlSchemaElem logging("Logging", - { - {"LogFromRemotes"}, - {"FlushLevel"}, - {"Sinks", { - {"Type"}, - {"Format"}, - {"Level"}, - {"LogName"}, - }, - }, + YamlSchemaElem replay("Replay", {{"UseTraceSource"}, + {"Direction"}, + {"MdfChannel", + { + {"ChannelName"}, + {"ChannelSource"}, + {"ChannelPath"}, + {"GroupName"}, + {"GroupSource"}, + {"GroupPath"}, + }}}); + YamlSchemaElem traceSinks("TraceSinks", { + {"Name"}, + {"OutputPath"}, + {"Type"}, + }); + YamlSchemaElem traceSources("TraceSources", { + {"Name"}, + {"InputPath"}, + {"Type"}, + }); + + YamlSchemaElem metricsSinks{"Sinks", + { + {"Type"}, + {"Name"}, + }}; + + YamlSchemaElem logging("Logging", { + {"LogFromRemotes"}, + {"FlushLevel"}, + { + "Sinks", + { + {"Type"}, + {"Format"}, + {"Level"}, + {"LogName"}, + }, + }, }); YamlSchemaElem clusterParameters("ClusterParameters", { @@ -199,10 +203,16 @@ auto MakeYamlSchema() -> YamlSchemaElem {"ExperimentalRemoteParticipantConnection"}, {"ConnectTimeoutSeconds"}, }}, - {"Experimental", - { {"TimeSynchronization", {{"AnimationFactor"}} } + {"Experimental", + { + {"TimeSynchronization", {{"AnimationFactor"}}}, + {"Metrics", + { + metricsSinks, + {"CollectFromRemote"}, + }}, }}, - }; + }; return yamlSchema; } diff --git a/SilKit/source/core/internal/CMakeLists.txt b/SilKit/source/core/internal/CMakeLists.txt index e898f8580..afc473634 100644 --- a/SilKit/source/core/internal/CMakeLists.txt +++ b/SilKit/source/core/internal/CMakeLists.txt @@ -44,6 +44,7 @@ target_link_libraries(I_SilKit_Core_Internal INTERFACE I_SilKit_Services_Flexray INTERFACE I_SilKit_Services_Ethernet INTERFACE I_SilKit_Experimental_NetworkSimulatorInternals + INTERFACE I_SilKit_Services_Metrics ) diff --git a/SilKit/source/core/internal/IParticipantInternal.hpp b/SilKit/source/core/internal/IParticipantInternal.hpp index 9796fd0dc..0cf210836 100644 --- a/SilKit/source/core/internal/IParticipantInternal.hpp +++ b/SilKit/source/core/internal/IParticipantInternal.hpp @@ -34,6 +34,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "RequestReplyDatatypes.hpp" #include "OrchestrationDatatypes.hpp" #include "LoggingDatatypesInternal.hpp" +#include "MetricsDatatypes.hpp" #include "WireCanMessages.hpp" #include "WireDataMessages.hpp" @@ -41,6 +42,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "WireFlexrayMessages.hpp" #include "WireLinMessages.hpp" #include "WireRpcMessages.hpp" +#include "Metrics.hpp" #include "ISimulator.hpp" #include "IReplayDataController.hpp" @@ -49,10 +51,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ // forwards namespace SilKit { namespace Services { -namespace Logging{ +namespace Logging { struct ILoggerInternal; } -} +} // namespace Services namespace Tracing { class ReplayScheduler; } // namespace Tracing @@ -166,6 +168,8 @@ class IParticipantInternal : public IParticipant virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const Services::Logging::LogMsg& msg) = 0; virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, Services::Logging::LogMsg&& msg) = 0; + virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const VSilKit::MetricsUpdate& msg) = 0; + virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const Discovery::ParticipantDiscoveryEvent& msg) = 0; virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const Discovery::ServiceDiscoveryEvent& msg) = 0; @@ -257,6 +261,9 @@ class IParticipantInternal : public IParticipant virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const std::string& targetParticipantName, Services::Logging::LogMsg&& msg) = 0; + virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const std::string& targetParticipantName, + const VSilKit::MetricsUpdate& msg) = 0; + virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const std::string& targetParticipantName, const Discovery::ParticipantDiscoveryEvent& msg) = 0; virtual void SendMsg(const SilKit::Core::IServiceEndpoint* from, const std::string& targetParticipantName, @@ -309,6 +316,8 @@ class IParticipantInternal : public IParticipant virtual auto CreateNetworkSimulator() -> Experimental::NetworkSimulation::INetworkSimulator* = 0; + virtual auto GetMetricsManager() -> IMetricsManager* = 0; + // Register handlers for completion of async service creation virtual void AddAsyncSubscriptionsCompletionHandler(std::function handler) = 0; @@ -329,8 +338,14 @@ class IParticipantInternal : public IParticipant const std::string& controllerName, const SilKit::Config::SimulatedNetwork& simulatedNetwork) = 0; virtual bool ParticipantHasCapability(const std::string& participantName, const std::string& capability) const = 0; - virtual std::string GetServiceDescriptorString(SilKit::Experimental::NetworkSimulation::ControllerDescriptor controllerDescriptor) = 0; + + virtual std::string GetServiceDescriptorString( + SilKit::Experimental::NetworkSimulation::ControllerDescriptor controllerDescriptor) = 0; + virtual auto GetLoggerInternal() -> Services::Logging::ILoggerInternal* = 0; + + virtual auto GetMetricsProcessor() -> IMetricsProcessor* = 0; + virtual auto GetMetricsSender() -> IMetricsSender* = 0; }; } // namespace Core diff --git a/SilKit/source/core/internal/ServiceConfigKeys.hpp b/SilKit/source/core/internal/ServiceConfigKeys.hpp index dbf7e543d..711d35f07 100644 --- a/SilKit/source/core/internal/ServiceConfigKeys.hpp +++ b/SilKit/source/core/internal/ServiceConfigKeys.hpp @@ -80,6 +80,8 @@ const std::string controllerTypeSystemMonitor = "SystemMonitor"; const std::string controllerTypeSystemController = "SystemController"; const std::string controllerTypeLifecycleService = "LifecycleService"; const std::string controllerTypeTimeSyncService = "TimeSyncService"; +const std::string controllerTypeMetricsReceiver = "MetricsReceiver"; +const std::string controllerTypeMetricsSender = "MetricsSender"; // misc / legacy controllers const std::string controllerTypeOther = "Other"; diff --git a/SilKit/source/core/internal/traits/SilKitMsgSerdesName.hpp b/SilKit/source/core/internal/traits/SilKitMsgSerdesName.hpp index 28d85cc83..aa2803e99 100644 --- a/SilKit/source/core/internal/traits/SilKitMsgSerdesName.hpp +++ b/SilKit/source/core/internal/traits/SilKitMsgSerdesName.hpp @@ -46,6 +46,7 @@ struct SilKitMsgTraitSerdesName } DefineSilKitMsgTrait_SerdesName(SilKit::Services::Logging::LogMsg, "LOGMSG"); +DefineSilKitMsgTrait_SerdesName(VSilKit::MetricsUpdate, "METRICSUPDATE"); DefineSilKitMsgTrait_SerdesName(SilKit::Services::Orchestration::SystemCommand, "SYSTEMCOMMAND"); DefineSilKitMsgTrait_SerdesName(SilKit::Services::Orchestration::ParticipantStatus, "PARTICIPANTSTATUS"); DefineSilKitMsgTrait_SerdesName(SilKit::Services::Orchestration::WorkflowConfiguration, "WORKFLOWCONFIGURATION"); diff --git a/SilKit/source/core/internal/traits/SilKitMsgTraits.hpp b/SilKit/source/core/internal/traits/SilKitMsgTraits.hpp index ff7ef776d..fca7a2edb 100644 --- a/SilKit/source/core/internal/traits/SilKitMsgTraits.hpp +++ b/SilKit/source/core/internal/traits/SilKitMsgTraits.hpp @@ -129,6 +129,7 @@ struct SilKitMsgTraits } DefineSilKitMsgTrait_TypeName(SilKit::Services::Logging, LogMsg); +DefineSilKitMsgTrait_TypeName(VSilKit, MetricsUpdate); DefineSilKitMsgTrait_TypeName(SilKit::Services::Orchestration, SystemCommand); DefineSilKitMsgTrait_TypeName(SilKit::Services::Orchestration, ParticipantStatus); DefineSilKitMsgTrait_TypeName(SilKit::Services::Orchestration, WorkflowConfiguration); diff --git a/SilKit/source/core/internal/traits/SilKitMsgVersion.hpp b/SilKit/source/core/internal/traits/SilKitMsgVersion.hpp index aba7ccda9..883908b0e 100644 --- a/SilKit/source/core/internal/traits/SilKitMsgVersion.hpp +++ b/SilKit/source/core/internal/traits/SilKitMsgVersion.hpp @@ -45,6 +45,7 @@ struct SilKitMsgTraitVersion } DefineSilKitMsgTrait_Version(SilKit::Services::Logging::LogMsg, 1); +DefineSilKitMsgTrait_Version(VSilKit::MetricsUpdate, 1); DefineSilKitMsgTrait_Version(SilKit::Services::Orchestration::SystemCommand, 1); DefineSilKitMsgTrait_Version(SilKit::Services::Orchestration::ParticipantStatus, 1); DefineSilKitMsgTrait_Version(SilKit::Services::Orchestration::WorkflowConfiguration, 1); diff --git a/SilKit/source/core/internal/traits/SilKitServiceConfigTraits.hpp b/SilKit/source/core/internal/traits/SilKitServiceConfigTraits.hpp index bd22da556..b66f77cb2 100644 --- a/SilKit/source/core/internal/traits/SilKitServiceConfigTraits.hpp +++ b/SilKit/source/core/internal/traits/SilKitServiceConfigTraits.hpp @@ -59,6 +59,8 @@ DefineSilKitServiceTrait_ConfigType(SilKit::Services::Rpc::RpcClient, SilKit::Co DefineSilKitServiceTrait_ConfigType(SilKit::Services::Rpc::RpcServer, SilKit::Config::RpcServer); DefineSilKitServiceTrait_ConfigType(SilKit::Services::Rpc::RpcServerInternal, SilKit::Config::RpcServer); +DefineSilKitServiceTrait_ConfigType(VSilKit::MetricsReceiver, SilKit::Config::InternalController); +DefineSilKitServiceTrait_ConfigType(VSilKit::MetricsSender, SilKit::Config::InternalController); DefineSilKitServiceTrait_ConfigType(SilKit::Services::Logging::LogMsgReceiver, SilKit::Config::InternalController); DefineSilKitServiceTrait_ConfigType(SilKit::Services::Logging::LogMsgSender, SilKit::Config::InternalController); DefineSilKitServiceTrait_ConfigType(SilKit::Services::Orchestration::TimeSyncService, @@ -106,6 +108,8 @@ DefineSilKitServiceTrait_ServiceType(SilKit::Services::Orchestration::TimeSyncSe SilKit::Core::ServiceType::InternalController); DefineSilKitServiceTrait_ServiceType(SilKit::Services::Orchestration::LifecycleService, SilKit::Core::ServiceType::InternalController); +DefineSilKitServiceTrait_ServiceType(VSilKit::MetricsReceiver, SilKit::Core::ServiceType::InternalController); +DefineSilKitServiceTrait_ServiceType(VSilKit::MetricsSender, SilKit::Core::ServiceType::InternalController); DefineSilKitServiceTrait_ServiceType(SilKit::Services::Logging::LogMsgReceiver, SilKit::Core::ServiceType::InternalController); DefineSilKitServiceTrait_ServiceType(SilKit::Services::Logging::LogMsgSender, diff --git a/SilKit/source/core/internal/traits/SilKitServices_fwd.hpp b/SilKit/source/core/internal/traits/SilKitServices_fwd.hpp index 65a18e3f1..0e741f871 100644 --- a/SilKit/source/core/internal/traits/SilKitServices_fwd.hpp +++ b/SilKit/source/core/internal/traits/SilKitServices_fwd.hpp @@ -82,3 +82,8 @@ class RpcServerInternal; } // namespace Services } // namespace SilKit + +namespace VSilKit { +class MetricsManager; +class MetricsSender; +} // namespace VSilKit diff --git a/SilKit/source/core/mock/nullconnection/NullConnectionParticipant.cpp b/SilKit/source/core/mock/nullconnection/NullConnectionParticipant.cpp index 365e79c86..8cfb245c8 100644 --- a/SilKit/source/core/mock/nullconnection/NullConnectionParticipant.cpp +++ b/SilKit/source/core/mock/nullconnection/NullConnectionParticipant.cpp @@ -32,7 +32,8 @@ namespace Core { namespace { struct NullConnection { - NullConnection(SilKit::Core::IParticipantInternal*, SilKit::Config::ParticipantConfiguration /*config*/, + NullConnection(SilKit::Core::IParticipantInternal*, VSilKit::IMetricsManager*, + SilKit::Config::ParticipantConfiguration /*config*/, std::string /*participantName*/, SilKit::Core::ParticipantId /*participantId*/, SilKit::Core::Orchestration::ITimeProvider* /*timeProvider*/, ProtocolVersion) { diff --git a/SilKit/source/core/mock/participant/MockParticipant.hpp b/SilKit/source/core/mock/participant/MockParticipant.hpp index bbf268538..2adbeb3f4 100644 --- a/SilKit/source/core/mock/participant/MockParticipant.hpp +++ b/SilKit/source/core/mock/participant/MockParticipant.hpp @@ -22,6 +22,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #pragma once #include +#include +#include #include "gtest/gtest.h" #include "gmock/gmock.h" @@ -176,6 +178,67 @@ class DummyNetworkSimulator : public Experimental::NetworkSimulation::INetworkSi void Start() override {} }; +class DummyMetricsManager : public IMetricsManager +{ + class DummyCounterMetric : public ICounterMetric + { + public: + void Add(uint64_t /* delta */) override {} + void Set(uint64_t /* value */) override {} + }; + + class DummyStatisticMetric : public IStatisticMetric + { + public: + void Take(double /* value */) override {} + }; + + class DummyStringListMetric : public IStringListMetric + { + public: + void Clear() override {} + void Add(const std::string&) override {} + }; + +public: + auto GetCounter(const std::string& name) -> ICounterMetric* override + { + auto it = _counters.find(name); + if (it == _counters.end()) + { + it = _counters.emplace().first; + } + return &(it->second); + } + + auto GetStatistic(const std::string& name) -> IStatisticMetric* override + { + auto it = _statistics.find(name); + if (it == _statistics.end()) + { + it = _statistics.emplace().first; + } + return &(it->second); + } + + auto GetStringList(const std::string& name) -> IStringListMetric* override + { + auto it = _stringLists.find(name); + if (it == _stringLists.end()) + { + it = _stringLists.emplace().first; + } + return &(it->second); + } + + void SubmitUpdates() override {} + +private: + std::unordered_map _counters; + std::unordered_map _statistics; + std::unordered_map _stringLists; +}; + class DummyParticipant : public IParticipantInternal { public: @@ -358,6 +421,8 @@ class DummyParticipant : public IParticipantInternal void SendMsg(const IServiceEndpoint* /*from*/, Services::Logging::LogMsg&& /*msg*/) override {} void SendMsg(const IServiceEndpoint* /*from*/, const Services::Logging::LogMsg& /*msg*/) override {} + void SendMsg(const IServiceEndpoint* /*from*/, const VSilKit::MetricsUpdate& /*msg*/) override {} + void SendMsg(const IServiceEndpoint* /*from*/, const Discovery::ParticipantDiscoveryEvent& /*msg*/) override {} void SendMsg(const IServiceEndpoint* /*from*/, const Discovery::ServiceDiscoveryEvent& /*msg*/) override {} @@ -522,6 +587,11 @@ class DummyParticipant : public IParticipantInternal { } + void SendMsg(const IServiceEndpoint* /*from*/, const std::string& /*targetParticipantName*/, + const VSilKit::MetricsUpdate& /*msg*/) override + { + } + void SendMsg(const IServiceEndpoint* /*from*/, const std::string& /*targetParticipantName*/, const Discovery::ParticipantDiscoveryEvent& /*msg*/) override { @@ -598,6 +668,10 @@ class DummyParticipant : public IParticipantInternal { return 0; }; + auto GetMetricsManager() -> IMetricsManager* override + { + return &mockMetricsManager; + } size_t GetNumberOfRemoteReceivers(const IServiceEndpoint* /*service*/, const std::string& /*msgTypeName*/) override { return 0; @@ -632,6 +706,16 @@ class DummyParticipant : public IParticipantInternal return &logger; } + auto GetMetricsProcessor() -> IMetricsProcessor* override + { + return nullptr; + } + + auto GetMetricsSender() -> IMetricsSender* override + { + return nullptr; + } + const std::string _name = "MockParticipant"; const std::string _registryUri = "silkit://mock.participant.silkit:0"; testing::NiceMock logger; @@ -644,6 +728,7 @@ class DummyParticipant : public IParticipantInternal MockRequestReplyService mockRequestReplyService; MockParticipantReplies mockParticipantReplies; DummyNetworkSimulator mockNetworkSimulator; + DummyMetricsManager mockMetricsManager; }; // ================================================================================ diff --git a/SilKit/source/core/participant/CMakeLists.txt b/SilKit/source/core/participant/CMakeLists.txt index f2dc4d168..99046477e 100644 --- a/SilKit/source/core/participant/CMakeLists.txt +++ b/SilKit/source/core/participant/CMakeLists.txt @@ -39,6 +39,7 @@ target_link_libraries(I_SilKit_Core_Participant INTERFACE I_SilKit_Services_PubSub INTERFACE I_SilKit_Services_Rpc INTERFACE I_SilKit_Tracing + INTERFACE I_SilKit_Experimental_NetworkSimulatorInternals # Due to Participant_impl.hpp being used directly in NullConnectionParticipant and # RpcTestUtilities, this interface library must also depend on the headers used by diff --git a/SilKit/source/core/participant/Participant.hpp b/SilKit/source/core/participant/Participant.hpp index 5ada814d6..5a2f59bca 100644 --- a/SilKit/source/core/participant/Participant.hpp +++ b/SilKit/source/core/participant/Participant.hpp @@ -34,6 +34,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "ParticipantConfiguration.hpp" #include "ReplayScheduler.hpp" +#include "Metrics.hpp" +#include "MetricsProcessor.hpp" +#include "IMetricsTimerThread.hpp" // Interfaces relying on I_SilKit_Core_Internal #include "IMsgForLogMsgSender.hpp" @@ -64,6 +67,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "IMsgForLifecycleService.hpp" #include "IMsgForTimeSyncService.hpp" +#include "IMsgForMetricsReceiver.hpp" +#include "IMsgForMetricsSender.hpp" + #include "ITraceMessageSink.hpp" #include "ITraceMessageSource.hpp" @@ -156,6 +162,7 @@ class Participant : public IParticipantInternal auto GetServiceDiscovery() -> Discovery::IServiceDiscovery* override; auto GetRequestReplyService() -> RequestReply::IRequestReplyService* override; auto GetParticipantRepliesProcedure() -> RequestReply::IParticipantReplies* override; + auto GetMetricsManager() -> IMetricsManager* override; auto GetLogger() -> Services::Logging::ILogger* override; auto CreateLifecycleService(Services::Orchestration::LifecycleConfiguration startConfiguration) @@ -213,6 +220,8 @@ class Participant : public IParticipantInternal void SendMsg(const IServiceEndpoint*, const Services::Logging::LogMsg& msg) override; void SendMsg(const IServiceEndpoint*, Services::Logging::LogMsg&& msg) override; + void SendMsg(const SilKit::Core::IServiceEndpoint* from, const VSilKit::MetricsUpdate& msg) override; + void SendMsg(const IServiceEndpoint* from, const Services::PubSub::WireDataMessageEvent& msg) override; void SendMsg(const IServiceEndpoint* from, const Services::Rpc::FunctionCall& msg) override; void SendMsg(const IServiceEndpoint* from, Services::Rpc::FunctionCall&& msg) override; @@ -296,6 +305,9 @@ class Participant : public IParticipantInternal void SendMsg(const IServiceEndpoint*, const std::string& targetParticipantName, Services::Logging::LogMsg&& msg) override; + void SendMsg(const IServiceEndpoint*, const std::string& targetParticipantName, + const VSilKit::MetricsUpdate& msg) override; + void SendMsg(const IServiceEndpoint* from, const std::string& targetParticipantName, const Services::PubSub::WireDataMessageEvent& msg) override; @@ -349,6 +361,9 @@ class Participant : public IParticipantInternal auto GetLoggerInternal() -> Services::Logging::ILoggerInternal* override; + auto GetMetricsProcessor() -> IMetricsProcessor* override; + auto GetMetricsSender() -> IMetricsSender* override; + public: // ---------------------------------------- // Public methods @@ -393,6 +408,7 @@ class Participant : public IParticipantInternal void OnSilKitSimulationJoined(); void SetupRemoteLogging(); + void SetupMetrics(); void SetTimeProvider(Services::Orchestration::ITimeProvider*); @@ -430,6 +446,12 @@ class Participant : public IParticipantInternal template void AddTraceSinksToSourceInternal(ITraceMessageSource* controller, ConfigT config); + auto GetOrCreateMetricsSender() -> VSilKit::IMetricsSender*; + + void CreateSystemInformationMetrics(); + + auto MakeTimerThread() -> std::unique_ptr; + private: // ---------------------------------------- // private members @@ -454,19 +476,28 @@ class Participant : public IParticipantInternal ControllerMap, ControllerMap, ControllerMap, ControllerMap, - ControllerMap> + ControllerMap, ControllerMap, + ControllerMap> _controllers; std::atomic _localEndpointId{0}; std::unique_ptr _networkSimulatorInternal; + std::unique_ptr _metricsProcessor; + std::unique_ptr _metricsManager; + + IMetricsSender* _metricsSender{nullptr}; + std::tuple _simulators{nullptr, nullptr, nullptr, nullptr}; SilKitConnectionT _connection; + // NB: Must be destroyed before _connection and _metricsManager + std::unique_ptr _metricsTimerThread; + // control variables to prevent multiple create accesses by public API std::atomic _isSystemMonitorCreated{false}; std::atomic _isSystemControllerCreated{false}; diff --git a/SilKit/source/core/participant/Participant_impl.hpp b/SilKit/source/core/participant/Participant_impl.hpp index 20c46f183..5fbfb6608 100644 --- a/SilKit/source/core/participant/Participant_impl.hpp +++ b/SilKit/source/core/participant/Participant_impl.hpp @@ -50,6 +50,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "YamlParser.hpp" #include "NetworkSimulatorInternal.hpp" +#include "MetricsManager.hpp" +#include "MetricsSender.hpp" +#include "MetricsTimerThread.hpp" +#include "CreateMetricsSinksFromParticipantConfiguration.hpp" + #include "tuple_tools/bind.hpp" #include "tuple_tools/for_each.hpp" #include "tuple_tools/predicative_get.hpp" @@ -62,6 +67,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "MessageTracing.hpp" #include "Uuid.hpp" #include "Assert.hpp" +#include "ExecutionEnvironment.hpp" #include "ILoggerInternal.hpp" @@ -92,11 +98,23 @@ template Participant::Participant(Config::ParticipantConfiguration participantConfig, ProtocolVersion version) : _participantConfig{participantConfig} , _participantId{Util::Hash::Hash(participantConfig.participantName)} - , _connection{this, _participantConfig, participantConfig.participantName, _participantId, &_timeProvider, version} + , _metricsProcessor{std::make_unique(GetParticipantName())} + , _metricsManager{std::make_unique(GetParticipantName(), *_metricsProcessor)} + , _connection{this, + _metricsManager.get(), + _participantConfig, + participantConfig.participantName, + _participantId, + &_timeProvider, + version} + , _metricsTimerThread{MakeTimerThread()} { // NB: do not create the _logger in the initializer list. If participantName is empty, // this will cause a fairly unintuitive exception in spdlog. _logger = std::make_unique(GetParticipantName(), _participantConfig.logging); + + dynamic_cast(*_metricsProcessor).SetLogger(*_logger); + dynamic_cast(*_metricsManager).SetLogger(*_logger); _connection.SetLogger(_logger.get()); Logging::Info(_logger.get(), "Creating participant '{}' at '{}', SIL Kit version: {}", GetParticipantName(), @@ -115,6 +133,7 @@ template void Participant::OnSilKitSimulationJoined() { SetupRemoteLogging(); + SetupMetrics(); // NB: Create the systemController to abort the simulation already in the startup phase (void)GetSystemController(); @@ -141,6 +160,13 @@ void Participant::OnSilKitSimulationJoined() _replayScheduler->ConfigureTimeProvider(&_timeProvider); _logger->Info("Replay Scheduler active."); } + + CreateSystemInformationMetrics(); + + if (_metricsTimerThread) + { + _metricsTimerThread->Start(); + } } template @@ -191,6 +217,36 @@ void Participant::SetupRemoteLogging() } } +template +void Participant::SetupMetrics() +{ + auto sender = GetOrCreateMetricsSender(); + + auto& processor = dynamic_cast(*_metricsProcessor); + { + auto sinks = VSilKit::CreateMetricsSinksFromParticipantConfiguration( + _logger.get(), sender, GetParticipantName(), _participantConfig.experimental.metrics.sinks); + processor.SetSinks(std::move(sinks)); + } + + // NB: Create the metrics manager prior to anything that might need it (possibly needs to be split into + // manager/sender explicitly) + (void)GetMetricsManager(); + + // NB: Create the metrics receiver if enabled in the configuration + if (_participantConfig.experimental.metrics.collectFromRemote) + { + Core::SupplementalData supplementalData; + supplementalData[SilKit::Core::Discovery::controllerType] = + SilKit::Core::Discovery::controllerTypeMetricsReceiver; + + SilKit::Config::InternalController config; + config.name = "MetricsReceiver"; + config.network = "default"; + CreateController(config, std::move(supplementalData), true, true, *_logger, processor); + } +} + template inline void Participant::SetTimeProvider(Orchestration::ITimeProvider* newClock) { @@ -904,6 +960,12 @@ auto Participant::GetParticipantRepliesProcedure() -> Request return _participantReplies.get(); } +template +auto Participant::GetMetricsManager() -> IMetricsManager* +{ + return _metricsManager.get(); +} + template bool Participant::GetIsSystemControllerCreated() { @@ -1219,6 +1281,12 @@ void Participant::SendMsg(const IServiceEndpoint* from, Servi SendMsgImpl(from, std::move(msg)); } +template +void Participant::SendMsg(const IServiceEndpoint* from, const VSilKit::MetricsUpdate& msg) +{ + SendMsgImpl(from, msg); +} + template void Participant::SendMsg(const IServiceEndpoint* from, const Discovery::ParticipantDiscoveryEvent& msg) @@ -1513,6 +1581,13 @@ void Participant::SendMsg(const IServiceEndpoint* from, const SendMsgImpl(from, targetParticipantName, std::move(msg)); } +template +void Participant::SendMsg(const IServiceEndpoint* from, const std::string& targetParticipantName, + const VSilKit::MetricsUpdate& msg) +{ + SendMsgImpl(from, targetParticipantName, msg); +} + template void Participant::SendMsg(const IServiceEndpoint* from, const std::string& targetParticipantName, const Discovery::ParticipantDiscoveryEvent& msg) @@ -1810,5 +1885,88 @@ std::string Participant::GetServiceDescriptorString( } +template +auto Participant::GetMetricsProcessor() -> IMetricsProcessor* +{ + return _metricsProcessor.get(); +} + + +template +auto Participant::GetMetricsSender() -> VSilKit::IMetricsSender* +{ + return _metricsSender; +} + + +template +auto Participant::GetOrCreateMetricsSender() -> VSilKit::IMetricsSender* +{ + if (_metricsSender == nullptr) + { + bool hasRemoteSinks{false}; + for (const auto& config : _participantConfig.experimental.metrics.sinks) + { + hasRemoteSinks = hasRemoteSinks || (config.type == Config::MetricsSink::Type::Remote); + } + + if (hasRemoteSinks) + { + auto* sender = GetController(SilKit::Core::Discovery::controllerTypeMetricsSender); + + if (sender == nullptr) + { + Core::SupplementalData supplementalData; + supplementalData[SilKit::Core::Discovery::controllerType] = + SilKit::Core::Discovery::controllerTypeMetricsSender; + + Config::InternalController config; + config.name = SilKit::Core::Discovery::controllerTypeMetricsSender; + config.network = "default"; + + sender = CreateController(config, std::move(supplementalData), true, true); + } + + _metricsSender = static_cast(static_cast(sender)); + } + else + { + _logger->Debug("Refusing to create MetricsSender because no remote sinks are configured"); + } + } + + return _metricsSender; +} + + +template +void Participant::CreateSystemInformationMetrics() +{ + auto ee = VSilKit::GetExecutionEnvironment(); + + GetMetricsManager()->GetStringList("SilKit/System/OperatingSystem")->Add(ee.operatingSystem); + GetMetricsManager()->GetStringList("SilKit/System/Hostname")->Add(ee.hostname); + GetMetricsManager()->GetStringList("SilKit/System/PageSize")->Add(ee.pageSize); + GetMetricsManager()->GetStringList("SilKit/System/ProcessorCount")->Add(ee.processorCount); + GetMetricsManager()->GetStringList("SilKit/System/ProcessorArchitecture")->Add(ee.processorArchitecture); + GetMetricsManager()->GetStringList("SilKit/System/PhysicalMemory")->Add(ee.physicalMemoryMiB + " MiB"); + GetMetricsManager()->GetStringList("SilKit/Process/Executable")->Add(ee.executable); + GetMetricsManager()->GetStringList("SilKit/Process/Username")->Add(ee.username); +} + + +template +auto Participant::MakeTimerThread() -> std::unique_ptr +{ + if (_participantConfig.experimental.metrics.sinks.empty()) + { + return nullptr; + } + + return std::make_unique( + [this] { ExecuteDeferred([this] { GetMetricsManager()->SubmitUpdates(); }); }); +} + + } // namespace Core } // namespace SilKit diff --git a/SilKit/source/core/vasio/CMakeLists.txt b/SilKit/source/core/vasio/CMakeLists.txt index 979a6d14b..160166557 100644 --- a/SilKit/source/core/vasio/CMakeLists.txt +++ b/SilKit/source/core/vasio/CMakeLists.txt @@ -41,6 +41,7 @@ target_link_libraries(I_SilKit_Core_VAsio INTERFACE I_SilKit_Services_Flexray INTERFACE I_SilKit_Services_Lin INTERFACE I_SilKit_Services_Logging + INTERFACE I_SilKit_Services_Metrics INTERFACE I_SilKit_Services_Orchestration INTERFACE I_SilKit_Services_PubSub INTERFACE I_SilKit_Services_Rpc diff --git a/SilKit/source/core/vasio/SerializedMessage.hpp b/SilKit/source/core/vasio/SerializedMessage.hpp index c8989f34e..687b69f46 100644 --- a/SilKit/source/core/vasio/SerializedMessage.hpp +++ b/SilKit/source/core/vasio/SerializedMessage.hpp @@ -37,6 +37,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "RequestReplySerdes.hpp" #include "LoggingSerdes.hpp" #include "DataSerdes.hpp" +#include "MetricsSerdes.hpp" namespace SilKit { namespace Core { diff --git a/SilKit/source/core/vasio/Test_VAsioConnection.cpp b/SilKit/source/core/vasio/Test_VAsioConnection.cpp index d6fb5f0fa..8170cd8e4 100644 --- a/SilKit/source/core/vasio/Test_VAsioConnection.cpp +++ b/SilKit/source/core/vasio/Test_VAsioConnection.cpp @@ -175,12 +175,13 @@ class Test_VAsioConnection : public testing::Test { protected: Test_VAsioConnection() - : _connection(nullptr, {}, "Test_VAsioConnection", 1, &_timeProvider) + : _connection(nullptr, &_dummyMetricsManager, {}, "Test_VAsioConnection", 1, &_timeProvider) { _connection.SetLogger(&_dummyLogger); } Tests::MockLogger _dummyLogger; + Tests::DummyMetricsManager _dummyMetricsManager; Services::Orchestration::TimeProvider _timeProvider; VAsioConnection _connection; MockVAsioPeer _from; diff --git a/SilKit/source/core/vasio/VAsioConnection.cpp b/SilKit/source/core/vasio/VAsioConnection.cpp index dd634f45d..430291c08 100644 --- a/SilKit/source/core/vasio/VAsioConnection.cpp +++ b/SilKit/source/core/vasio/VAsioConnection.cpp @@ -262,9 +262,10 @@ namespace Core { namespace tt = Util::tuple_tools; -VAsioConnection::VAsioConnection(IParticipantInternal* participant, SilKit::Config::ParticipantConfiguration config, - std::string participantName, ParticipantId participantId, - Services::Orchestration::ITimeProvider* timeProvider, ProtocolVersion version) +VAsioConnection::VAsioConnection(IParticipantInternal* participant, IMetricsManager* metricsManager, + SilKit::Config::ParticipantConfiguration config, std::string participantName, + ParticipantId participantId, Services::Orchestration::ITimeProvider* timeProvider, + ProtocolVersion version) : _config{std::move(config)} , _participantName{std::move(participantName)} , _participantId{participantId} @@ -274,6 +275,7 @@ VAsioConnection::VAsioConnection(IParticipantInternal* participant, SilKit::Conf , _connectKnownParticipants{*_ioContext, *this, *this, MakeConnectKnownParticipantsSettings(_config)} , _remoteConnectionManager{*this, MakeRemoteConnectionManagerSettings(_config)} , _version{version} + , _metricsManager{metricsManager} , _participant{participant} { } @@ -352,6 +354,9 @@ auto VAsioConnection::PrepareAcceptorEndpointUris(const std::string& connectUri) void VAsioConnection::OpenTcpAcceptors(const std::vector& acceptorEndpointUris) { + auto metric = _metricsManager->GetStringList("TcpAcceptors"); + metric->Clear(); + for (const auto& uriString : acceptorEndpointUris) { const auto uri = Uri::Parse(uriString); @@ -381,6 +386,8 @@ void VAsioConnection::OpenTcpAcceptors(const std::vector& acceptorE Services::Logging::Error(_logger, "Unable to accept TCP connections on {}:{}: {}", host, uri.Port(), exception.what()); } + + metric->Add(fmt::format("{}:{}", host, uri.Port())); } } else if (uri.Type() == Uri::UriType::Local && uri.Scheme() == "local") @@ -396,6 +403,9 @@ void VAsioConnection::OpenTcpAcceptors(const std::vector& acceptorE void VAsioConnection::OpenLocalAcceptors(const std::vector& acceptorEndpointUris) { + auto metric = _metricsManager->GetStringList("LocalAcceptors"); + metric->Clear(); + for (const auto& uriString : acceptorEndpointUris) { const auto uri = Uri::Parse(uriString); @@ -424,6 +434,8 @@ void VAsioConnection::OpenLocalAcceptors(const std::vector& accepto Services::Logging::Error(_logger, "Unable to accept local domain connections on '{}': {}", uri.Path(), exception.what()); } + + metric->Add(fmt::format("{}", uri.Path())); } else if (uri.Type() == Uri::UriType::Tcp && uri.Scheme() == "tcp") { @@ -1076,8 +1088,19 @@ void VAsioConnection::HandleConnectedPeer(IVAsioPeer* peer) void VAsioConnection::AssociateParticipantNameAndPeer(const std::string& simulationName, const std::string& participantName, IVAsioPeer* peer) { - std::lock_guard lock{_mutex}; - _participantNameToPeer[simulationName].insert({participantName, peer}); + { + std::lock_guard lock{_mutex}; + _participantNameToPeer[simulationName].insert({participantName, peer}); + } + + IStringListMetric* metric; + auto metricNameBase = "Peer/" + simulationName + "/" + participantName; + + metric = _metricsManager->GetStringList(metricNameBase + "/LocalEndpoint"); + metric->Add(peer->GetLocalAddress()); + + metric = _metricsManager->GetStringList(metricNameBase + "/RemoteEndpoint"); + metric->Add(peer->GetRemoteAddress()); } auto VAsioConnection::FindPeerByName(const std::string& simulationName, diff --git a/SilKit/source/core/vasio/VAsioConnection.hpp b/SilKit/source/core/vasio/VAsioConnection.hpp index a3e382473..d7b388bb6 100644 --- a/SilKit/source/core/vasio/VAsioConnection.hpp +++ b/SilKit/source/core/vasio/VAsioConnection.hpp @@ -61,6 +61,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "VAsioCapabilities.hpp" #include "WireLinMessages.hpp" #include "Uri.hpp" +#include "Metrics.hpp" #include "IIoContext.hpp" #include "IConnectionMethods.hpp" @@ -89,7 +90,8 @@ class VAsioConnection // Constructors and Destructor VAsioConnection(const VAsioConnection&) = delete; VAsioConnection(VAsioConnection&&) = delete; - VAsioConnection(IParticipantInternal* participant, SilKit::Config::ParticipantConfiguration config, + VAsioConnection(IParticipantInternal* participant, IMetricsManager* metricsManager, + SilKit::Config::ParticipantConfiguration config, std::string participantName, ParticipantId participantId, Services::Orchestration::ITimeProvider* timeProvider, ProtocolVersion version = CurrentProtocolVersion()); @@ -249,7 +251,7 @@ class VAsioConnection Services::Flexray::FlexrayTxBufferConfigUpdate, Services::Flexray::WireFlexrayTxBufferUpdate, Services::Flexray::FlexrayPocStatusEvent, Core::Discovery::ParticipantDiscoveryEvent, Core::Discovery::ServiceDiscoveryEvent, Core::RequestReply::RequestReplyCall, - Core::RequestReply::RequestReplyCallReturn, + Core::RequestReply::RequestReplyCallReturn, VSilKit::MetricsUpdate, // Private testing data types Core::Tests::Version1::TestMessage, Core::Tests::Version2::TestMessage, Core::Tests::TestFrameEvent>; @@ -563,6 +565,9 @@ class VAsioConnection ProtocolVersion _version; friend class Test_VAsioConnection; + // metrics + IMetricsManager* _metricsManager; + // for debugging purposes: IParticipantInternal* _participant{nullptr}; diff --git a/SilKit/source/core/vasio/VAsioRegistry.cpp b/SilKit/source/core/vasio/VAsioRegistry.cpp index 4a048a338..9e1754ae7 100644 --- a/SilKit/source/core/vasio/VAsioRegistry.cpp +++ b/SilKit/source/core/vasio/VAsioRegistry.cpp @@ -28,6 +28,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "TransformAcceptorUris.hpp" #include "VAsioConstants.hpp" +#include "MetricsReceiver.hpp" +#include "MetricsProcessor.hpp" +#include "MetricsManager.hpp" +#include "CreateMetricsSinksFromParticipantConfiguration.hpp" + +#include "traits/SilKitServiceConfigTraits.hpp" + namespace Log = SilKit::Services::Logging; @@ -45,7 +52,15 @@ VAsioRegistry::VAsioRegistry(std::shared_ptr(cfg)} - , _connection{nullptr, *_vasioConfig, REGISTRY_PARTICIPANT_NAME, REGISTRY_PARTICIPANT_ID, &_timeProvider, version} + , _metricsProcessor{std::make_unique(REGISTRY_PARTICIPANT_NAME)} + , _metricsManager{std::make_unique(REGISTRY_PARTICIPANT_NAME, *_metricsProcessor)} + , _connection{nullptr, + _metricsManager.get(), + *_vasioConfig, + REGISTRY_PARTICIPANT_NAME, + REGISTRY_PARTICIPANT_ID, + &_timeProvider, + version} { _logger = std::make_unique(REGISTRY_PARTICIPANT_NAME, _vasioConfig->logging); @@ -54,6 +69,8 @@ VAsioRegistry::VAsioRegistry(std::shared_ptrOnLoggerCreated(_logger.get()); } + dynamic_cast(*_metricsProcessor).SetLogger(*_logger); + dynamic_cast(*_metricsManager).SetLogger(*_logger); _connection.SetLogger(_logger.get()); _connection.RegisterMessageReceiver([this](IVAsioPeer* from, const ParticipantAnnouncement& announcement) { @@ -65,6 +82,9 @@ VAsioRegistry::VAsioRegistry(std::shared_ptr std::string @@ -155,6 +175,10 @@ auto VAsioRegistry::StartListening(const std::string& listenUri) -> std::string _connection.StartIoWorker(); _connection.RegisterSilKitService(this); + if (_vasioConfig->experimental.metrics.collectFromRemote) + { + _connection.RegisterSilKitService(_metricsReceiver.get()); + } return uri.EncodedString(); } @@ -297,6 +321,40 @@ bool VAsioRegistry::AllParticipantsAreConnected() const return false; } +void VAsioRegistry::SetupMetrics() +{ + auto& processor = dynamic_cast(*_metricsProcessor); + { + // the registry has no MetricsSender + auto sinks = VSilKit::CreateMetricsSinksFromParticipantConfiguration( + _logger.get(), nullptr, REGISTRY_PARTICIPANT_NAME, _vasioConfig->experimental.metrics.sinks); + processor.SetSinks(std::move(sinks)); + } + + if (_vasioConfig->experimental.metrics.collectFromRemote) + { + auto metricsReceiver = std::make_unique(nullptr, *_logger, processor); + + SilKit::Core::SupplementalData supplementalData; + supplementalData[SilKit::Core::Discovery::controllerType] = + SilKit::Core::Discovery::controllerTypeMetricsReceiver; + + auto sd = metricsReceiver->GetServiceDescriptor(); + sd.SetParticipantNameAndComputeId(_serviceDescriptor.GetParticipantName()); + sd.SetParticipantId(_serviceDescriptor.GetParticipantId()); + sd.SetNetworkName("default"); + sd.SetNetworkType(SilKit::Config::NetworkType::Undefined); + sd.SetServiceName("MetricsReceiver"); + sd.SetServiceId(_localEndpointId++); + sd.SetServiceType(SilKitServiceTraitServiceType::GetServiceType()); + sd.SetSupplementalData(std::move(supplementalData)); + + metricsReceiver->SetServiceDescriptor(sd); + + _metricsReceiver = std::move(metricsReceiver); + } +} + void VAsioRegistry::ReceiveMsg(const SilKit::Core::IServiceEndpoint* from, const SilKit::Services::Orchestration::ParticipantStatus& msg) { @@ -350,5 +408,16 @@ auto VAsioRegistry::GetServiceDescriptor() const -> const ServiceDescriptor& } +void VAsioRegistry::OnMetricsUpdate(const std::string& participantName, const VSilKit::MetricsUpdate& metricsUpdate) +{ + Log::Info(GetLogger(), "Participant {} updates {} metrics", participantName, metricsUpdate.metrics.size()); + for (const auto& data : metricsUpdate.metrics) + { + Log::Info(GetLogger(), "Metric Update: {} {} {} {} ({})", data.name, data.kind, data.value, data.timestamp, + participantName); + } +} + + } // namespace Core } // namespace SilKit diff --git a/SilKit/source/core/vasio/VAsioRegistry.hpp b/SilKit/source/core/vasio/VAsioRegistry.hpp index 31df4617e..2dece0f13 100644 --- a/SilKit/source/core/vasio/VAsioRegistry.hpp +++ b/SilKit/source/core/vasio/VAsioRegistry.hpp @@ -31,6 +31,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "ProtocolVersion.hpp" #include "TimeProvider.hpp" +#include "MetricsReceiver.hpp" + namespace SilKit { namespace Core { @@ -64,6 +66,7 @@ class VAsioRegistry : public SilKit::Vendor::Vector::ISilKitRegistry , public IMsgForVAsioRegistry , public Core::IServiceEndpoint + , public VSilKit::IMetricsReceiverListener { public: // CTor VAsioRegistry() = delete; @@ -102,6 +105,8 @@ class VAsioRegistry bool AllParticipantsAreConnected() const; + void SetupMetrics(); + private: // IReceiver<...> void ReceiveMsg(const IServiceEndpoint* from, const SilKit::Services::Orchestration::ParticipantStatus& msg) override; @@ -113,6 +118,9 @@ class VAsioRegistry void SetServiceDescriptor(const ServiceDescriptor& serviceDescriptor) override; auto GetServiceDescriptor() const -> const ServiceDescriptor& override; +private: // IMetricsReceiverListener + void OnMetricsUpdate(const std::string& participantName, const VSilKit::MetricsUpdate& metricsUpdate) override; + private: // ---------------------------------------- // private members @@ -123,8 +131,13 @@ class VAsioRegistry std::function _onAllParticipantsDisconnected; std::shared_ptr _vasioConfig; Services::Orchestration::TimeProvider _timeProvider; + std::unique_ptr _metricsReceiver; + std::unique_ptr _metricsProcessor; + std::unique_ptr _metricsManager; VAsioConnection _connection; + std::atomic _localEndpointId{1}; + ServiceDescriptor _serviceDescriptor; }; diff --git a/SilKit/source/services/CMakeLists.txt b/SilKit/source/services/CMakeLists.txt index 6327c69ee..2df5503f9 100644 --- a/SilKit/source/services/CMakeLists.txt +++ b/SilKit/source/services/CMakeLists.txt @@ -27,4 +27,5 @@ add_subdirectory(pubsub) add_subdirectory(rpc) add_subdirectory(lin) add_subdirectory(logging) +add_subdirectory(metrics) add_subdirectory(orchestration) \ No newline at end of file diff --git a/SilKit/source/services/logging/Logger.cpp b/SilKit/source/services/logging/Logger.cpp index 5f9b1225b..475f97069 100644 --- a/SilKit/source/services/logging/Logger.cpp +++ b/SilKit/source/services/logging/Logger.cpp @@ -274,13 +274,7 @@ Logger::Logger(const std::string& participantName, Config::Logging config) // gets created, the first default logger will be dropped from the registry as well. // Generate a tm object for the timestamp once, so that all file loggers will have the very same timestamp. - auto timeNow = std::time(nullptr); - std::tm tmBuffer{}; -#if defined(_WIN32) - localtime_s(&tmBuffer, &timeNow); -#else - localtime_r(&timeNow, &tmBuffer); -#endif + const auto logFileTimestamp = SilKit::Util::CurrentTimestampString(); // Defined JSON pattern for the logger output std::string jsonpattern{R"({"ts":"%E","log":"%n","lvl":"%l", %v })"}; @@ -296,7 +290,7 @@ Logger::Logger(const std::string& participantName, Config::Logging config) } } if (sink.format == Config::Sink::Format::Simple && sink.type != Config::Sink::Type::Remote) - { + { if (log_level < _loggerSimple->level()) { _loggerSimple->set_level(log_level); @@ -342,7 +336,7 @@ Logger::Logger(const std::string& participantName, Config::Logging config) if (sink.format == Config::Sink::Format::Json) { - auto filename = fmt::format("{}_{:%FT%H-%M-%S}.jsonl", sink.logName, tmBuffer); + auto filename = fmt::format("{}_{}.jsonl", sink.logName, logFileTimestamp); auto fileSink = std::make_shared(filename); using spdlog::details::make_unique; // for pre c++14 auto formatter = make_unique(); @@ -354,7 +348,7 @@ Logger::Logger(const std::string& participantName, Config::Logging config) } else { - auto filename = fmt::format("{}_{:%FT%H-%M-%S}.txt", sink.logName, tmBuffer); + auto filename = fmt::format("{}_{}.txt", sink.logName, logFileTimestamp); auto fileSink = std::make_shared(filename); fileSink->set_level(log_level); _loggerSimple->sinks().push_back(fileSink); diff --git a/SilKit/source/services/logging/SilKitFmtFormatters.hpp b/SilKit/source/services/logging/SilKitFmtFormatters.hpp index 8bea70863..6b8621aff 100644 --- a/SilKit/source/services/logging/SilKitFmtFormatters.hpp +++ b/SilKit/source/services/logging/SilKitFmtFormatters.hpp @@ -38,6 +38,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "string_utils_internal.hpp" #include "ServiceDatatypes.hpp" #include "LoggingDatatypesInternal.hpp" +#include "MetricsDatatypes.hpp" #include "RequestReplyDatatypes.hpp" @@ -96,6 +97,9 @@ MAKE_FORMATTER(SilKit::Services::Lin::WireLinControllerConfig); MAKE_FORMATTER(SilKit::Services::Logging::LogMsg); +MAKE_FORMATTER(VSilKit::MetricKind); +MAKE_FORMATTER(VSilKit::MetricsUpdate); + MAKE_FORMATTER(SilKit::Services::Orchestration::NextSimTask); MAKE_FORMATTER(SilKit::Services::Orchestration::ParticipantState); MAKE_FORMATTER(SilKit::Services::Orchestration::ParticipantStatus); diff --git a/SilKit/source/services/metrics/CMakeLists.txt b/SilKit/source/services/metrics/CMakeLists.txt new file mode 100644 index 000000000..7393ee916 --- /dev/null +++ b/SilKit/source/services/metrics/CMakeLists.txt @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +# +# SPDX-License-Identifier: MIT + + +add_library(I_SilKit_Services_Metrics INTERFACE) + +target_include_directories(I_SilKit_Services_Metrics + INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}" +) + +target_link_libraries(I_SilKit_Services_Metrics + INTERFACE SilKitInterface + + INTERFACE I_SilKit_Core_Internal +) + + +add_library(O_SilKit_Services_Metrics OBJECT + MetricsDatatypes.cpp + MetricsManager.cpp + MetricsProcessor.cpp + MetricsReceiver.cpp + MetricsSender.cpp + MetricsSerdes.cpp + + MetricsJsonSink.cpp + MetricsRemoteSink.cpp + + MetricsTimerThread.cpp + + CreateMetricsSinksFromParticipantConfiguration.cpp +) + +target_link_libraries(O_SilKit_Services_Metrics + PUBLIC I_SilKit_Services_Metrics + PRIVATE I_SilKit_Util_StringHelpers +) + + +add_silkit_test_to_executable(SilKitUnitTests + SOURCES Test_MetricsManager.cpp + LIBS S_SilKitImpl +) + +add_silkit_test_to_executable(SilKitUnitTests + SOURCES Test_MetricsProcessor.cpp + LIBS S_SilKitImpl +) + +add_silkit_test_to_executable(SilKitUnitTests + SOURCES Test_MetricsSerdes.cpp + LIBS S_SilKitImpl +) + +add_silkit_test_to_executable(SilKitUnitTests + SOURCES Test_MetricsJsonSink.cpp + LIBS S_SilKitImpl +) + +add_silkit_test_to_executable(SilKitUnitTests + SOURCES Test_MetricsRemoteSink.cpp + LIBS S_SilKitImpl +) diff --git a/SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.cpp b/SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.cpp new file mode 100644 index 000000000..2ecabaf59 --- /dev/null +++ b/SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.cpp @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "CreateMetricsSinksFromParticipantConfiguration.hpp" + +#include "MetricsJsonSink.hpp" +#include "MetricsRemoteSink.hpp" + +#include "Assert.hpp" +#include "StringHelpers.hpp" +#include "ILoggerInternal.hpp" + +#include + +#include "fmt/format.h" + +namespace VSilKit { + +namespace Log = SilKit::Services::Logging; + +auto CreateMetricsSinksFromParticipantConfiguration( + SilKit::Services::Logging::ILogger *logger, IMetricsSender *sender, const std::string &participantName, + const std::vector &configuredSinks) -> std::vector> +{ + std::vector> sinks; + + auto metricsFileTimestamp = SilKit::Util::CurrentTimestampString(); + + for (const auto &config : configuredSinks) + { + std::unique_ptr sink; + + if (config.type == SilKit::Config::MetricsSink::Type::JsonFile) + { + auto filename = fmt::format("{}_{}.txt", config.name, metricsFileTimestamp); + auto ostream = std::make_unique(filename); + auto realSink = std::make_unique(std::move(ostream)); + sink = std::move(realSink); + } + + if (config.type == SilKit::Config::MetricsSink::Type::Remote) + { + SILKIT_ASSERT(sender != nullptr); + + auto realSink = std::make_unique(participantName, *sender); + sink = std::move(realSink); + } + + if (sink == nullptr) + { + Log::Error(logger, "Failed to create metrics sink {}", config.name); + continue; + } + + sinks.emplace_back(std::move(sink)); + } + + return sinks; +} + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.hpp b/SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.hpp new file mode 100644 index 000000000..5b1158265 --- /dev/null +++ b/SilKit/source/services/metrics/CreateMetricsSinksFromParticipantConfiguration.hpp @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMetricsSink.hpp" +#include "IMetricsSender.hpp" +#include "silkit/services/logging/ILogger.hpp" + +#include "ParticipantConfiguration.hpp" + +#include +#include +#include + +namespace VSilKit { + +auto CreateMetricsSinksFromParticipantConfiguration( + SilKit::Services::Logging::ILogger* logger, IMetricsSender* sender, const std::string& participantName, + const std::vector& configuredSinks) -> std::vector>; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/ICounterMetric.hpp b/SilKit/source/services/metrics/ICounterMetric.hpp new file mode 100644 index 000000000..5c3e80868 --- /dev/null +++ b/SilKit/source/services/metrics/ICounterMetric.hpp @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace VSilKit { + +struct ICounterMetric +{ + virtual ~ICounterMetric() = default; + virtual void Add(std::uint64_t delta) = 0; + virtual void Set(std::uint64_t value) = 0; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/IMetricsManager.hpp b/SilKit/source/services/metrics/IMetricsManager.hpp new file mode 100644 index 000000000..8095d14ec --- /dev/null +++ b/SilKit/source/services/metrics/IMetricsManager.hpp @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace VSilKit { + +struct ICounterMetric; +struct IStatisticMetric; +struct IStringListMetric; + +struct IMetricsManager +{ + virtual ~IMetricsManager() = default; + virtual void SubmitUpdates() = 0; + virtual auto GetCounter(const std::string& name) -> ICounterMetric* = 0; + virtual auto GetStatistic(const std::string& name) -> IStatisticMetric* = 0; + virtual auto GetStringList(const std::string& name) -> IStringListMetric* = 0; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/IMetricsProcessor.hpp b/SilKit/source/services/metrics/IMetricsProcessor.hpp new file mode 100644 index 000000000..33cc380c4 --- /dev/null +++ b/SilKit/source/services/metrics/IMetricsProcessor.hpp @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace VSilKit { + +struct MetricsUpdate; + +struct IMetricsProcessor +{ + virtual ~IMetricsProcessor() = default; + virtual void Process(const std::string& origin, const MetricsUpdate& metricsUpdate) = 0; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/IMetricsSender.hpp b/SilKit/source/services/metrics/IMetricsSender.hpp new file mode 100644 index 000000000..c707f171d --- /dev/null +++ b/SilKit/source/services/metrics/IMetricsSender.hpp @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +namespace VSilKit { + +struct MetricsUpdate; + +struct IMetricsSender +{ + virtual ~IMetricsSender() = default; + virtual void Send(const MetricsUpdate& msg) = 0; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/IMetricsSink.hpp b/SilKit/source/services/metrics/IMetricsSink.hpp new file mode 100644 index 000000000..d715823de --- /dev/null +++ b/SilKit/source/services/metrics/IMetricsSink.hpp @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "silkit/util/Span.hpp" + +#include "MetricsDatatypes.hpp" + +#include + +namespace VSilKit { + +struct IMetricsSink +{ + virtual ~IMetricsSink() = default; + + virtual void Process(const std::string& origin, const MetricsUpdate& metricsUpdate) = 0; +}; + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/IMetricsTimerThread.hpp b/SilKit/source/services/metrics/IMetricsTimerThread.hpp new file mode 100644 index 000000000..e9348e773 --- /dev/null +++ b/SilKit/source/services/metrics/IMetricsTimerThread.hpp @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +namespace VSilKit { + +struct IMetricsTimerThread +{ + virtual ~IMetricsTimerThread() = default; + virtual void Start() = 0; +}; + +} // namespace VSilKit + +namespace SilKit { +namespace Core { +using VSilKit::IMetricsTimerThread; +} // namespace Core +} // namespace SilKit diff --git a/SilKit/source/services/metrics/IMsgForMetricsReceiver.hpp b/SilKit/source/services/metrics/IMsgForMetricsReceiver.hpp new file mode 100644 index 000000000..2f86ac2a1 --- /dev/null +++ b/SilKit/source/services/metrics/IMsgForMetricsReceiver.hpp @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "MetricsDatatypes.hpp" + +#include "IReceiver.hpp" +#include "ISender.hpp" + + +namespace VSilKit { + + +struct IMsgForMetricsReceiver + : SilKit::Core::ISender<> + , SilKit::Core::IReceiver +{ +}; + + +} // namespace VSilKit + + +namespace SilKit { +namespace Core { +using VSilKit::IMsgForMetricsReceiver; +} // namespace Core +} // namespace SilKit diff --git a/SilKit/source/services/metrics/IMsgForMetricsSender.hpp b/SilKit/source/services/metrics/IMsgForMetricsSender.hpp new file mode 100644 index 000000000..12c51eab3 --- /dev/null +++ b/SilKit/source/services/metrics/IMsgForMetricsSender.hpp @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "MetricsDatatypes.hpp" + +#include "IReceiver.hpp" +#include "ISender.hpp" + + +namespace VSilKit { + + +struct IMsgForMetricsSender + : SilKit::Core::ISender + , SilKit::Core::IReceiver<> +{ +}; + + +} // namespace VSilKit + + +namespace SilKit { +namespace Core { +using VSilKit::IMsgForMetricsSender; +} // namespace Core +} // namespace SilKit diff --git a/SilKit/source/services/metrics/IStatisticMetric.hpp b/SilKit/source/services/metrics/IStatisticMetric.hpp new file mode 100644 index 000000000..537d76093 --- /dev/null +++ b/SilKit/source/services/metrics/IStatisticMetric.hpp @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +namespace VSilKit { + +struct IStatisticMetric +{ + virtual ~IStatisticMetric() = default; + virtual void Take(double value) = 0; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/IStringListMetric.hpp b/SilKit/source/services/metrics/IStringListMetric.hpp new file mode 100644 index 000000000..ee449ab13 --- /dev/null +++ b/SilKit/source/services/metrics/IStringListMetric.hpp @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace VSilKit { + +struct IStringListMetric +{ + virtual ~IStringListMetric() = default; + virtual void Clear() = 0; + virtual void Add(const std::string& value) = 0; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/Metrics.hpp b/SilKit/source/services/metrics/Metrics.hpp new file mode 100644 index 000000000..30b5214cc --- /dev/null +++ b/SilKit/source/services/metrics/Metrics.hpp @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include + +#include "ICounterMetric.hpp" +#include "IStatisticMetric.hpp" +#include "IStringListMetric.hpp" +#include "IMetricsManager.hpp" +#include "IMetricsSender.hpp" +#include "IMetricsProcessor.hpp" + + +namespace SilKit { +namespace Core { +using VSilKit::ICounterMetric; +using VSilKit::IStatisticMetric; +using VSilKit::IStringListMetric; +using VSilKit::IMetricsManager; +using VSilKit::IMetricsProcessor; +using VSilKit::IMetricsSender; +} // namespace Core +} // namespace SilKit diff --git a/SilKit/source/services/metrics/MetricsDatatypes.cpp b/SilKit/source/services/metrics/MetricsDatatypes.cpp new file mode 100644 index 000000000..47dfe1335 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsDatatypes.cpp @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsDatatypes.hpp" + +#include +#include + + +namespace VSilKit { + + +auto operator==(const MetricData& lhs, const MetricData& rhs) -> bool +{ + return lhs.timestamp == rhs.timestamp && lhs.name == rhs.name && lhs.kind == rhs.kind && lhs.value == rhs.value; +} + +auto operator==(const MetricsUpdate& lhs, const MetricsUpdate& rhs) -> bool +{ + return lhs.metrics == rhs.metrics; +} + + +auto operator<<(std::ostream& os, const MetricKind& metricKind) -> std::ostream& +{ + switch (metricKind) + { + case MetricKind::COUNTER: + return os << "MetricKind::COUNTER"; + case MetricKind::STATISTIC: + return os << "MetricKind::STATISTIC"; + case MetricKind::STRING_LIST: + return os << "MetricKind::STRING_LIST"; + default: + return os << "MetricKind(" << static_cast>(metricKind) << ")"; + } +} + +auto operator<<(std::ostream& os, const MetricData& metricData) -> std::ostream& +{ + return os << "MetricData{timestamp=" << metricData.timestamp << ", name=" << metricData.name + << ", kind=" << metricData.kind << ", value=" << metricData.value << "}"; +} + +auto operator<<(std::ostream& os, const MetricsUpdate& metricsUpdate) -> std::ostream& +{ + os << "MetricsUpdate{metrics=["; + + const char* separator = ""; + for (const auto& metricData : metricsUpdate.metrics) + { + os << separator << metricData; + separator = ", "; + } + + os << "]}"; + return os; +} + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsDatatypes.hpp b/SilKit/source/services/metrics/MetricsDatatypes.hpp new file mode 100644 index 000000000..d2fb82572 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsDatatypes.hpp @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include +#include +#include + + +namespace VSilKit { + + +using MetricId = uint64_t; +using MetricTimestamp = uint64_t; + + +enum struct MetricKind +{ + COUNTER, + STATISTIC, + STRING_LIST, +}; + + +struct MetricData +{ + MetricTimestamp timestamp; + std::string name; + MetricKind kind; + std::string value; +}; + + +struct MetricsUpdate +{ + std::vector metrics; +}; + + +auto operator==(const MetricData& lhs, const MetricData& rhs) -> bool; + +auto operator==(const MetricsUpdate& lhs, const MetricsUpdate& rhs) -> bool; + + +auto operator<<(std::ostream& os, const MetricKind& metricKind) -> std::ostream&; + +auto operator<<(std::ostream& os, const MetricData& metricData) -> std::ostream&; + +auto operator<<(std::ostream& os, const MetricsUpdate& metricValue) -> std::ostream&; + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsJsonSink.cpp b/SilKit/source/services/metrics/MetricsJsonSink.cpp new file mode 100644 index 000000000..e56756527 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsJsonSink.cpp @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsJsonSink.hpp" + +#include "StringHelpers.hpp" + +#include + +namespace { + +struct MetricKindString +{ + VSilKit::MetricKind kind; + + friend auto operator<<(std::ostream& ostream, const MetricKindString& self) -> std::ostream& + { + switch (self.kind) + { + case VSilKit::MetricKind::COUNTER: + return ostream << "COUNTER"; + case VSilKit::MetricKind::STATISTIC: + return ostream << "STATISTIC"; + case VSilKit::MetricKind::STRING_LIST: + return ostream << "STRING_LIST"; + default: + return ostream << static_cast>(self.kind); + } + } +}; + +} // namespace + +namespace VSilKit { + +MetricsJsonSink::MetricsJsonSink(std::unique_ptr ostream) + : _ostream{std::move(ostream)} +{ +} + +void MetricsJsonSink::Process(const std::string& origin, const MetricsUpdate& metricsUpdate) +{ + std::lock_guard lock{_mx}; + + for (const auto& data : metricsUpdate.metrics) + { + *_ostream << R"({"ts":)" << data.timestamp << R"(,"pn":")" << SilKit::Util::EscapedJsonString{origin} + << R"(","mn":")" << SilKit::Util::EscapedJsonString{data.name} << R"(","mk":")" + << MetricKindString{data.kind} << R"(","mv":)" << data.value << R"(})" << '\n'; + } + + *_ostream << std::flush; +} + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsJsonSink.hpp b/SilKit/source/services/metrics/MetricsJsonSink.hpp new file mode 100644 index 000000000..12967d8a8 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsJsonSink.hpp @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMetricsSink.hpp" + +#include +#include +#include +#include + +namespace VSilKit { + +class MetricsJsonSink : public IMetricsSink +{ + std::mutex _mx; + std::unique_ptr _ostream; + +public: + explicit MetricsJsonSink(std::unique_ptr ostream); + + void Process(const std::string& origin, const MetricsUpdate& metricsUpdate) override; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/MetricsManager.cpp b/SilKit/source/services/metrics/MetricsManager.cpp new file mode 100644 index 000000000..f6bb1388e --- /dev/null +++ b/SilKit/source/services/metrics/MetricsManager.cpp @@ -0,0 +1,338 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsManager.hpp" + +#include "Assert.hpp" +#include "MetricsProcessor.hpp" + +#include +#include + +#include +#include +#include +#include + +#include "fmt/format.h" + + +namespace VSilKit { + + +class MetricsManager::CounterMetric + : public ICounterMetric + , public IMetric +{ +public: + CounterMetric(); + +public: // ICounterMetric + void Add(uint64_t delta) override; + void Set(uint64_t value) override; + +public: // MetricsManager::IMetric + auto GetMetricKind() const -> MetricKind override; + auto GetUpdateTime() const -> MetricTimePoint override; + auto FormatValue() const -> std::string override; + +private: + TimePoint _timestamp; + uint64_t _value{0}; +}; + + +class MetricsManager::StatisticMetric + : public IStatisticMetric + , public IMetric +{ +public: + StatisticMetric(); + +public: // IStatisticMetric + void Take(double value) override; + +public: // MetricsManager::IMetric + auto GetMetricKind() const -> MetricKind override; + auto GetUpdateTime() const -> MetricTimePoint override; + auto FormatValue() const -> std::string override; + +private: + TimePoint _timestamp; + double _mean{0.0}; + double _variance{0.0}; + double _minimum{std::numeric_limits::max()}; + double _maximum{std::numeric_limits::lowest()}; + + // see https://fanf2.user.srcf.net/hermes/doc/antiforgery/stats.pdf + // Incremental calculation of weighted mean and variance + // by Tony Finch (fanf2@cam.ac.uk) University of Cambridge Computing Service +}; + + +class MetricsManager::StringListMetric + : public IStringListMetric + , public IMetric +{ +public: + StringListMetric(); + +public: // IStringListMetric + void Clear() override; + void Add(std::string const &value) override; + +public: // MetricsManager::IMetric + auto GetMetricKind() const -> MetricKind override; + auto GetUpdateTime() const -> MetricTimePoint override; + auto FormatValue() const -> std::string override; + +private: + TimePoint _timestamp; + std::vector _strings; +}; + + +MetricsManager::MetricsManager(std::string participantName, IMetricsProcessor &processor) + : _participantName{std::move(participantName)} + , _processor{&processor} +{ +} + + +void MetricsManager::SetLogger(SilKit::Services::Logging::ILogger &logger) +{ + _logger = &logger; +} + +void MetricsManager::SubmitUpdates() +{ + MetricsUpdate msg{}; + + { + std::lock_guard lock{_mutex}; + + msg.metrics.reserve(_metrics.size()); + + const auto to_ns = [](const auto timepoint) { + return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); + }; + + for (const auto &pair : _metrics) + { + const auto &name = pair.first; + const auto *metric = pair.second.get(); + + const auto timepoint = metric->GetUpdateTime(); + if (timepoint <= _lastSubmitUpdate) + { + continue; + } + + MetricData data{}; + data.timestamp = to_ns(timepoint); + data.name = name; + data.kind = metric->GetMetricKind(); + data.value = metric->FormatValue(); + + msg.metrics.emplace_back(std::move(data)); + } + + _lastSubmitUpdate = Clock::now(); + } + + if (!msg.metrics.empty()) + { + _processor->Process(_participantName, msg); + } +} + + +// IMetricsManager + +auto MetricsManager::GetCounter(const std::string &name) -> ICounterMetric * +{ + return &dynamic_cast(*GetOrCreateMetric(name, MetricKind::COUNTER)); +} + +auto MetricsManager::GetStatistic(const std::string &name) -> IStatisticMetric * +{ + return &dynamic_cast(*GetOrCreateMetric(name, MetricKind::STATISTIC)); +} + +auto MetricsManager::GetStringList(const std::string &name) -> IStringListMetric * +{ + return &dynamic_cast(*GetOrCreateMetric(name, MetricKind::STRING_LIST)); +} + + +// MetricsManager + +auto MetricsManager::GetOrCreateMetric(std::string name, MetricKind kind) -> IMetric * +{ + std::lock_guard lock{_mutex}; + + auto it = _metrics.find(name); + + if (it == _metrics.end()) + { + switch (kind) + { + case MetricKind::COUNTER: + it = _metrics.emplace(name, std::make_unique()).first; + break; + case MetricKind::STATISTIC: + it = _metrics.emplace(name, std::make_unique()).first; + break; + case MetricKind::STRING_LIST: + it = _metrics.emplace(name, std::make_unique()).first; + break; + default: + throw SilKit::SilKitError{fmt::format("Invalid MetricKind ({})", kind)}; + } + } + + const auto existingKind = it->second->GetMetricKind(); + if (existingKind != kind) + { + throw SilKit::SilKitError{ + fmt::format("Existing metric {} has different kind {} than the requested {}", name, existingKind, kind)}; + } + + return it->second.get(); +} + + +// CounterMetric + +MetricsManager::CounterMetric::CounterMetric() = default; + +void MetricsManager::CounterMetric::Add(uint64_t delta) +{ + _timestamp = MetricsManager::Clock::now(); + _value += delta; +} + +void MetricsManager::CounterMetric::Set(uint64_t value) +{ + _timestamp = MetricsManager::Clock::now(); + _value = value; +} + +auto MetricsManager::CounterMetric::GetMetricKind() const -> MetricKind +{ + return MetricKind::COUNTER; +} + +auto MetricsManager::CounterMetric::GetUpdateTime() const -> MetricTimePoint +{ + return _timestamp; +} + +auto MetricsManager::CounterMetric::FormatValue() const -> std::string +{ + return std::to_string(_value); +} + + +// StatisticMetric + +MetricsManager::StatisticMetric::StatisticMetric() = default; + +void MetricsManager::StatisticMetric::Take(double value) +{ + _timestamp = MetricsManager::Clock::now(); + + constexpr double alpha = 0.5; + + const auto difference = value - _mean; + const auto increment = alpha * difference; + + _mean += increment; + _variance = (1.0 - alpha) * (_variance + difference * increment); + _minimum = std::min(_minimum, value); + _maximum = std::max(_maximum, value); +} + +auto MetricsManager::StatisticMetric::GetMetricKind() const -> MetricKind +{ + return MetricKind::STATISTIC; +} + +auto MetricsManager::StatisticMetric::GetUpdateTime() const -> MetricTimePoint +{ + return _timestamp; +} + +auto MetricsManager::StatisticMetric::FormatValue() const -> std::string +{ + return fmt::format(R"([{},{},{},{}])", _mean, std::sqrt(_variance), _minimum, _maximum); +} + + +// StringListMetric + +MetricsManager::StringListMetric::StringListMetric() = default; + +void MetricsManager::StringListMetric::Clear() +{ + _timestamp = MetricsManager::Clock::now(); + _strings.clear(); +} + +void MetricsManager::StringListMetric::Add(std::string const &value) +{ + _timestamp = MetricsManager::Clock::now(); + _strings.emplace_back(value); +} + +auto MetricsManager::StringListMetric::GetMetricKind() const -> MetricKind +{ + return MetricKind::STRING_LIST; +} + +auto MetricsManager::StringListMetric::GetUpdateTime() const -> MetricTimePoint +{ + return _timestamp; +} + +auto MetricsManager::StringListMetric::FormatValue() const -> std::string +{ + std::string result; + const auto formatString = [](std::string &result, const std::string &string) { + result.push_back('"'); + for (const char ch : string) + { + switch (ch) + { + case '\\': + result.push_back(ch); + result.push_back(ch); + break; + case '"': + result.push_back('\\'); + result.push_back(ch); + break; + default: + result.push_back(ch); + break; + } + } + result.push_back('"'); + }; + + result.push_back('['); + for (size_t index = 0; index != _strings.size(); ++index) + { + if (index != 0) + { + result.push_back(','); + } + formatString(result, _strings[index]); + } + result.push_back(']'); + return result; +} + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsManager.hpp b/SilKit/source/services/metrics/MetricsManager.hpp new file mode 100644 index 000000000..ceb348461 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsManager.hpp @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMsgForMetricsSender.hpp" +#include "IMetricsManager.hpp" +#include "Metrics.hpp" + +#include "ILoggerInternal.hpp" +#include "IServiceEndpoint.hpp" +#include "IParticipantInternal.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +namespace VSilKit { + +using MetricClock = std::chrono::steady_clock; +using MetricTimePoint = MetricClock::time_point; + + +class MetricsManager : public IMetricsManager +{ +public: + using Clock = MetricClock; + using TimePoint = MetricTimePoint; + +private: + struct IMetric + { + virtual ~IMetric() = default; + virtual auto GetMetricKind() const -> MetricKind = 0; + virtual auto GetUpdateTime() const -> TimePoint = 0; + virtual auto FormatValue() const -> std::string = 0; + }; + + class CounterMetric; + class StatisticMetric; + class StringListMetric; + +public: + MetricsManager(std::string participantName, IMetricsProcessor& processor); + + void SetLogger(SilKit::Services::Logging::ILogger& logger); + +public: // IMetricsManager + void SubmitUpdates() override; + auto GetCounter(const std::string& name) -> ICounterMetric* override; + auto GetStatistic(const std::string& name) -> IStatisticMetric* override; + auto GetStringList(const std::string& name) -> IStringListMetric* override; + +private: + auto GetOrCreateMetric(std::string name, MetricKind kind) -> IMetric*; + +private: + std::string _participantName; + IMetricsProcessor* _processor{nullptr}; + SilKit::Services::Logging::ILogger* _logger{nullptr}; + + // Metrics + + std::mutex _mutex; + std::unordered_map> _metrics; + TimePoint _lastSubmitUpdate{Clock::now()}; +}; + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsProcessor.cpp b/SilKit/source/services/metrics/MetricsProcessor.cpp new file mode 100644 index 000000000..f2dd412c5 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsProcessor.cpp @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsProcessor.hpp" + +#include "IParticipantInternal.hpp" +#include "ILoggerInternal.hpp" + +namespace Log = SilKit::Services::Logging; + +namespace VSilKit { + +MetricsProcessor::MetricsProcessor(std::string participantName) + : _participantName{std::move(participantName)} +{ +} + +void MetricsProcessor::SetLogger(SilKit::Services::Logging::ILogger &logger) +{ + _logger = &logger; +} + +void MetricsProcessor::SetSinks(std::vector> sinks) +{ + std::lock_guard lock{_mutex}; + + if (_sinksSetUp) + { + Log::Error(_logger, "Refusing to setup metrics sinks again"); + return; + } + + _sinks = std::move(sinks); + + for (const auto &sink : _sinks) + { + for (const auto &pair : _updateCache) + { + const auto &origin = pair.first; + const auto &update = pair.second; + sink->Process(origin, update); + } + } + + _updateCache.clear(); + + _sinksSetUp = true; +} + +void MetricsProcessor::Process(const std::string &origin, const VSilKit::MetricsUpdate &metricsUpdate) +{ + if (!_sinksSetUp) + { + std::lock_guard lock{_mutex}; + + if (!_sinksSetUp) + { + auto &cache = _updateCache[origin].metrics; + const auto &metrics = metricsUpdate.metrics; + + cache.insert(cache.end(), metrics.begin(), metrics.end()); + + return; + } + } + + for (const auto &sink : _sinks) + { + sink->Process(origin, metricsUpdate); + } +} + +void MetricsProcessor::OnMetricsUpdate(const std::string &participantName, const MetricsUpdate &metricsUpdate) +{ + if (participantName == _participantName) + { + return; // ignore metrics received from ourself (avoids infinite loops) + } + + Process(participantName, metricsUpdate); +} + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsProcessor.hpp b/SilKit/source/services/metrics/MetricsProcessor.hpp new file mode 100644 index 000000000..c5e654a55 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsProcessor.hpp @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMetricsSink.hpp" +#include "MetricsDatatypes.hpp" +#include "MetricsReceiver.hpp" + +#include +#include +#include +#include +#include + +namespace VSilKit { + +class MetricsProcessor + : public IMetricsProcessor + , public IMetricsReceiverListener +{ + std::mutex _mutex; + + std::string _participantName; + SilKit::Services::Logging::ILogger* _logger{nullptr}; + + std::atomic _sinksSetUp{false}; + std::vector> _sinks; + + std::unordered_map _updateCache; + +public: + explicit MetricsProcessor(std::string participantName); + +public: + void SetLogger(SilKit::Services::Logging::ILogger& logger); + void SetSinks(std::vector> sinks); + +public: // IMetricsProcessor + void Process(const std::string& origin, const MetricsUpdate& metricsUpdate) override; + +public: // IMetricsReceiverListener + void OnMetricsUpdate(std::string const& participantName, const MetricsUpdate& metricsUpdate) override; +}; + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsReceiver.cpp b/SilKit/source/services/metrics/MetricsReceiver.cpp new file mode 100644 index 000000000..607d35b83 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsReceiver.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsReceiver.hpp" + +#include "ILoggerInternal.hpp" + + +namespace Log = SilKit::Services::Logging; + + +namespace VSilKit { + + +MetricsReceiver::MetricsReceiver(SilKit::Core::IParticipantInternal *, SilKit::Services::Logging::ILogger &logger, + IMetricsReceiverListener &listener) + : _logger{&logger} + , _listener{&listener} +{ + _serviceDescriptor.SetNetworkName("default"); +} + + +// IMsgForMetricsReceiver + +void MetricsReceiver::ReceiveMsg(const SilKit::Core::IServiceEndpoint *from, const VSilKit::MetricsUpdate &msg) +{ + if (_listener == nullptr) + { + return; + } + + _listener->OnMetricsUpdate(from->GetServiceDescriptor().GetParticipantName(), msg); +} + + +// IServiceEndpoint + +void MetricsReceiver::SetServiceDescriptor(const SilKit::Core::ServiceDescriptor &serviceDescriptor) +{ + _serviceDescriptor = serviceDescriptor; +} + +auto MetricsReceiver::GetServiceDescriptor() const -> const SilKit::Core::ServiceDescriptor & +{ + return _serviceDescriptor; +} + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsReceiver.hpp b/SilKit/source/services/metrics/MetricsReceiver.hpp new file mode 100644 index 000000000..d7ea5e721 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsReceiver.hpp @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMsgForMetricsReceiver.hpp" + +#include "ILoggerInternal.hpp" +#include "IServiceEndpoint.hpp" +#include "IParticipantInternal.hpp" + + +namespace VSilKit { + + +struct IMetricsReceiverListener +{ + virtual ~IMetricsReceiverListener() = default; + virtual void OnMetricsUpdate(const std::string& participantName, const MetricsUpdate& metricsUpdate) = 0; +}; + + +class MetricsReceiver + : public IMsgForMetricsReceiver + , public SilKit::Core::IServiceEndpoint +{ +public: + MetricsReceiver(SilKit::Core::IParticipantInternal*, SilKit::Services::Logging::ILogger& logger, + IMetricsReceiverListener& listener); + + // NB: The first constructor argument is present to enable using the CreateController function template. It is + // allowed to be nullptr. + +public: // IMsgForMetricsReceiver + void ReceiveMsg(const SilKit::Core::IServiceEndpoint* from, const MetricsUpdate& msg) override; + +public: // IServiceEndpoint + void SetServiceDescriptor(const SilKit::Core::ServiceDescriptor& serviceDescriptor) override; + auto GetServiceDescriptor() const -> const SilKit::Core::ServiceDescriptor& override; + +private: + SilKit::Services::Logging::ILogger* _logger{nullptr}; + IMetricsReceiverListener* _listener{nullptr}; + + SilKit::Core::ServiceDescriptor _serviceDescriptor; +}; + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsRemoteSink.cpp b/SilKit/source/services/metrics/MetricsRemoteSink.cpp new file mode 100644 index 000000000..3e9c79ee3 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsRemoteSink.cpp @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsRemoteSink.hpp" + +#include "IParticipantInternal.hpp" +#include "MetricsSender.hpp" + +namespace VSilKit { + +MetricsRemoteSink::MetricsRemoteSink(std::string participantName, IMetricsSender &sender) + : _participantName{std::move(participantName)} + , _sender{&sender} +{ +} + +void MetricsRemoteSink::Process(const std::string &origin, const VSilKit::MetricsUpdate &metricsUpdate) +{ + // do not forward metrics from other participants again + if (origin != _participantName) + { + return; + } + + _sender->Send(metricsUpdate); +} + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsRemoteSink.hpp b/SilKit/source/services/metrics/MetricsRemoteSink.hpp new file mode 100644 index 000000000..a248db6b4 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsRemoteSink.hpp @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMetricsSink.hpp" + +namespace VSilKit { +struct IMetricsSender; +} // namespace VSilKit + +namespace VSilKit { + +class MetricsRemoteSink : public IMetricsSink +{ + std::string _participantName; + IMetricsSender* _sender{nullptr}; + +public: + explicit MetricsRemoteSink(std::string participantName, IMetricsSender& sender); + + void Process(const std::string& origin, const MetricsUpdate& metricsUpdate) override; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/MetricsSender.cpp b/SilKit/source/services/metrics/MetricsSender.cpp new file mode 100644 index 000000000..a1d9a1e50 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsSender.cpp @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsSender.hpp" + +#include "ILoggerInternal.hpp" + + +namespace Log = SilKit::Services::Logging; + + +namespace VSilKit { + + +MetricsSender::MetricsSender(SilKit::Core::IParticipantInternal *participant) + : _participant{participant} + , _logger{participant->GetLogger()} +{ + _serviceDescriptor.SetNetworkName("default"); +} + + +// IMetricsSender + +void MetricsSender::Send(const VSilKit::MetricsUpdate &msg) +{ + _participant->SendMsg(this, msg); +} + + +// IServiceEndpoint + +void MetricsSender::SetServiceDescriptor(const SilKit::Core::ServiceDescriptor &serviceDescriptor) +{ + _serviceDescriptor = serviceDescriptor; +} + +auto MetricsSender::GetServiceDescriptor() const -> const SilKit::Core::ServiceDescriptor & +{ + return _serviceDescriptor; +} + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsSender.hpp b/SilKit/source/services/metrics/MetricsSender.hpp new file mode 100644 index 000000000..1336db288 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsSender.hpp @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMsgForMetricsSender.hpp" + +#include "ILoggerInternal.hpp" +#include "IServiceEndpoint.hpp" +#include "IParticipantInternal.hpp" + + +namespace VSilKit { + + +class MetricsSender + : public IMetricsSender + , public IMsgForMetricsSender + , public SilKit::Core::IServiceEndpoint +{ +public: + explicit MetricsSender(SilKit::Core::IParticipantInternal*); + + // NB: The first constructor argument is present to enable using the CreateController function template. It is + // allowed to be nullptr. + +public: // IMetricsSender + void Send(const MetricsUpdate& msg) override; + +public: // IServiceEndpoint + void SetServiceDescriptor(const SilKit::Core::ServiceDescriptor& serviceDescriptor) override; + auto GetServiceDescriptor() const -> const SilKit::Core::ServiceDescriptor& override; + +private: + SilKit::Core::IParticipantInternal* _participant{nullptr}; + SilKit::Services::Logging::ILogger* _logger{nullptr}; + + SilKit::Core::ServiceDescriptor _serviceDescriptor; +}; + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsSerdes.cpp b/SilKit/source/services/metrics/MetricsSerdes.cpp new file mode 100644 index 000000000..56acf9113 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsSerdes.cpp @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsSerdes.hpp" + + +namespace VSilKit { + + +inline auto operator<<(SilKit::Core::MessageBuffer& buffer, const MetricData& msg) -> SilKit::Core::MessageBuffer& +{ + return buffer << msg.timestamp << msg.name << msg.kind << msg.value; +} + +inline auto operator>>(SilKit::Core::MessageBuffer& buffer, MetricData& out) -> SilKit::Core::MessageBuffer& +{ + return buffer >> out.timestamp >> out.name >> out.kind >> out.value; +} + + +void Serialize(SilKit::Core::MessageBuffer& buffer, const MetricsUpdate& msg) +{ + buffer << msg.metrics; +} + +void Deserialize(SilKit::Core::MessageBuffer& buffer, MetricsUpdate& out) +{ + buffer >> out.metrics; +} + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsSerdes.hpp b/SilKit/source/services/metrics/MetricsSerdes.hpp new file mode 100644 index 000000000..4913783cc --- /dev/null +++ b/SilKit/source/services/metrics/MetricsSerdes.hpp @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "MetricsDatatypes.hpp" + +#include "MessageBuffer.hpp" + + +namespace VSilKit { + + +void Serialize(SilKit::Core::MessageBuffer& buffer, const MetricsUpdate& msg); +void Deserialize(SilKit::Core::MessageBuffer& buffer, MetricsUpdate& out); + + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsTimerThread.cpp b/SilKit/source/services/metrics/MetricsTimerThread.cpp new file mode 100644 index 000000000..a104c025e --- /dev/null +++ b/SilKit/source/services/metrics/MetricsTimerThread.cpp @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "MetricsTimerThread.hpp" + +#include "SetThreadName.hpp" + +namespace VSilKit { + +MetricsTimerThread::MetricsTimerThread(std::function callback) + : _callback{std::move(callback)} + , _thread{MakeThread()} +{ +} + +MetricsTimerThread::~MetricsTimerThread() +{ + if (!_started.exchange(true)) + { + try + { + _go.set_value(); + } + catch (const std::future_error &) + { + } + } + + try + { + _done.set_value(); + } + catch (const std::future_error &) + { + } + + if (_thread.joinable()) + { + _thread.join(); + } +} + +void MetricsTimerThread::Start() +{ + if (!_started.exchange(true)) + { + try + { + _go.set_value(); + } + catch (const std::future_error &) + { + } + } +} + +auto MetricsTimerThread::MakeThread() -> std::thread +{ + auto go = _go.get_future(); + auto done = _done.get_future(); + return std::thread{[go = std::move(go), done = std::move(done), callback = &_callback]() mutable { + try + { + SilKit::Util::SetThreadName("SK Metrics"); + + go.get(); + + while (true) + { + if (done.wait_for(std::chrono::seconds{1}) != std::future_status::timeout) + { + break; + } + + if (*callback) + { + try + { + (*callback)(); + } + catch (...) + { + // ignore exceptions thrown in the callback + } + } + } + } + catch (...) + { + // leaking an exception here can result in a hard crash + } + }}; +} + +} // namespace VSilKit diff --git a/SilKit/source/services/metrics/MetricsTimerThread.hpp b/SilKit/source/services/metrics/MetricsTimerThread.hpp new file mode 100644 index 000000000..180189934 --- /dev/null +++ b/SilKit/source/services/metrics/MetricsTimerThread.hpp @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "IMetricsTimerThread.hpp" + +#include +#include +#include +#include + +namespace VSilKit { + +class MetricsTimerThread : public IMetricsTimerThread +{ + std::atomic _started{false}; + std::promise _go; + std::promise _done; + std::function _callback; + + std::thread _thread; + +public: + explicit MetricsTimerThread(std::function callback); + + ~MetricsTimerThread() override; + + void Start() override; + +private: + auto MakeThread() -> std::thread; +}; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/services/metrics/Test_MetricsJsonSink.cpp b/SilKit/source/services/metrics/Test_MetricsJsonSink.cpp new file mode 100644 index 000000000..1c4d145be --- /dev/null +++ b/SilKit/source/services/metrics/Test_MetricsJsonSink.cpp @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "MetricsDatatypes.hpp" +#include "MetricsJsonSink.hpp" +#include "StringHelpers.hpp" + +#include +#include + +#include "fmt/format.h" + +#include "yaml-cpp/yaml.h" + +namespace { + +using VSilKit::MetricData; +using VSilKit::MetricKind; +using VSilKit::MetricTimestamp; +using VSilKit::MetricsJsonSink; +using VSilKit::MetricsUpdate; + +using SilKit::Util::EscapeString; + + +TEST(Test_MetricsJsonSink, test_json_escaping_and_structure) +{ + const std::string origin{"Participant Name"}; + + const MetricTimestamp ts1{1}; + const std::string mn1{"Metric\rName\n1"}; + const auto mk1 = MetricKind::COUNTER; + const std::string mv1{"1"}; + + const MetricTimestamp ts2{2}; + const std::string mn2{"Metric\rName\n2"}; + const auto mk2 = MetricKind::STATISTIC; + const std::string mv2{"[1,2,3,4]"}; + + const MetricTimestamp ts3{3}; + const std::string mn3{"Metric\rName\n3"}; + const auto mk3 = MetricKind::STRING_LIST; + const std::string mv3a{"A\r\nB\nC"}; + const std::string mv3b{"D\tE\nF\tG"}; + const std::string mv3{fmt::format(R"(["{}","{}"])", EscapeString(mv3a), EscapeString(mv3b))}; + + MetricsUpdate update; + update.metrics.emplace_back(MetricData{ts1, mn1, mk1, mv1}); + update.metrics.emplace_back(MetricData{ts2, mn2, mk2, mv2}); + update.metrics.emplace_back(MetricData{ts3, mn3, mk3, mv3}); + + auto ownedOstream = std::make_unique(); + auto& ostream = *ownedOstream; + + // create the sink and process the update + + MetricsJsonSink sink{std::move(ownedOstream)}; + sink.Process(origin, update); + + // extract the resulting json from the string-stream + + auto result = ostream.str(); + + // split the result into lines and parse them + + std::vector nodes; + + std::string::size_type pos; + while ((pos = result.find('\n')) != std::string::npos) + { + auto line = result.substr(0, pos); + result = result.substr(pos + 1); + + nodes.emplace_back(YAML::Load(line)); + } + + // checks + + ASSERT_EQ(nodes.size(), 3); + + ASSERT_TRUE(nodes[0].IsMap()); + ASSERT_EQ(nodes[0]["ts"].as(), ts1); + ASSERT_EQ(nodes[0]["mn"].as(), mn1); + ASSERT_EQ(nodes[0]["mk"].as(), "COUNTER"); + ASSERT_EQ(nodes[0]["mv"].as(), mv1); + + ASSERT_TRUE(nodes[1].IsMap()); + ASSERT_EQ(nodes[1]["ts"].as(), ts2); + ASSERT_EQ(nodes[1]["mn"].as(), mn2); + ASSERT_EQ(nodes[1]["mk"].as(), "STATISTIC"); + ASSERT_EQ(nodes[1]["mv"].as>(), (std::vector{1, 2, 3, 4})); + + ASSERT_TRUE(nodes[2].IsMap()); + ASSERT_EQ(nodes[2]["ts"].as(), ts3); + ASSERT_EQ(nodes[2]["mn"].as(), mn3); + ASSERT_EQ(nodes[2]["mk"].as(), "STRING_LIST"); + ASSERT_EQ(nodes[2]["mv"].as>(), (std::vector{mv3a, mv3b})); +} + + +} // anonymous namespace diff --git a/SilKit/source/services/metrics/Test_MetricsManager.cpp b/SilKit/source/services/metrics/Test_MetricsManager.cpp new file mode 100644 index 000000000..2a228c089 --- /dev/null +++ b/SilKit/source/services/metrics/Test_MetricsManager.cpp @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: 2023 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include + +#include "IMetricsProcessor.hpp" +#include "MetricsDatatypes.hpp" +#include "MetricsManager.hpp" + +namespace { + +using VSilKit::MetricData; +using VSilKit::MetricKind; +using VSilKit::MetricsManager; +using VSilKit::MetricsUpdate; +using VSilKit::IMetricsProcessor; + +using testing::Contains; +using testing::Matches; + +struct MockMetricsProcessor : IMetricsProcessor +{ + MOCK_METHOD(void, Process, (const std::string&, const MetricsUpdate&), (override)); +}; + +MATCHER_P2(MetricDataWithNameAndKind, metricName, metricKind, "") +{ + return arg.name == metricName && arg.kind == metricKind; +} + +MATCHER_P2(MetricsUpdateWithSingleMetricWithNameAndKind, metricName, metricKind, "") +{ + return arg.metrics.size() == 1 && Matches(MetricDataWithNameAndKind(metricName, metricKind))(arg.metrics.front()); +} + + +TEST(Test_MetricsManager, counter_metric_create_and_update_only_submits_after_change) +{ + const std::string participantName{"Participant Name"}; + const std::string metricName{"Counter Metric"}; + + MetricsUpdate metricsUpdate; + metricsUpdate.metrics.emplace_back(MetricData{}); + + MockMetricsProcessor mockMetricsProcessor; + EXPECT_CALL(mockMetricsProcessor, + Process(participantName, MetricsUpdateWithSingleMetricWithNameAndKind(metricName, MetricKind::COUNTER))) + .Times(1); + + MetricsManager metricsManager{participantName, mockMetricsProcessor}; + // no metrics to report, no Process call should be made + metricsManager.SubmitUpdates(); + + auto metric = metricsManager.GetCounter(metricName); + // no metric value to report, no Process call should be made + metricsManager.SubmitUpdates(); + + metric->Set(12345); + // metric value to report, single Process call + metricsManager.SubmitUpdates(); +} + + +TEST(Test_MetricsManager, statistic_metric_create_and_update_only_submits_after_change) +{ + const std::string participantName{"Participant Name"}; + const std::string metricName{"Statistic Metric"}; + + MetricsUpdate metricsUpdate; + metricsUpdate.metrics.emplace_back(MetricData{}); + + MockMetricsProcessor mockMetricsProcessor; + EXPECT_CALL(mockMetricsProcessor, Process(participantName, MetricsUpdateWithSingleMetricWithNameAndKind( + metricName, MetricKind::STATISTIC))) + .Times(1); + + MetricsManager metricsManager{participantName, mockMetricsProcessor}; + // no metrics to report, no Process call should be made + metricsManager.SubmitUpdates(); + + auto metric = metricsManager.GetStatistic(metricName); + // no metric value to report, no Process call should be made + metricsManager.SubmitUpdates(); + + metric->Take(123.456); + // metric value to report, single Process call + metricsManager.SubmitUpdates(); +} + + +TEST(Test_MetricsManager, string_list_metric_create_and_update_only_submits_after_change) +{ + const std::string participantName{"Participant Name"}; + const std::string metricName{"String-List Metric"}; + + MetricsUpdate metricsUpdate; + metricsUpdate.metrics.emplace_back(MetricData{}); + + MockMetricsProcessor mockMetricsProcessor; + EXPECT_CALL(mockMetricsProcessor, Process(participantName, MetricsUpdateWithSingleMetricWithNameAndKind( + metricName, MetricKind::STRING_LIST))) + .Times(1); + + MetricsManager metricsManager{participantName, mockMetricsProcessor}; + // no metrics to report, no Process call should be made + metricsManager.SubmitUpdates(); + + auto metric = metricsManager.GetStringList(metricName); + // no metric value to report, no Process call should be made + metricsManager.SubmitUpdates(); + + metric->Add("Woweeee!"); + // metric value to report, single Process call + metricsManager.SubmitUpdates(); +} + + +} // anonymous namespace diff --git a/SilKit/source/services/metrics/Test_MetricsProcessor.cpp b/SilKit/source/services/metrics/Test_MetricsProcessor.cpp new file mode 100644 index 000000000..b7f8813db --- /dev/null +++ b/SilKit/source/services/metrics/Test_MetricsProcessor.cpp @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "IMetricsSink.hpp" +#include "MetricsDatatypes.hpp" +#include "MetricsProcessor.hpp" + +#include +#include +#include +#include + +namespace { + +using VSilKit::IMetricsSink; +using VSilKit::MetricsProcessor; +using VSilKit::MetricsUpdate; +using VSilKit::MetricData; +using VSilKit::MetricKind; + +using testing::ContainerEq; + +struct MockMetricsSink : IMetricsSink +{ + MOCK_METHOD(void, Process, (const std::string&, const MetricsUpdate&), (override)); +}; + +MATCHER_P(MetricsUpdateWithMetrics, metrics, "") +{ + return Matches(ContainerEq(metrics))(arg.metrics); +} + + +TEST(Test_MetricsProcessor, metrics_are_cached_until_sinks_are_set) +{ + const std::string participantName{"Participant Name"}; + + MetricsUpdate updateOne; + updateOne.metrics.emplace_back(MetricData{1, "1", MetricKind::COUNTER, "1"}); + + MetricsUpdate updateTwo; + updateTwo.metrics.emplace_back(MetricData{2, "2", MetricKind::COUNTER, "2"}); + + std::vector metrics; + std::copy(updateOne.metrics.begin(), updateOne.metrics.end(), std::back_inserter(metrics)); + std::copy(updateTwo.metrics.begin(), updateTwo.metrics.end(), std::back_inserter(metrics)); + + std::vector> sinks; + sinks.emplace_back(std::make_unique()); + + auto& sink = dynamic_cast(*sinks[0]); + + EXPECT_CALL(sink, Process(participantName, MetricsUpdateWithMetrics(metrics))).Times(1); + + MetricsProcessor processor{participantName}; + processor.Process(participantName, updateOne); + processor.Process(participantName, updateTwo); + processor.SetSinks(std::move(sinks)); +} + + +TEST(Test_MetricsProcessor, metrics_are_cached_until_sinks_are_set_per_origin) +{ + const std::string participantName{"Participant Name"}; + const std::string updateOneOrigin{"One"}; + const std::string updateTwoOrigin{"Two"}; + + MetricsUpdate updateOneA; + updateOneA.metrics.emplace_back(MetricData{1, "1A", MetricKind::COUNTER, "1"}); + + MetricsUpdate updateOneB; + updateOneB.metrics.emplace_back(MetricData{2, "1B", MetricKind::COUNTER, "1"}); + + MetricsUpdate updateTwoA; + updateTwoA.metrics.emplace_back(MetricData{2, "2", MetricKind::COUNTER, "2"}); + + MetricsUpdate updateTwoB; + updateTwoB.metrics.emplace_back(MetricData{2, "2", MetricKind::COUNTER, "2"}); + + std::vector metricsOne; + std::copy(updateOneA.metrics.begin(), updateOneA.metrics.end(), std::back_inserter(metricsOne)); + std::copy(updateOneB.metrics.begin(), updateOneB.metrics.end(), std::back_inserter(metricsOne)); + + std::vector metricsTwo; + std::copy(updateTwoA.metrics.begin(), updateTwoA.metrics.end(), std::back_inserter(metricsTwo)); + std::copy(updateTwoB.metrics.begin(), updateTwoB.metrics.end(), std::back_inserter(metricsTwo)); + + std::vector> sinks; + sinks.emplace_back(std::make_unique()); + + auto& sink = dynamic_cast(*sinks[0]); + + EXPECT_CALL(sink, Process(updateOneOrigin, MetricsUpdateWithMetrics(metricsOne))).Times(1); + EXPECT_CALL(sink, Process(updateTwoOrigin, MetricsUpdateWithMetrics(metricsTwo))).Times(1); + + MetricsProcessor processor{participantName}; + processor.Process(updateOneOrigin, updateOneA); + processor.Process(updateTwoOrigin, updateTwoA); + processor.Process(updateOneOrigin, updateOneB); + processor.Process(updateTwoOrigin, updateTwoB); + processor.SetSinks(std::move(sinks)); +} + + +TEST(Test_MetricsProcessor, metrics_are_processed_directly_after_sinks_are_set) +{ + const std::string participantName{"Participant Name"}; + + MetricsUpdate updateOne; + updateOne.metrics.emplace_back(MetricData{1, "1", MetricKind::COUNTER, "1"}); + + MetricsUpdate updateTwo; + updateTwo.metrics.emplace_back(MetricData{2, "2", MetricKind::COUNTER, "2"}); + + std::vector> sinks; + sinks.emplace_back(std::make_unique()); + + auto& sink = dynamic_cast(*sinks[0]); + + EXPECT_CALL(sink, Process(participantName, MetricsUpdateWithMetrics(updateOne.metrics))).Times(1); + EXPECT_CALL(sink, Process(participantName, MetricsUpdateWithMetrics(updateTwo.metrics))).Times(1); + + MetricsProcessor processor{participantName}; + processor.SetSinks(std::move(sinks)); + processor.Process(participantName, updateOne); + processor.Process(participantName, updateTwo); +} + + +TEST(Test_MetricsProcessor, receive_handler_ignores_metrics_from_own_participant) +{ + const std::string participantName{"Participant Name"}; + + MetricsUpdate updateOne; + updateOne.metrics.emplace_back(MetricData{1, "1", MetricKind::COUNTER, "1"}); + + MetricsUpdate updateTwo; + updateTwo.metrics.emplace_back(MetricData{2, "2", MetricKind::COUNTER, "2"}); + + std::vector> sinks; + sinks.emplace_back(std::make_unique()); + + auto& sink = dynamic_cast(*sinks[0]); + + EXPECT_CALL(sink, Process(participantName, MetricsUpdateWithMetrics(updateOne.metrics))).Times(1); + EXPECT_CALL(sink, Process(participantName, MetricsUpdateWithMetrics(updateTwo.metrics))).Times(0); + + MetricsProcessor processor{participantName}; + processor.SetSinks(std::move(sinks)); + processor.Process(participantName, updateOne); + processor.OnMetricsUpdate(participantName, updateTwo); +} + + +} // anonymous namespace \ No newline at end of file diff --git a/SilKit/source/services/metrics/Test_MetricsRemoteSink.cpp b/SilKit/source/services/metrics/Test_MetricsRemoteSink.cpp new file mode 100644 index 000000000..6e5112922 --- /dev/null +++ b/SilKit/source/services/metrics/Test_MetricsRemoteSink.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "IMetricsSender.hpp" +#include "MetricsDatatypes.hpp" +#include "MetricsRemoteSink.hpp" + +namespace { + +using VSilKit::IMetricsSender; +using VSilKit::MetricData; +using VSilKit::MetricKind; +using VSilKit::MetricTimestamp; +using VSilKit::MetricsRemoteSink; +using VSilKit::MetricsUpdate; + +using testing::_; + +struct MockMetricsSender : IMetricsSender +{ + MOCK_METHOD(void, Send, (const MetricsUpdate&), (override)); +}; + + +TEST(Test_MetricsRemoteSink, test_forward_to_sender_only_if_local_origin) +{ + const std::string origin{"Participant Name"}; + const std::string otherOrigin{"Another Participant Name"}; + + MetricsUpdate updateOne; + updateOne.metrics.emplace_back(MetricData{1, "One", MetricKind::COUNTER, "1"}); + + MetricsUpdate updateTwo; + updateTwo.metrics.emplace_back(MetricData{2, "Two", MetricKind::COUNTER, "2"}); + + MockMetricsSender sender; + + EXPECT_CALL(sender, Send(updateOne)).Times(1); + EXPECT_CALL(sender, Send(updateTwo)).Times(0); + + MetricsRemoteSink sink{origin, sender}; + sink.Process(origin, updateOne); + sink.Process(otherOrigin, updateTwo); +} + + +} // anonymous namespace diff --git a/SilKit/source/services/metrics/Test_MetricsSerdes.cpp b/SilKit/source/services/metrics/Test_MetricsSerdes.cpp new file mode 100644 index 000000000..833ccb4ea --- /dev/null +++ b/SilKit/source/services/metrics/Test_MetricsSerdes.cpp @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "MetricsDatatypes.hpp" +#include "MetricsSerdes.hpp" +#include "MessageBuffer.hpp" + +namespace { + +using VSilKit::MetricsUpdate; +using VSilKit::MetricData; +using VSilKit::MetricKind; + +using testing::ContainerEq; + + +TEST(Test_MetricsSerdes, serialize_deserialize_roundtrip) +{ + MetricsUpdate original; + original.metrics.emplace_back(MetricData{1, "1", MetricKind::COUNTER, "1"}); + original.metrics.emplace_back(MetricData{2, "2", MetricKind::STATISTIC, "[1,2,3,4]"}); + original.metrics.emplace_back(MetricData{3, "3", MetricKind::STRING_LIST, R"(["1","2","3","4"])"}); + + SilKit::Core::MessageBuffer buffer; + VSilKit::Serialize(buffer, original); + + MetricsUpdate copy; + VSilKit::Deserialize(buffer, copy); + + ASSERT_THAT(copy.metrics, ContainerEq(original.metrics)); +} + + +} // anonymous namespace \ No newline at end of file diff --git a/SilKit/source/services/orchestration/TimeSyncService.cpp b/SilKit/source/services/orchestration/TimeSyncService.cpp index 47e98e3a2..b140aa119 100644 --- a/SilKit/source/services/orchestration/TimeSyncService.cpp +++ b/SilKit/source/services/orchestration/TimeSyncService.cpp @@ -287,6 +287,9 @@ TimeSyncService::TimeSyncService(Core::IParticipantInternal* participant, ITimeP , _logger{participant->GetLogger()} , _timeProvider{timeProvider} , _timeConfiguration{participant->GetLogger()} + , _simStepCounterMetric{participant->GetMetricsManager()->GetCounter("SimStepCount")} + , _simStepExecutionTimeStatisticMetric{participant->GetMetricsManager()->GetStatistic("SimStepExecutionDuration")} + , _simStepWaitingTimeStatisticMetric{participant->GetMetricsManager()->GetStatistic("SimStepWaitingDuration")} , _watchDog{healthCheckConfig} , _animationFactor{animationFactor} { @@ -475,10 +478,19 @@ void TimeSyncService::ExecuteSimStep(std::chrono::nanoseconds timePoint, std::ch { SILKIT_ASSERT(_simTask); using DoubleMSecs = std::chrono::duration; + using DoubleSecs = std::chrono::duration; _waitTimeMonitor.StopMeasurement(); - Trace(_logger, "Starting next Simulation Task. Waiting time was: {}ms", - std::chrono::duration_cast(_waitTimeMonitor.CurrentDuration()).count()); + const auto waitingDuration = _waitTimeMonitor.CurrentDuration(); + const auto waitingDurationMs = std::chrono::duration_cast(waitingDuration); + const auto waitingDurationS = std::chrono::duration_cast(waitingDuration); + + Trace(_logger, "Starting next Simulation Task. Waiting time was: {}ms", waitingDurationMs.count()); + if (_waitTimeMonitor.SampleCount() > 1) + { + // skip the first sample, since it was never 'started' (it is always the current epoch of the underlying clock) + _simStepWaitingTimeStatisticMetric->Take(waitingDurationS.count()); + } _timeProvider->SetTime(timePoint, duration); @@ -488,8 +500,14 @@ void TimeSyncService::ExecuteSimStep(std::chrono::nanoseconds timePoint, std::ch _watchDog.Reset(); _execTimeMonitor.StopMeasurement(); - Trace(_logger, "Finished Simulation Step. Execution time was: {}ms", - std::chrono::duration_cast(_execTimeMonitor.CurrentDuration()).count()); + const auto executionDuration = _execTimeMonitor.CurrentDuration(); + const auto executionDurationMs = std::chrono::duration_cast(executionDuration); + const auto executionDurationS = std::chrono::duration_cast(executionDuration); + + _simStepCounterMetric->Add(1); + _simStepExecutionTimeStatisticMetric->Take(executionDurationS.count()); + + Trace(_logger, "Finished Simulation Step. Execution time was: {}ms", executionDurationMs.count()); _waitTimeMonitor.StartMeasurement(); } diff --git a/SilKit/source/services/orchestration/TimeSyncService.hpp b/SilKit/source/services/orchestration/TimeSyncService.hpp index 1d280b727..d8a68dc4d 100644 --- a/SilKit/source/services/orchestration/TimeSyncService.hpp +++ b/SilKit/source/services/orchestration/TimeSyncService.hpp @@ -36,6 +36,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "TimeProvider.hpp" #include "TimeConfiguration.hpp" #include "WatchDog.hpp" +#include "Metrics.hpp" namespace SilKit { namespace Services { @@ -132,6 +133,10 @@ class TimeSyncService ITimeProvider* _timeProvider{nullptr}; TimeConfiguration _timeConfiguration; + VSilKit::ICounterMetric* _simStepCounterMetric; + VSilKit::IStatisticMetric* _simStepExecutionTimeStatisticMetric; + VSilKit::IStatisticMetric* _simStepWaitingTimeStatisticMetric; + mutable std::mutex _timeSyncPolicyMx; std::shared_ptr _timeSyncPolicy{nullptr}; diff --git a/SilKit/source/services/rpc/RpcTestUtilities.hpp b/SilKit/source/services/rpc/RpcTestUtilities.hpp index f361cc383..7f3ff4e2c 100644 --- a/SilKit/source/services/rpc/RpcTestUtilities.hpp +++ b/SilKit/source/services/rpc/RpcTestUtilities.hpp @@ -43,7 +43,8 @@ namespace Tests { struct MockConnection { - MockConnection(SilKit::Core::IParticipantInternal*, SilKit::Config::ParticipantConfiguration /*config*/, + MockConnection(SilKit::Core::IParticipantInternal*, VSilKit::IMetricsManager*, + SilKit::Config::ParticipantConfiguration /*config*/, std::string /*participantName*/, SilKit::Core::ParticipantId /*participantId*/, SilKit::Services::Orchestration::ITimeProvider* /*timeProvider*/, SilKit::Core::ProtocolVersion) { diff --git a/SilKit/source/util/CMakeLists.txt b/SilKit/source/util/CMakeLists.txt index 88da87518..dfe913575 100644 --- a/SilKit/source/util/CMakeLists.txt +++ b/SilKit/source/util/CMakeLists.txt @@ -22,6 +22,17 @@ add_library(I_SilKit_Util INTERFACE) target_include_directories(I_SilKit_Util INTERFACE ${CMAKE_CURRENT_LIST_DIR}) +add_Library(O_SilKit_Util OBJECT + ExecutionEnvironment.cpp +) +target_link_libraries(O_SilKit_Util + PUBLIC I_SilKit_Util + PRIVATE SilKitInterface + PRIVATE asio + PRIVATE fmt-header-only +) + + add_subdirectory(tests) @@ -63,7 +74,10 @@ add_library(O_SilKit_Util_StringHelpers OBJECT StringHelpers.cpp ) target_include_directories(O_SilKit_Util_StringHelpers INTERFACE ${CMAKE_CURRENT_LIST_DIR}) -target_link_libraries(O_SilKit_Util_StringHelpers PUBLIC I_SilKit_Util_StringHelpers) +target_link_libraries(O_SilKit_Util_StringHelpers + PUBLIC I_SilKit_Util_StringHelpers + PRIVATE fmt::fmt-header-only +) diff --git a/SilKit/source/util/ExecutionEnvironment.cpp b/SilKit/source/util/ExecutionEnvironment.cpp new file mode 100644 index 000000000..c26d04f4c --- /dev/null +++ b/SilKit/source/util/ExecutionEnvironment.cpp @@ -0,0 +1,346 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#include "ExecutionEnvironment.hpp" + +#include "Assert.hpp" +#include "FileHelpers.hpp" + +#include +#include +#include +#include + +#include "asio.hpp" + +#include "fmt/format.h" + +#ifdef __unix__ +#include +#include +#endif + +#ifdef __linux__ +#include +#include +#include +#endif + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#endif + +namespace VSilKit { + +namespace { + +constexpr const char* UNKNOWN_VALUE = ""; + + +// Utilities + +#ifdef __unix__ + +auto GetUtsname() -> const struct ::utsname& +{ + static const auto result = [] { + struct ::utsname buffer = {}; + SILKIT_ASSERT(::uname(&buffer) == 0); + return buffer; + }(); + + return result; +} + +#endif + +#ifdef _WIN32 + +struct Win32ProcessorInformation +{ + std::string pageSize; + std::string architecture; + + static auto Get() -> const Win32ProcessorInformation& + { + static const auto result = [] { + ::SYSTEM_INFO systemInfo{}; + ::GetNativeSystemInfo(&systemInfo); + + Win32ProcessorInformation info; + + info.pageSize = std::to_string(systemInfo.dwPageSize); + + switch (systemInfo.wProcessorArchitecture) + { + case PROCESSOR_ARCHITECTURE_AMD64: + info.architecture = "x86_64"; + break; + case PROCESSOR_ARCHITECTURE_ARM: + info.architecture = "arm"; + break; + case PROCESSOR_ARCHITECTURE_ARM64: + info.architecture = "arm64"; + break; + case PROCESSOR_ARCHITECTURE_INTEL: + info.architecture = "x86"; + break; + default: + info.architecture = ""; + break; + } + + return info; + }(); + + return result; + } +}; + +#endif + + +// Function: GetUsername + +#ifdef __unix__ + +auto GetUsername() -> std::string +{ + static const auto result = []() -> std::string { + const auto uid = ::getuid(); + const auto pwd = ::getpwuid(uid); + if (pwd == nullptr) + { + return UNKNOWN_VALUE; + } + + return pwd->pw_name; + }(); + + return result; +} + +#endif + +#ifdef _WIN32 + +auto GetUsername() -> std::string +{ + static const auto result = []() -> std::string { + std::array username{}; + auto usernameLength = static_cast(username.size()); + if (::GetUserNameA(username.data(), &usernameLength) == 0) + { + return UNKNOWN_VALUE; + } + return std::string{username.data()}; + }(); + + return result; +} + +#endif + + +// Function: GetProcessExecutable + +#ifdef __unix__ +#ifdef __linux__ + +auto GetProcessExecutable() -> std::string +{ + static const auto result = []() -> std::string { + std::vector buffer; + buffer.resize(4096, '\0'); + + auto pathLength = ::readlink("/proc/self/exe", buffer.data(), buffer.size() - 1); + if (pathLength <= 0) + { + return UNKNOWN_VALUE; + } + + buffer.resize(static_cast(pathLength)); + return std::string{buffer.begin(), buffer.end()}; + }(); + + return result; +} + +#else + +auto GetProcessExecutable() -> std::string +{ + // should be getprogname on xBSD and macOS, and qh_get_progname on QNX + return UNKNOWN_VALUE; +} + +#endif +#endif + +#ifdef _WIN32 + +auto GetProcessExecutable() -> std::string +{ + static const auto result = []() -> std::string { + std::vector buffer; + buffer.resize(4096, '\0'); + + const auto pathLength = ::GetModuleFileNameA(NULL, buffer.data(), static_cast(buffer.size())); + if (pathLength <= 0 || pathLength > buffer.size()) + { + return UNKNOWN_VALUE; + } + + buffer.resize(static_cast(pathLength)); + return std::string{buffer.begin(), buffer.end()}; + }(); + + return result; +} + +#endif + + +// Function: GetPageSize + +#ifdef __unix__ + +auto GetPageSize() -> std::string +{ + static const auto result = std::to_string(::sysconf(_SC_PAGESIZE)); + return result; +} + +#endif + +#ifdef _WIN32 + +auto GetPageSize() -> std::string +{ + return Win32ProcessorInformation::Get().pageSize; +} + +#endif + + +// Function: GetPhysicalMemoryMB + +#ifdef __unix__ +#ifdef __linux__ + +auto GetPhysicalMemoryMB() -> std::string +{ + static const auto result = [] { + const auto pageSize = static_cast(::sysconf(_SC_PAGESIZE)); + const auto pageCount = static_cast(::sysconf(_SC_PHYS_PAGES)); + const auto physicalMemoryMiB = ((pageCount / 1024) * static_cast(pageSize)) / 1024; + return std::to_string(physicalMemoryMiB); + }(); + + return result; +} + +#else + +auto GetPhysicalMemoryMB() -> std::string +{ + return UNKNOWN_VALUE; +} + +#endif +#endif + +#ifdef _WIN32 + +auto GetPhysicalMemoryMB() -> std::string +{ + const auto result = []() -> std::string { + ULONGLONG value{}; + if (::GetPhysicallyInstalledSystemMemory(&value) != TRUE) + { + return UNKNOWN_VALUE; + } + return std::to_string(value / 1024); + }(); + + return result; +} + +#endif + + +// Function: GetProcessorArchitecture + +#ifdef __unix__ + +auto GetProcessorArchitecture() -> std::string +{ + return GetUtsname().machine; +} + +#endif + +#ifdef _WIN32 + +auto GetProcessorArchitecture() -> std::string +{ + return Win32ProcessorInformation::Get().architecture; +} + +#endif + + +// Function: GetProcessorArchitecture + +#ifdef __unix__ + +auto GetOperatingSystem() -> std::string +{ + return fmt::format("{} {}", GetUtsname().sysname, GetUtsname().release); +} + +#endif + +#ifdef _WIN32 + +auto GetOperatingSystem() -> std::string +{ + return "Windows"; +} + +#endif + + +// Function: GetProcessorCount + +auto GetProcessorCount() -> std::string +{ + static const auto result = std::to_string(std::thread::hardware_concurrency()); + return result; +} + + +} // namespace + + +auto GetExecutionEnvironment() -> ExecutionEnvironment +{ + ExecutionEnvironment ee; + ee.hostname = asio::ip::host_name(); + ee.username = GetUsername(); + ee.executable = GetProcessExecutable(); + ee.processorArchitecture = GetProcessorArchitecture(); + ee.processorCount = GetProcessorCount(); + ee.pageSize = GetPageSize(); + ee.physicalMemoryMiB = GetPhysicalMemoryMB(); + ee.operatingSystem = GetOperatingSystem(); + + return ee; +} + + +} // namespace VSilKit diff --git a/SilKit/source/util/ExecutionEnvironment.hpp b/SilKit/source/util/ExecutionEnvironment.hpp new file mode 100644 index 000000000..61ebeffca --- /dev/null +++ b/SilKit/source/util/ExecutionEnvironment.hpp @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2024 Vector Informatik GmbH +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace VSilKit { + +struct ExecutionEnvironment +{ + std::string operatingSystem; + std::string hostname; + std::string pageSize; + std::string processorCount; + std::string processorArchitecture; + std::string physicalMemoryMiB; + std::string executable; + std::string username; +}; + +auto GetExecutionEnvironment() -> ExecutionEnvironment; + +} // namespace VSilKit \ No newline at end of file diff --git a/SilKit/source/util/StringHelpers.cpp b/SilKit/source/util/StringHelpers.cpp index d80b1d8cd..fe7042a75 100644 --- a/SilKit/source/util/StringHelpers.cpp +++ b/SilKit/source/util/StringHelpers.cpp @@ -21,74 +21,97 @@ #include "StringHelpers.hpp" +#include +#include + +#include "fmt/chrono.h" + namespace SilKit { namespace Util { -auto EscapeString(const std::string& input) -> std::string -{ - std::string result; - result.reserve(input.size() * 2); // Reserve enough memory for the result +namespace { - size_t i = 0; - while (i < input.size()) +template +void DoEscape(const std::string& string, F f) +{ + for (const char ch : string) { - char character = input[i]; - switch (character) + switch (ch) { - case '\b': - { - result += "\\"; - result += 'b'; - break; - } - case '\t': - { - result += "\\"; - result += 't'; - break; - } - case '\n': - { - result += "\\"; - result += 'n'; - break; - } - case '\f': - { - result += "\\"; - result += 'f'; - break; - } - case '\r': - { - result += "\\"; - result += 'r'; - break; - } - case '\\': - { - result += "\\"; - result += character; - break; - } - case '\"': - { - result += "\\"; - result += character; - break; - } - default: - { - result += character; - break; - } + case '\b': + f('\\'); + f('b'); + break; + case '\t': + f('\\'); + f('t'); + break; + case '\n': + f('\\'); + f('n'); + break; + case '\f': + f('\\'); + f('f'); + break; + case '\r': + f('\\'); + f('r'); + break; + case '"': + f('\\'); + f('"'); + break; + case '\\': + f('\\'); + f('\\'); + break; + default: + f(ch); + break; } - i++; } +} + +} // namespace + + +auto EscapeString(const std::string& input) -> std::string +{ + std::size_t count{0}; + DoEscape(input, [&count](const auto) { ++count; }); + + std::string result; + result.reserve(count); + DoEscape(input, [&result](const auto ch) { result.push_back(ch); }); + return result; } + +auto operator<<(std::ostream& ostream, const EscapedJsonString& self) -> std::ostream& +{ + DoEscape(self.string, [&ostream](const auto ch) { ostream.put(ch); }); + return ostream; +} + + +auto CurrentTimestampString() -> std::string +{ + auto time = std::time(nullptr); + + std::tm tm{}; +#if defined(_WIN32) + localtime_s(&tm, &time); +#else + localtime_r(&time, &tm); +#endif + + return fmt::format("{:%FT%H-%M-%S}", tm); +} + + } // namespace Util } // namespace SilKit diff --git a/SilKit/source/util/StringHelpers.hpp b/SilKit/source/util/StringHelpers.hpp index 0badaaaa4..785318be9 100644 --- a/SilKit/source/util/StringHelpers.hpp +++ b/SilKit/source/util/StringHelpers.hpp @@ -28,8 +28,17 @@ namespace SilKit { namespace Util { +struct EscapedJsonString +{ + const std::string& string; + + friend auto operator<<(std::ostream& ostream, const EscapedJsonString& self) -> std::ostream&; +}; + auto EscapeString(const std::string& input) -> std::string; +auto CurrentTimestampString() -> std::string; + } // namespace Util } // namespace SilKit diff --git a/SilKit/source/util/tests/Test_Filesystem.cpp b/SilKit/source/util/tests/Test_Filesystem.cpp index 40a560fdf..45a044d07 100644 --- a/SilKit/source/util/tests/Test_Filesystem.cpp +++ b/SilKit/source/util/tests/Test_Filesystem.cpp @@ -20,6 +20,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "Filesystem.hpp" +#include #include #include "gtest/gtest.h" @@ -28,25 +29,33 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ namespace SilKit { namespace Util { namespace tests { +namespace { -TEST(UtilsFilesystemTest, get_parent_path) +auto FixSep(std::string path) -> std::string { - const auto some_path = Filesystem::path("/This/Is/Some/Path/To/A/File/"); + static const char sep = Filesystem::path::preferred_separator; + std::replace(path.begin(), path.end(), '/', sep); + return path; +} + +TEST(Test_UtilsFilesystem, get_parent_path) +{ + const auto some_path = Filesystem::path{FixSep("/This/Is/Some/Path/To/A/File/")}; auto parent_path = Filesystem::parent_path(some_path); - const auto parent_path_ref1 = std::string("/This/Is/Some/Path/To/A/File"); + const auto parent_path_ref1 = FixSep("/This/Is/Some/Path/To/A/File"); ASSERT_EQ(parent_path_ref1, parent_path.string()); parent_path = Filesystem::parent_path(parent_path.string()); - const auto parent_path_ref2 = std::string("/This/Is/Some/Path/To/A"); + const auto parent_path_ref2 = FixSep("/This/Is/Some/Path/To/A"); ASSERT_EQ(parent_path_ref2, parent_path.string()); parent_path = Filesystem::parent_path(parent_path.string()); - const auto parent_path_ref3 = std::string("/This/Is/Some/Path/To"); + const auto parent_path_ref3 = FixSep("/This/Is/Some/Path/To"); ASSERT_EQ(parent_path_ref3, parent_path.string()); } -TEST(UtilsFilesystemTest, test_root_parent) +TEST(Test_UtilsFilesystem, test_root_parent) { const Filesystem::path root_path{"RootFile"}; const auto parent_path = Filesystem::parent_path(root_path); @@ -54,21 +63,23 @@ TEST(UtilsFilesystemTest, test_root_parent) ASSERT_EQ("", parent_path.string()); } -TEST(UtilsFilesystemTest, test_concat_paths) +TEST(Test_UtilsFilesystem, test_concat_paths) { const Filesystem::path file_name{"File"}; - const Filesystem::path root_path{"/Path/To/"}; + const Filesystem::path root_path{FixSep("/Path/To/")}; const auto file_path = Filesystem::concatenate_paths(root_path, file_name); - ASSERT_EQ("/Path/To/File", file_path.string()); + ASSERT_EQ(FixSep("/Path/To/File"), file_path.string()); } -TEST(UtilsFilesystemTest, test_concat_path_strings) +TEST(Test_UtilsFilesystem, test_concat_path_strings) { - const auto file_path = Filesystem::concatenate_paths("/Path/To", "File"); + const auto file_path = Filesystem::concatenate_paths(FixSep("/Path/To"), "File"); - ASSERT_EQ("/Path/To/File", file_path.string()); + ASSERT_EQ(FixSep("/Path/To/File"), file_path.string()); } + +} // namespace } // namespace tests } // namespace Util } // namespace SilKit diff --git a/Utilities/SilKitRegistry/Registry.cpp b/Utilities/SilKitRegistry/Registry.cpp index cb2044f02..5ec38e998 100644 --- a/Utilities/SilKitRegistry/Registry.cpp +++ b/Utilities/SilKitRegistry/Registry.cpp @@ -147,6 +147,8 @@ void OverrideFromRegistryConfiguration(std::shared_ptrmiddleware.enableDomainSockets = registryConfiguration.enableDomainSockets.value(); } + + config->experimental.metrics = registryConfiguration.experimental.metrics; } void OverrideRegistryUri(std::shared_ptr configuration, diff --git a/Utilities/SilKitRegistry/config/RegistryConfiguration.cpp b/Utilities/SilKitRegistry/config/RegistryConfiguration.cpp index 8aa60677e..55dffd4e8 100644 --- a/Utilities/SilKitRegistry/config/RegistryConfiguration.cpp +++ b/Utilities/SilKitRegistry/config/RegistryConfiguration.cpp @@ -31,6 +31,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "fmt/format.h" #include "yaml-cpp/yaml.h" +namespace SilKitRegistry { +namespace Config { +namespace V1 { + +bool operator==(const Experimental& lhs, const Experimental& rhs) +{ + return lhs.metrics == rhs.metrics; +} + +} // namespace V1 +} // namespace Config +} // namespace SilKitRegistry namespace SilKitRegistry { namespace Config { diff --git a/Utilities/SilKitRegistry/config/RegistryConfiguration.hpp b/Utilities/SilKitRegistry/config/RegistryConfiguration.hpp index 655fed21f..0d22d8b81 100644 --- a/Utilities/SilKitRegistry/config/RegistryConfiguration.hpp +++ b/Utilities/SilKitRegistry/config/RegistryConfiguration.hpp @@ -25,6 +25,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ // Internal SIL Kit Headers #include "Configuration.hpp" +#include "ParticipantConfiguration.hpp" namespace SilKitRegistry { @@ -36,6 +37,11 @@ constexpr inline auto GetSchemaVersion() -> const char* return "1"; } +struct Experimental +{ + SilKit::Config::v1::Metrics metrics; +}; + struct RegistryConfiguration { std::string description{""}; @@ -43,8 +49,11 @@ struct RegistryConfiguration SilKit::Util::Optional enableDomainSockets; SilKit::Util::Optional dashboardUri; SilKit::Config::Logging logging{}; + Experimental experimental{}; }; +bool operator==(const Experimental& lhs, const Experimental& rhs); + } // namespace V1 } // namespace Config } // namespace SilKitRegistry diff --git a/Utilities/SilKitRegistry/config/RegistryYamlConversion.cpp b/Utilities/SilKitRegistry/config/RegistryYamlConversion.cpp index 6aa554a3b..82428be1b 100644 --- a/Utilities/SilKitRegistry/config/RegistryYamlConversion.cpp +++ b/Utilities/SilKitRegistry/config/RegistryYamlConversion.cpp @@ -37,6 +37,34 @@ using SilKit::Config::non_default_encode; namespace YAML { +template <> +Node Converter::encode(const V1::Experimental& obj) +{ + static const V1::Experimental defaultObj{}; + + Node node; + + non_default_encode(obj.metrics, node, "Metrics", defaultObj.metrics); + + return node; +} + +template <> +bool Converter::decode(const Node& node, V1::Experimental& obj) +{ + optional_decode(obj.metrics, node, "Metrics"); + + for (const auto& sink : obj.metrics.sinks) + { + if (sink.type == SilKit::Config::MetricsSink::Type::Remote) + { + throw SilKit::ConfigurationError{"SIL Kit Registry does not support remote metrics sinks"}; + } + } + + return true; +} + template <> Node Converter::encode(const V1::RegistryConfiguration& obj) { @@ -50,6 +78,7 @@ Node Converter::encode(const V1::RegistryConfiguration& obj) optional_encode(obj.enableDomainSockets, node, "EnableDomainSockets"); optional_encode(obj.dashboardUri, node, "DashboardUri"); non_default_encode(obj.logging, node, "Logging", defaultObj.logging); + non_default_encode(obj.experimental, node, "Experimental", defaultObj.experimental); return node; } @@ -62,6 +91,7 @@ bool Converter::decode(const Node& node, V1::RegistryConfiguration& obj) optional_decode(obj.enableDomainSockets, node, "EnableDomainSockets"); optional_decode(obj.dashboardUri, node, "DashboardUri"); optional_decode(obj.logging, node, "Logging"); + optional_decode(obj.experimental, node, "Experimental"); if (obj.logging.logFromRemotes) { diff --git a/Utilities/SilKitRegistry/config/RegistryYamlConversion.hpp b/Utilities/SilKitRegistry/config/RegistryYamlConversion.hpp index 72becdb8b..4ebd28abc 100644 --- a/Utilities/SilKitRegistry/config/RegistryYamlConversion.hpp +++ b/Utilities/SilKitRegistry/config/RegistryYamlConversion.hpp @@ -31,6 +31,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ namespace YAML { +DEFINE_SILKIT_CONVERT(SilKitRegistry::Config::V1::Experimental); DEFINE_SILKIT_CONVERT(SilKitRegistry::Config::V1::RegistryConfiguration); } // namespace YAML