Add more metrics and smooth out prometheus implementation (#181)

Summary: ## Details 1. Add hostname and refactor everyone to common function. 2. Make prometheus-cpp a git submodule and update build scripts 3. Add more metrics, excluding per network and per GPU metrics. 4. Bump version. # TestPlan Ran on my laptop using a docker container. ## How to Run Please install Docker desktop. 1. Download this docker file in a directory https://gist.github.com/briancoutinho/c5faaa60e49a5ad796b972e6b3ef175d 2. Build using docker build . -t prometheus:v2 3. Run docker container forwarding port and mounting dynolog open source repo `docker run -p 9090:9090 -it -v ~/Work/dynolog_oss/dynolog:/workspace/dynolog prometheus:v2 /bin/bash` 4. Build dynolog ./scripts/build.sh To get the logging setup add the following in prometheus.yml ``` - job_name: "dynolog" static_configs: - targets: ["localhost:8080"] ``` Then run prometheus and dynolog. ```cd /workspace/prometheus/prometheus-2.44.0.linux-amd64; ./prometheus --config.file prometheus.yml & cd - ./build/bin/dynolog -kernel_monitor_reporting_interval_s 10 -use_JSON -use_prometheus & ``` Open [https://localhost:9090](http://localhost:9090/metrics)/ ![Screenshot 2023-10-16 at 5 52 05 PM](https://github.com/facebookincubator/dynolog/assets/6922212/bf19cb2d-654c-4f5f-9f06-e9d2f46c77ef) ![Screenshot 2023-10-16 at 5 53 13 PM](https://github.com/facebookincubator/dynolog/assets/6922212/cf63e741-df68-424f-900e-2bb8d6d8c221) ![Screenshot 2023-10-16 at 5 53 25 PM](https://github.com/facebookincubator/dynolog/assets/6922212/717d0704-4f4d-4c66-8dc4-d7b1c70dc40e) ![Screenshot 2023-10-16 at 5 53 29 PM](https://github.com/facebookincubator/dynolog/assets/6922212/81fd1fa5-e487-423c-9096-926120978032) Pull Request resolved: #181 Test Plan: Imported from GitHub, without a `Test Plan:` line. Brian: Come on Test plan was there in the doc :( Differential Revision: D50393435 Pulled By: briancoutinho fbshipit-source-id: 379df41b2e88f4e1c0e84d902978be375a6d0e3d
facebookincubator · Oct 19, 2023 · 5a138ae · 5a138ae
1 parent 15116e5
commit 5a138ae
Show file tree

Hide file tree

Showing 11 changed files with 87 additions and 34 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -22,3 +22,7 @@
 [submodule "third_party/DCGM"]
 	path = third_party/DCGM
 	url = https://github.com/NVIDIA/DCGM.git
+[submodule "third_party/prometheus-cpp"]
+	path = third_party/prometheus-cpp
+	url = https://github.com/jupp0r/prometheus-cpp.git
+	branch = v1.0.2
diff --git a/dynolog/src/FBRelayLogger.cpp b/dynolog/src/FBRelayLogger.cpp
@@ -11,8 +11,8 @@
 #include <nlohmann/json.hpp>
 #include <stdio.h>
 #include <sys/socket.h>
-#include <unistd.h>
 #include <cerrno>
+#include "hbt/src/common/System.h"
 
 using json = nlohmann::json;
 
@@ -24,18 +24,6 @@ DEFINE_string(
 
 namespace dynolog {
 
-namespace {
-
-std::string hostname = []() {
-  static std::string hostname;
-  char buf[1024];
-  gethostname(buf, sizeof(buf));
-  hostname = buf;
-  return hostname;
-}();
-
-};
-
 int setup_ipv4_socket(const std::string& addr, int port) {
   int sock_fd, domain = AF_INET;
   struct sockaddr_in serv_addr;
@@ -139,7 +127,7 @@ void FBRelayLogger::initSocket() {
       FLAGS_fbrelay_address, FLAGS_fbrelay_port);
 }
 
-FBRelayLogger::FBRelayLogger() {
+FBRelayLogger::FBRelayLogger() : hostname_(facebook::hbt::getHostName()) {
   initSocket();
 }
 
@@ -167,8 +155,8 @@ void FBRelayLogger::finalize() {
       {"@timestamp", timestampStr()},
       {"agent",
        {
-           {"hostname", hostname},
-           {"name", hostname},
+           {"hostname", hostname_},
+           {"name", hostname_},
            {"type", entity},
            {"version", "0.1.0"},
        }},

diff --git a/dynolog/src/FBRelayLogger.h b/dynolog/src/FBRelayLogger.h
@@ -43,6 +43,7 @@ class FBRelayLogger : public JsonLogger {
   void initSocket();
 
   std::unique_ptr<SocketWrapper> socket;
+  std::string hostname_;
 };
 
 } // namespace dynolog
diff --git a/dynolog/src/Metrics.cpp b/dynolog/src/Metrics.cpp
@@ -5,18 +5,58 @@
 
 #include "dynolog/src/Metrics.h"
 
+#include <fmt/format.h>
+#include <map>
+
 namespace dynolog {
 
-const std::vector<MetricDesc>& getAllMetrics() {
+const std::vector<MetricDesc> getAllMetrics() {
   static std::vector<MetricDesc> metrics_ = {
       {.name = "cpu_util",
        .type = MetricType::Ratio,
        .desc = "Fraction of total CPU time spend on user or system mode."},
+      {.name = "cpu_u",
+       .type = MetricType::Ratio,
+       .desc = "Fraction of total CPU time spent in user mode."},
+      {.name = "cpu_s",
+       .type = MetricType::Ratio,
+       .desc = "Fraction of total CPU time spent in system mode."},
+      {.name = "cpu_i",
+       .type = MetricType::Ratio,
+       .desc = "Fraction of total CPU time spent in idle mode."},
+      {.name = "mips",
+       .type = MetricType::Ratio,
+       .desc = "Number of million instructions executed per second."},
+      {.name = "mega_cycles_per_second",
+       .type = MetricType::Ratio,
+       .desc = "Number of active CPU clock cycles per second."},
       {.name = "uptime",
        .type = MetricType::Instant,
        .desc = "How long the system has been running in seconds."},
   };
-  return metrics_;
+  static std::map<std::string, std::string> cpustats = {
+      {"cpu_u_ms", "user"},
+      {"cpu_s_ms", "system"},
+      {"cpu_n_ms", "nice"},
+      {"cpu_w_ms", "iowait"},
+      {"cpu_x_ms", "irq"},
+      {"cpu_y_ms", "softirq"},
+  };
+
+  auto metrics = metrics_;
+
+  for (const auto& [name, cpu_mode] : cpustats) {
+    MetricDesc m{
+        .name = name,
+        .type = MetricType::Delta,
+        .desc = fmt::format(
+            "Total CPU time in milliseconds spent in {} mode. "
+            "For more details please see man page for /proc/stat",
+            cpu_mode)};
+    metrics.push_back(m);
+  }
+
+  return metrics;
 }
 
 } // namespace dynolog
diff --git a/dynolog/src/Metrics.h b/dynolog/src/Metrics.h
@@ -23,6 +23,6 @@ struct MetricDesc {
   std::string desc;
 };
 
-const std::vector<MetricDesc>& getAllMetrics();
+const std::vector<MetricDesc> getAllMetrics();
 
 } // namespace dynolog
diff --git a/dynolog/src/ODSJsonLogger.cpp b/dynolog/src/ODSJsonLogger.cpp
@@ -4,27 +4,23 @@
 // LICENSE file in the root directory of this source tree.
 
 #include "dynolog/src/ODSJsonLogger.h"
+#include "hbt/src/common/System.h"
+
 #ifdef USE_GRAPH_ENDPOINT
 #include <cpr/cpr.h> // @manual
 #include <curl/curl.h> // @manual
 #endif
 #include <fmt/format.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
-#include <unistd.h>
 
 DEFINE_string(category_id, "", "The category id of the ODS endpoint");
 DEFINE_string(ods_entity_prefix, "", "The prefix for ODS entity name");
 
 namespace dynolog {
-constexpr int HOSTNAME_MAX = 50;
 constexpr char kODSUrl[] = "https://graph.facebook.com/v2.2/ods_metrics";
 
-ODSJsonLogger::ODSJsonLogger() {
-  char hostname[HOSTNAME_MAX];
-  gethostname(hostname, HOSTNAME_MAX);
-  hostname_ = std::string(hostname);
-}
+ODSJsonLogger::ODSJsonLogger() : hostname_(facebook::hbt::getHostName()) {}
 
 void ODSJsonLogger::finalize() {
   std::string entity = FLAGS_ods_entity_prefix + hostname_;

diff --git a/dynolog/src/PrometheusLogger.cpp b/dynolog/src/PrometheusLogger.cpp
@@ -5,6 +5,7 @@
 
 #include "dynolog/src/PrometheusLogger.h"
 #include "dynolog/src/Metrics.h"
+#include "hbt/src/common/System.h"
 
 #include <fmt/format.h>
 #include <glog/logging.h>
@@ -30,11 +31,13 @@ PrometheusManager::PrometheusManager()
 
   // setup registry
   registry_ = std::make_shared<Registry>();
+  const std::string hostname = facebook::hbt::getHostName();
 
   // setup counters and gauges
   for (const auto& m : getAllMetrics()) {
     // all metric types fit with Gauges so far.
-    auto& g = buildGaugeFromMetric(m, *registry_).Add({{"host_name", "test"}});
+    auto& g =
+        buildGaugeFromMetric(m, *registry_).Add({{"host_name", hostname}});
     gauges_[m.name] = &g;
   }
 

diff --git a/dynolog/src/ScubaLogger.cpp b/dynolog/src/ScubaLogger.cpp
@@ -5,6 +5,7 @@
 
 #include "dynolog/src/ScubaLogger.h"
 #include <string>
+#include "hbt/src/common/System.h"
 #ifdef USE_GRAPH_ENDPOINT
 #include <cpr/cpr.h> // @manual
 #include <curl/curl.h> // @manual
@@ -24,11 +25,8 @@ DEFINE_string(
     "The scribe category name for scuba logging");
 
 ScubaLogger::ScubaLogger(const std::string& scribe_category)
-    : scribe_category_(scribe_category) {
-  char hostname[HOSTNAME_MAX] = "";
-  gethostname(hostname, HOSTNAME_MAX);
-  hostname_ = std::string(hostname);
-}
+    : scribe_category_(scribe_category),
+      hostname_(facebook::hbt::getHostName()) {}
 
 void ScubaLogger::logInt(const std::string& key, int64_t val) {
   metrics_int_[key] = val;

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -8,19 +8,41 @@
 #  ./scripts/build.sh <optional cmake args>
 
 set -eux -o pipefail
+BUILD_PROMETHEUS="${BUILD_PROMETHEUS:-0}"
+USE_PROMETHEUS="OFF"
 
 # Check dependencies
 cmake --version || echo "Please install cmake for your platform using dnf/apt-get etc."
 ninja --version || echo "Please install ninja for your platform using dnf/apt-get etc."
 rustc --version || echo "Please install Rust and Cargo - see https://www.rust-lang.org/tools/install"
 cargo --version || echo "Please install Rust and Cargo - see https://www.rust-lang.org/tools/install"
 
+## Build Prometheus if enabled
+if [ "${BUILD_PROMETHEUS}" -eq 1 ]
+then
+  mkdir -p ./third_party/prometheus-cpp/_build
+  pushd ./third_party/prometheus-cpp/
+
+  git submodule init
+  git submodule update
+
+  cd ./_build
+  cmake .. -DBUILD_SHARED_LIBS=ON -DENABLE_PUSH=OFF -DENABLE_COMPRESSION=OFF \
+     -DENABLE_TESTING=OFF
+  cmake --build . --parallel 4
+  cmake --install .
+
+  USE_PROMETHEUS="ON"
+  popd
+fi
+
 ## Build dynolog
 echo "Running cmake"
 mkdir -p build; cd build;
 
 # note we can build without ninja if not available on this system
-cmake -DCMAKE_BUILD_TYPE=Release -G Ninja "$@" ..
+cmake "-DUSE_PROMETHEUS=${USE_PROMETHEUS}" \
+  -DCMAKE_BUILD_TYPE=Release -G Ninja "$@" ..
 cmake --build .
 
 mkdir -p bin

diff --git a/third_party/prometheus-cpp b/third_party/prometheus-cpp
diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-0.3.0
+0.3.1