Merge pull request #4362 from MrAnno/healthcheck

syslog-ng health metrics and `syslog-ng-ctl healthcheck`
bshifter · Mar 21, 2023 · 197f8df · 197f8df
2 parents b0565cb + d82b817
commit 197f8df
Show file tree

Hide file tree

Showing 36 changed files with 879 additions and 15 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -23,6 +23,6 @@ EXPOSE 514/udp
 EXPOSE 601/tcp
 EXPOSE 6514/tcp
 
-HEALTHCHECK --interval=2m --timeout=3s --start-period=30s CMD /usr/sbin/syslog-ng-ctl stats || exit 1
+HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5
 ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 ENTRYPOINT ["/usr/sbin/syslog-ng", "-F"]
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
@@ -12,6 +12,7 @@ add_flex_bison_dependency(CfgLexer CfgGrammar)
 add_subdirectory(ack-tracker)
 add_subdirectory(compat)
 add_subdirectory(control)
+add_subdirectory(healthcheck)
 add_subdirectory(debugger)
 add_subdirectory(filter)
 add_subdirectory(logmsg)
@@ -36,6 +37,7 @@ set(LIB_SUBDIR_HEADERS
     ${ACK_TRACKER_HEADERS}
     ${COMPAT_HEADERS}
     ${CONTROL_HEADERS}
+    ${HEALTHCHECK_HEADERS}
     ${DEBUGGER_HEADERS}
     ${FILTER_HEADERS}
     ${LOGMSG_HEADERS}
@@ -239,6 +241,7 @@ set(LIB_SOURCES
     ${ACK_TRACKER_SOURCES}
     ${COMPAT_SOURCES}
     ${CONTROL_SOURCES}
+    ${HEALTHCHECK_SOURCES}
     ${DEBUGGER_SOURCES}
     ${FILTER_SOURCES}
     ${LOGMSG_SOURCES}
@@ -355,6 +358,7 @@ install(FILES ${LIB_HEADERS} DESTINATION include/syslog-ng)
 install(FILES ${VALUE_PAIRS_HEADERS} DESTINATION include/syslog-ng/value-pairs)
 install(FILES ${COMPAT_HEADERS} DESTINATION include/syslog-ng/compat)
 install(FILES ${CONTROL_HEADERS} DESTINATION include/syslog-ng/control)
+install(FILES ${HEALTHCHECK_HEADERS} DESTINATION include/syslog-ng/healthcheck)
 install(FILES ${DEBUGGER_HEADERS} DESTINATION include/syslog-ng/debugger)
 install(FILES ${FILTER_HEADERS} DESTINATION include/syslog-ng/filter)
 install(FILES ${LOGMSG_HEADERS} DESTINATION include/syslog-ng/logmsg)

diff --git a/lib/Makefile.am b/lib/Makefile.am
@@ -12,6 +12,7 @@ include lib/template/Makefile.am
 include lib/value-pairs/Makefile.am
 include lib/stats/Makefile.am
 include lib/control/Makefile.am
+include lib/healthcheck/Makefile.am
 include lib/debugger/Makefile.am
 include lib/compat/Makefile.am
 include lib/logmsg/Makefile.am
@@ -280,6 +281,7 @@ lib_libsyslog_ng_la_SOURCES		= \
 	$(value_pairs_sources)		\
 	$(stats_sources)		\
 	$(control_sources)		\
+	$(healthcheck_sources) \
 	$(debugger_sources)		\
 	$(compat_sources)     		\
 	$(logmsg_sources)		\

diff --git a/lib/apphook.c b/lib/apphook.c
@@ -28,6 +28,7 @@
 #include "dnscache.h"
 #include "alarms.h"
 #include "stats/stats-registry.h"
+#include "healthcheck/healthcheck-stats.h"
 #include "logmsg/logmsg.h"
 #include "logsource.h"
 #include "logwriter.h"
@@ -224,6 +225,7 @@ app_startup(void)
   alarm_init();
   main_loop_thread_resource_init();
   stats_init();
+  healthcheck_stats_global_init();
   tzset();
   log_msg_global_init();
   log_tags_global_init();

diff --git a/lib/cfg-grammar-internal.c b/lib/cfg-grammar-internal.c
@@ -43,6 +43,7 @@ ValuePairsTransformSet *last_vp_transset;
 LogMatcherOptions *last_matcher_options;
 HostResolveOptions *last_host_resolve_options;
 StatsOptions *last_stats_options;
+HealthCheckStatsOptions *last_healthcheck_options;
 DNSCacheOptions *last_dns_cache_options;
 LogRewrite *last_rewrite;
 CfgArgs *last_block_args;

diff --git a/lib/cfg-grammar-internal.h b/lib/cfg-grammar-internal.h
@@ -51,6 +51,9 @@
 #include "logthrsource/logthrfetcherdrv.h"
 #include "logthrdest/logthrdestdrv.h"
 
+#include "stats/stats.h"
+#include "healthcheck/healthcheck-stats.h"
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -77,6 +80,7 @@ extern ValuePairsTransformSet *last_vp_transset;
 extern LogMatcherOptions *last_matcher_options;
 extern HostResolveOptions *last_host_resolve_options;
 extern StatsOptions *last_stats_options;
+extern HealthCheckStatsOptions *last_healthcheck_options;
 extern LogRewrite *last_rewrite;
 extern CfgArgs *last_block_args;
 extern DNSCacheOptions *last_dns_cache_options;

diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y
@@ -192,6 +192,7 @@
 %token KW_LIFETIME                    10403
 %token KW_MAX_DYNAMIC                 10404
 %token KW_SYSLOG_STATS                10405
+%token KW_HEALTHCHECK_FREQ            10406
 
 %token KW_CHAIN_HOSTNAMES             10090
 %token KW_NORMALIZE_HOSTNAMES         10091
@@ -961,7 +962,7 @@ options_item
 	| KW_LOG_LEVEL '(' string ')'		{ CHECK_ERROR(cfg_set_log_level(configuration, $3), @3, "Unknown log-level() option"); free($3); }
 	| { last_template_options = &configuration->template_options; } template_option
 	| { last_host_resolve_options = &configuration->host_resolve_options; } host_resolve_option
-	| { last_stats_options = &configuration->stats_options; } stat_option
+	| { last_stats_options = &configuration->stats_options; last_healthcheck_options = &configuration->healthcheck_options; } stat_option
 	| { last_dns_cache_options = &configuration->dns_cache_options; } dns_cache_option
 	| { last_file_perm_options = &configuration->file_perm_options; } file_perm_option
 	| LL_PLUGIN
@@ -996,6 +997,7 @@ stats_group_option
 	| KW_LIFETIME '(' positive_integer ')'      { last_stats_options->lifetime = $3; }
 	| KW_MAX_DYNAMIC '(' nonnegative_integer ')'   { last_stats_options->max_dynamic = $3; }
 	| KW_SYSLOG_STATS '(' yesnoauto ')'     { last_stats_options->syslog_stats = $3; }
+	| KW_HEALTHCHECK_FREQ '(' nonnegative_integer ')' { last_healthcheck_options->freq = $3; }
 	;
 
 dns_cache_option

diff --git a/lib/cfg-parser.c b/lib/cfg-parser.c
@@ -98,6 +98,7 @@ static CfgLexerKeyword main_keywords[] =
   { "lifetime",           KW_LIFETIME },
   { "max_dynamics",       KW_MAX_DYNAMIC },
   { "syslog_stats",       KW_SYSLOG_STATS },
+  { "healthcheck_freq",   KW_HEALTHCHECK_FREQ},
   { "min_iw_size_per_reader", KW_MIN_IW_SIZE_PER_READER },
   { "flush_lines",        KW_FLUSH_LINES },
   { "flush_timeout",      KW_FLUSH_TIMEOUT, KWS_OBSOLETE, "Some drivers support batch-timeout() instead that you can specify at the destination level." },

diff --git a/lib/cfg.c b/lib/cfg.c
@@ -526,6 +526,7 @@ cfg_new(gint version)
     self->use_uniqid = TRUE;
 
   stats_options_defaults(&self->stats_options);
+  healthcheck_stats_options_defaults(&self->healthcheck_options);
 
   self->min_iw_size_per_reader = 100;
 

diff --git a/lib/cfg.h b/lib/cfg.h
@@ -35,6 +35,7 @@
 #include "host-resolve.h"
 #include "logmsg/type-hinting.h"
 #include "stats/stats.h"
+#include "healthcheck/healthcheck-stats.h"
 #include "dnscache.h"
 #include "file-perms.h"
 
@@ -70,6 +71,7 @@ struct _GlobalConfig
   CfgArgs *globals;
 
   StatsOptions stats_options;
+  HealthCheckStatsOptions healthcheck_options;
   gint mark_freq;
   gint flush_lines;
   gint mark_mode;

diff --git a/lib/healthcheck/CMakeLists.txt b/lib/healthcheck/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(HEALTHCHECK_HEADERS
+    healthcheck/healthcheck.h
+    healthcheck/healthcheck-control.h
+    healthcheck/healthcheck-stats.h
+    healthcheck/stopwatch.h
+    PARENT_SCOPE)
+
+set(HEALTHCHECK_SOURCES
+    healthcheck/healthcheck.c
+    healthcheck/healthcheck-control.c
+    healthcheck/healthcheck-stats.c
+    PARENT_SCOPE)
diff --git a/lib/healthcheck/Makefile.am b/lib/healthcheck/Makefile.am
@@ -0,0 +1,15 @@
+healthcheckincludedir = ${pkgincludedir}/healthcheck
+
+EXTRA_DIST += \
+	lib/healthcheck/CMakeLists.txt
+
+healthcheckinclude_HEADERS = \
+	lib/healthcheck/healthcheck.h \
+	lib/healthcheck/healthcheck-control.h \
+	lib/healthcheck/healthcheck-stats.h \
+	lib/healthcheck/stopwatch.h
+
+healthcheck_sources = \
+	lib/healthcheck/healthcheck.c \
+	lib/healthcheck/healthcheck-control.c \
+	lib/healthcheck/healthcheck-stats.c
diff --git a/lib/healthcheck/healthcheck-control.c b/lib/healthcheck/healthcheck-control.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2023 László Várady <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#include "healthcheck-control.h"
+#include "healthcheck.h"
+
+#include "control/control.h"
+#include "control/control-connection.h"
+#include "control/control-commands.h"
+#include "stats/stats-prometheus.h"
+
+static void
+_send_healthcheck_reply(HealthCheckResult result, gpointer c)
+{
+  ControlConnection *cc = (ControlConnection *) c;
+  gchar double_buf[G_ASCII_DTOSTR_BUF_SIZE];
+
+  gdouble io_worker_latency = result.io_worker_latency / 1e9;
+  gdouble mainloop_io_worker_roundtrip_latency = result.mainloop_io_worker_roundtrip_latency / 1e9;
+
+  GString *reply = g_string_new("OK ");
+  g_string_append_printf(reply, PROMETHEUS_METRIC_PREFIX
+                         "io_worker_latency_seconds %s\n",
+                         g_ascii_dtostr(double_buf, G_N_ELEMENTS(double_buf), io_worker_latency));
+  g_string_append_printf(reply, PROMETHEUS_METRIC_PREFIX
+                         "mainloop_io_worker_roundtrip_latency_seconds %s\n",
+                         g_ascii_dtostr(double_buf, G_N_ELEMENTS(double_buf), mainloop_io_worker_roundtrip_latency));
+
+
+  control_connection_send_reply(cc, reply);
+  control_connection_unref(cc);
+}
+
+static void
+control_connection_healthcheck(ControlConnection *cc, GString *command, gpointer user_data, gboolean *cancelled)
+{
+  HealthCheck *hc = healthcheck_new();
+
+  if (!healthcheck_run(hc, _send_healthcheck_reply, control_connection_ref(cc)))
+    {
+      GString *reply = g_string_new("FAIL Another healthcheck command is already running");
+      control_connection_send_reply(cc, reply);
+      control_connection_unref(cc);
+    }
+
+  healthcheck_unref(hc);
+}
+
+void
+healthcheck_register_control_commands(void)
+{
+  control_register_command("HEALTHCHECK", control_connection_healthcheck, NULL, FALSE);
+}
diff --git a/lib/healthcheck/healthcheck-control.h b/lib/healthcheck/healthcheck-control.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2023 László Várady <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#ifndef HEALTHCHECK_CONTROL_H
+#define HEALTHCHECK_CONTROL_H
+
+#include "syslog-ng.h"
+
+void healthcheck_register_control_commands(void);
+
+#endif