Add fatal signal backtrace

cmake/Modules/FindUNWIND.cmake

@@ -0,0 +1,25 @@
+#############################################################################
+# Copyright (c) 2016 Balabit
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# As an additional exemption you are allowed to compile & link against the
+# OpenSSL libraries as published by the OpenSSL project. See the file
+# COPYING for details.
+#
+#############################################################################
+
+find_package(PkgConfig)
+pkg_check_modules(LIBUNWIND QUIET libunwind>=1.6.2)

cmake/syslog-ng-config.h.in

@@ -106,3 +106,4 @@
 #cmakedefine01 SYSLOG_NG_HAVE_SO_MEMINFO
 #cmakedefine01 SYSLOG_NG_ENABLE_AFSOCKET_MEMINFO_METRICS
 #cmakedefine01 SYSLOG_NG_HAVE_IV_WORK_POOL_SUBMIT_CONTINUATION
+#cmakedefine01 SYSLOG_NG_ENABLE_LIBUNWIND

configure.ac

@@ -1396,6 +1396,12 @@ AC_CHECK_DECLS([MONGOC_URI_SERVERSELECTIONTIMEOUTMS], [], [], [[#include <mongoc
 CPPFLAGS="$CPPFLAGS_SAVE"
 fi
 
+dnl ***************************************************************************
+dnl libunwind headers/libraries
+dnl ***************************************************************************
+
+PKG_CHECK_MODULES(LIBUNWIND, libunwind >= 1.6.2, enable_libunwind="yes", enable_libunwind="no")
+
 dnl ***************************************************************************
 dnl libesmtp headers/libraries
 dnl ***************************************************************************
@@ -2097,7 +2103,7 @@ fi
 python_moduledir="$moduledir"/python
 python_sysconf_moduledir="${sysconfdir}/python"
 
-CPPFLAGS="$CPPFLAGS $GLIB_CFLAGS $EVTLOG_CFLAGS $PCRE2_CFLAGS $OPENSSL_CFLAGS $LIBNET_CFLAGS $LIBDBI_CFLAGS $IVYKIS_CFLAGS $JSON_CFLAGS $LIBCAP_CFLAGS -D_GNU_SOURCE -D_DEFAULT_SOURCE -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
+CPPFLAGS="$CPPFLAGS $GLIB_CFLAGS $EVTLOG_CFLAGS $PCRE2_CFLAGS $OPENSSL_CFLAGS $LIBNET_CFLAGS $LIBUNWIND_CFLAGS $LIBDBI_CFLAGS $IVYKIS_CFLAGS $JSON_CFLAGS $LIBCAP_CFLAGS -D_GNU_SOURCE -D_DEFAULT_SOURCE -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
 
 ########################################################
 ## NOTES: on how syslog-ng is linked
@@ -2145,7 +2151,7 @@ fi
 MODULE_DEPS_LIBS="\$(top_builddir)/lib/libsyslog-ng.la"
 
 if test "x$linking_mode" = "xdynamic"; then
-	SYSLOGNG_DEPS_LIBS="$LIBS $BASE_LIBS $GLIB_LIBS $EVTLOG_LIBS $SECRETSTORAGE_LIBS $RESOLV_LIBS $LIBCAP_LIBS $PCRE2_LIBS $REGEX_LIBS $DL_LIBS"
+	SYSLOGNG_DEPS_LIBS="$LIBS $BASE_LIBS $GLIB_LIBS $EVTLOG_LIBS $SECRETSTORAGE_LIBS $RESOLV_LIBS $LIBCAP_LIBS $PCRE2_LIBS $REGEX_LIBS $DL_LIBS $LIBUNWIND_LIBS"
 
 	if test "x$with_ivykis" = "xinternal"; then
 		# when using the internal ivykis, we're linking it statically into libsyslog-ng.so
@@ -2164,7 +2170,7 @@ if test "x$linking_mode" = "xdynamic"; then
 	# syslog-ng binary is linked with the default link command (e.g. libtool)
 	SYSLOGNG_LINK='$(LINK)'
 else
-	SYSLOGNG_DEPS_LIBS="$LIBS $BASE_LIBS $RESOLV_LIBS $EVTLOG_NO_LIBTOOL_LIBS $SECRETSTORAGE_NO_LIBTOOL_LIBS $LD_START_STATIC -Wl,${WHOLE_ARCHIVE_OPT} $GLIB_LIBS $PCRE2_LIBS $REGEX_LIBS  -Wl,${NO_WHOLE_ARCHIVE_OPT} $IVYKIS_NO_LIBTOOL_LIBS $LD_END_STATIC $LIBCAP_LIBS $DL_LIBS"
+	SYSLOGNG_DEPS_LIBS="$LIBS $BASE_LIBS $RESOLV_LIBS $EVTLOG_NO_LIBTOOL_LIBS $SECRETSTORAGE_NO_LIBTOOL_LIBS $LD_START_STATIC -Wl,${WHOLE_ARCHIVE_OPT} $GLIB_LIBS $PCRE2_LIBS $REGEX_LIBS  -Wl,${NO_WHOLE_ARCHIVE_OPT} $IVYKIS_NO_LIBTOOL_LIBS $LD_END_STATIC $LIBCAP_LIBS $DL_LIBS $LIBUNWIND_LIBS"
 	TOOL_DEPS_LIBS="$LIBS $BASE_LIBS $GLIB_LIBS $EVTLOG_LIBS $SECRETSTORAGE_LIBS $RESOLV_LIBS $LIBCAP_LIBS $PCRE2_LIBS $REGEX_LIBS $IVYKIS_LIBS $DL_LIBS"
 	CORE_DEPS_LIBS=""
 
@@ -2247,6 +2253,7 @@ AC_DEFINE_UNQUOTED(ENABLE_ENV_WRAPPER, `enable_value $enable_env_wrapper`, [Enab
 AC_DEFINE_UNQUOTED(ENABLE_SYSTEMD, `enable_value $enable_systemd`, [Enable systemd support])
 AC_DEFINE_UNQUOTED(ENABLE_KAFKA, `enable_value $enable_kafka`, [Enable kafka support])
 AC_DEFINE_UNQUOTED(ENABLE_CPP, `enable_value $enable_cpp`, [Enable C++ support])
+AC_DEFINE_UNQUOTED(ENABLE_LIBUNWIND, `enable_value $enable_libunwind`, [Enable stackdump using libunwind])
 AC_DEFINE_UNQUOTED(SYSTEMD_JOURNAL_MODE, `journald_mode`, [Systemd-journal support mode])
 AC_DEFINE_UNQUOTED(HAVE_INOTIFY, `enable_value $ac_cv_func_inotify_init`, [Have inotify])
 AC_DEFINE_UNQUOTED(USE_CONST_IVYKIS_MOCK, `enable_value $IVYKIS_VERSION_UPDATED`, [ivykis version is greater than $IVYKIS_UPDATED_VERSION])
@@ -2414,6 +2421,7 @@ echo "  Env wrapper support         : ${enable_env_wrapper:=no}"
 echo "  systemd support             : ${enable_systemd:=no} (unit dir: ${systemdsystemunitdir:=none})"
 echo "  systemd-journal support     : ${with_systemd_journal:=no}"
 echo "  JSON support                : $with_jsonc"
+echo "  unwind support              : ${enable_libunwind:=no}"
 echo " Build options:"
 echo "  Generate manual pages       : ${enable_manpages:=no}"
 echo "  Install manual pages        : ${enable_manpages_install:=no}"

lib/CMakeLists.txt

@@ -153,6 +153,7 @@ set (LIB_HEADERS
     serialize.h
     service-management.h
     seqnum.h
+    stackdump.h
     str-format.h
     str-utils.h
     syslog-names.h
@@ -249,6 +250,7 @@ set(LIB_SOURCES
     scratch-buffers.c
     serialize.c
     service-management.c
+    stackdump.c
     str-format.c
     str-utils.c
     syslog-names.c

lib/Makefile.am

@@ -172,6 +172,7 @@ pkginclude_HEADERS			+= \
 	lib/service-management.h	\
 	lib/seqnum.h			\
 	lib/signal-handler.h		\
+	lib/stackdump.h			\
 	lib/str-format.h		\
 	lib/str-utils.h			\
 	lib/syslog-names.h		\
@@ -264,6 +265,7 @@ lib_libsyslog_ng_la_SOURCES		= \
 	lib/scratch-buffers.c		\
 	lib/serialize.c			\
 	lib/service-management.c	\
+	lib/stackdump.c			\
 	lib/str-format.c		\
 	lib/str-utils.c			\
 	lib/syslog-names.c		\

lib/gprocess.c

@@ -27,6 +27,7 @@
 #include "messages.h"
 #include "reloc.h"
 #include "console.h"
+#include "stackdump.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -1302,6 +1303,14 @@ g_process_perform_supervise(void)
   exit(0);
 }
 
+
+static void
+g_process_setup_fatal_signal_handler(void)
+{
+  stackdump_setup_signal(SIGSEGV);
+  stackdump_setup_signal(SIGABRT);
+}
+
 /**
  * g_process_start:
  *
@@ -1405,6 +1414,8 @@ g_process_start(void)
       g_assert_not_reached();
     }
 
+  g_process_setup_fatal_signal_handler();
+
   /* daemon process, we should return to the caller to perform work */
   /* Only call setsid() for backgrounded processes. */
   if (process_opts.mode != G_PM_FOREGROUND)

lib/stackdump.c

@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2024 Balázs Scheidler <[email protected]>
+ * Copyright (c) 2024 Axoflow
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+#include "stackdump.h"
+#include "console.h"
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#if SYSLOG_NG_ENABLE_LIBUNWIND
+
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+#include <dlfcn.h>
+
+/* this is Linux only for now */
+
+static void
+_stackdump_print_stack(gpointer stack_pointer)
+{
+  guint8 *p = stack_pointer;
+
+  for (gint i = 0; i < 16; i++)
+    {
+      gchar line[51] = {0};
+      for (gint j = 0; j < 8; j++)
+        {
+          sprintf(&line[j*3], "%02x ", (guint) *(p+j));
+        }
+      line[8*3] = ' ';
+      for (gint j = 8; j < 16; j++)
+        {
+          sprintf(&line[j*3 + 1], "%02x ", (guint) *(p+j));
+        }
+
+      console_printf("Stack %p: %s", p, line);
+      p += 16;
+    }
+}
+
+static void
+_stackdump_print_backtrace(void)
+{
+  void *bt[256];
+  gint count;
+
+  count = unw_backtrace(bt, 256);
+  console_printf("Backtrace dump, count=%d", count);
+  for (gint i = 0; i < count; i++)
+    {
+      Dl_info dlinfo;
+      struct link_map *linkmap;
+
+      if (!dladdr1(bt[i], &dlinfo, (void **) &linkmap, RTLD_DL_LINKMAP))
+        {
+          console_printf("[%d]: %p", i, bt[i]);
+        }
+      else
+        {
+          if (dlinfo.dli_sname == NULL || dlinfo.dli_sname[0] == '\0')
+            {
+              ptrdiff_t addr_rel_to_base = (char *) bt[i] - (char *) linkmap->l_addr;
+              console_printf("[%d]: %s(+0x%lx) [%p]", i, dlinfo.dli_fname, (guint64) addr_rel_to_base, bt[i]);
+            }
+          else
+            {
+              ptrdiff_t addr_rel_to_sym = (char *) bt[i] - (char *) dlinfo.dli_saddr;
+              console_printf("[%d]: %s(%s+0x%lx) [%p]", i, dlinfo.dli_fname, dlinfo.dli_sname, addr_rel_to_sym, bt[i]);
+            }
+        }
+    }
+}
+
+
+#ifdef __x86_64__
+/****************************************************************************
+ *
+ *
+ * x86_64 support
+ *
+ *
+ ****************************************************************************/
+
+void
+_stackdump_print_registers(struct sigcontext *p)
+{
+  console_printf(
+    "Registers: "
+    "rax=%016lx rbx=%016lx rcx=%016lx rdx=%016lx rsi=%016lx rdi=%016lx "
+    "rbp=%016lx rsp=%016lx r8=%016lx r9=%016lx r10=%016lx r11=%016lx "
+    "r12=%016lx r13=%016lx r14=%016lx r15=%016lx rip=%016lx",
+    p->rax, p->rbx, p->rcx, p->rdx, p->rsi, p->rdi, p->rbp, p->rsp, p->r8, p->r9, p->r10, p->r11, p->r12, p->r13, p->r14,
+    p->r15, p->rip);
+  _stackdump_print_stack((gpointer) p->rsp);
+}
+
+#elif __x86__
+/****************************************************************************
+ *
+ *
+ * i386 support
+ *
+ *
+ ****************************************************************************/
+
+void
+_stackdump_print_registers(struct sigcontext *p)
+{
+  console_printf(
+    "Registers: eax=%08lx ebx=%08lx ecx=%08lx edx=%08lx esi=%08lx edi=%08lx ebp=%08lx esp=%08lx eip=%08lx",
+    p->eax, p->ebx, p->ecx, p->edx, p->esi, p->edi, p->ebp, p->esp, p->eip);
+  _stackdump_print_stack((gpointer) p->esp);
+}
+
+#else
+/****************************************************************************
+ *
+ *
+ * unsupported platform
+ *
+ *
+ ****************************************************************************/
+
+static void
+_stackdump_print_registers(struct sigcontext *p)
+{
+}
+
+#endif
+
+
+static inline void
+_stackdump_log(struct sigcontext *p)
+{
+  /* the order is important here, even if it might be illogical.  The
+   * backtrace function is the most fragile (as backtrace_symbols() may
+   * allocate memory).  Let's log everything else first, and then we can
+   * produce the backtrace, which is potentially causing another crash.  */
+
+  _stackdump_print_registers(p);
+  _stackdump_print_backtrace();
+}
+
+static void
+_fatal_signal_handler(int signo, siginfo_t *info, void *uc)
+{
+  struct ucontext_t *ucontext = (struct ucontext_t *) uc;
+  struct sigcontext *p = (struct sigcontext *) &ucontext->uc_mcontext;
+  struct sigaction act;
+
+  memset(&act, 0, sizeof(act));
+  act.sa_handler = SIG_DFL;
+  sigaction(signo, &act, NULL);
+
+  console_printf("Fatal signal received, stackdump follows, signal=%d", signo);
+  _stackdump_log(p);
+  /* let's get a stacktrace as well */
+  kill(getpid(), signo);
+}
+
+void
+stackdump_setup_signal(gint signal_number)
+{
+  struct sigaction act;
+
+  memset(&act, 0, sizeof(act));
+  act.sa_flags = SA_SIGINFO;
+  act.sa_sigaction = _fatal_signal_handler;
+
+  sigaction(signal_number, &act, NULL);
+}
+
+#else
+
+void
+stackdump_setup_signal(gint signal_number)
+{
+}
+
+#endif

lib/stackdump.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2024 Balázs Scheidler <[email protected]>
+ * Copyright (c) 2024 Axoflow
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+#ifndef STACKDUMP_H_INCLUDED
+#define STACKDUMP_H_INCLUDED
+
+#include "syslog-ng.h"
+
+void stackdump_setup_signal(gint signal_number);
+
+#endif

tests/copyright/policy

@@ -110,6 +110,7 @@ lib/stats/stats-compat\.h
 lib/stats/tests/test_stats_prometheus\.c
 lib/stats/tests/test_stats_cluster_key_builder\.c
 lib/timeutils/timeutils\.h
+lib/stackdump\.[ch]
 lib/logscheduler\.[ch]
 lib/logscheduler-pipe\.[ch]
 lib/logmsg/tests/dump_logmsg\.c