From a32b7a9ec00790cf0467f90b03818b651075e2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 21 Feb 2025 16:28:16 +0100 Subject: [PATCH 1/3] webhook: set source IP accordingly when behind proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- .../syslogng/modules/webhook/scl/webhook.conf | 2 ++ .../syslogng/modules/webhook/source.py | 21 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf b/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf index e20c8aff28..d689272ed9 100644 --- a/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf +++ b/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf @@ -32,6 +32,7 @@ block source webhook( tls_use_system_cert_store(no) tls_ca_file("") tls_ca_dir("") + proxy_header("") ... ) { @@ -48,6 +49,7 @@ block source webhook( "tls_use_system_cert_store" => `tls_use_system_cert_store` "tls_ca_file" => "`tls_ca_file`" "tls_ca_dir" => "`tls_ca_dir`" + "proxy_header" => "`proxy_header`" ) `__VARARGS__` ); diff --git a/modules/python-modules/syslogng/modules/webhook/source.py b/modules/python-modules/syslogng/modules/webhook/source.py index d6fb936ec7..2ea28bfd35 100644 --- a/modules/python-modules/syslogng/modules/webhook/source.py +++ b/modules/python-modules/syslogng/modules/webhook/source.py @@ -52,6 +52,17 @@ async def post(self, **path_arguments) -> None: await self.finish({"status": "received"}) + def _set_proxied_ip(self, msg: LogMessage) -> None: + proxy_headers = self.request.headers.get_list(self.source.proxy_header) + + if proxy_headers and len(proxy_headers) > 0: + # the closest/last IP (the proxy_header flag implies that the last one can be trusted) + msg.set_source_ipaddress(proxy_headers[-1]) + msg["PEERIP"] = self.request.remote_ip + return + + msg.set_source_ipaddress(self.request.remote_ip) + def _construct_msg(self, request, path_arguments) -> LogMessage: msg = LogMessage(self.request.body) msg.set_recvd_rawmsg_size(len(self.request.body)) @@ -63,7 +74,10 @@ def _construct_msg(self, request, path_arguments) -> LogMessage: for key, value in path_arguments.items(): msg[key] = value - msg.set_source_ipaddress(self.request.remote_ip) + if self.source.proxy_header: + self._set_proxied_ip(msg) + else: + msg.set_source_ipaddress(self.request.remote_ip) return msg @@ -184,6 +198,11 @@ def init_options(self, options: dict[str, Any]) -> bool: self.tls_use_system_cert_store = bool(options.get("tls_use_system_cert_store", False)) self.tls_ca_file = options.get("tls_ca_file") self.tls_ca_dir = options.get("tls_ca_dir") + + self.proxy_header = options.get("proxy_header") + if self.proxy_header == "yes": + self.proxy_header = "x-forwarded-for" + return True except KeyError as e: self.logger.error(f"Missing option '{e.args[0]}'") From 15b2771f66a9ee79d1aed76e3346d8bc94786bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 21 Feb 2025 18:23:42 +0100 Subject: [PATCH 2/3] webhook: support adding request headers to the message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- .../syslogng/modules/webhook/scl/webhook.conf | 2 ++ .../syslogng/modules/webhook/source.py | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf b/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf index d689272ed9..a637c129fe 100644 --- a/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf +++ b/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf @@ -33,6 +33,7 @@ block source webhook( tls_ca_file("") tls_ca_dir("") proxy_header("") + include_request_headers(no) ... ) { @@ -50,6 +51,7 @@ block source webhook( "tls_ca_file" => "`tls_ca_file`" "tls_ca_dir" => "`tls_ca_dir`" "proxy_header" => "`proxy_header`" + "include_request_headers" => `include_request_headers` ) `__VARARGS__` ); diff --git a/modules/python-modules/syslogng/modules/webhook/source.py b/modules/python-modules/syslogng/modules/webhook/source.py index 2ea28bfd35..1097f98f81 100644 --- a/modules/python-modules/syslogng/modules/webhook/source.py +++ b/modules/python-modules/syslogng/modules/webhook/source.py @@ -21,6 +21,7 @@ ############################################################################# from syslogng import LogSource, LogMessage +from collections import defaultdict import logging import asyncio @@ -28,11 +29,14 @@ import tornado import ssl import signal +import json from typing import Any signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) -WEBHOOK_QUERY_NV_PREFIX = "webhook.query." +WEBHOOK_NV_PREFIX = "webhook." +WEBHOOK_QUERY_NV_PREFIX = WEBHOOK_NV_PREFIX + "query." +WEBHOOK_HEADERS_KEY = WEBHOOK_NV_PREFIX + "headers" class Handler(tornado.web.RequestHandler): @@ -63,6 +67,15 @@ def _set_proxied_ip(self, msg: LogMessage) -> None: msg.set_source_ipaddress(self.request.remote_ip) + def _set_request_headers(self, msg: LogMessage) -> None: + headers = defaultdict(list) + for h in self.request.headers.get_all(): + name = h[0] + if name: + headers[name].append(h[1]) + + msg[WEBHOOK_HEADERS_KEY] = json.dumps(headers) + def _construct_msg(self, request, path_arguments) -> LogMessage: msg = LogMessage(self.request.body) msg.set_recvd_rawmsg_size(len(self.request.body)) @@ -74,6 +87,9 @@ def _construct_msg(self, request, path_arguments) -> LogMessage: for key, value in path_arguments.items(): msg[key] = value + if self.source.include_request_headers: + self._set_request_headers(msg) + if self.source.proxy_header: self._set_proxied_ip(msg) else: @@ -203,6 +219,8 @@ def init_options(self, options: dict[str, Any]) -> bool: if self.proxy_header == "yes": self.proxy_header = "x-forwarded-for" + self.include_request_headers = bool(options.get("include_request_headers", False)) + return True except KeyError as e: self.logger.error(f"Missing option '{e.args[0]}'") From 1b2911cb4dcf0157c170b6b71753f60d5205e019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 28 Feb 2025 13:41:04 +0100 Subject: [PATCH 3/3] news: add entry for #524 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- news/feature-524.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 news/feature-524.md diff --git a/news/feature-524.md b/news/feature-524.md new file mode 100644 index 0000000000..0d68b51c9e --- /dev/null +++ b/news/feature-524.md @@ -0,0 +1,14 @@ +`webhook()`: headers support + +`include-request-headers(yes)` stores request headers under the `${webhook.headers}` key, +allowing further processing, for example, in FilterX: + +``` +filterx { + headers = json(${webhook.headers}); + $type = headers["Content-Type"][-1]; +}; +``` + +`proxy-header("x-forwarded-for")` helps retain the sender's original IP and the proxy's IP address +(`$SOURCEIP`, `$PEERIP`).