Merge pull request axoflow#178 from axoflow/scl-elasticsearch-datastream

scl: add elasticsearch-datastream destination

news/feature-178.md

@@ -0,0 +1,12 @@
+### Send log messages to Elasticsearch data stream
+The `elasticsearch-datastream()` destination can be used to feed Elasticsearch [data streams](https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html).
+
+Example config:
+
+```
+elasticsearch-datastream(
+  url("https://elastic-endpoint:9200/my-data-stream/_bulk")
+  user("elastic")
+  password("ba3DI8u5qX61We7EP748V8RZ") 
+);
+```

scl/elasticsearch/elastic-datastream.conf

@@ -0,0 +1,50 @@
+#############################################################################
+# Copyright (c) 2024 Axoflow
+# Copyright (c) 2024 <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 as published
+# by the Free Software Foundation, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# As an additional exemption you are allowed to compile & link against the
+# OpenSSL libraries as published by the OpenSSL project. See the file
+# COPYING for details.
+#
+#############################################################################
+
+@requires json-plugin
+
+block destination elasticsearch-datastream(
+  url()
+  workers(4)
+  batch_lines(100)
+  timeout(10)
+  record("--scope rfc5424 --exclude DATE --key ISODATE @timestamp=${ISODATE}")
+  headers("Content-Type: application/x-ndjson")
+  body_suffix("\n")
+  ...)
+{
+
+@requires http "The elasticsearch-datastream() driver depends on the AxoSyslog http module, please install the axosyslog-mod-http (Debian & derivatives) or the axosyslog-http (RHEL & co) package"
+
+    http(
+        url(`url`)
+        headers(`headers`)
+        workers(`workers`)
+        batch_lines(`batch_lines`)
+        timeout(`timeout`)
+        body_suffix(`body_suffix`)
+        body("{\"create\":{ }}\n$(format-json --scope none `record`)")
+        method("PUT")
+        `__VARARGS__`
+    );
+};

tests/copyright/policy

@@ -224,6 +224,7 @@ modules/json/filterx-cache-json-file\.[ch]$
 modules/json/tests/test_filterx_format_json\.c$
 scl/fortigate/.*\.conf$
 scl/cee/.*\.conf$
+scl/elasticsearch/elastic-datastream.conf$
 scl/logscale/logscale\.conf$
 scl/mariadb/.*\.conf$
 scl/python/python-modules\.conf$