From 58957af471e7947b1f478473fe27142096f59b4f Mon Sep 17 00:00:00 2001 From: shifter Date: Fri, 4 Oct 2024 11:56:29 +0200 Subject: [PATCH 1/9] scanner: extend csv-scanner with a new dialect to handle escaped delimiters in unquoted values Signed-off-by: shifter --- lib/scanner/csv-scanner/csv-scanner.c | 12 ++++++++ lib/scanner/csv-scanner/csv-scanner.h | 1 + .../csv-scanner/tests/test_csv_scanner.c | 30 +++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/lib/scanner/csv-scanner/csv-scanner.c b/lib/scanner/csv-scanner/csv-scanner.c index 0debb35b72..402fc52b55 100644 --- a/lib/scanner/csv-scanner/csv-scanner.c +++ b/lib/scanner/csv-scanner/csv-scanner.c @@ -344,11 +344,23 @@ _parse_string_delimiters_at_current_position(CSVScanner *self) static gboolean _parse_character_delimiters_at_current_position(CSVScanner *self) { + gboolean escaped = FALSE; + if (self->options->dialect == CSV_SCANNER_ESCAPE_UNQUOTED_DELIMITER && + *self->src == '\\' && + *(self->src + 1)) + { + self->src++; + escaped = TRUE; + } if (_strchr_optimized_for_single_char_haystack(self->options->delimiters, *self->src) != NULL) { + if (escaped) + return FALSE; self->src++; return TRUE; } + if (escaped) + self->src--; return FALSE; } diff --git a/lib/scanner/csv-scanner/csv-scanner.h b/lib/scanner/csv-scanner/csv-scanner.h index 59533a5b14..649e8610e9 100644 --- a/lib/scanner/csv-scanner/csv-scanner.h +++ b/lib/scanner/csv-scanner/csv-scanner.h @@ -34,6 +34,7 @@ typedef enum CSV_SCANNER_ESCAPE_BACKSLASH, CSV_SCANNER_ESCAPE_BACKSLASH_WITH_SEQUENCES, CSV_SCANNER_ESCAPE_DOUBLE_CHAR, + CSV_SCANNER_ESCAPE_UNQUOTED_DELIMITER, } CSVScannerDialect; #define CSV_SCANNER_STRIP_WHITESPACE 0x0001 diff --git a/lib/scanner/csv-scanner/tests/test_csv_scanner.c b/lib/scanner/csv-scanner/tests/test_csv_scanner.c index f6b184f3c1..0cbab53c99 100644 --- a/lib/scanner/csv-scanner/tests/test_csv_scanner.c +++ b/lib/scanner/csv-scanner/tests/test_csv_scanner.c @@ -21,6 +21,7 @@ * */ #include +#include #include "scratch-buffers.h" #include "apphook.h" @@ -427,6 +428,7 @@ Test(csv_scanner, escape_backslash_x_sequences) csv_scanner_deinit(&scanner); } + Test(csv_scanner, escape_backslash_invalid_x_sequence) { _default_options_with_flags(2, CSV_SCANNER_STRIP_WHITESPACE); @@ -476,6 +478,34 @@ Test(csv_scanner, columnless_no_flags) csv_scanner_deinit(&scanner); } +Test(csv_scanner, escaped_unquoted_delimiter) +{ + _default_options_with_flags(3, CSV_SCANNER_STRIP_WHITESPACE); + + csv_scanner_options_set_dialect(&options, CSV_SCANNER_ESCAPE_UNQUOTED_DELIMITER); + csv_scanner_options_set_delimiters(&options, "|"); + csv_scanner_init(&scanner, &options, "first|foo\\|bar\\|ba\\z|last"); + + cr_expect(_column_index_equals(0)); + cr_expect(!_scan_complete()); + + cr_expect(_scan_next()); + cr_expect(_column_equals(0, "first")); + cr_expect(!_scan_complete()); + + cr_expect(_scan_next()); + cr_expect(_column_equals(1, "foo|bar|ba\\z")); + cr_expect(!_scan_complete()); + + cr_expect(_scan_next()); + cr_expect(_column_equals(2, "last")); + cr_expect(!_scan_complete()); + + cr_expect(!_scan_next()); + cr_expect(_scan_complete()); + csv_scanner_deinit(&scanner); +} + static void setup(void) { From 8b1e848a462adc4e84b6184feb6fc3dcf2cb92a9 Mon Sep 17 00:00:00 2001 From: shifter Date: Fri, 4 Oct 2024 11:59:38 +0200 Subject: [PATCH 2/9] modules/cef: add event-format-parser base class for filterx CEF/LEEF parser Signed-off-by: shifter --- modules/cef/CMakeLists.txt | 3 + modules/cef/Makefile.am | 3 + modules/cef/event-format-parser-cfg.h | 61 +++++ modules/cef/event-format-parser.c | 312 ++++++++++++++++++++++++++ modules/cef/event-format-parser.h | 86 +++++++ 5 files changed, 465 insertions(+) create mode 100644 modules/cef/event-format-parser-cfg.h create mode 100644 modules/cef/event-format-parser.c create mode 100644 modules/cef/event-format-parser.h diff --git a/modules/cef/CMakeLists.txt b/modules/cef/CMakeLists.txt index 482250018c..26a63e6a0d 100644 --- a/modules/cef/CMakeLists.txt +++ b/modules/cef/CMakeLists.txt @@ -2,6 +2,9 @@ set(CEF_SOURCES format-cef-extension.c format-cef-extension.h cef-plugin.c + event-format-parser.c + event-format-parser.h + event-format-parser-cfg.h ) add_module( diff --git a/modules/cef/Makefile.am b/modules/cef/Makefile.am index 6ec1fc2a7c..e8f3a02a46 100644 --- a/modules/cef/Makefile.am +++ b/modules/cef/Makefile.am @@ -5,6 +5,9 @@ EXTRA_DIST += modules/cef/CMakeLists.txt modules_cef_libcef_la_SOURCES = \ modules/cef/format-cef-extension.c \ modules/cef/format-cef-extension.h \ + modules/cef/event-format-parser-cfg.h \ + modules/cef/event-format-parser.c \ + modules/cef/event-format-parser.h \ modules/cef/cef-plugin.c modules_cef_libcef_la_CFLAGS = \ diff --git a/modules/cef/event-format-parser-cfg.h b/modules/cef/event-format-parser-cfg.h new file mode 100644 index 0000000000..fb9f57736f --- /dev/null +++ b/modules/cef/event-format-parser-cfg.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef EVENT_FORMAT_PARSER_CFG_H_INCLUDED +#define EVENT_FORMAT_PARSER_CFG_H_INCLUDED + +#include "filterx/filterx-object.h" + +typedef struct _FilterXFunctionEventFormatParser FilterXFunctionEventFormatParser; + +typedef FilterXObject *(*FieldParser)(FilterXFunctionEventFormatParser *parser, const gchar *value, gint value_len, + GError **error, + gpointer user_data); + +typedef struct _Field +{ + const gchar *name; + FieldParser field_parser; +} Field; + +typedef struct _Header +{ + const gchar *delimiters; + size_t num_fields; + Field *fields; +} Header; + +typedef struct _Extensions +{ + gchar value_separator; + const gchar *pair_separator; +} Extensions; + +typedef struct _Config +{ + const gchar *signature; + Header header; + Extensions extensions; +} Config; + +#endif diff --git a/modules/cef/event-format-parser.c b/modules/cef/event-format-parser.c new file mode 100644 index 0000000000..507bab6e12 --- /dev/null +++ b/modules/cef/event-format-parser.c @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include +#include "event-format-parser.h" +#include "filterx-func-parse-cef.h" +#include "filterx/object-string.h" +#include "filterx/object-primitive.h" +#include "filterx/expr-literal.h" +#include "filterx/expr-literal-generator.h" +#include "filterx/filterx-eval.h" +#include "filterx/filterx-globals.h" +#include "filterx/object-extractor.h" +#include "filterx/object-json.h" +#include "filterx/object-message-value.h" +#include "filterx/object-null.h" +#include "filterx/filterx-object.h" +#include "filterx/object-dict-interface.h" +#include "filterx/object-list-interface.h" +#include "filterx/object-string.h" + +GQuark +event_format_parser_error_quark(void) +{ + return g_quark_from_static_string("event-parser-error-quark"); +} + +Field +field(FilterXFunctionEventFormatParser *self, int index) +{ + g_assert(index >= 0 && index < self->config.header.num_fields); + return self->config.header.fields[index]; +} + +static FilterXObject * +parse_default(FilterXFunctionEventFormatParser *self, const gchar *value, gint value_len, GError **error, + gpointer user_data) +{ + return filterx_string_new(value, value_len); +} + +FilterXObject * +parse_version(FilterXFunctionEventFormatParser *self, const gchar *value, gint value_len, GError **error, + gpointer user_data) +{ + const gchar *log_signature = self->config.signature; + gchar *colon_pos = memchr(value, ':', value_len); + if (!colon_pos || colon_pos == value) + { + g_set_error(error, EVENT_FORMAT_PARSER_ERROR, EVENT_FORMAT_PARSER_ERR_NO_LOG_SIGN, + EVENT_FORMAT_PARSER_ERR_NO_LOG_SIGN_MSG, log_signature); + return FALSE; + } + gint sign_len = colon_pos - value; + if (!(strncmp(value, log_signature, sign_len) == 0)) + { + g_set_error(error, EVENT_FORMAT_PARSER_ERROR, EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS, + EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS_MSG, value, log_signature); + return FALSE; + } + return filterx_string_new(++colon_pos, value_len - sign_len - 1); +} + +gboolean +_set_dict_value(FilterXObject *out, + const gchar *key, gsize key_len, + const gchar *value, gsize value_len) +{ + FilterXObject *dict_key = filterx_string_new(key, key_len); + FilterXObject *dict_val = filterx_string_new(value, value_len); + + gboolean ok = filterx_object_set_subscript(out, dict_key, &dict_val); + + filterx_object_unref(dict_key); + filterx_object_unref(dict_val); + return ok; +} + +static gboolean +_unescape_value_separators(KVScanner *self) +{ + gchar escaped_separator[3] = {'\\', self->value_separator, 0}; + + const gchar *start = self->value->str; + const gchar *pos = start; + + while ((pos = g_strstr_len(start, self->value->len - (pos - start), escaped_separator)) != NULL) + { + g_string_append_len(self->decoded_value, start, pos - start); + g_string_append_c(self->decoded_value, self->value_separator); + start = pos + 2; + } + g_string_append(self->decoded_value, start); + return TRUE; +} + + +FilterXObject * +parse_extensions(FilterXFunctionEventFormatParser *self, const gchar *input, gint input_len, GError **error, + gpointer user_data) +{ + FilterXObject *fillable = (FilterXObject *)user_data; + FilterXObject *output = filterx_object_create_dict(fillable); + + KVScanner kv_scanner; + kv_scanner_init(&kv_scanner, self->config.extensions.value_separator, self->config.extensions.pair_separator, FALSE); + kv_scanner_set_transform_value(&kv_scanner, _unescape_value_separators); + kv_scanner_input(&kv_scanner, input); + while (kv_scanner_scan_next(&kv_scanner)) + { + const gchar *name = kv_scanner_get_current_key(&kv_scanner); + gsize name_len = kv_scanner_get_current_key_len(&kv_scanner); + const gchar *value = kv_scanner_get_current_value(&kv_scanner); + gsize value_len = kv_scanner_get_current_value_len(&kv_scanner); + + if (!_set_dict_value(output, name, name_len, value, value_len)) + goto exit; + } + +exit: + kv_scanner_deinit(&kv_scanner); + return output; +} + +static inline gboolean +_fill_object_col(FilterXFunctionEventFormatParser *self, gint64 index, const gchar *input, gint input_len, + FilterXObject *fillable, + GError **error) +{ + Field f = field(self, index); + FilterXObject *key = filterx_string_new(f.name, -1); + FilterXObject *val = NULL; + + if (!f.field_parser) + val = parse_default(self, input, input_len, error, fillable); + else + val = f.field_parser(self, input, input_len, error, fillable); + + gboolean ok = FALSE; + if (!*error) + ok = filterx_object_set_subscript(fillable, key, &val); + + filterx_object_unref(val); + filterx_object_unref(key); + return ok; +} + +static gboolean +parse(FilterXFunctionEventFormatParser *self, const gchar *log, gsize len, FilterXObject *fillable, GError **error) +{ + gboolean ok = FALSE; + gsize num_fields = self->config.header.num_fields; + + CSVScanner csv_scanner; + csv_scanner_init(&csv_scanner, &self->csv_opts, log); + + guint64 i = 0; + while (csv_scanner_scan_next(&csv_scanner)) + { + if (i >= num_fields) + break; + + const gchar *input = csv_scanner_get_current_value(&csv_scanner); + gint input_len = csv_scanner_get_current_value_len(&csv_scanner); + + ok = _fill_object_col(self, i, input, input_len, fillable, error); + if(!ok || *error) + goto exit; + + i++; + } + + if (i < self->csv_opts.expected_columns) + { + g_set_error(error, EVENT_FORMAT_PARSER_ERROR, EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS, + EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS_MSG, i, self->config.header.num_fields); + } + + +exit: + csv_scanner_deinit(&csv_scanner); + + return ok; +} + +static gboolean +_generate(FilterXExprGenerator *s, FilterXObject *fillable) +{ + FilterXFunctionEventFormatParser *self = (FilterXFunctionEventFormatParser *) s; + gboolean ok = FALSE; + + FilterXObject *obj = filterx_expr_eval(self->msg); + if (!obj) + return FALSE; + + gsize len; + const gchar *input; + if (!filterx_object_extract_string_ref(obj, &input, &len)) + { + filterx_eval_push_error_info(EVENT_FORMAT_PARSER_ERR_NOT_STRING_INPUT_MSG, &self->super.super.super, + NULL, TRUE); + ok = FALSE; + goto exit; + } + + GError *error = NULL; + + ok = parse(self, input, len, fillable, &error); + + if (error) + { + filterx_eval_push_error_info(g_strdup(error->message), &self->super.super.super, + NULL, TRUE); + ok = FALSE; + g_error_free(error); + } + +exit: + filterx_object_unref(obj); + return ok; +} + +static void +_free(FilterXExpr *s) +{ + FilterXFunctionEventFormatParser *self = (FilterXFunctionEventFormatParser *) s; + filterx_expr_unref(self->msg); + csv_scanner_options_clean(&self->csv_opts); + filterx_generator_function_free_method(&self->super); + +} + +static FilterXExpr * +_extract_msg_expr(FilterXFunctionArgs *args, GError **error) +{ + FilterXExpr *msg_expr = filterx_function_args_get_expr(args, 0); + if (!msg_expr) + { + g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL, + "argument must be set: msg_str. "); + return NULL; + } + + return msg_expr; +} + +static gboolean +_extract_args(FilterXFunctionEventFormatParser *self, FilterXFunctionArgs *args, GError **error) +{ + gsize args_len = filterx_function_args_len(args); + if (args_len != 1) + { + g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL, + "invalid number of arguments. "); + return FALSE; + } + + self->msg = _extract_msg_expr(args, error); + if (!self->msg) + return FALSE; + + return TRUE; +} + +static void +_set_config(FilterXFunctionEventFormatParser *self, Config *cfg) +{ + g_assert(cfg); + self->config = *cfg; + csv_scanner_options_set_delimiters(&self->csv_opts, cfg->header.delimiters); + csv_scanner_options_set_quote_pairs(&self->csv_opts, ""); + csv_scanner_options_set_dialect(&self->csv_opts, CSV_SCANNER_ESCAPE_UNQUOTED_DELIMITER); + csv_scanner_options_set_expected_columns(&self->csv_opts, cfg->header.num_fields); + self->csv_opts.flags |= CSV_SCANNER_GREEDY; +} + +gboolean +filterx_function_parser_init_instance(FilterXFunctionEventFormatParser *self, const gchar *fn_name, + FilterXFunctionArgs *args, Config *cfg, GError **error) +{ + filterx_generator_function_init_instance(&self->super, fn_name); + self->super.super.generate = _generate; + self->super.super.create_container = filterx_generator_create_dict_container; + self->super.super.super.free_fn = _free; + + _set_config(self, cfg); + + if (!_extract_args(self, args, error) || + !filterx_function_args_check(args, error)) + return FALSE; + return TRUE; +} diff --git a/modules/cef/event-format-parser.h b/modules/cef/event-format-parser.h new file mode 100644 index 0000000000..b833e9f365 --- /dev/null +++ b/modules/cef/event-format-parser.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef EVENT_FORMAT_PARSER_H_INCLUDED +#define EVENT_FORMAT_PARSER_H_INCLUDED + +#include "syslog-ng.h" +#include "str-utils.h" +#include "filterx/filterx-object.h" +#include "filterx/expr-function.h" + +#include "scanner/csv-scanner/csv-scanner.h" +#include "parser/parser-expr.h" +#include "scanner/kv-scanner/kv-scanner.h" + +#include "event-format-parser-cfg.h" + +#define EVENT_FORMAT_PARSER_ERR_NO_LOG_SIGN_MSG "no log signature %s found" +#define EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS_MSG "the log signature differs. actual:%s expected:%s" +#define EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS_MSG "not enough header columns provided. actual:%ld expected:%ld" +#define EVENT_FORMAT_PARSER_ERR_NOT_STRING_INPUT_MSG "input argument must be string" + +#define EVENT_FORMAT_PARSER_ERROR event_format_parser_error_quark() +GQuark event_format_parser_error_quark(void); + +enum EventFormatParserError +{ + EVENT_FORMAT_PARSER_ERR_NO_LOG_SIGN, + EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS, + EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS, + EVENT_FORMAT_PARSER_ERR_NOT_STRING_INPUT, +}; + +struct _FilterXFunctionEventFormatParser +{ + FilterXGeneratorFunction super; + FilterXExpr *msg; + CSVScannerOptions csv_opts; + Config config; +}; + +gboolean filterx_function_parser_init_instance(FilterXFunctionEventFormatParser *s, const gchar *fn_name, + FilterXFunctionArgs *args, Config *cfg, GError **error); + +FilterXObject *parse_version(FilterXFunctionEventFormatParser *parser, const gchar *value, gint value_len, + GError **error, + gpointer user_data); +FilterXObject *parse_extensions(FilterXFunctionEventFormatParser *parser, const gchar *value, gint value_len, + GError **error, + gpointer user_data); + +static inline void append_error_message(GError **error, const char *extra_info) +{ + if (error == NULL || *error == NULL) + return; + + gchar *new_message = g_strdup_printf("%s: %s", (*error)->message, extra_info); + GError *new_error = g_error_new((*error)->domain, (*error)->code, "%s", new_message); + + g_error_free(*error); + *error = new_error; + + g_free(new_message); +} + +#endif From 9646fbf43024e431a6adf2dc9374e4f63701f80a Mon Sep 17 00:00:00 2001 From: shifter Date: Fri, 4 Oct 2024 12:02:19 +0200 Subject: [PATCH 3/9] filterx/modules/cef: add filterx-func-parse-cef() based on event-format-parser Signed-off-by: shifter --- modules/cef/CMakeLists.txt | 2 + modules/cef/Makefile.am | 2 + modules/cef/cef-plugin.c | 3 ++ modules/cef/filterx-func-parse-cef.c | 80 ++++++++++++++++++++++++++++ modules/cef/filterx-func-parse-cef.h | 36 +++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 modules/cef/filterx-func-parse-cef.c create mode 100644 modules/cef/filterx-func-parse-cef.h diff --git a/modules/cef/CMakeLists.txt b/modules/cef/CMakeLists.txt index 26a63e6a0d..6489838da5 100644 --- a/modules/cef/CMakeLists.txt +++ b/modules/cef/CMakeLists.txt @@ -5,6 +5,8 @@ set(CEF_SOURCES event-format-parser.c event-format-parser.h event-format-parser-cfg.h + filterx-func-parse-cef.c + filterx-func-parse-cef.h ) add_module( diff --git a/modules/cef/Makefile.am b/modules/cef/Makefile.am index e8f3a02a46..cc7d3b8644 100644 --- a/modules/cef/Makefile.am +++ b/modules/cef/Makefile.am @@ -8,6 +8,8 @@ modules_cef_libcef_la_SOURCES = \ modules/cef/event-format-parser-cfg.h \ modules/cef/event-format-parser.c \ modules/cef/event-format-parser.h \ + modules/cef/filterx-func-parse-cef.c \ + modules/cef/filterx-func-parse-cef.h \ modules/cef/cef-plugin.c modules_cef_libcef_la_CFLAGS = \ diff --git a/modules/cef/cef-plugin.c b/modules/cef/cef-plugin.c index 58e2fd3006..1fb33a5a4f 100644 --- a/modules/cef/cef-plugin.c +++ b/modules/cef/cef-plugin.c @@ -22,10 +22,13 @@ #include "format-cef-extension.h" #include "plugin.h" #include "plugin-types.h" +#include "filterx-func-parse-cef.h" +#include "filterx/expr-function.h" static Plugin cef_plugins[] = { TEMPLATE_FUNCTION_PLUGIN(tf_cef, "format-cef-extension"), + FILTERX_GENERATOR_FUNCTION_PLUGIN(parse_cef), }; gboolean diff --git a/modules/cef/filterx-func-parse-cef.c b/modules/cef/filterx-func-parse-cef.c new file mode 100644 index 0000000000..a1abfd3554 --- /dev/null +++ b/modules/cef/filterx-func-parse-cef.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 shifter + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "filterx-func-parse-cef.h" +#include "event-format-parser.h" + +#include "scanner/csv-scanner/csv-scanner.h" +#include "scanner/kv-scanner/kv-scanner.h" + +static Field cef_fields[] = +{ + { .name = "version", .field_parser = parse_version}, + { .name = "device_vendor"}, + { .name = "device_product"}, + { .name = "device_version"}, + { .name = "device_event_class_id"}, + { .name = "name"}, + { .name = "agent_severity"}, + { .name = "extensions", .field_parser = parse_extensions}, +}; + +typedef struct FilterXFunctionParseCEF_ +{ + FilterXFunctionEventFormatParser super; +} FilterXFunctionParseCEF; + +FilterXExpr * +filterx_function_parse_cef_new(FilterXFunctionArgs *args, GError **err) +{ + FilterXFunctionParseCEF *self = g_new0(FilterXFunctionParseCEF, 1); + + Config cfg = + { + .signature = "CEF", + .header = { + .num_fields = 8, + .delimiters = "|", + .fields = cef_fields, + }, + .extensions = { + .pair_separator = " ", + .value_separator = '=', + }, + }; + + if (!filterx_function_parser_init_instance(&self->super, "parse_cef", args, &cfg, err)) + goto error; + + filterx_function_args_free(args); + return &self->super.super.super.super; + +error: + append_error_message(err, FILTERX_FUNC_PARSE_CEF_USAGE); + filterx_function_args_free(args); + filterx_expr_unref(&self->super.super.super.super); + return NULL; +} + +FILTERX_GENERATOR_FUNCTION(parse_cef, filterx_function_parse_cef_new); diff --git a/modules/cef/filterx-func-parse-cef.h b/modules/cef/filterx-func-parse-cef.h new file mode 100644 index 0000000000..42e98e591e --- /dev/null +++ b/modules/cef/filterx-func-parse-cef.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef FILTERX_FUNC_PARSE_CEF_H_INCLUDED +#define FILTERX_FUNC_PARSE_CEF_H_INCLUDED + +#include "plugin.h" +#include "filterx/expr-function.h" + +#define FILTERX_FUNC_PARSE_CEF_USAGE "Usage: parse_cef(str)" + +FILTERX_GENERATOR_FUNCTION_DECLARE(parse_cef); + +FilterXExpr *filterx_function_parse_cef_new(FilterXFunctionArgs *args, GError **error); + +#endif From 57101f76c42cca1e23dee000670ff1c62d3ae38d Mon Sep 17 00:00:00 2001 From: shifter Date: Fri, 4 Oct 2024 12:03:04 +0200 Subject: [PATCH 4/9] filterx/modules/cef: add unit tests for filterx-func-parse-cef() Signed-off-by: shifter --- modules/cef/tests/CMakeLists.txt | 1 + modules/cef/tests/Makefile.am | 11 + .../tests/test-filterx-function-parse-cef.c | 237 ++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 modules/cef/tests/test-filterx-function-parse-cef.c diff --git a/modules/cef/tests/CMakeLists.txt b/modules/cef/tests/CMakeLists.txt index d861ea6146..d39865ed6c 100644 --- a/modules/cef/tests/CMakeLists.txt +++ b/modules/cef/tests/CMakeLists.txt @@ -1 +1,2 @@ add_unit_test(LIBTEST CRITERION TARGET test-format-cef-extension DEPENDS cef) +add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-cef DEPENDS cef) diff --git a/modules/cef/tests/Makefile.am b/modules/cef/tests/Makefile.am index c0efea9a74..d2a89956fb 100644 --- a/modules/cef/tests/Makefile.am +++ b/modules/cef/tests/Makefile.am @@ -1,4 +1,6 @@ modules_cef_tests_TESTS = \ + modules/cef/tests/test-format-cef-extension \ + modules/cef/tests/test-filterx-function-parse-cef \ modules/cef/tests/test-format-cef-extension check_PROGRAMS += ${modules_cef_tests_TESTS} @@ -13,3 +15,12 @@ modules_cef_tests_test_format_cef_extension_LDFLAGS = \ EXTRA_modules_cef_tests_test_format_cef_extension_DEPENDENCIES = \ $(top_builddir)/modules/cef/libcef.la + +modules_cef_tests_test_filterx_function_parse_cef_CFLAGS = $(TEST_CFLAGS) -I$(top_srcdir)/modules/cef +modules_cef_tests_test_filterx_function_parse_cef_LDADD = $(TEST_LDADD) +modules_cef_tests_test_filterx_function_parse_cef_LDFLAGS = \ + $(PREOPEN_SYSLOGFORMAT) \ + -dlpreopen $(top_builddir)/modules/cef/libcef.la + +EXTRA_modules_cef_tests_test_filterx_function_parse_cef_DEPENDENCIES = \ + $(top_builddir)/modules/cef/libcef.la diff --git a/modules/cef/tests/test-filterx-function-parse-cef.c b/modules/cef/tests/test-filterx-function-parse-cef.c new file mode 100644 index 0000000000..ceaa976010 --- /dev/null +++ b/modules/cef/tests/test-filterx-function-parse-cef.c @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#include +#include "libtest/filterx-lib.h" + +#include "apphook.h" +#include "scratch-buffers.h" + +#include "filterx-func-parse-cef.h" +#include "event-format-parser.h" + +#include "filterx/object-string.h" +#include "filterx/object-null.h" +#include "filterx/expr-literal.h" +#include "filterx/object-json.h" +#include "filterx/object-list-interface.h" +#include "filterx/object-dict-interface.h" +#include "filterx/object-primitive.h" +#include "filterx/filterx-eval.h" + + +FilterXExpr * +_new_cef_parser(FilterXFunctionArgs *args, GError **error, FilterXObject *fillable) +{ + FilterXExpr *func = filterx_function_parse_cef_new(args, error); + + if (!func) + return NULL; + + FilterXExpr *fillable_expr = filterx_non_literal_new(fillable); + filterx_generator_set_fillable(func, fillable_expr); + + return func; +} + +FilterXFunctionArgs * +_assert_create_args(const gchar *input) +{ + GList *args = NULL; + args = g_list_append(args, filterx_function_arg_new(NULL, filterx_literal_new(filterx_string_new(input, -1)))); + GError *args_err = NULL; + FilterXFunctionArgs *result = filterx_function_args_new(args, &args_err); + cr_assert_null(args_err); + g_error_free(args_err); + return result; +} + +FilterXObject * +_eval_cef_input(const gchar *input, GError **error) +{ + FilterXExpr *func = _new_cef_parser(_assert_create_args(input), error, filterx_json_object_new_empty()); + FilterXObject *obj = filterx_expr_eval(func); + filterx_expr_unref(func); + return obj; +} + +void +_assert_cef_parser_result(const gchar *input, const gchar *expected_result) +{ + + GError *err = NULL; + FilterXObject *obj = _eval_cef_input(input, &err); + cr_assert_null(err); + cr_assert_not_null(obj); + + cr_assert(filterx_object_is_type(obj, &FILTERX_TYPE_NAME(dict))); + + GString *repr = scratch_buffers_alloc(); + + LogMessageValueType lmvt; + cr_assert(filterx_object_marshal(obj, repr, &lmvt)); + + cr_assert_str_eq(repr->str, expected_result); + + filterx_object_unref(obj); + g_error_free(err); +} + +Test(filterx_func_parse_cef, test_invalid_input) +{ + GError *error = NULL; + GList *args = NULL; + args = g_list_append(args, filterx_function_arg_new(NULL, filterx_literal_new(filterx_integer_new(33)))); + GError *args_err = NULL; + FilterXFunctionArgs *fx_args = filterx_function_args_new(args, &args_err); + cr_assert_null(args_err); + g_error_free(args_err); + + FilterXExpr *func = _new_cef_parser(fx_args, &error, filterx_json_object_new_empty()); + cr_assert_null(error); + FilterXObject *obj = filterx_expr_eval(func); + cr_assert_null(obj); + + cr_assert_str_eq(EVENT_FORMAT_PARSER_ERR_NOT_STRING_INPUT_MSG, filterx_eval_get_last_error()); + + filterx_expr_unref(func); + g_error_free(error); +} + +Test(filterx_func_parse_cef, test_invalid_version) +{ + GError *init_err = NULL; + cr_assert_null( + _eval_cef_input("INVALID|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|rt=1647626887000 cs9=site location Bldg cs9Label=GroupName dhost=WS6465 dst=10.55.203.12 cs2=KES cs2Label=ProductName cs3=11.0.0.0 cs3Label=ProductVersion cs10=Uninstall EDR cs10Label=TaskName cs4=885 cs4Label=TaskId cn2=4 cn2Label=TaskNewState cn1=0 cn1Label=TaskOldState", + &init_err)); + cr_assert_null(init_err); + const gchar *last_error = filterx_eval_get_last_error(); + cr_assert_not_null(last_error); + GString *expected_err_msg = scratch_buffers_alloc(); + g_string_append_printf(expected_err_msg, EVENT_FORMAT_PARSER_ERR_NO_LOG_SIGN_MSG, "CEF"); + cr_assert_str_eq(expected_err_msg->str, last_error); +} + +Test(filterx_func_parse_cef, test_invalid_log_signature) +{ + GError *init_err = NULL; + cr_assert_null( + _eval_cef_input("BAD_SIGN:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|rt=1647626887000 cs9=site location Bldg cs9Label=GroupName dhost=WS6465 dst=10.55.203.12 cs2=KES cs2Label=ProductName cs3=11.0.0.0 cs3Label=ProductVersion cs10=Uninstall EDR cs10Label=TaskName cs4=885 cs4Label=TaskId cn2=4 cn2Label=TaskNewState cn1=0 cn1Label=TaskOldState", + &init_err)); + cr_assert_null(init_err); + const gchar *last_error = filterx_eval_get_last_error(); + cr_assert_not_null(last_error); + GString *expected_err_msg = scratch_buffers_alloc(); + g_string_append_printf(expected_err_msg, EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS_MSG, "BAD_SIGN:0", "CEF"); + cr_assert_str_eq(expected_err_msg->str, last_error); +} + +Test(filterx_func_parse_cef, test_header_missing_field) +{ + GError *init_err = NULL; + cr_assert_null(_eval_cef_input("CEF:0|KasperskyLab|SecurityCenter|", &init_err)); + cr_assert_null(init_err); + const gchar *last_error = filterx_eval_get_last_error(); + cr_assert_not_null(last_error); + GString *expected_err_msg = scratch_buffers_alloc(); + g_string_append_printf(expected_err_msg, EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS_MSG, (guint64)3, (guint64)8); + cr_assert_str_eq(expected_err_msg->str, last_error); +} + +Test(filterx_func_parse_cef, test_basic_cef_message) +{ + _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|rt=1647626887000 cs9=site location Bldg cs9Label=GroupName dhost=WS6465 dst=10.55.203.12 cs2=KES cs2Label=ProductName cs3=11.0.0.0 cs3Label=ProductVersion cs10=Uninstall EDR cs10Label=TaskName cs4=885 cs4Label=TaskId cn2=4 cn2Label=TaskNewState cn1=0 cn1Label=TaskOldState", + "{\"version\":\"0\",\"device_vendor\":\"KasperskyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{\"rt\":\"1647626887000\",\"cs9\":\"site location Bldg\",\"cs9Label\":\"GroupName\",\"dhost\":\"WS6465\",\"dst\":\"10.55.203.12\",\"cs2\":\"KES\",\"cs2Label\":\"ProductName\",\"cs3\":\"11.0.0.0\",\"cs3Label\":\"ProductVersion\",\"cs10\":\"Uninstall EDR\",\"cs10Label\":\"TaskName\",\"cs4\":\"885\",\"cs4Label\":\"TaskId\",\"cn2\":\"4\",\"cn2Label\":\"TaskNewState\",\"cn1\":\"0\",\"cn1Label\":\"TaskOldState\"}}"); +} + +Test(filterx_func_parse_cef, test_extensions_empty) +{ + _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|", + "{\"version\":\"0\",\"device_vendor\":\"KasperskyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{}}"); +} + +Test(filterx_func_parse_cef, test_header_escaped_delimiter) +{ + _assert_cef_parser_result("CEF:0|Kaspers\\|kyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|rt=1647626887000", + "{\"version\":\"0\",\"device_vendor\":\"Kaspers|kyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{\"rt\":\"1647626887000\"}}"); +} + + +Test(filterx_func_parse_cef, test_exteansion_escaped_delimiter) +{ + _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|escaped=foo\\=bar\\=baz", + "{\"version\":\"0\",\"device_vendor\":\"KasperskyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{\"escaped\":\"foo=bar=baz\"}}"); +} + +Test(filterx_func_parse_cef, test_header_do_not_strip_whitespaces) +{ + _assert_cef_parser_result("CEF:0| KasperskyLab | SecurityCenter | 13.2.0.1511 | KLPRCI_TaskState | Completed successfully | 1 |", + "{\"version\":\"0\",\"device_vendor\":\" KasperskyLab \",\"device_product\":\" SecurityCenter \",\"device_version\":\" 13.2.0.1511 \",\"device_event_class_id\":\" KLPRCI_TaskState \",\"name\":\" Completed successfully \",\"agent_severity\":\" 1 \",\"extensions\":{}}"); +} + +Test(filterx_func_parse_cef, test_extensions_space_in_value) +{ + _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|foo=bar baz tik=tak toe", + "{\"version\":\"0\",\"device_vendor\":\"KasperskyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{\"foo\":\"bar baz\",\"tik\":\"tak toe\"}}"); +} + +// TODO: fix spaces? +// Test(filterx_func_parse_cef, test_extensions_trailing_space_in_key) +// { +// _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|foo bar=bar baz", +// "{\"version\":\"0\",\"device_vendor\":\"KasperskyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{\"foo bar\":\"bar baz\"}}"); +// } + +// Test(filterx_func_parse_cef, test_extensions_trailing_space_in_value) +// { +// _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|foo= bar baz tik= tak toe ", +// "{\"version\":\"0\",\"device_vendor\":\"KasperskyLab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{\"foo\":\"bar baz\"}}"); +// } + +Test(filterx_func_parse_cef, test_header_whitespaces) +{ + _assert_cef_parser_result("CEF:0|Kasper sky Lab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|", + "{\"version\":\"0\",\"device_vendor\":\"Kasper sky Lab\",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{}}"); +} + +Test(filterx_func_parse_cef, test_header_leading_trailing_whitespaces) +{ + _assert_cef_parser_result("CEF:0| KasperskyLab |SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|", + "{\"version\":\"0\",\"device_vendor\":\" KasperskyLab \",\"device_product\":\"SecurityCenter\",\"device_version\":\"13.2.0.1511\",\"device_event_class_id\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agent_severity\":\"1\",\"extensions\":{}}"); +} + +static void +setup(void) +{ + app_startup(); + init_libtest_filterx(); +} + +static void +teardown(void) +{ + scratch_buffers_explicit_gc(); + deinit_libtest_filterx(); + app_shutdown(); +} + +TestSuite(filterx_func_parse_cef, .init = setup, .fini = teardown); From 87d6ec630fcc6eb26be5cbede6b8422a55771d37 Mon Sep 17 00:00:00 2001 From: shifter Date: Fri, 4 Oct 2024 12:04:32 +0200 Subject: [PATCH 5/9] filterx/modules/cef: add filterx-func-parse-leef() based on event-format-parser this version supports only LEEF:1.0 atm, LEEF:2.0 will be added as a separate feature Signed-off-by: shifter --- modules/cef/CMakeLists.txt | 2 + modules/cef/Makefile.am | 2 + modules/cef/cef-plugin.c | 2 + modules/cef/filterx-func-parse-leef.c | 79 +++++++++++++++++++++++++++ modules/cef/filterx-func-parse-leef.h | 36 ++++++++++++ 5 files changed, 121 insertions(+) create mode 100644 modules/cef/filterx-func-parse-leef.c create mode 100644 modules/cef/filterx-func-parse-leef.h diff --git a/modules/cef/CMakeLists.txt b/modules/cef/CMakeLists.txt index 6489838da5..56b4c92be1 100644 --- a/modules/cef/CMakeLists.txt +++ b/modules/cef/CMakeLists.txt @@ -7,6 +7,8 @@ set(CEF_SOURCES event-format-parser-cfg.h filterx-func-parse-cef.c filterx-func-parse-cef.h + filterx-func-parse-leef.c + filterx-func-parse-leef.h ) add_module( diff --git a/modules/cef/Makefile.am b/modules/cef/Makefile.am index cc7d3b8644..309e6f3b06 100644 --- a/modules/cef/Makefile.am +++ b/modules/cef/Makefile.am @@ -10,6 +10,8 @@ modules_cef_libcef_la_SOURCES = \ modules/cef/event-format-parser.h \ modules/cef/filterx-func-parse-cef.c \ modules/cef/filterx-func-parse-cef.h \ + modules/cef/filterx-func-parse-leef.c \ + modules/cef/filterx-func-parse-leef.h \ modules/cef/cef-plugin.c modules_cef_libcef_la_CFLAGS = \ diff --git a/modules/cef/cef-plugin.c b/modules/cef/cef-plugin.c index 1fb33a5a4f..b4083a52cb 100644 --- a/modules/cef/cef-plugin.c +++ b/modules/cef/cef-plugin.c @@ -23,12 +23,14 @@ #include "plugin.h" #include "plugin-types.h" #include "filterx-func-parse-cef.h" +#include "filterx-func-parse-leef.h" #include "filterx/expr-function.h" static Plugin cef_plugins[] = { TEMPLATE_FUNCTION_PLUGIN(tf_cef, "format-cef-extension"), FILTERX_GENERATOR_FUNCTION_PLUGIN(parse_cef), + FILTERX_GENERATOR_FUNCTION_PLUGIN(parse_leef), }; gboolean diff --git a/modules/cef/filterx-func-parse-leef.c b/modules/cef/filterx-func-parse-leef.c new file mode 100644 index 0000000000..f7f56a0656 --- /dev/null +++ b/modules/cef/filterx-func-parse-leef.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 shifter + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "filterx-func-parse-leef.h" +#include "event-format-parser.h" + +#include "scanner/csv-scanner/csv-scanner.h" +#include "scanner/kv-scanner/kv-scanner.h" + +static Field leef_fields[] = +{ + { .name = "version", .field_parser = parse_version}, + { .name = "vendor"}, + { .name = "product_name"}, + { .name = "product_version"}, + { .name = "event_id"}, + { .name = "extensions", .field_parser = parse_extensions}, +}; + +typedef struct FilterXFunctionParseLEEF_ +{ + FilterXFunctionEventFormatParser super; +} FilterXFunctionParseLEEF; + + +FilterXExpr * +filterx_function_parse_leef_new(FilterXFunctionArgs *args, GError **err) +{ + FilterXFunctionParseLEEF *self = g_new0(FilterXFunctionParseLEEF, 1); + + Config cfg = + { + .signature = "LEEF", + .header = { + .num_fields = 6, + .delimiters = "|", + .fields = leef_fields, + }, + .extensions = { + .pair_separator = "\t", + .value_separator = '=', + }, + }; + + if (!filterx_function_parser_init_instance(&self->super, "parse_leef", args, &cfg, err)) + goto error; + + filterx_function_args_free(args); + return &self->super.super.super.super; + +error: + append_error_message(err, FILTERX_FUNC_PARSE_LEEF_USAGE); + filterx_function_args_free(args); + filterx_expr_unref(&self->super.super.super.super); + return NULL; +} + +FILTERX_GENERATOR_FUNCTION(parse_leef, filterx_function_parse_leef_new); diff --git a/modules/cef/filterx-func-parse-leef.h b/modules/cef/filterx-func-parse-leef.h new file mode 100644 index 0000000000..c5017ce056 --- /dev/null +++ b/modules/cef/filterx-func-parse-leef.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef FILTERX_FUNC_PARSE_LEEF_H_INCLUDED +#define FILTERX_FUNC_PARSE_LEEF_H_INCLUDED + +#include "plugin.h" +#include "filterx/expr-function.h" + +#define FILTERX_FUNC_PARSE_LEEF_USAGE "Usage: parse_leef(str)" + +FILTERX_GENERATOR_FUNCTION_DECLARE(parse_leef); + +FilterXExpr *filterx_function_parse_leef_new(FilterXFunctionArgs *args, GError **error); + +#endif From 92d466442b51cf12f97324abe5c42dd5c4d938e4 Mon Sep 17 00:00:00 2001 From: shifter Date: Fri, 4 Oct 2024 12:05:19 +0200 Subject: [PATCH 6/9] filterx/modules/leef: add filter-func-parse-leef() unit tests Signed-off-by: shifter --- modules/cef/tests/CMakeLists.txt | 1 + modules/cef/tests/Makefile.am | 11 +- .../tests/test-filterx-function-parse-leef.c | 237 ++++++++++++++++++ 3 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 modules/cef/tests/test-filterx-function-parse-leef.c diff --git a/modules/cef/tests/CMakeLists.txt b/modules/cef/tests/CMakeLists.txt index d39865ed6c..8032140a48 100644 --- a/modules/cef/tests/CMakeLists.txt +++ b/modules/cef/tests/CMakeLists.txt @@ -1,2 +1,3 @@ add_unit_test(LIBTEST CRITERION TARGET test-format-cef-extension DEPENDS cef) add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-cef DEPENDS cef) +add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-leef DEPENDS cef) diff --git a/modules/cef/tests/Makefile.am b/modules/cef/tests/Makefile.am index d2a89956fb..648318f755 100644 --- a/modules/cef/tests/Makefile.am +++ b/modules/cef/tests/Makefile.am @@ -1,7 +1,7 @@ modules_cef_tests_TESTS = \ modules/cef/tests/test-format-cef-extension \ modules/cef/tests/test-filterx-function-parse-cef \ - modules/cef/tests/test-format-cef-extension + modules/cef/tests/test-filterx-function-parse-leef check_PROGRAMS += ${modules_cef_tests_TESTS} @@ -24,3 +24,12 @@ modules_cef_tests_test_filterx_function_parse_cef_LDFLAGS = \ EXTRA_modules_cef_tests_test_filterx_function_parse_cef_DEPENDENCIES = \ $(top_builddir)/modules/cef/libcef.la + +modules_cef_tests_test_filterx_function_parse_leef_CFLAGS = $(TEST_CFLAGS) -I$(top_srcdir)/modules/cef +modules_cef_tests_test_filterx_function_parse_leef_LDADD = $(TEST_LDADD) +modules_cef_tests_test_filterx_function_parse_leef_LDFLAGS = \ + $(PREOPEN_SYSLOGFORMAT) \ + -dlpreopen $(top_builddir)/modules/cef/libcef.la + +EXTRA_modules_cef_tests_test_filterx_function_parse_leef_DEPENDENCIES = \ + $(top_builddir)/modules/cef/libcef.la diff --git a/modules/cef/tests/test-filterx-function-parse-leef.c b/modules/cef/tests/test-filterx-function-parse-leef.c new file mode 100644 index 0000000000..e880f8b7fe --- /dev/null +++ b/modules/cef/tests/test-filterx-function-parse-leef.c @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Axoflow + * Copyright (c) 2024 shifter + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#include +#include "libtest/filterx-lib.h" + +#include "apphook.h" +#include "scratch-buffers.h" + +#include "filterx-func-parse-leef.h" +#include "event-format-parser.h" + +#include "filterx/object-string.h" +#include "filterx/object-null.h" +#include "filterx/expr-literal.h" +#include "filterx/object-json.h" +#include "filterx/object-list-interface.h" +#include "filterx/object-dict-interface.h" +#include "filterx/object-primitive.h" +#include "filterx/filterx-eval.h" + + +FilterXExpr * +_new_leef_parser(FilterXFunctionArgs *args, GError **error, FilterXObject *fillable) +{ + FilterXExpr *func = filterx_function_parse_leef_new(args, error); + + if (!func) + return NULL; + + FilterXExpr *fillable_expr = filterx_non_literal_new(fillable); + filterx_generator_set_fillable(func, fillable_expr); + + return func; +} + +FilterXFunctionArgs * +_assert_create_args(const gchar *input) +{ + GList *args = NULL; + args = g_list_append(args, filterx_function_arg_new(NULL, filterx_literal_new(filterx_string_new(input, -1)))); + GError *args_err = NULL; + FilterXFunctionArgs *result = filterx_function_args_new(args, &args_err); + cr_assert_null(args_err); + g_error_free(args_err); + return result; +} + +FilterXObject * +_eval_leef_input(const gchar *input, GError **error) +{ + FilterXExpr *func = _new_leef_parser(_assert_create_args(input), error, filterx_json_object_new_empty()); + FilterXObject *obj = filterx_expr_eval(func); + filterx_expr_unref(func); + return obj; +} + +void +_assert_leef_parser_result(const gchar *input, const gchar *expected_result) +{ + + GError *err = NULL; + FilterXObject *obj = _eval_leef_input(input, &err); + cr_assert_null(err); + cr_assert_not_null(obj); + + cr_assert(filterx_object_is_type(obj, &FILTERX_TYPE_NAME(dict))); + + GString *repr = scratch_buffers_alloc(); + + LogMessageValueType lmvt; + cr_assert(filterx_object_marshal(obj, repr, &lmvt)); + + cr_assert_str_eq(repr->str, expected_result); + + filterx_object_unref(obj); + g_error_free(err); +} + +Test(filterx_func_parse_leef, test_invalid_input) +{ + GError *error = NULL; + GList *args = NULL; + args = g_list_append(args, filterx_function_arg_new(NULL, filterx_literal_new(filterx_integer_new(33)))); + GError *args_err = NULL; + FilterXFunctionArgs *fx_args = filterx_function_args_new(args, &args_err); + cr_assert_null(args_err); + g_error_free(args_err); + + FilterXExpr *func = _new_leef_parser(fx_args, &error, filterx_json_object_new_empty()); + cr_assert_null(error); + FilterXObject *obj = filterx_expr_eval(func); + cr_assert_null(obj); + + cr_assert_str_eq(EVENT_FORMAT_PARSER_ERR_NOT_STRING_INPUT_MSG, filterx_eval_get_last_error()); + + filterx_expr_unref(func); + g_error_free(error); +} + +Test(filterx_func_parse_leef, test_invalid_version) +{ + GError *init_err = NULL; + cr_assert_null( + _eval_leef_input("INVALID|Microsoft|MSExchange|4.0 SP1|15345|src=192.0.2.0 dst=172.50.123.1 sev=5cat=anomaly srcPort=81 dstPort=21 usrName=joe.black", + &init_err)); + cr_assert_null(init_err); + const gchar *last_error = filterx_eval_get_last_error(); + cr_assert_not_null(last_error); + GString *expected_err_msg = scratch_buffers_alloc(); + g_string_append_printf(expected_err_msg, EVENT_FORMAT_PARSER_ERR_NO_LOG_SIGN_MSG, "LEEF"); + cr_assert_str_eq(expected_err_msg->str, last_error); +} + +Test(filterx_func_parse_leef, test_invalid_log_signature) +{ + GError *init_err = NULL; + cr_assert_null( + _eval_leef_input("BAD_SIGN:1.0|Microsoft|MSExchange|4.0 SP1|15345|src=192.0.2.0 dst=172.50.123.1 sev=5cat=anomaly srcPort=81 dstPort=21 usrName=joe.black", + &init_err)); + cr_assert_null(init_err); + const gchar *last_error = filterx_eval_get_last_error(); + cr_assert_not_null(last_error); + GString *expected_err_msg = scratch_buffers_alloc(); + g_string_append_printf(expected_err_msg, EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS_MSG, "BAD_SIGN:1.0", "LEEF"); + cr_assert_str_eq(expected_err_msg->str, last_error); +} + +Test(filterx_func_parse_leef, test_header_missing_field) +{ + GError *init_err = NULL; + cr_assert_null(_eval_leef_input("LEEF:1.0|Microsoft|MSExchange", &init_err)); + cr_assert_null(init_err); + const gchar *last_error = filterx_eval_get_last_error(); + cr_assert_not_null(last_error); + GString *expected_err_msg = scratch_buffers_alloc(); + g_string_append_printf(expected_err_msg, EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS_MSG, (guint64)3, (guint64)6); + cr_assert_str_eq(expected_err_msg->str, last_error); +} + +Test(filterx_func_parse_leef, test_basic_leef_message) +{ + _assert_leef_parser_result("LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345|src=192.0.2.0 dst=172.50.123.1 sev=5cat=anomaly srcPort=81 dstPort=21 usrName=joe.black", + "{\"version\":\"1.0\",\"vendor\":\"Microsoft\",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{\"src\":\"192.0.2.0\",\"dst\":\"172.50.123.1\",\"sev\":\"5cat=anomaly\",\"srcPort\":\"81\",\"dstPort\":\"21\",\"usrName\":\"joe.black\"}}"); +} + +Test(filterx_func_parse_leef, test_extensions_empty) +{ + _assert_leef_parser_result("LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345|", + "{\"version\":\"1.0\",\"vendor\":\"Microsoft\",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{}}"); +} + +Test(filterx_func_parse_leef, test_header_escaped_delimiter) +{ + _assert_leef_parser_result("LEEF:1.0|Micro\\|soft|MSExchange|4.0 SP1|15345|src=192.0.2.0 dst=172.50.123.1 sev=5cat=anomaly srcPort=81 dstPort=21 usrName=joe.black", + "{\"version\":\"1.0\",\"vendor\":\"Micro|soft\",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{\"src\":\"192.0.2.0\",\"dst\":\"172.50.123.1\",\"sev\":\"5cat=anomaly\",\"srcPort\":\"81\",\"dstPort\":\"21\",\"usrName\":\"joe.black\"}}"); +} + + +Test(filterx_func_parse_leef, test_extension_escaped_delimiter) +{ + _assert_leef_parser_result("LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345|foo=foo\\=bar\\=baz tik=tik\\=tak\\=toe", + "{\"version\":\"1.0\",\"vendor\":\"Microsoft\",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{\"foo\":\"foo=bar=baz\",\"tik\":\"tik=tak=toe\"}}"); +} + +Test(filterx_func_parse_leef, test_header_do_not_strip_whitespaces) +{ + _assert_leef_parser_result("LEEF:1.0| Microsoft | MSExchange | 4.0 SP1 | 15345 |", + "{\"version\":\"1.0\",\"vendor\":\" Microsoft \",\"product_name\":\" MSExchange \",\"product_version\":\" 4.0 SP1 \",\"event_id\":\" 15345 \",\"extensions\":{}}"); +} + +Test(filterx_func_parse_leef, test_extensions_space_in_value) +{ + _assert_leef_parser_result("LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345|foo=bar baz tik=tak toe", + "{\"version\":\"1.0\",\"vendor\":\"Microsoft\",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{\"foo\":\"bar baz\",\"tik\":\"tak toe\"}}"); +} + +// TODO: fix spaces? +// Test(filterx_func_parse_cef, test_extensions_trailing_space_in_key) +// { +// _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|foo bar=bar baz", +// "{\"version\":\"0\",\"deviceVendor\":\"KasperskyLab\",\"deviceProduct\":\"SecurityCenter\",\"deviceVersion\":\"13.2.0.1511\",\"deviceEventClassId\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agentSeverity\":\"1\",\"extensions\":{\"foo bar\":\"bar baz\"}}"); +// } + +// Test(filterx_func_parse_cef, test_extensions_trailing_space_in_value) +// { +// _assert_cef_parser_result("CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|foo= bar baz tik= tak toe ", +// "{\"version\":\"0\",\"deviceVendor\":\"KasperskyLab\",\"deviceProduct\":\"SecurityCenter\",\"deviceVersion\":\"13.2.0.1511\",\"deviceEventClassId\":\"KLPRCI_TaskState\",\"name\":\"Completed successfully\",\"agentSeverity\":\"1\",\"extensions\":{\"foo\":\"bar baz\"}}"); +// } + +Test(filterx_func_parse_leef, test_header_whitespaces) +{ + _assert_leef_parser_result("LEEF:1.0|Mic roso ft|MSExchange|4.0 SP1|15345|", + "{\"version\":\"1.0\",\"vendor\":\"Mic roso ft\",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{}}"); +} + +Test(filterx_func_parse_leef, test_header_leading_trailing_whitespaces) +{ + _assert_leef_parser_result("LEEF:1.0| Microsoft |MSExchange|4.0 SP1|15345|foo=foo\\=bar\\=baz tik=tik\\=tak\\=toe", + "{\"version\":\"1.0\",\"vendor\":\" Microsoft \",\"product_name\":\"MSExchange\",\"product_version\":\"4.0 SP1\",\"event_id\":\"15345\",\"extensions\":{\"foo\":\"foo=bar=baz\",\"tik\":\"tik=tak=toe\"}}"); +} + +static void +setup(void) +{ + app_startup(); + init_libtest_filterx(); +} + +static void +teardown(void) +{ + scratch_buffers_explicit_gc(); + deinit_libtest_filterx(); + app_shutdown(); +} + +TestSuite(filterx_func_parse_leef, .init = setup, .fini = teardown); From 774eefd0f939e873f37f061964b4cd588cc673b4 Mon Sep 17 00:00:00 2001 From: shifter Date: Sat, 5 Oct 2024 10:39:38 +0200 Subject: [PATCH 7/9] modules/cef: policy update Signed-off-by: shifter --- tests/copyright/policy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/copyright/policy b/tests/copyright/policy index c40c166e24..58b98c85e8 100644 --- a/tests/copyright/policy +++ b/tests/copyright/policy @@ -293,6 +293,12 @@ modules/csvparser/filterx-func-parse-csv\.[ch] modules/csvparser/filterx-func-format-csv\.[ch] modules/csvparser/tests/test_filterx_func_parse_csv.c modules/csvparser/tests/test_filterx_func_format_csv.c +modules/cef/event-format-parser.[ch] +modules/cef/event-format-parser-cfg.h +modules/cef/filterx-func-parse-cef.[ch] +modules/cef/filterx-func-parse-leef.[ch] +modules/cef/tests/test-filterx-function-parse-cef.c +modules/cef/tests/test-filterx-function-parse-leef.c ########################################################################### # These files are GPLd with Balabit origin. From c7cffe15681bb6baacaf4075d8faf76dbbe293fd Mon Sep 17 00:00:00 2001 From: shifter Date: Sat, 5 Oct 2024 11:01:15 +0200 Subject: [PATCH 8/9] filterx/modules/cef: add parse_cef light test Signed-off-by: shifter --- .../functional_tests/filterx/test_filterx.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index e9ef1b5b96..69e5716a6a 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -2338,3 +2338,31 @@ def test_startswith_endswith_includes(config, syslog_ng): assert "processed" not in file_false.get_stats() assert file_true.read_log() == '{"startswith_foo":true,"contains_bar":true,"endswith_baz":true,"works_with_message_value":true}\n' + + +def test_parse_cef(config, syslog_ng): + (file_true, file_false) = create_config( + config, r""" + custom_message = "CEF:0|KasperskyLab|SecurityCenter|13.2.0.1511|KLPRCI_TaskState|Completed successfully|1|foo=foo\\=bar bar=bar\\=baz baz=test"; + $MSG = json(parse_cef(custom_message)); + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + exp = ( + r"""{"version":"0",""" + r""""device_vendor":"KasperskyLab",""" + r""""device_product":"SecurityCenter",""" + r""""device_version":"13.2.0.1511",""" + r""""device_event_class_id":"KLPRCI_TaskState",""" + r""""name":"Completed successfully",""" + r""""agent_severity":"1",""" + r""""extensions":{""" + r""""foo":"foo=bar",""" + r""""bar":"bar=baz",""" + r""""baz":"test"}""" + r"""}""" + "\n" + ) + assert file_true.read_log() == exp From 3d16c1c48ff85ed16fb227075aed6ea7136c37a7 Mon Sep 17 00:00:00 2001 From: shifter Date: Sat, 5 Oct 2024 11:02:33 +0200 Subject: [PATCH 9/9] filterx/modules/cef: add parse_leef light test Signed-off-by: shifter --- .../functional_tests/filterx/test_filterx.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index 69e5716a6a..046b584655 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -2366,3 +2366,32 @@ def test_parse_cef(config, syslog_ng): r"""}""" + "\n" ) assert file_true.read_log() == exp + + +def test_parse_leef(config, syslog_ng): + (file_true, file_false) = create_config( + config, r""" + custom_message = "LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345|src=192.0.2.0 dst=172.50.123.1 sev=5cat=anomaly srcPort=81 dstPort=21 usrName=joe.black"; + $MSG = json(parse_leef(custom_message)); + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + exp = ( + r"""{"version":"1.0",""" + r""""vendor":"Microsoft",""" + r""""product_name":"MSExchange",""" + r""""product_version":"4.0 SP1",""" + r""""event_id":"15345",""" + r""""extensions":{""" + r""""src":"192.0.2.0",""" + r""""dst":"172.50.123.1",""" + r""""sev":"5cat=anomaly",""" + r""""srcPort":"81",""" + r""""dstPort":"21",""" + r""""usrName":"joe.black"}""" + r"""}""" + "\n" + ) + assert file_true.read_log() == exp