diff --git a/lib/filterx/CMakeLists.txt b/lib/filterx/CMakeLists.txt index fe2890dd14..3db0f51b14 100644 --- a/lib/filterx/CMakeLists.txt +++ b/lib/filterx/CMakeLists.txt @@ -45,6 +45,7 @@ set(FILTERX_HEADERS filterx/func-unset-empties.h filterx/func-str-transform.h filterx/func-flatten.h + filterx/func-sdata.h filterx/expr-plus.h filterx/expr-null-coalesce.h filterx/expr-plus-generator.h @@ -98,6 +99,7 @@ set(FILTERX_SOURCES filterx/func-unset-empties.c filterx/func-str-transform.c filterx/func-flatten.c + filterx/func-sdata.c filterx/expr-plus.c filterx/filterx-private.c filterx/expr-null-coalesce.c diff --git a/lib/filterx/Makefile.am b/lib/filterx/Makefile.am index e4d1d47f97..7cd6b732e9 100644 --- a/lib/filterx/Makefile.am +++ b/lib/filterx/Makefile.am @@ -47,6 +47,7 @@ filterxinclude_HEADERS = \ lib/filterx/func-unset-empties.h \ lib/filterx/func-str-transform.h \ lib/filterx/func-flatten.h \ + lib/filterx/func-sdata.h \ lib/filterx/filterx-private.h \ lib/filterx/expr-null-coalesce.h \ lib/filterx/expr-plus-generator.h @@ -100,6 +101,7 @@ filterx_sources = \ lib/filterx/func-unset-empties.c \ lib/filterx/func-str-transform.c \ lib/filterx/func-flatten.c \ + lib/filterx/func-sdata.c \ lib/filterx/filterx-private.c \ lib/filterx/expr-null-coalesce.c \ lib/filterx/expr-plus-generator.c \ diff --git a/lib/filterx/filterx-globals.c b/lib/filterx/filterx-globals.c index 15840114a7..133198ea38 100644 --- a/lib/filterx/filterx-globals.c +++ b/lib/filterx/filterx-globals.c @@ -36,6 +36,7 @@ #include "filterx/func-unset-empties.h" #include "filterx/func-str-transform.h" #include "filterx/func-flatten.h" +#include "filterx/func-sdata.h" #include "filterx/expr-regexp.h" #include "filterx/expr-unset.h" #include "filterx/filterx-eval.h" @@ -99,6 +100,8 @@ _simple_init(void) g_assert(filterx_builtin_simple_function_register("vars", filterx_simple_function_vars)); g_assert(filterx_builtin_simple_function_register("lower", filterx_simple_function_lower)); g_assert(filterx_builtin_simple_function_register("upper", filterx_simple_function_upper)); + g_assert(filterx_builtin_simple_function_register("has_sdata", + filterx_simple_function_has_sdata)); } static void @@ -129,6 +132,8 @@ _ctors_init(void) g_assert(filterx_builtin_function_ctor_register("regexp_subst", filterx_function_regexp_subst_new)); g_assert(filterx_builtin_function_ctor_register("unset", filterx_function_unset_new)); g_assert(filterx_builtin_function_ctor_register("flatten", filterx_function_flatten_new)); + g_assert(filterx_builtin_function_ctor_register("is_sdata_from_enterprise", + filterx_function_is_sdata_from_enterprise_new)); } static void @@ -161,6 +166,8 @@ _generator_ctors_init(void) filterx_builtin_function_ctors_init_private(&filterx_builtin_generator_function_ctors); g_assert(filterx_builtin_generator_function_ctor_register("regexp_search", filterx_generator_function_regexp_search_new)); + g_assert(filterx_builtin_generator_function_ctor_register("get_sdata", + filterx_generator_function_get_sdata_new)); } static void diff --git a/lib/filterx/func-sdata.c b/lib/filterx/func-sdata.c new file mode 100644 index 0000000000..1aac8ddf89 --- /dev/null +++ b/lib/filterx/func-sdata.c @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 Szilard Parrag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "func-sdata.h" +#include "filterx/object-primitive.h" +#include "filterx/filterx-eval.h" +#include "filterx/object-string.h" +#include "filterx/object-extractor.h" +#include "filterx/object-json.h" + +#include + +#define SDATA_PREFIX_LEN (7) + +#define FILTERX_FUNC_IS_SDATA_FROM_ENTERPRISE_USAGE "Usage: is_sdata_from_enteprise(\"32473\")" + +typedef struct FilterXFunctionIsSdataFromEnteprise_ +{ + FilterXFunction super; + gchar *str_literal; + gsize str_literal_len; +} FilterXFunctionIsSdataFromEnteprise; + +static gboolean +_extract_args(FilterXFunctionIsSdataFromEnteprise *self, FilterXFunctionArgs *args, GError **error) +{ + gsize len = filterx_function_args_len(args); + if (len != 1) + { + g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL, + "invalid number of arguments. " FILTERX_FUNC_IS_SDATA_FROM_ENTERPRISE_USAGE); + return FALSE; + } + + gsize str_literal_len; + const gchar *str_literal = filterx_function_args_get_literal_string(args, 0, &str_literal_len); + if (!str_literal) + { + g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL, + "argument must be a string literal. " FILTERX_FUNC_IS_SDATA_FROM_ENTERPRISE_USAGE); + return FALSE; + } + self->str_literal = g_strdup(str_literal); + self->str_literal_len = str_literal_len; + return TRUE; +} + +static FilterXObject * +_eval(FilterXExpr *s) +{ + FilterXFunctionIsSdataFromEnteprise *self = (FilterXFunctionIsSdataFromEnteprise *) s; + + gboolean contains = FALSE; + FilterXEvalContext *context = filterx_eval_get_context(); + LogMessage *msg = context->msgs[0]; + + for (guint8 i = 0; i < msg->num_sdata && !contains; i++) + { + const gchar *value = log_msg_get_value_name(msg->sdata[i], NULL); + gchar *at_sign = strchr(value, '@'); + if (!at_sign) + continue; + contains = strncmp(at_sign+1, self->str_literal, self->str_literal_len) == 0; + } + + return filterx_boolean_new(contains); +} + +static void +_free(FilterXExpr *s) +{ + FilterXFunctionIsSdataFromEnteprise *self = (FilterXFunctionIsSdataFromEnteprise *) s; + + g_free(self->str_literal); + filterx_function_free_method(&self->super); +} + + +FilterXExpr * +filterx_function_is_sdata_from_enterprise_new(const gchar *function_name, FilterXFunctionArgs *args, GError **error) +{ + FilterXFunctionIsSdataFromEnteprise *self = g_new0(FilterXFunctionIsSdataFromEnteprise, 1); + filterx_function_init_instance(&self->super, function_name); + + if (!_extract_args(self, args, error) || !filterx_function_args_check(args, error)) + goto error; + self->super.super.eval = _eval; + self->super.super.free_fn = _free; + filterx_function_args_free(args); + return &self->super.super; + +error: + filterx_function_args_free(args); + filterx_expr_unref(&self->super.super); + return NULL; +} + + +FilterXObject * +filterx_simple_function_has_sdata(FilterXExpr *s, GPtrArray *args) +{ + if (args && args->len != 0) + { + filterx_simple_function_argument_error(s, "Incorrect number of arguments", FALSE); + return NULL; + } + + FilterXEvalContext *context = filterx_eval_get_context(); + LogMessage *msg = context->msgs[0]; + return filterx_boolean_new(msg->num_sdata != 0); +} + +static gboolean +_should_create_new_dict(const gchar *previous_sd_id, gsize previous_sd_id_len, const gchar *current_sd_id, + gsize current_sd_id_len) +{ + if (previous_sd_id_len != current_sd_id_len) + return TRUE; + + return strncmp(previous_sd_id, current_sd_id, previous_sd_id_len) != 0; +} + + +static gboolean +_extract_sd_components(const gchar *key, gssize key_len, const gchar **sd_id_start, gsize *sd_id_len, + const gchar **sd_param_name_start, gsize *sd_param_name_len) +{ + /* skip '.SDATA.' */ + if (key_len < SDATA_PREFIX_LEN) + return FALSE; + + const gchar *last_dot = NULL; + for(gsize i = key_len; i > 0 && !last_dot; i--) + { + if (key[i-1] == '.') + last_dot = key + i - 1; + } + if (!last_dot) + return FALSE; + + *sd_id_start = key + SDATA_PREFIX_LEN; + *sd_id_len = last_dot - *sd_id_start; + + *sd_param_name_start = last_dot + 1; + *sd_param_name_len = key + key_len - *sd_param_name_start; + + return TRUE; +} + + +static gboolean +_insert_into_dict(FilterXObject *dict, const gchar *key, gssize key_len, const gchar *value, gssize value_len) +{ + FilterXObject *fob_key = filterx_string_new(key, key_len); + FilterXObject *fob_value = filterx_string_new(value, value_len); + gboolean res = filterx_object_set_subscript(dict, fob_key, &fob_value); + + filterx_object_unref(fob_key); + filterx_object_unref(fob_value); + + return res; +} + +static gboolean +_insert_while_same_sd_id(LogMessage *msg, guint8 index, guint8 num_sdata, guint8 *num_insertions, + FilterXObject *inner_dict, + const gchar *current_sd_id_start, gsize current_sd_id_len) +{ + const gchar *next_sd_id_start; + gsize next_sd_id_len; + const gchar *param_name_start; + gsize param_name_len; + + *num_insertions = 0; + + for (guint8 j = index + 1; j < num_sdata; j++) + { + gssize next_name_len; + const gchar *next_name = log_msg_get_value_name(msg->sdata[j], &next_name_len); + gssize next_value_len; + const gchar *next_value = log_msg_get_value(msg, msg->sdata[j], &next_value_len); + + if (!_extract_sd_components(next_name, next_name_len, &next_sd_id_start, &next_sd_id_len, ¶m_name_start, + ¶m_name_len)) + { + filterx_object_unref(inner_dict); + return FALSE; + } + + if (_should_create_new_dict(current_sd_id_start, current_sd_id_len, next_sd_id_start, next_sd_id_len)) + break; + + if (!_insert_into_dict(inner_dict, param_name_start, param_name_len, next_value, next_value_len)) + return FALSE; + + (*num_insertions)++; + } + + return TRUE; +} + +static gboolean +_generate(FilterXExprGenerator *s, FilterXObject *fillable) +{ + FilterXEvalContext *context = filterx_eval_get_context(); + LogMessage *msg = context->msgs[0]; + + const gchar *current_sd_id_start; + gsize current_sd_id_len; + const gchar *param_name_start; + gsize param_name_len; + + for (guint8 i = 0; i < msg->num_sdata;) + { + gssize name_len; + const gchar *name = log_msg_get_value_name(msg->sdata[i], &name_len); + gssize value_len; + const gchar *value = log_msg_get_value(msg, msg->sdata[i], &value_len); + + if (!_extract_sd_components(name, name_len, ¤t_sd_id_start, ¤t_sd_id_len, ¶m_name_start, + ¶m_name_len)) + { + return FALSE; + } + + FilterXObject *inner_dict = filterx_object_create_dict(fillable); + if (!_insert_into_dict(inner_dict, param_name_start, param_name_len, value, value_len)) + return FALSE; + + guint8 num_insertions; + if(!_insert_while_same_sd_id(msg, i, msg->num_sdata, &num_insertions, inner_dict, current_sd_id_start, + current_sd_id_len)) + return FALSE; + i += num_insertions + 1; + + FilterXObject *sd_id_key = filterx_string_new(current_sd_id_start, current_sd_id_len); + filterx_object_set_subscript(fillable, sd_id_key, &inner_dict); + filterx_object_unref(inner_dict); + filterx_object_unref(sd_id_key); + } + + return TRUE; +} + + +static FilterXObject * +_create_container(FilterXExprGenerator *s, FilterXExpr *fillable_parent) +{ + FilterXObject *fillable_parent_obj = filterx_expr_eval_typed(fillable_parent); + if (!fillable_parent_obj) + return NULL; + + FilterXObject *result = filterx_object_create_dict(fillable_parent_obj); + filterx_object_unref(fillable_parent_obj); + + return result; +} + +static void +_get_sdata_free(FilterXExpr *s) +{ + FilterXGenFuncGetSdata *self = (FilterXGenFuncGetSdata *) s; + filterx_generator_function_free_method(&self->super); +} + +FilterXExpr * +filterx_generator_function_get_sdata_new(const gchar *function_name, FilterXFunctionArgs *args, GError **error) +{ + FilterXGenFuncGetSdata *self = g_new0(FilterXGenFuncGetSdata, 1); + + filterx_generator_function_init_instance(&self->super, function_name); + self->super.super.generate = _generate; + self->super.super.super.free_fn = _get_sdata_free; + self->super.super.create_container = _create_container; + + if (filterx_function_args_len(args) != 0) + { + g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL, "invalid number of arguments."); + goto error; + } + + filterx_function_args_free(args); + return &self->super.super.super; + +error: + filterx_function_args_free(args); + filterx_expr_unref(&self->super.super.super); + return NULL; +} diff --git a/lib/filterx/func-sdata.h b/lib/filterx/func-sdata.h new file mode 100644 index 0000000000..24a63f915a --- /dev/null +++ b/lib/filterx/func-sdata.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 Szilard Parrag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef FILTERX_FUNC_SDATA_H_INCLUDED +#define FILTERX_FUNC_SDATA_H_INCLUDED + +#include "filterx/expr-function.h" + +FilterXExpr *filterx_function_is_sdata_from_enterprise_new(const gchar *function_name, FilterXFunctionArgs *args, + GError **error); +FilterXObject *filterx_simple_function_has_sdata(FilterXExpr *s, GPtrArray *args); +FilterXExpr *filterx_generator_function_get_sdata_new(const gchar *function_name, FilterXFunctionArgs *args, + GError **error); +typedef struct FilterXGenFuncGetSdata_ +{ + FilterXGeneratorFunction super; +} FilterXGenFuncGetSdata; + +#endif diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index 36dd47d554..c022b77831 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -1948,3 +1948,63 @@ def test_plus_equal_grammar_rules(config, syslog_ng): r""""attr_gen_var":["some","basic","foo","bar","and","add","to","plus"]}""" + "\n" ) assert file_true.read_log() == exp + + +def test_get_sdata(config, syslog_ng): + file_true = config.create_file_destination(file_name="dest-true.log", template="'$MSG\n'") + file_false = config.create_file_destination(file_name="dest-false.log", template="'$MSG\n'") + + raw_conf = f""" +@version: {config.get_version()} + +options {{ stats(level(1)); }}; + +source genmsg {{ + example-msg-generator( + num(1) + template("[Originator@6876 sub=Vimsvc.ha-eventmgr opID=esxui-13c6-6b16 sid=5214bde6 user=root][anotherSDID@32473 iut=4 eventSource=Application eventID=1012]") + ); +}}; + +destination dest_true {{ + {render_statement(file_true)}; +}}; + +destination dest_false {{ + {render_statement(file_false)}; +}}; + +parser my_sdata_parser {{ + sdata-parser(); +}}; + +log {{ + source(genmsg); + if {{ + parser(my_sdata_parser); + filterx {{ {"$MSG = {}; $MSG.got_sdata = get_sdata();"} \n}}; + destination(dest_true); + }} else {{ + destination(dest_false); + }}; +}}; +""" + config.set_raw_config(raw_conf) + + syslog_ng.start(config) + + assert json.loads(file_true.read_log()) == { + "got_sdata": { + "anotherSDID@32473": { + "iut": "4", + "eventSource": "Application", + "eventID": "1012", + }, + "Originator@6876": { + "sub": "Vimsvc.ha-eventmgr", + "opID": "esxui-13c6-6b16", + "sid": "5214bde6", + "user": "root", + }, + }, + }