From 3fac25758c30270c0709da762a0718b9468da121 Mon Sep 17 00:00:00 2001 From: shifter Date: Wed, 24 Jul 2024 23:32:43 +0200 Subject: [PATCH 1/7] filterx: extend expr-generator with filterx_expr_is_generator function Helper function to decide about a FilterXExpression whether is a generator expression or not.Added a new helper function to evaluate whether a given FilterXExpression is a generator expression. This enhancement helps in differentiating FilterXExpressions based on their type, which is crucial for processing and handling expressions correctly in the codebase. Signed-off-by: shifter --- lib/filterx/expr-generator.c | 6 ++++++ lib/filterx/expr-generator.h | 1 + 2 files changed, 7 insertions(+) diff --git a/lib/filterx/expr-generator.c b/lib/filterx/expr-generator.c index e8344a4a62..882c386113 100644 --- a/lib/filterx/expr-generator.c +++ b/lib/filterx/expr-generator.c @@ -48,6 +48,12 @@ _eval(FilterXExpr *s) return NULL; } +gboolean +filterx_expr_is_generator(FilterXExpr *s) +{ + return s->eval == _eval; +} + void filterx_generator_init_instance(FilterXExpr *s) { diff --git a/lib/filterx/expr-generator.h b/lib/filterx/expr-generator.h index 2a67a64de1..5e810f8782 100644 --- a/lib/filterx/expr-generator.h +++ b/lib/filterx/expr-generator.h @@ -38,6 +38,7 @@ struct FilterXExprGenerator_ void filterx_generator_set_fillable(FilterXExpr *s, FilterXExpr *fillable); void filterx_generator_init_instance(FilterXExpr *s); void filterx_generator_free_method(FilterXExpr *s); +gboolean filterx_expr_is_generator(FilterXExpr *s); FilterXExpr *filterx_generator_create_container_new(FilterXExpr *g, FilterXExpr *fillable_parent); From 4e3b5986000e6372af626c554d3b869e288f25e4 Mon Sep 17 00:00:00 2001 From: shifter Date: Wed, 24 Jul 2024 23:38:54 +0200 Subject: [PATCH 2/7] filterx: introduce expr-plus-generator operator Introduced expr-plus-generator as an alternative to the expr-plus operator, designed to work with generators. This new operator functions as a wrapper around the LHS and RHS generators, which can be either generators or variables. At least one of them needs to be a generator. Both generator and variable elements will be handled as a single generator in further processing. Signed-off-by: shifter --- lib/filterx/CMakeLists.txt | 2 + lib/filterx/Makefile.am | 4 +- lib/filterx/expr-plus-generator.c | 120 ++++++++++++++++++++++++++++++ lib/filterx/expr-plus-generator.h | 31 ++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 lib/filterx/expr-plus-generator.c create mode 100644 lib/filterx/expr-plus-generator.h diff --git a/lib/filterx/CMakeLists.txt b/lib/filterx/CMakeLists.txt index 5d70c15f7c..ab73435394 100644 --- a/lib/filterx/CMakeLists.txt +++ b/lib/filterx/CMakeLists.txt @@ -44,6 +44,7 @@ set(FILTERX_HEADERS filterx/func-flatten.h filterx/expr-plus.h filterx/expr-null-coalesce.h + filterx/expr-plus-generator.h PARENT_SCOPE ) @@ -94,6 +95,7 @@ set(FILTERX_SOURCES filterx/expr-plus.c filterx/filterx-private.c filterx/expr-null-coalesce.c + filterx/expr-plus-generator.c PARENT_SCOPE ) diff --git a/lib/filterx/Makefile.am b/lib/filterx/Makefile.am index 867a49d86f..266ad4b05b 100644 --- a/lib/filterx/Makefile.am +++ b/lib/filterx/Makefile.am @@ -45,7 +45,8 @@ filterxinclude_HEADERS = \ lib/filterx/func-str-transform.h \ lib/filterx/func-flatten.h \ lib/filterx/filterx-private.h \ - lib/filterx/expr-null-coalesce.h + lib/filterx/expr-null-coalesce.h \ + lib/filterx/expr-plus-generator.h filterx_sources = \ @@ -95,6 +96,7 @@ filterx_sources = \ lib/filterx/func-flatten.c \ lib/filterx/filterx-private.c \ lib/filterx/expr-null-coalesce.c \ + lib/filterx/expr-plus-generator.c \ lib/filterx/filterx-grammar.y BUILT_SOURCES += \ diff --git a/lib/filterx/expr-plus-generator.c b/lib/filterx/expr-plus-generator.c new file mode 100644 index 0000000000..79c305a864 --- /dev/null +++ b/lib/filterx/expr-plus-generator.c @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 shifter + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ +#include "expr-plus.h" +#include "object-string.h" +#include "filterx-eval.h" +#include "scratch-buffers.h" +#include "expr-generator.h" +#include "object-list-interface.h" +#include "object-dict-interface.h" + +typedef struct FilterXOperatorPlusGenerator +{ + FilterXExprGenerator super; + FilterXExpr *lhs; + FilterXExpr *rhs; +} FilterXOperatorPlusGenerator; + +static gboolean +_generate_obj(FilterXOperatorPlusGenerator *self, FilterXObject *obj, FilterXObject *fillable) +{ + if (filterx_object_is_type(fillable, &FILTERX_TYPE_NAME(list))) + return filterx_list_merge(fillable, obj); + + if (filterx_object_is_type(fillable, &FILTERX_TYPE_NAME(dict))) + return filterx_dict_merge(fillable, obj); + + filterx_eval_push_error("Failed to merge objects, invalid fillable type", &self->super.super, fillable); + + return FALSE; +} + +static gboolean +_handle_object_or_generator(FilterXOperatorPlusGenerator *self, FilterXExpr *expr, FilterXObject *fillable) +{ + FilterXObject *obj = NULL; + if (filterx_expr_is_generator(expr)) + { + filterx_generator_set_fillable(expr, filterx_expr_ref(self->super.fillable)); + obj = filterx_expr_eval(expr); + filterx_object_unref(obj); + return !!obj; + } + + obj = filterx_expr_eval(expr); + if (!obj) + return FALSE; + if (!_generate_obj(self, obj, fillable)) + { + filterx_object_unref(obj); + return FALSE; + } + + filterx_object_unref(obj); + return TRUE; +} + +static gboolean +_expr_plus_generator_generate(FilterXExprGenerator *s, FilterXObject *fillable) +{ + FilterXOperatorPlusGenerator *self = (FilterXOperatorPlusGenerator *) s; + + if (!_handle_object_or_generator(self, self->lhs, fillable)) + return FALSE; + if (!_handle_object_or_generator(self, self->rhs, fillable)) + return FALSE; + return TRUE; +} + +static FilterXObject * +_expr_plus_generator_create_container(FilterXExprGenerator *s, FilterXExpr *fillable_parent) +{ + FilterXOperatorPlusGenerator *self = (FilterXOperatorPlusGenerator *) s; + FilterXExprGenerator *generator = (FilterXExprGenerator *)(filterx_expr_is_generator( + self->rhs) ? self->rhs : self->lhs); + return generator->create_container(generator, fillable_parent); +} + +static void +_expr_plus_generator_free(FilterXExpr *s) +{ + FilterXOperatorPlusGenerator *self = (FilterXOperatorPlusGenerator *) s; + filterx_expr_unref(self->lhs); + filterx_expr_unref(self->rhs); + filterx_generator_free_method(s); +} + +FilterXExpr * +filterx_operator_plus_generator_new(FilterXExpr *lhs, FilterXExpr *rhs) +{ + FilterXOperatorPlusGenerator *self = g_new0(FilterXOperatorPlusGenerator, 1); + filterx_generator_init_instance(&self->super.super); + self->lhs = lhs; + self->rhs = rhs; + self->super.generate = _expr_plus_generator_generate; + self->super.super.free_fn = _expr_plus_generator_free; + self->super.create_container = _expr_plus_generator_create_container; + + return &self->super.super; +} diff --git a/lib/filterx/expr-plus-generator.h b/lib/filterx/expr-plus-generator.h new file mode 100644 index 0000000000..32157be2a7 --- /dev/null +++ b/lib/filterx/expr-plus-generator.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 shifter + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ +#ifndef FILTERX_EXPR_PLUS_GENERATOR_H_INCLUDED +#define FILTERX_EXPR_PLUS_GENERATOR_H_INCLUDED + +#include "filterx-expr.h" + +FilterXExpr *filterx_operator_plus_generator_new(FilterXExpr *lhs, FilterXExpr *rhs); + +#endif From a64e5f985dd9a5de28cae1e2123ab82cde2cd288 Mon Sep 17 00:00:00 2001 From: shifter Date: Wed, 24 Jul 2024 23:39:41 +0200 Subject: [PATCH 3/7] filterx: grammar changes for expr-plus-generator Signed-off-by: shifter --- lib/filterx/filterx-grammar.ym | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/filterx/filterx-grammar.ym b/lib/filterx/filterx-grammar.ym index 88c76eaf4f..3414292f2c 100644 --- a/lib/filterx/filterx-grammar.ym +++ b/lib/filterx/filterx-grammar.ym @@ -54,6 +54,7 @@ #include "filterx/expr-regexp.h" #include "filterx/expr-plus.h" #include "filterx/expr-null-coalesce.h" +#include "filterx/expr-plus-generator.h" #include "template/templates.h" @@ -296,7 +297,6 @@ declaration : KW_DECLARE filterx_variable KW_ASSIGN expr { filterx_variable_expr_declare($2); $$ = filterx_assign_new($2, $4); } ; - expr : expr_value { $$ = $1; } | function_call { $$ = $1; } @@ -346,6 +346,9 @@ expr_generator_unchecked | list_generator | regexp_search | '(' expr_generator ')' { $$ = $2; } + | expr '+' expr_generator { $$ = filterx_operator_plus_generator_new($1, $3); } + | expr_generator '+' expr { $$ = filterx_operator_plus_generator_new($1, $3); } + | expr_generator '+' expr_generator { $$ = filterx_operator_plus_generator_new($1, $3); } ; function_call From 4b8cd81ba5ae8509cd97d71e751ce70f89617904 Mon Sep 17 00:00:00 2001 From: shifter Date: Wed, 24 Jul 2024 23:40:29 +0200 Subject: [PATCH 4/7] filterx: expr-plus-generator unit tests Signed-off-by: shifter --- lib/filterx/tests/CMakeLists.txt | 1 + lib/filterx/tests/Makefile.am | 6 +- lib/filterx/tests/test_expr_plus_generator.c | 323 +++++++++++++++++++ 3 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 lib/filterx/tests/test_expr_plus_generator.c diff --git a/lib/filterx/tests/CMakeLists.txt b/lib/filterx/tests/CMakeLists.txt index 0965a9c9dd..da188737b4 100644 --- a/lib/filterx/tests/CMakeLists.txt +++ b/lib/filterx/tests/CMakeLists.txt @@ -22,3 +22,4 @@ add_unit_test(LIBTEST CRITERION TARGET test_expr_function DEPENDS json-plugin ${ add_unit_test(LIBTEST CRITERION TARGET test_expr_regexp DEPENDS json-plugin ${JSONC_LIBRARY}) add_unit_test(LIBTEST CRITERION TARGET test_expr_null_coalesce DEPENDS json-plugin ${JSONC_LIBRARY}) add_unit_test(LIBTEST CRITERION TARGET test_expr_plus DEPENDS json-plugin ${JSONC_LIBRARY}) +add_unit_test(LIBTEST CRITERION TARGET test_expr_plus_generator DEPENDS json-plugin ${JSONC_LIBRARY}) diff --git a/lib/filterx/tests/Makefile.am b/lib/filterx/tests/Makefile.am index 4f0eb9eb92..98b7074fcd 100644 --- a/lib/filterx/tests/Makefile.am +++ b/lib/filterx/tests/Makefile.am @@ -23,7 +23,8 @@ lib_filterx_tests_TESTS = \ lib/filterx/tests/test_func_flatten \ lib/filterx/tests/test_expr_regexp \ lib/filterx/tests/test_expr_null_coalesce \ - lib/filterx/tests/test_expr_plus + lib/filterx/tests/test_expr_plus \ + lib/filterx/tests/test_expr_plus_generator EXTRA_DIST += lib/filterx/tests/CMakeLists.txt @@ -100,3 +101,6 @@ lib_filterx_tests_test_expr_null_coalesce_LDADD = $(TEST_LDADD) $(JSON_LIBS) lib_filterx_tests_test_expr_plus_CFLAGS = $(TEST_CFLAGS) lib_filterx_tests_test_expr_plus_LDADD = $(TEST_LDADD) $(JSON_LIBS) + +lib_filterx_tests_test_expr_plus_generator_CFLAGS = $(TEST_CFLAGS) +lib_filterx_tests_test_expr_plus_generator_LDADD = $(TEST_LDADD) $(JSON_LIBS) diff --git a/lib/filterx/tests/test_expr_plus_generator.c b/lib/filterx/tests/test_expr_plus_generator.c new file mode 100644 index 0000000000..f5c560f8fb --- /dev/null +++ b/lib/filterx/tests/test_expr_plus_generator.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 shifter + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include +#include "libtest/cr_template.h" +#include "libtest/filterx-lib.h" + +#include "filterx/filterx-object.h" +#include "filterx/object-primitive.h" +#include "filterx/expr-comparison.h" +#include "filterx/filterx-expr.h" +#include "filterx/expr-literal.h" +#include "filterx/object-string.h" +#include "filterx/object-null.h" +#include "filterx/object-datetime.h" +#include "filterx/object-message-value.h" +#include "filterx/expr-null-coalesce.h" +#include "filterx/func-istype.h" +#include "filterx/filterx-eval.h" +#include "filterx/func-len.h" +#include "filterx/expr-function.h" +#include "filterx/expr-plus-generator.h" +#include "filterx/object-json.h" +#include "filterx/object-list-interface.h" +#include "filterx/object-dict-interface.h" +#include "filterx/expr-generator.h" +#include "filterx/expr-literal-generator.h" + +#include "apphook.h" +#include "scratch-buffers.h" + +void +_assert_cmp_lists(FilterXObject *expected, FilterXObject *provided) +{ + cr_assert(filterx_object_is_type(expected, &FILTERX_TYPE_NAME(list))); + cr_assert(filterx_object_is_type(provided, &FILTERX_TYPE_NAME(list))); + guint64 expected_len, provided_len; + cr_assert(filterx_object_len(expected, &expected_len)); + cr_assert(filterx_object_len(provided, &provided_len)); + cr_assert(expected_len == provided_len, "expected len:%lu provided len:%lu", expected_len, provided_len); + for (guint64 i = 0; i < expected_len; i ++) + { + FilterXObject *expected_value_obj = filterx_list_get_subscript(expected, i); + FilterXObject *provided_value_obj = filterx_list_get_subscript(provided, i); + cr_assert(expected_value_obj->type == provided_value_obj->type, "expected type:%s provided type:%s", + expected_value_obj->type->name, provided_value_obj->type->name); + + GString *expected_val = scratch_buffers_alloc(); + GString *provided_val = scratch_buffers_alloc(); + cr_assert(filterx_object_repr(expected_value_obj, expected_val), "repr call failure on expected val"); + cr_assert(filterx_object_repr(provided_value_obj, provided_val), "repr call failure on provided val"); + + cr_assert_str_eq(expected_val->str, provided_val->str, "values of nth(%lu) elt differs: expected:%s provided:%s", i, + expected_val->str, provided_val->str); + + filterx_object_unref(expected_value_obj); + filterx_object_unref(provided_value_obj); + } +} + +Test(expr_plus_generator, test_list_add_two_generators_with_post_set_fillable) +{ + FilterXExpr *lhs = filterx_literal_list_generator_new(); + GList *lhs_vals = NULL; + lhs_vals = g_list_append(lhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("foo", -1)), FALSE)); + lhs_vals = g_list_append(lhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("bar", -1)), FALSE)); + filterx_literal_generator_set_elements(lhs, lhs_vals); + + FilterXExpr *rhs = filterx_literal_list_generator_new(); + GList *rhs_vals = NULL; + rhs_vals = g_list_append(rhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("baz", -1)), FALSE)); + rhs_vals = g_list_append(rhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("other", -1)), FALSE)); + filterx_literal_generator_set_elements(rhs, rhs_vals); + + + FilterXExpr *expr = filterx_operator_plus_generator_new(lhs, rhs); + FilterXObject *json_array = filterx_json_array_new_empty(); + FilterXExpr *fillable = filterx_literal_new(json_array); + filterx_generator_set_fillable(expr, filterx_expr_ref(fillable)); + + FilterXObject *res_object = filterx_expr_eval(expr); + cr_assert_not_null(res_object); + cr_assert(filterx_object_is_type(res_object, &FILTERX_TYPE_NAME(list))); + + FilterXObject *expected = filterx_json_array_new_from_repr("[\"foo\",\"bar\",\"baz\",\"other\"]", -1); + + _assert_cmp_lists(expected, res_object); + + filterx_expr_unref(expr); + filterx_expr_unref(fillable); + filterx_object_unref(expected); + filterx_object_unref(json_array); +} + +Test(expr_plus_generator, test_list_add_variable_to_generator_with_post_set_fillable) +{ + FilterXExpr *lhs = filterx_literal_list_generator_new(); + GList *lhs_vals = NULL; + lhs_vals = g_list_append(lhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("foo", -1)), FALSE)); + lhs_vals = g_list_append(lhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("bar", -1)), FALSE)); + filterx_literal_generator_set_elements(lhs, lhs_vals); + + FilterXExpr *rhs = filterx_non_literal_new(filterx_json_array_new_from_repr("[\"baz\", \"other\"]", -1)); + + FilterXExpr *expr = filterx_operator_plus_generator_new(lhs, rhs); + FilterXObject *json_array = filterx_json_array_new_empty(); + FilterXExpr *fillable = filterx_literal_new(json_array); + filterx_generator_set_fillable(expr, filterx_expr_ref(fillable)); + + FilterXObject *res_object = filterx_expr_eval(expr); + cr_assert_not_null(res_object); + cr_assert(filterx_object_is_type(res_object, &FILTERX_TYPE_NAME(list))); + + FilterXObject *expected = filterx_json_array_new_from_repr("[\"foo\",\"bar\",\"baz\",\"other\"]", -1); + + _assert_cmp_lists(expected, res_object); + + filterx_expr_unref(expr); + filterx_expr_unref(fillable); + filterx_object_unref(expected); + filterx_object_unref(json_array); +} + +Test(expr_plus_generator, test_list_add_generator_to_variable_with_post_set_fillable) +{ + FilterXExpr *lhs = filterx_non_literal_new(filterx_json_array_new_from_repr("[\"foo\", \"bar\"]", -1)); + + FilterXExpr *rhs = filterx_literal_list_generator_new(); + GList *rhs_vals = NULL; + rhs_vals = g_list_append(rhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("baz", -1)), FALSE)); + rhs_vals = g_list_append(rhs_vals, + filterx_literal_generator_elem_new(NULL, filterx_literal_new(filterx_string_new("other", -1)), FALSE)); + filterx_literal_generator_set_elements(rhs, rhs_vals); + + FilterXExpr *expr = filterx_operator_plus_generator_new(lhs, rhs); + FilterXObject *json_array = filterx_json_array_new_empty(); + FilterXExpr *fillable = filterx_literal_new(json_array); + filterx_generator_set_fillable(expr, filterx_expr_ref(fillable)); + + FilterXObject *res_object = filterx_expr_eval(expr); + cr_assert_not_null(res_object); + cr_assert(filterx_object_is_type(res_object, &FILTERX_TYPE_NAME(list))); + + FilterXObject *expected = filterx_json_array_new_from_repr("[\"foo\",\"bar\",\"baz\",\"other\"]", -1); + + _assert_cmp_lists(expected, res_object); + + filterx_expr_unref(expr); + filterx_expr_unref(fillable); + filterx_object_unref(expected); + filterx_object_unref(json_array); +} + + +Test(expr_plus_generator, test_nested_dict_add_two_generators_with_post_set_fillable) +{ + FilterXExpr *lhs = filterx_literal_dict_generator_new(); + GList *lhs_vals = NULL; + GList *lhs_inner = NULL; + lhs_inner = g_list_append(lhs_inner, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("bar", -1)), + filterx_literal_new(filterx_string_new("baz", -1)), TRUE)); + lhs_vals = g_list_append(lhs_vals, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("foo", -1)), + filterx_literal_inner_dict_generator_new(lhs, lhs_inner), FALSE)); + filterx_literal_generator_set_elements(lhs, lhs_vals); + + FilterXExpr *rhs = filterx_literal_dict_generator_new(); + GList *rhs_vals = NULL; + GList *rhs_inner = NULL; + rhs_inner = g_list_append(rhs_inner, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("tak", -1)), + filterx_literal_new(filterx_string_new("toe", -1)), TRUE)); + rhs_vals = g_list_append(rhs_vals, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("tik", -1)), + filterx_literal_inner_dict_generator_new(rhs, rhs_inner), FALSE)); + filterx_literal_generator_set_elements(rhs, rhs_vals); + + FilterXExpr *expr = filterx_operator_plus_generator_new(lhs, rhs); + FilterXObject *json_dict = filterx_json_object_new_empty(); + FilterXExpr *fillable = filterx_literal_new(json_dict); + filterx_generator_set_fillable(expr, filterx_expr_ref(fillable)); + + FilterXObject *res_object = filterx_expr_eval(expr); + cr_assert_not_null(res_object); + cr_assert(filterx_object_is_type(res_object, &FILTERX_TYPE_NAME(dict))); + + struct json_object *jso = NULL; + FilterXObject *assoc_object = NULL; + cr_assert(filterx_object_map_to_json(res_object, &jso, &assoc_object)); + + const gchar *json_repr = json_object_to_json_string_ext(jso, JSON_C_TO_STRING_PLAIN); + cr_assert_str_eq(json_repr, "{\"foo\":{\"bar\":\"baz\"},\"tik\":{\"tak\":\"toe\"}}"); + json_object_put(jso); + filterx_object_unref(assoc_object); + + filterx_expr_unref(expr); + filterx_expr_unref(fillable); + filterx_object_unref(json_dict); +} + +Test(expr_plus_generator, test_nested_dict_add_variable_to_generator_with_post_set_fillable) +{ + FilterXExpr *lhs = filterx_literal_dict_generator_new(); + GList *lhs_vals = NULL; + GList *lhs_inner = NULL; + lhs_inner = g_list_append(lhs_inner, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("bar", -1)), + filterx_literal_new(filterx_string_new("baz", -1)), TRUE)); + lhs_vals = g_list_append(lhs_vals, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("foo", -1)), + filterx_literal_inner_dict_generator_new(lhs, lhs_inner), FALSE)); + filterx_literal_generator_set_elements(lhs, lhs_vals); + + FilterXExpr *rhs = filterx_non_literal_new(filterx_json_object_new_from_repr("{\"tik\":{\"tak\":\"toe\"}}", -1)); + + FilterXExpr *expr = filterx_operator_plus_generator_new(lhs, rhs); + FilterXObject *json_dict = filterx_json_object_new_empty(); + FilterXExpr *fillable = filterx_literal_new(json_dict); + filterx_generator_set_fillable(expr, filterx_expr_ref(fillable)); + + FilterXObject *res_object = filterx_expr_eval(expr); + cr_assert_not_null(res_object); + cr_assert(filterx_object_is_type(res_object, &FILTERX_TYPE_NAME(dict))); + + struct json_object *jso = NULL; + FilterXObject *assoc_object = NULL; + cr_assert(filterx_object_map_to_json(res_object, &jso, &assoc_object)); + + const gchar *json_repr = json_object_to_json_string_ext(jso, JSON_C_TO_STRING_PLAIN); + cr_assert_str_eq(json_repr, "{\"foo\":{\"bar\":\"baz\"},\"tik\":{\"tak\":\"toe\"}}"); + json_object_put(jso); + filterx_object_unref(assoc_object); + + filterx_expr_unref(expr); + filterx_expr_unref(fillable); + filterx_object_unref(json_dict); +} + +Test(expr_plus_generator, test_nested_dict_add_generator_to_variable_with_post_set_fillable) +{ + + FilterXExpr *lhs = filterx_non_literal_new(filterx_json_object_new_from_repr("{\"foo\":{\"bar\":\"baz\"}}", -1)); + + FilterXExpr *rhs = filterx_literal_dict_generator_new(); + GList *rhs_vals = NULL; + GList *rhs_inner = NULL; + rhs_inner = g_list_append(rhs_inner, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("tak", -1)), + filterx_literal_new(filterx_string_new("toe", -1)), TRUE)); + rhs_vals = g_list_append(rhs_vals, + filterx_literal_generator_elem_new(filterx_literal_new(filterx_string_new("tik", -1)), + filterx_literal_inner_dict_generator_new(rhs, rhs_inner), FALSE)); + filterx_literal_generator_set_elements(rhs, rhs_vals); + + FilterXExpr *expr = filterx_operator_plus_generator_new(lhs, rhs); + FilterXObject *json_dict = filterx_json_object_new_empty(); + FilterXExpr *fillable = filterx_literal_new(json_dict); + filterx_generator_set_fillable(expr, filterx_expr_ref(fillable)); + + FilterXObject *res_object = filterx_expr_eval(expr); + cr_assert_not_null(res_object); + cr_assert(filterx_object_is_type(res_object, &FILTERX_TYPE_NAME(dict))); + + struct json_object *jso = NULL; + FilterXObject *assoc_object = NULL; + cr_assert(filterx_object_map_to_json(res_object, &jso, &assoc_object)); + + const gchar *json_repr = json_object_to_json_string_ext(jso, JSON_C_TO_STRING_PLAIN); + cr_assert_str_eq(json_repr, "{\"foo\":{\"bar\":\"baz\"},\"tik\":{\"tak\":\"toe\"}}"); + json_object_put(jso); + filterx_object_unref(assoc_object); + + filterx_expr_unref(expr); + filterx_expr_unref(fillable); + filterx_object_unref(json_dict); +} + +static void +setup(void) +{ + app_startup(); + init_libtest_filterx(); +} + +static void +teardown(void) +{ + scratch_buffers_explicit_gc(); + deinit_libtest_filterx(); + app_shutdown(); +} + + +TestSuite(expr_plus_generator, .init = setup, .fini = teardown); From f40ab6c71f36bc1458ddbeb278cbcdabb4b295d5 Mon Sep 17 00:00:00 2001 From: shifter Date: Wed, 24 Jul 2024 23:40:50 +0200 Subject: [PATCH 5/7] filterx: expr-plus-generator light tests Signed-off-by: shifter --- .../functional_tests/filterx/test_filterx.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index 9e609bcd8d..27b90a0dfd 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -1830,3 +1830,34 @@ def test_flatten(config, syslog_ng): '{"top_level_field":42,"top_level_dict.inner_dict.inner_inner_field":1,"top_level_dict.inner_field":1337},' \ '{"top_level_field":42,"top_level_dict->inner_dict->inner_inner_field":1,"top_level_dict->inner_field":1337}' \ ']\n' + + +def test_add_operator_for_generators(config, syslog_ng): + (file_true, file_false) = create_config( + config, r""" + $MSG = json(); + js1 = json_array(["foo","bar"]); + js2 = json_array(["baz","other"]); + $MSG.list_var_gen = js1 + ["baz1","other1"]; + $MSG.list_gen_var = ["foo2", "bar2"] + js2; + $MSG.list_gen_gen = ["foo3", "bar3"] + ["baz3", "other3"]; + dict1 = json({"foo":{"bar":"baz"}}); + dict2 = json({"tik":{"tak":"toe"}}); + $MSG.dict_var_gen = dict1 + {"tik1":{"tak1":"toe1"}}; + $MSG.dict_gen_var = {"foo2":{"bar2":"baz2"}} + dict2; + $MSG.dict_gen_gen = {"foo3":{"bar3":"baz3"}} + {"tik3":{"tak3":"toe3"}}; + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + exp = ( + r"""{"list_var_gen":["foo","bar","baz1","other1"],""" + r""""list_gen_var":["foo2","bar2","baz","other"],""" + r""""list_gen_gen":["foo3","bar3","baz3","other3"],""" + r""""dict_var_gen":{"foo":{"bar":"baz"},"tik1":{"tak1":"toe1"}},""" + r""""dict_gen_var":{"foo2":{"bar2":"baz2"},"tik":{"tak":"toe"}},""" + r""""dict_gen_gen":{"foo3":{"bar3":"baz3"},"tik3":{"tak3":"toe3"}}}""" + "\n" + ) + assert file_true.read_log() == exp From 022c0e3abc9e0d1bebc29ce059db9f11f42de0cf Mon Sep 17 00:00:00 2001 From: shifter Date: Thu, 25 Jul 2024 13:46:23 +0200 Subject: [PATCH 6/7] filterx: changing grammar to override += operator. Further refactor of generator_plus_assignment needed to avoid grammar conflicts. Signed-off-by: shifter --- lib/filterx/filterx-grammar.ym | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/filterx/filterx-grammar.ym b/lib/filterx/filterx-grammar.ym index 3414292f2c..c6cea683db 100644 --- a/lib/filterx/filterx-grammar.ym +++ b/lib/filterx/filterx-grammar.ym @@ -112,6 +112,8 @@ construct_template_expr(LogTemplate *template) %type stmt %type stmt_expr %type assignment +%type plus_assignment +%type generator_plus_assignment %type generator_assignment %type generator_casted_assignment %type declaration @@ -181,6 +183,17 @@ stmt_expr | declaration { $$ = $1; } ; +plus_assignment + : variable KW_PLUS_ASSIGN expr { $$ = filterx_assign_new(filterx_expr_ref($1), filterx_operator_plus_new($1, $3)); } + | expr '[' expr ']' KW_PLUS_ASSIGN expr { $$ = filterx_set_subscript_new(filterx_expr_ref($1), filterx_expr_ref($3), filterx_operator_plus_new(filterx_get_subscript_new($1, $3), $6)); } + | expr '.' LL_IDENTIFIER KW_PLUS_ASSIGN expr + { + $$ = filterx_setattr_new(filterx_expr_ref($1), $3, filterx_operator_plus_new(filterx_getattr_new($1, $3), $5)); + free($3); + } + ; + + assignment /* TODO extract lvalues */ : variable KW_ASSIGN expr { $$ = filterx_assign_new($1, $3); } @@ -188,8 +201,11 @@ assignment | expr '[' expr ']' KW_ASSIGN expr { $$ = filterx_set_subscript_new($1, $3, $6); } | expr '[' ']' KW_ASSIGN expr { $$ = filterx_set_subscript_new($1, NULL, $5); } | generator_assignment + | plus_assignment { $$ = $1; } ; + + generator_assignment /* TODO extract lvalues */ : expr '.' LL_IDENTIFIER KW_ASSIGN expr_generator @@ -224,10 +240,15 @@ generator_assignment NULL ); } - | expr KW_PLUS_ASSIGN expr_generator { $$ = $3; filterx_generator_set_fillable($3, $1); } + | generator_plus_assignment | generator_casted_assignment ; +generator_plus_assignment + : variable KW_PLUS_ASSIGN expr_generator { $$ = $3; filterx_generator_set_fillable($3, $1); } + | expr '[' expr ']' KW_PLUS_ASSIGN expr_generator { $$ = $6; filterx_generator_set_fillable($6, filterx_get_subscript_new($1, $3)); } + | expr '.' LL_IDENTIFIER KW_PLUS_ASSIGN expr_generator { $$ = $5; filterx_generator_set_fillable($5, filterx_getattr_new($1, $3)); free($3);} + generator_casted_assignment /* TODO extract lvalues */ : variable KW_ASSIGN LL_IDENTIFIER '(' expr_generator ')' From a9c69e1ff5c1d549ef43a84b1cc72495d361dac8 Mon Sep 17 00:00:00 2001 From: shifter Date: Thu, 25 Jul 2024 16:10:11 +0200 Subject: [PATCH 7/7] filterx: grammar += operator light tests Signed-off-by: shifter --- .../functional_tests/filterx/test_filterx.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index 27b90a0dfd..f8e96cd9d4 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -1861,3 +1861,64 @@ def test_add_operator_for_generators(config, syslog_ng): r""""dict_gen_gen":{"foo3":{"bar3":"baz3"},"tik3":{"tak3":"toe3"}}}""" + "\n" ) assert file_true.read_log() == exp + + +def test_plus_equal_grammar_rules(config, syslog_ng): + (file_true, file_false) = create_config( + config, r""" + $MSG = json(); + js1 = json_array(["foo","bar"]); + js2 = json_array(["baz","other"]); + + a = 3; + a += 2; + $MSG.var_int = a; + + b = "foo"; + b += "bar"; + $MSG.var_string = b; + + c = 0.44; + c += 0.58; + $MSG.var_double = c; + + d = strptime("2000-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z"); + d += 3600000000; + $MSG.var_datetime_integer = string(d); + + e = strptime("2000-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z"); + e += 3600.000000; + $MSG.var_datetime_double = string(e); + + $MSG.attr = "tik"; + $MSG.attr += "tak"; + + $MSG["subs"] = "bar"; + $MSG["subs"] += "baz"; + + $MSG["sub_gen_var"] = ["control"]; + $MSG["sub_gen_var"] += js2; + $MSG["sub_gen_var"] += ["wtf", "happened"]; + + $MSG.attr_gen_var = ["some", "basic"]; + $MSG.attr_gen_var += js1; + $MSG.attr_gen_var += ["and"]; + $MSG.attr_gen_var += ["add", "to", "plus"]; + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + exp = ( + r"""{"var_int":5,""" + r""""var_string":"foobar",""" + r""""var_double":1.02,""" + r""""var_datetime_integer":"2000-01-01T01:00:00.000+00:00",""" + r""""var_datetime_double":"2000-01-01T01:00:00.000+00:00",""" + r""""attr":"tiktak",""" + r""""subs":"barbaz",""" + r""""sub_gen_var":["control","baz","other","wtf","happened"],""" + r""""attr_gen_var":["some","basic","foo","bar","and","add","to","plus"]}""" + "\n" + ) + assert file_true.read_log() == exp