From 4087b3fa51af0178a8833d69d614a98ed94733eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 27 Sep 2024 14:20:04 +0200 Subject: [PATCH 01/16] filterx: reorder (C)Make source file list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/CMakeLists.txt | 128 +++++++++++----------- lib/filterx/Makefile.am | 218 ++++++++++++++++++------------------- 2 files changed, 173 insertions(+), 173 deletions(-) diff --git a/lib/filterx/CMakeLists.txt b/lib/filterx/CMakeLists.txt index 32ec9c249..c36fde5ae 100644 --- a/lib/filterx/CMakeLists.txt +++ b/lib/filterx/CMakeLists.txt @@ -2,58 +2,58 @@ set(FILTERX_HEADERS filterx/expr-assign.h filterx/expr-boolalg.h filterx/expr-comparison.h + filterx/expr-compound.h + filterx/expr-condition.h + filterx/expr-done.h + filterx/expr-drop.h filterx/expr-function.h - filterx/expr-getattr.h + filterx/expr-generator.h filterx/expr-get-subscript.h + filterx/expr-getattr.h + filterx/expr-isset.h + filterx/expr-literal-generator.h filterx/expr-literal.h - filterx/expr-variable.h - filterx/expr-setattr.h + filterx/expr-null-coalesce.h + filterx/expr-plus-generator.h + filterx/expr-plus.h + filterx/expr-regexp.h filterx/expr-set-subscript.h + filterx/expr-setattr.h filterx/expr-template.h - filterx/expr-drop.h - filterx/expr-done.h + filterx/expr-unset.h + filterx/expr-variable.h filterx/filterx-config.h - filterx/filterx-eval.h filterx/filterx-error.h + filterx/filterx-eval.h filterx/filterx-expr.h filterx/filterx-globals.h + filterx/filterx-metrics-labels.h + filterx/filterx-metrics.h filterx/filterx-object.h filterx/filterx-parser.h filterx/filterx-pipe.h + filterx/filterx-private.h filterx/filterx-scope.h filterx/filterx-weakrefs.h - filterx/filterx-metrics.h - filterx/filterx-metrics-labels.h - filterx/object-extractor.h + filterx/func-flags.h + filterx/func-flatten.h + filterx/func-istype.h + filterx/func-len.h + filterx/func-sdata.h + filterx/func-str-transform.h + filterx/func-str.h + filterx/func-unset-empties.h + filterx/func-vars.h filterx/object-datetime.h - filterx/object-json.h + filterx/object-dict-interface.h + filterx/object-extractor.h filterx/object-json-internal.h + filterx/object-json.h + filterx/object-list-interface.h filterx/object-message-value.h filterx/object-null.h filterx/object-primitive.h filterx/object-string.h - filterx/object-list-interface.h - filterx/object-dict-interface.h - filterx/expr-condition.h - filterx/expr-isset.h - filterx/expr-unset.h - filterx/expr-compound.h - filterx/expr-generator.h - filterx/expr-literal-generator.h - filterx/expr-regexp.h - filterx/filterx-private.h - filterx/func-istype.h - filterx/func-len.h - filterx/func-vars.h - filterx/func-unset-empties.h - filterx/func-str.h - filterx/func-str-transform.h - filterx/func-flatten.h - filterx/func-sdata.h - filterx/expr-plus.h - filterx/expr-null-coalesce.h - filterx/expr-plus-generator.h - filterx/func-flags.h PARENT_SCOPE ) @@ -61,58 +61,58 @@ set(FILTERX_SOURCES filterx/expr-assign.c filterx/expr-boolalg.c filterx/expr-comparison.c + filterx/expr-compound.c + filterx/expr-condition.c + filterx/expr-done.c + filterx/expr-drop.c filterx/expr-function.c - filterx/expr-getattr.c + filterx/expr-generator.c filterx/expr-get-subscript.c + filterx/expr-getattr.c + filterx/expr-isset.c + filterx/expr-literal-generator.c filterx/expr-literal.c - filterx/expr-variable.c - filterx/expr-setattr.c + filterx/expr-null-coalesce.c + filterx/expr-plus-generator.c + filterx/expr-plus.c + filterx/expr-regexp.c filterx/expr-set-subscript.c + filterx/expr-setattr.c filterx/expr-template.c - filterx/expr-drop.c - filterx/expr-done.c + filterx/expr-unset.c + filterx/expr-variable.c filterx/filterx-config.c - filterx/filterx-eval.c filterx/filterx-error.c + filterx/filterx-eval.c filterx/filterx-expr.c filterx/filterx-globals.c + filterx/filterx-metrics-labels.c + filterx/filterx-metrics.c filterx/filterx-object.c filterx/filterx-parser.c filterx/filterx-pipe.c + filterx/filterx-private.c filterx/filterx-scope.c filterx/filterx-weakrefs.c - filterx/filterx-metrics.c - filterx/filterx-metrics-labels.c - filterx/object-extractor.c + filterx/func-flatten.c + filterx/func-istype.c + filterx/func-len.c + filterx/func-sdata.c + filterx/func-str-transform.c + filterx/func-str.c + filterx/func-unset-empties.c + filterx/func-vars.c filterx/object-datetime.c - filterx/object-json.c - filterx/object-json-object.c + filterx/object-dict-interface.c + filterx/object-extractor.c filterx/object-json-array.c + filterx/object-json-object.c + filterx/object-json.c + filterx/object-list-interface.c filterx/object-message-value.c filterx/object-null.c filterx/object-primitive.c filterx/object-string.c - filterx/object-list-interface.c - filterx/object-dict-interface.c - filterx/expr-condition.c - filterx/expr-isset.c - filterx/expr-unset.c - filterx/expr-compound.c - filterx/expr-generator.c - filterx/expr-literal-generator.c - filterx/expr-regexp.c - filterx/func-istype.c - filterx/func-len.c - filterx/func-vars.c - filterx/func-unset-empties.c - filterx/func-str.c - filterx/func-str-transform.c - filterx/func-flatten.c - filterx/func-sdata.c - filterx/expr-plus.c - filterx/filterx-private.c - filterx/expr-null-coalesce.c - filterx/expr-plus-generator.c PARENT_SCOPE ) diff --git a/lib/filterx/Makefile.am b/lib/filterx/Makefile.am index b5f92e210..d9b30a3a7 100644 --- a/lib/filterx/Makefile.am +++ b/lib/filterx/Makefile.am @@ -1,120 +1,120 @@ filterxincludedir = ${pkgincludedir}/filterx filterxinclude_HEADERS = \ - lib/filterx/filterx-parser.h \ - lib/filterx/filterx-globals.h \ - lib/filterx/filterx-expr.h \ - lib/filterx/expr-literal.h \ - lib/filterx/expr-template.h \ - lib/filterx/expr-boolalg.h \ - lib/filterx/expr-assign.h \ - lib/filterx/expr-getattr.h \ - lib/filterx/expr-setattr.h \ - lib/filterx/expr-get-subscript.h \ - lib/filterx/expr-set-subscript.h \ - lib/filterx/expr-plus.h \ - lib/filterx/expr-variable.h \ - lib/filterx/expr-comparison.h \ - lib/filterx/expr-drop.h \ - lib/filterx/expr-done.h \ - lib/filterx/filterx-object.h \ - lib/filterx/filterx-weakrefs.h \ - lib/filterx/object-primitive.h \ - lib/filterx/filterx-scope.h \ - lib/filterx/filterx-eval.h \ - lib/filterx/filterx-error.h \ - lib/filterx/object-extractor.h \ - lib/filterx/object-json.h \ - lib/filterx/object-json-internal.h \ - lib/filterx/object-string.h \ - lib/filterx/object-datetime.h \ - lib/filterx/object-null.h \ - lib/filterx/object-message-value.h \ - lib/filterx/object-list-interface.h \ - lib/filterx/object-dict-interface.h \ - lib/filterx/filterx-config.h \ - lib/filterx/filterx-pipe.h \ - lib/filterx/filterx-metrics.h \ - lib/filterx/filterx-metrics-labels.h \ - lib/filterx/expr-function.h \ + lib/filterx/expr-assign.h \ + lib/filterx/expr-boolalg.h \ + lib/filterx/expr-comparison.h \ + lib/filterx/expr-compound.h \ lib/filterx/expr-condition.h \ + lib/filterx/expr-done.h \ + lib/filterx/expr-drop.h \ + lib/filterx/expr-function.h \ + lib/filterx/expr-generator.h \ + lib/filterx/expr-get-subscript.h \ + lib/filterx/expr-getattr.h \ lib/filterx/expr-isset.h \ - lib/filterx/expr-unset.h \ - lib/filterx/expr-compound.h \ - lib/filterx/expr-generator.h \ - lib/filterx/expr-literal-generator.h \ - lib/filterx/expr-regexp.h \ - lib/filterx/func-istype.h \ - lib/filterx/func-len.h \ - lib/filterx/func-vars.h \ - lib/filterx/func-unset-empties.h \ - lib/filterx/func-str-transform.h \ - lib/filterx/func-str.h \ - lib/filterx/func-flatten.h \ - lib/filterx/func-sdata.h \ - lib/filterx/filterx-private.h \ - lib/filterx/expr-null-coalesce.h \ - lib/filterx/expr-plus-generator.h \ - lib/filterx/func-flags.h + lib/filterx/expr-literal-generator.h \ + lib/filterx/expr-literal.h \ + lib/filterx/expr-null-coalesce.h \ + lib/filterx/expr-plus-generator.h \ + lib/filterx/expr-plus.h \ + lib/filterx/expr-regexp.h \ + lib/filterx/expr-set-subscript.h \ + lib/filterx/expr-setattr.h \ + lib/filterx/expr-template.h \ + lib/filterx/expr-unset.h \ + lib/filterx/expr-variable.h \ + lib/filterx/filterx-config.h \ + lib/filterx/filterx-error.h \ + lib/filterx/filterx-eval.h \ + lib/filterx/filterx-expr.h \ + lib/filterx/filterx-globals.h \ + lib/filterx/filterx-metrics-labels.h \ + lib/filterx/filterx-metrics.h \ + lib/filterx/filterx-object.h \ + lib/filterx/filterx-parser.h \ + lib/filterx/filterx-pipe.h \ + lib/filterx/filterx-private.h \ + lib/filterx/filterx-scope.h \ + lib/filterx/filterx-weakrefs.h \ + lib/filterx/func-flags.h \ + lib/filterx/func-flatten.h \ + lib/filterx/func-istype.h \ + lib/filterx/func-len.h \ + lib/filterx/func-sdata.h \ + lib/filterx/func-str-transform.h \ + lib/filterx/func-str.h \ + lib/filterx/func-unset-empties.h \ + lib/filterx/func-vars.h \ + lib/filterx/object-datetime.h \ + lib/filterx/object-dict-interface.h \ + lib/filterx/object-extractor.h \ + lib/filterx/object-json-internal.h \ + lib/filterx/object-json.h \ + lib/filterx/object-list-interface.h \ + lib/filterx/object-message-value.h \ + lib/filterx/object-null.h \ + lib/filterx/object-primitive.h \ + lib/filterx/object-string.h filterx_sources = \ - lib/filterx/filterx-parser.c \ - lib/filterx/filterx-globals.c \ - lib/filterx/filterx-expr.c \ - lib/filterx/expr-literal.c \ - lib/filterx/expr-template.c \ - lib/filterx/expr-boolalg.c \ - lib/filterx/expr-assign.c \ - lib/filterx/expr-getattr.c \ - lib/filterx/expr-setattr.c \ - lib/filterx/expr-get-subscript.c \ - lib/filterx/expr-set-subscript.c \ - lib/filterx/expr-plus.c \ - lib/filterx/expr-variable.c \ - lib/filterx/expr-comparison.c \ - lib/filterx/expr-drop.c \ - lib/filterx/expr-done.c \ - lib/filterx/filterx-object.c \ - lib/filterx/filterx-weakrefs.c \ - lib/filterx/object-primitive.c \ - lib/filterx/filterx-scope.c \ - lib/filterx/filterx-eval.c \ - lib/filterx/filterx-error.c \ - lib/filterx/object-extractor.c \ - lib/filterx/object-json.c \ - lib/filterx/object-json-object.c \ - lib/filterx/object-json-array.c \ - lib/filterx/object-string.c \ - lib/filterx/object-datetime.c \ - lib/filterx/object-null.c \ - lib/filterx/object-message-value.c \ - lib/filterx/object-list-interface.c \ - lib/filterx/object-dict-interface.c \ - lib/filterx/filterx-config.c \ - lib/filterx/filterx-pipe.c \ - lib/filterx/filterx-metrics.c \ - lib/filterx/filterx-metrics-labels.c \ - lib/filterx/expr-function.c \ - lib/filterx/expr-condition.c \ - lib/filterx/expr-isset.c \ - lib/filterx/expr-unset.c \ - lib/filterx/expr-compound.c \ - lib/filterx/expr-generator.c \ - lib/filterx/expr-literal-generator.c \ - lib/filterx/expr-regexp.c \ - lib/filterx/func-istype.c \ - lib/filterx/func-len.c \ - lib/filterx/func-vars.c \ - lib/filterx/func-unset-empties.c \ - lib/filterx/func-str-transform.c \ - lib/filterx/func-str.c \ - lib/filterx/func-flatten.c \ - lib/filterx/func-sdata.c \ - lib/filterx/filterx-private.c \ - lib/filterx/expr-null-coalesce.c \ - lib/filterx/expr-plus-generator.c \ - lib/filterx/filterx-grammar.y + lib/filterx/expr-assign.c \ + lib/filterx/expr-boolalg.c \ + lib/filterx/expr-comparison.c \ + lib/filterx/expr-compound.c \ + lib/filterx/expr-condition.c \ + lib/filterx/expr-done.c \ + lib/filterx/expr-drop.c \ + lib/filterx/expr-function.c \ + lib/filterx/expr-generator.c \ + lib/filterx/expr-get-subscript.c \ + lib/filterx/expr-getattr.c \ + lib/filterx/expr-isset.c \ + lib/filterx/expr-literal-generator.c \ + lib/filterx/expr-literal.c \ + lib/filterx/expr-null-coalesce.c \ + lib/filterx/expr-plus-generator.c \ + lib/filterx/expr-plus.c \ + lib/filterx/expr-regexp.c \ + lib/filterx/expr-set-subscript.c \ + lib/filterx/expr-setattr.c \ + lib/filterx/expr-template.c \ + lib/filterx/expr-unset.c \ + lib/filterx/expr-variable.c \ + lib/filterx/filterx-config.c \ + lib/filterx/filterx-error.c \ + lib/filterx/filterx-eval.c \ + lib/filterx/filterx-expr.c \ + lib/filterx/filterx-globals.c \ + lib/filterx/filterx-grammar.y \ + lib/filterx/filterx-metrics-labels.c \ + lib/filterx/filterx-metrics.c \ + lib/filterx/filterx-object.c \ + lib/filterx/filterx-parser.c \ + lib/filterx/filterx-pipe.c \ + lib/filterx/filterx-private.c \ + lib/filterx/filterx-scope.c \ + lib/filterx/filterx-weakrefs.c \ + lib/filterx/func-flatten.c \ + lib/filterx/func-istype.c \ + lib/filterx/func-len.c \ + lib/filterx/func-sdata.c \ + lib/filterx/func-str-transform.c \ + lib/filterx/func-str.c \ + lib/filterx/func-unset-empties.c \ + lib/filterx/func-vars.c \ + lib/filterx/object-datetime.c \ + lib/filterx/object-dict-interface.c \ + lib/filterx/object-extractor.c \ + lib/filterx/object-json-array.c \ + lib/filterx/object-json-object.c \ + lib/filterx/object-json.c \ + lib/filterx/object-list-interface.c \ + lib/filterx/object-message-value.c \ + lib/filterx/object-null.c \ + lib/filterx/object-primitive.c \ + lib/filterx/object-string.c BUILT_SOURCES += \ lib/filterx/filterx-grammar.y \ From 3fb8877bcaf99444b69d092a8cff7fbf81472145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 27 Sep 2024 14:20:58 +0200 Subject: [PATCH 02/16] filterx: extract FilterXVariable to a separate unit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/CMakeLists.txt | 2 + lib/filterx/Makefile.am | 2 + lib/filterx/expr-variable.c | 3 +- lib/filterx/filterx-scope.c | 134 ++++++--------------------------- lib/filterx/filterx-scope.h | 19 +---- lib/filterx/filterx-variable.c | 108 ++++++++++++++++++++++++++ lib/filterx/filterx-variable.h | 110 +++++++++++++++++++++++++++ 7 files changed, 249 insertions(+), 129 deletions(-) create mode 100644 lib/filterx/filterx-variable.c create mode 100644 lib/filterx/filterx-variable.h diff --git a/lib/filterx/CMakeLists.txt b/lib/filterx/CMakeLists.txt index c36fde5ae..39979c0c6 100644 --- a/lib/filterx/CMakeLists.txt +++ b/lib/filterx/CMakeLists.txt @@ -34,6 +34,7 @@ set(FILTERX_HEADERS filterx/filterx-pipe.h filterx/filterx-private.h filterx/filterx-scope.h + filterx/filterx-variable.h filterx/filterx-weakrefs.h filterx/func-flags.h filterx/func-flatten.h @@ -93,6 +94,7 @@ set(FILTERX_SOURCES filterx/filterx-pipe.c filterx/filterx-private.c filterx/filterx-scope.c + filterx/filterx-variable.c filterx/filterx-weakrefs.c filterx/func-flatten.c filterx/func-istype.c diff --git a/lib/filterx/Makefile.am b/lib/filterx/Makefile.am index d9b30a3a7..8b57577df 100644 --- a/lib/filterx/Makefile.am +++ b/lib/filterx/Makefile.am @@ -36,6 +36,7 @@ filterxinclude_HEADERS = \ lib/filterx/filterx-pipe.h \ lib/filterx/filterx-private.h \ lib/filterx/filterx-scope.h \ + lib/filterx/filterx-variable.h \ lib/filterx/filterx-weakrefs.h \ lib/filterx/func-flags.h \ lib/filterx/func-flatten.h \ @@ -95,6 +96,7 @@ filterx_sources = \ lib/filterx/filterx-pipe.c \ lib/filterx/filterx-private.c \ lib/filterx/filterx-scope.c \ + lib/filterx/filterx-variable.c \ lib/filterx/filterx-weakrefs.c \ lib/filterx/func-flatten.c \ lib/filterx/func-istype.c \ diff --git a/lib/filterx/expr-variable.c b/lib/filterx/expr-variable.c index ba84a5fe5..fba862149 100644 --- a/lib/filterx/expr-variable.c +++ b/lib/filterx/expr-variable.c @@ -25,6 +25,7 @@ #include "filterx/object-string.h" #include "filterx/filterx-scope.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-variable.h" #include "logmsg/logmsg.h" @@ -186,7 +187,7 @@ filterx_variable_expr_new(FilterXString *name, FilterXVariableType type) self->super.unset = _unset; self->variable_name = (FilterXObject *) name; - self->handle = filterx_scope_map_variable_to_handle(filterx_string_get_value_ref(self->variable_name, NULL), type); + self->handle = filterx_map_varname_to_handle(filterx_string_get_value_ref(self->variable_name, NULL), type); return &self->super; } diff --git a/lib/filterx/filterx-scope.c b/lib/filterx/filterx-scope.c index c240222a2..a57708f57 100644 --- a/lib/filterx/filterx-scope.c +++ b/lib/filterx/filterx-scope.c @@ -23,81 +23,6 @@ #include "filterx/filterx-scope.h" #include "scratch-buffers.h" -#define FILTERX_HANDLE_FLOATING_BIT (1UL << 31) -#define FILTERX_SCOPE_MAX_GENERATION ((1UL << 20) - 1) - -struct _FilterXVariable -{ - /* the MSB indicates that the variable is a floating one */ - FilterXVariableHandle handle; - /* - * assigned -- Indicates that the variable was assigned to a new value - * - * declared -- this variable is declared (e.g. retained for the entire input pipeline) - */ - guint32 assigned:1, - declared:1, - generation:20; - FilterXObject *value; -}; - -gboolean -filterx_variable_handle_is_floating(FilterXVariableHandle handle) -{ - return !!(handle & FILTERX_HANDLE_FLOATING_BIT); -} - -gboolean -filterx_variable_is_floating(FilterXVariable *v) -{ - return filterx_variable_handle_is_floating(v->handle); -} - -static NVHandle -filterx_variable_get_nv_handle(FilterXVariable *v) -{ - return v->handle & ~FILTERX_HANDLE_FLOATING_BIT; -} - -const gchar * -filterx_variable_get_name(FilterXVariable *v, gssize *len) -{ - return log_msg_get_handle_name(filterx_variable_get_nv_handle(v), len); -} - -FilterXObject * -filterx_variable_get_value(FilterXVariable *v) -{ - return filterx_object_ref(v->value); -} - -void -filterx_variable_set_value(FilterXVariable *v, FilterXObject *new_value) -{ - filterx_object_unref(v->value); - v->value = filterx_object_ref(new_value); - v->assigned = TRUE; -} - -void -filterx_variable_unset_value(FilterXVariable *v) -{ - filterx_variable_set_value(v, NULL); -} - -gboolean -filterx_variable_is_set(FilterXVariable *v) -{ - return v->value != NULL; -} - - -static void -_variable_free(FilterXVariable *v) -{ - filterx_object_unref(v->value); -} - struct _FilterXScope { GAtomicCounter ref_cnt; @@ -168,19 +93,6 @@ filterx_scope_is_dirty(FilterXScope *self) return self->dirty; } -FilterXVariableHandle -filterx_scope_map_variable_to_handle(const gchar *name, FilterXVariableType type) -{ - if (type == FX_VAR_MESSAGE) - name++; - - NVHandle nv_handle = log_msg_get_value_handle(name); - - if (type == FX_VAR_MESSAGE) - return (FilterXVariableHandle) nv_handle; - return (FilterXVariableHandle) nv_handle | FILTERX_HANDLE_FLOATING_BIT; -} - FilterXVariable * filterx_scope_lookup_variable(FilterXScope *self, FilterXVariableHandle handle) { @@ -189,7 +101,8 @@ filterx_scope_lookup_variable(FilterXScope *self, FilterXVariableHandle handle) if (_lookup_variable(self, handle, &v)) { if (filterx_variable_handle_is_floating(handle) && - !v->declared && v->generation != self->generation) + !filterx_variable_is_declared(v) && + !filterx_variable_is_same_generation(v, self->generation)) return NULL; if (!filterx_variable_handle_is_floating(handle) && v->generation == 0 && self->syncable) return NULL; @@ -203,20 +116,20 @@ _register_variable(FilterXScope *self, FilterXVariableHandle handle, FilterXObject *initial_value) { - FilterXVariable v, *v_slot; + FilterXVariable *v_slot; if (_lookup_variable(self, handle, &v_slot)) { /* already present */ - if (v_slot->generation != self->generation) + if (!filterx_variable_is_same_generation(v_slot, self->generation)) { /* existing value is from a previous generation, override it as if * it was a new value */ - v_slot->generation = self->generation; + filterx_variable_set_generation(v_slot, self->generation); filterx_variable_set_value(v_slot, initial_value); /* consider this to be unset just as an initial registration is */ - v_slot->assigned = FALSE; + filterx_variable_unassign(v_slot); } return v_slot; } @@ -225,10 +138,9 @@ _register_variable(FilterXScope *self, g_assert(v_index <= self->variables->len); g_assert(&g_array_index(self->variables, FilterXVariable, v_index) == v_slot); - v.handle = handle; - v.assigned = FALSE; - v.value = filterx_object_ref(initial_value); - v.generation = self->generation; + + FilterXVariable v; + filterx_variable_init_instance(&v, handle, initial_value, self->generation); g_array_insert_val(self->variables, v_index, v); return &g_array_index(self->variables, FilterXVariable, v_index); @@ -240,7 +152,7 @@ filterx_scope_register_variable(FilterXScope *self, FilterXObject *initial_value) { FilterXVariable *v = _register_variable(self, handle, initial_value); - v->declared = FALSE; + filterx_variable_set_declared(v, FALSE); /* the scope needs to be synced with the message if it holds a * message-tied variable (e.g. $MSG) */ @@ -258,7 +170,8 @@ filterx_scope_register_declared_variable(FilterXScope *self, g_assert(filterx_variable_handle_is_floating(handle)); FilterXVariable *v = _register_variable(self, handle, initial_value); - v->declared = TRUE; + filterx_variable_set_declared(v, TRUE); + return v; } @@ -272,8 +185,9 @@ filterx_scope_foreach_variable(FilterXScope *self, FilterXScopeForeachFunc func, if (!variable->value) continue; - if (filterx_variable_handle_is_floating(variable->handle) && - !variable->declared && variable->generation != self->generation) + if (filterx_variable_is_floating(variable) && + !filterx_variable_is_declared(variable) && + !filterx_variable_is_same_generation(variable, self->generation)) continue; if (!func(variable, user_data)) @@ -337,10 +251,10 @@ filterx_scope_sync(FilterXScope *self, LogMessage *msg) msg_trace("Filterx sync: whiteout variable, unsetting in message", evt_tag_str("variable", log_msg_get_value_name(filterx_variable_get_nv_handle(v), NULL))); /* we need to unset */ - log_msg_unset_value(msg, v->handle); - v->assigned = FALSE; + log_msg_unset_value(msg, filterx_variable_get_nv_handle(v)); + filterx_variable_unassign(v); } - else if (v->assigned || v->value->modified_in_place) + else if (filterx_variable_is_assigned(v) || v->value->modified_in_place) { LogMessageValueType t; @@ -350,9 +264,9 @@ filterx_scope_sync(FilterXScope *self, LogMessage *msg) g_string_truncate(buffer, 0); if (!filterx_object_marshal(v->value, buffer, &t)) g_assert_not_reached(); - log_msg_set_value_with_type(msg, v->handle, buffer->str, buffer->len, t); + log_msg_set_value_with_type(msg, filterx_variable_get_nv_handle(v), buffer->str, buffer->len, t); v->value->modified_in_place = FALSE; - v->assigned = FALSE; + filterx_variable_unassign(v); } else { @@ -370,7 +284,7 @@ filterx_scope_new(void) g_atomic_counter_set(&self->ref_cnt, 1); self->variables = g_array_sized_new(FALSE, TRUE, sizeof(FilterXVariable), 16); - g_array_set_clear_func(self->variables, (GDestroyNotify) _variable_free); + g_array_set_clear_func(self->variables, (GDestroyNotify) filterx_variable_free_method); return self; } @@ -383,19 +297,19 @@ filterx_scope_clone(FilterXScope *other) { FilterXVariable *v = &g_array_index(other->variables, FilterXVariable, src_index); - if (v->declared || !filterx_variable_is_floating(v)) + if (filterx_variable_is_declared(v) || !filterx_variable_is_floating(v)) { g_array_append_val(self->variables, *v); FilterXVariable *v_clone = &g_array_index(self->variables, FilterXVariable, dst_index); - v_clone->generation = 0; + filterx_variable_set_generation(v_clone, 0); if (v->value) v_clone->value = filterx_object_clone(v->value); else v_clone->value = NULL; dst_index++; msg_trace("Filterx scope, cloning scope variable", - evt_tag_str("variable", log_msg_get_value_name((v->handle & ~FILTERX_HANDLE_FLOATING_BIT), NULL))); + evt_tag_str("variable", log_msg_get_value_name((filterx_variable_get_nv_handle(v)), NULL))); } } diff --git a/lib/filterx/filterx-scope.h b/lib/filterx/filterx-scope.h index 07ad1e520..89f12c366 100644 --- a/lib/filterx/filterx-scope.h +++ b/lib/filterx/filterx-scope.h @@ -24,25 +24,9 @@ #define FILTERX_SCOPE_H_INCLUDED #include "filterx-object.h" +#include "filterx-variable.h" #include "logmsg/logmsg.h" -typedef struct _FilterXVariable FilterXVariable; -typedef guint32 FilterXVariableHandle; -typedef enum -{ - FX_VAR_MESSAGE, - FX_VAR_FLOATING, - FX_VAR_DECLARED, -} FilterXVariableType; - -gboolean filterx_variable_is_floating(FilterXVariable *v); -gboolean filterx_variable_handle_is_floating(FilterXVariableHandle handle); -const gchar *filterx_variable_get_name(FilterXVariable *v, gssize *len); -FilterXObject *filterx_variable_get_value(FilterXVariable *v); -void filterx_variable_set_value(FilterXVariable *v, FilterXObject *new_value); -void filterx_variable_unset_value(FilterXVariable *v); -gboolean filterx_variable_is_set(FilterXVariable *v); - /* * FilterXScope represents variables in a filterx scope. * @@ -66,7 +50,6 @@ void filterx_scope_set_dirty(FilterXScope *self); gboolean filterx_scope_is_dirty(FilterXScope *self); void filterx_scope_sync(FilterXScope *self, LogMessage *msg); -FilterXVariableHandle filterx_scope_map_variable_to_handle(const gchar *name, FilterXVariableType type); FilterXVariable *filterx_scope_lookup_variable(FilterXScope *self, FilterXVariableHandle handle); FilterXVariable *filterx_scope_register_variable(FilterXScope *self, FilterXVariableHandle handle, diff --git a/lib/filterx/filterx-variable.c b/lib/filterx/filterx-variable.c new file mode 100644 index 000000000..f8b30f77f --- /dev/null +++ b/lib/filterx/filterx-variable.c @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 László Várady + * Copyright (c) 2023 Balazs Scheidler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "filterx-variable.h" + +#define FILTERX_HANDLE_FLOATING_BIT (1UL << 31) + +gboolean +filterx_variable_handle_is_floating(FilterXVariableHandle handle) +{ + return !!(handle & FILTERX_HANDLE_FLOATING_BIT); +} + +gboolean +filterx_variable_is_floating(FilterXVariable *v) +{ + return filterx_variable_handle_is_floating(v->handle); +} + +NVHandle +filterx_variable_get_nv_handle(FilterXVariable *v) +{ + return v->handle & ~FILTERX_HANDLE_FLOATING_BIT; +} + +const gchar * +filterx_variable_get_name(FilterXVariable *v, gssize *len) +{ + return log_msg_get_handle_name(filterx_variable_get_nv_handle(v), len); +} + +FilterXObject * +filterx_variable_get_value(FilterXVariable *v) +{ + return filterx_object_ref(v->value); +} + +void +filterx_variable_set_value(FilterXVariable *v, FilterXObject *new_value) +{ + filterx_object_unref(v->value); + v->value = filterx_object_ref(new_value); + v->assigned = TRUE; +} + +void +filterx_variable_unset_value(FilterXVariable *v) +{ + filterx_variable_set_value(v, NULL); +} + +gboolean +filterx_variable_is_set(FilterXVariable *v) +{ + return v->value != NULL; +} + +FilterXVariableHandle +filterx_map_varname_to_handle(const gchar *name, FilterXVariableType type) +{ + if (type == FX_VAR_MESSAGE) + name++; + + NVHandle nv_handle = log_msg_get_value_handle(name); + + if (type == FX_VAR_MESSAGE) + return (FilterXVariableHandle) nv_handle; + return (FilterXVariableHandle) nv_handle | FILTERX_HANDLE_FLOATING_BIT; +} + +void +filterx_variable_free_method(FilterXVariable *v) +{ + filterx_object_unref(v->value); +} + +void +filterx_variable_init_instance(FilterXVariable *v, FilterXVariableHandle handle, + FilterXObject *initial_value, guint32 generation) +{ + v->handle = handle; + v->assigned = FALSE; + v->declared = FALSE; + v->generation = generation; + v->value = filterx_object_ref(initial_value); +} diff --git a/lib/filterx/filterx-variable.h b/lib/filterx/filterx-variable.h new file mode 100644 index 000000000..38754dad0 --- /dev/null +++ b/lib/filterx/filterx-variable.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 László Várady + * Copyright (c) 2023 Balazs Scheidler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef FILTERX_VARIABLE_H +#define FILTERX_VARIABLE_H + +#include "syslog-ng.h" +#include "filterx-object.h" + +#define FILTERX_SCOPE_MAX_GENERATION ((1UL << 20) - 1) + +typedef guint32 FilterXVariableHandle; + +typedef struct _FilterXVariable +{ + /* the MSB indicates that the variable is a floating one */ + FilterXVariableHandle handle; + /* + * assigned -- Indicates that the variable was assigned to a new value + * + * declared -- this variable is declared (e.g. retained for the entire input pipeline) + */ + guint32 assigned:1, + declared:1, + generation:20; + FilterXObject *value; +} FilterXVariable; + +typedef enum +{ + FX_VAR_MESSAGE, + FX_VAR_FLOATING, + FX_VAR_DECLARED, +} FilterXVariableType; + + +void filterx_variable_init_instance(FilterXVariable *v, FilterXVariableHandle handle, + FilterXObject *initial_value, guint32 generation); +void filterx_variable_free_method(FilterXVariable *v); + +gboolean filterx_variable_is_floating(FilterXVariable *v); +gboolean filterx_variable_handle_is_floating(FilterXVariableHandle handle); +const gchar *filterx_variable_get_name(FilterXVariable *v, gssize *len); +NVHandle filterx_variable_get_nv_handle(FilterXVariable *v); +FilterXObject *filterx_variable_get_value(FilterXVariable *v); +void filterx_variable_set_value(FilterXVariable *v, FilterXObject *new_value); +void filterx_variable_unset_value(FilterXVariable *v); +gboolean filterx_variable_is_set(FilterXVariable *v); + +static inline gboolean +filterx_variable_is_declared(FilterXVariable *v) +{ + return v->declared; +} + +static inline gboolean +filterx_variable_is_assigned(FilterXVariable *v) +{ + return v->assigned; +} + +static inline gboolean +filterx_variable_is_same_generation(FilterXVariable *v, guint32 generation) +{ + return v->generation == generation; +} + +static inline void +filterx_variable_set_generation(FilterXVariable *v, guint32 generation) +{ + v->generation = generation; +} + +static inline void +filterx_variable_unassign(FilterXVariable *v) +{ + v->assigned = FALSE; +} + +static inline void +filterx_variable_set_declared(FilterXVariable *v, gboolean declared) +{ + v->declared = declared; +} + +FilterXVariableHandle filterx_map_varname_to_handle(const gchar *name, FilterXVariableType type); + +#endif From 1b0ac10d52ffbc2f78b4f1a3e0886fb6909bc1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Tue, 1 Oct 2024 23:50:53 +0200 Subject: [PATCH 03/16] filterx-object: extract _filterx_type_init_methods() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/filterx-object.c | 8 +++++++- lib/filterx/filterx-object.h | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/filterx/filterx-object.c b/lib/filterx/filterx-object.c index 5e34cb5fb..0faa2912d 100644 --- a/lib/filterx/filterx-object.c +++ b/lib/filterx/filterx-object.c @@ -61,7 +61,7 @@ filterx_object_setattr_string(FilterXObject *self, const gchar *attr_name, Filte } while (0) void -filterx_type_init(FilterXType *type) +_filterx_type_init_methods(FilterXType *type) { INIT_TYPE_METHOD(type, unmarshal); INIT_TYPE_METHOD(type, marshal); @@ -80,6 +80,12 @@ filterx_type_init(FilterXType *type) INIT_TYPE_METHOD(type, len); INIT_TYPE_METHOD(type, add); INIT_TYPE_METHOD(type, free_fn); +} + +void +filterx_type_init(FilterXType *type) +{ + _filterx_type_init_methods(type); if (!filterx_type_register(type->name, type)) msg_error("Reregistering filterx type", evt_tag_str("name", type->name)); diff --git a/lib/filterx/filterx-object.h b/lib/filterx/filterx-object.h index 418f3ab98..10ae25ff2 100644 --- a/lib/filterx/filterx-object.h +++ b/lib/filterx/filterx-object.h @@ -58,6 +58,7 @@ struct _FilterXType }; void filterx_type_init(FilterXType *type); +void _filterx_type_init_methods(FilterXType *type); #define FILTERX_TYPE_NAME(_name) filterx_type_ ## _name #define FILTERX_DECLARE_TYPE(_name) \ From 036418b8f9a1c1885594b59c1a56baecb958a495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Wed, 2 Oct 2024 00:44:23 +0200 Subject: [PATCH 04/16] filterx: add missing filterx_object_free_method() calls to destructors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/object-json-array.c | 2 ++ lib/filterx/object-json-object.c | 2 ++ lib/filterx/object-message-value.c | 2 ++ modules/grpc/otel/filterx/object-otel-array.cpp | 2 ++ modules/grpc/otel/filterx/object-otel-kvlist.cpp | 2 ++ modules/grpc/otel/filterx/object-otel-logrecord.cpp | 2 ++ modules/grpc/otel/filterx/object-otel-resource.cpp | 2 ++ modules/grpc/otel/filterx/object-otel-scope.cpp | 2 ++ 8 files changed, 16 insertions(+) diff --git a/lib/filterx/object-json-array.c b/lib/filterx/object-json-array.c index 87119101d..b955d6e06 100644 --- a/lib/filterx/object-json-array.c +++ b/lib/filterx/object-json-array.c @@ -283,6 +283,8 @@ _free(FilterXObject *s) filterx_weakref_clear(&self->root_container); g_mutex_clear(&self->lock); + + filterx_object_free_method(s); } FilterXObject * diff --git a/lib/filterx/object-json-object.c b/lib/filterx/object-json-object.c index e80f10bec..405d5488b 100644 --- a/lib/filterx/object-json-object.c +++ b/lib/filterx/object-json-object.c @@ -269,6 +269,8 @@ _free(FilterXObject *s) filterx_weakref_clear(&self->root_container); g_mutex_clear(&self->lock); + + filterx_object_free_method(s); } FilterXObject * diff --git a/lib/filterx/object-message-value.c b/lib/filterx/object-message-value.c index 8ba67874b..cb13a6c0c 100644 --- a/lib/filterx/object-message-value.c +++ b/lib/filterx/object-message-value.c @@ -361,6 +361,8 @@ _free(FilterXObject *s) FilterXMessageValue *self = (FilterXMessageValue *) s; g_free(self->buf); + + filterx_object_free_method(s); } static gboolean diff --git a/modules/grpc/otel/filterx/object-otel-array.cpp b/modules/grpc/otel/filterx/object-otel-array.cpp index 03120f195..ddeba4d3b 100644 --- a/modules/grpc/otel/filterx/object-otel-array.cpp +++ b/modules/grpc/otel/filterx/object-otel-array.cpp @@ -151,6 +151,8 @@ _free(FilterXObject *s) delete self->cpp; self->cpp = NULL; + + filterx_object_free_method(s); } static gboolean diff --git a/modules/grpc/otel/filterx/object-otel-kvlist.cpp b/modules/grpc/otel/filterx/object-otel-kvlist.cpp index 4eb533f5c..19d9354b7 100644 --- a/modules/grpc/otel/filterx/object-otel-kvlist.cpp +++ b/modules/grpc/otel/filterx/object-otel-kvlist.cpp @@ -257,6 +257,8 @@ _free(FilterXObject *s) delete self->cpp; self->cpp = NULL; + + filterx_object_free_method(s); } static gboolean diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index cd071b607..37f438072 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -174,6 +174,8 @@ _free(FilterXObject *s) delete self->cpp; self->cpp = NULL; + + filterx_object_free_method(s); } static gboolean diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 57b49cd03..9f3919e56 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -160,6 +160,8 @@ _free(FilterXObject *s) delete self->cpp; self->cpp = NULL; + + filterx_object_free_method(s); } static gboolean diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index 00bef980f..d1a7751ec 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -160,6 +160,8 @@ _free(FilterXObject *s) delete self->cpp; self->cpp = NULL; + + filterx_object_free_method(s); } static gboolean From 8b0f353ccb12f84197e65a24aa8fea3dece0e820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Wed, 2 Oct 2024 23:23:53 +0200 Subject: [PATCH 05/16] filterx: change signature of list_factory() and dict_factory() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Needed for introducing FilterXRef. Signed-off-by: László Várady --- lib/filterx/filterx-object.h | 8 ++++---- lib/filterx/object-json-array.c | 16 ++++++++++++++-- lib/filterx/object-json-object.c | 16 ++++++++++++++-- libtest/filterx-lib.c | 16 ++++++++++++++-- modules/grpc/otel/filterx/object-otel-array.cpp | 16 ++++++++++++++-- modules/grpc/otel/filterx/object-otel-kvlist.cpp | 16 ++++++++++++++-- .../grpc/otel/filterx/object-otel-logrecord.cpp | 16 ++++++++++++++-- .../grpc/otel/filterx/object-otel-resource.cpp | 16 ++++++++++++++-- modules/grpc/otel/filterx/object-otel-scope.cpp | 16 ++++++++++++++-- 9 files changed, 116 insertions(+), 20 deletions(-) diff --git a/lib/filterx/filterx-object.h b/lib/filterx/filterx-object.h index 10ae25ff2..1539d7908 100644 --- a/lib/filterx/filterx-object.h +++ b/lib/filterx/filterx-object.h @@ -48,8 +48,8 @@ struct _FilterXType gboolean (*set_subscript)(FilterXObject *self, FilterXObject *key, FilterXObject **new_value); gboolean (*is_key_set)(FilterXObject *self, FilterXObject *key); gboolean (*unset_key)(FilterXObject *self, FilterXObject *key); - FilterXObject *(*list_factory)(void); - FilterXObject *(*dict_factory)(void); + FilterXObject *(*list_factory)(FilterXObject *self); + FilterXObject *(*dict_factory)(FilterXObject *self); gboolean (*repr)(FilterXObject *self, GString *repr); gboolean (*len)(FilterXObject *self, guint64 *len); FilterXObject *(*add)(FilterXObject *self, FilterXObject *object); @@ -282,7 +282,7 @@ filterx_object_create_list(FilterXObject *self) if (!self->type->list_factory) return NULL; - return self->type->list_factory(); + return self->type->list_factory(self); } static inline FilterXObject * @@ -291,7 +291,7 @@ filterx_object_create_dict(FilterXObject *self) if (!self->type->dict_factory) return NULL; - return self->type->dict_factory(); + return self->type->dict_factory(self); } static inline FilterXObject * diff --git a/lib/filterx/object-json-array.c b/lib/filterx/object-json-array.c index b955d6e06..b0c075ef2 100644 --- a/lib/filterx/object-json-array.c +++ b/lib/filterx/object-json-array.c @@ -372,6 +372,18 @@ filterx_json_array_get_value(FilterXObject *s) return self->jso; } +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_json_array_new_empty(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_json_object_new_empty(); +} + FILTERX_DEFINE_TYPE(json_array, FILTERX_TYPE_NAME(list), .is_mutable = TRUE, .truthy = _truthy, @@ -380,7 +392,7 @@ FILTERX_DEFINE_TYPE(json_array, FILTERX_TYPE_NAME(list), .repr = _repr, .map_to_json = _map_to_json, .clone = _clone, - .list_factory = filterx_json_array_new_empty, - .dict_factory = filterx_json_object_new_empty, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .make_readonly = _make_readonly, ); diff --git a/lib/filterx/object-json-object.c b/lib/filterx/object-json-object.c index 405d5488b..b224a229d 100644 --- a/lib/filterx/object-json-object.c +++ b/lib/filterx/object-json-object.c @@ -311,6 +311,18 @@ filterx_json_object_get_value(FilterXObject *s) return self->jso; } +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_json_array_new_empty(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_json_object_new_empty(); +} + FILTERX_DEFINE_TYPE(json_object, FILTERX_TYPE_NAME(dict), .is_mutable = TRUE, .truthy = _truthy, @@ -319,7 +331,7 @@ FILTERX_DEFINE_TYPE(json_object, FILTERX_TYPE_NAME(dict), .repr = _repr, .map_to_json = _map_to_json, .clone = _clone, - .list_factory = filterx_json_array_new_empty, - .dict_factory = filterx_json_object_new_empty, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .make_readonly = _make_readonly, ); diff --git a/libtest/filterx-lib.c b/libtest/filterx-lib.c index d06e3eb2a..2b46c74e9 100644 --- a/libtest/filterx-lib.c +++ b/libtest/filterx-lib.c @@ -206,6 +206,18 @@ deinit_libtest_filterx(void) filterx_eval_deinit_context(&filterx_env.context); } +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_test_list_new(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_test_dict_new(); +} + FILTERX_DEFINE_TYPE(test_dict, FILTERX_TYPE_NAME(object)); FILTERX_DEFINE_TYPE(test_list, FILTERX_TYPE_NAME(object)); FILTERX_DEFINE_TYPE(test_unknown_object, FILTERX_TYPE_NAME(object), @@ -214,5 +226,5 @@ FILTERX_DEFINE_TYPE(test_unknown_object, FILTERX_TYPE_NAME(object), .marshal = _unknown_marshal, .repr = _unknown_repr, .map_to_json = _unknown_map_to_json, - .list_factory = filterx_test_list_new, - .dict_factory = filterx_test_dict_new); + .list_factory = _list_factory, + .dict_factory = _dict_factory); diff --git a/modules/grpc/otel/filterx/object-otel-array.cpp b/modules/grpc/otel/filterx/object-otel-array.cpp index ddeba4d3b..f572bd15f 100644 --- a/modules/grpc/otel/filterx/object-otel-array.cpp +++ b/modules/grpc/otel/filterx/object-otel-array.cpp @@ -406,12 +406,24 @@ OtelArrayField::FilterXObjectSetter(google::protobuf::Message *message, ProtoRef OtelArrayField syslogng::grpc::otel::filterx::otel_array_converter; +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_otel_array_new(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_otel_kvlist_new(); +} + FILTERX_DEFINE_TYPE(otel_array, FILTERX_TYPE_NAME(list), .is_mutable = TRUE, .marshal = _marshal, .clone = _filterx_otel_array_clone, .truthy = _truthy, - .list_factory = filterx_otel_array_new, - .dict_factory = filterx_otel_kvlist_new, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-kvlist.cpp b/modules/grpc/otel/filterx/object-otel-kvlist.cpp index 19d9354b7..3f0644e84 100644 --- a/modules/grpc/otel/filterx/object-otel-kvlist.cpp +++ b/modules/grpc/otel/filterx/object-otel-kvlist.cpp @@ -550,12 +550,24 @@ OtelKVListField::FilterXObjectSetter(google::protobuf::Message *message, ProtoRe OtelKVListField syslogng::grpc::otel::filterx::otel_kvlist_converter; +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_otel_array_new(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_otel_kvlist_new(); +} + FILTERX_DEFINE_TYPE(otel_kvlist, FILTERX_TYPE_NAME(dict), .is_mutable = TRUE, .marshal = _marshal, .clone = _filterx_otel_kvlist_clone, .truthy = _truthy, - .list_factory = filterx_otel_array_new, - .dict_factory = filterx_otel_kvlist_new, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 37f438072..19789bd40 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -319,6 +319,18 @@ filterx_otel_logrecord_new_from_args(FilterXExpr *s, GPtrArray *args) return &self->super.super; } +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_otel_array_new(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_otel_kvlist_new(); +} + FILTERX_SIMPLE_FUNCTION(otel_logrecord, filterx_otel_logrecord_new_from_args); FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(dict), @@ -326,7 +338,7 @@ FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(dict), .marshal = _marshal, .clone = _filterx_otel_logrecord_clone, .truthy = _truthy, - .list_factory = filterx_otel_array_new, - .dict_factory = filterx_otel_kvlist_new, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 9f3919e56..58ac31e05 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -305,6 +305,18 @@ filterx_otel_resource_new_from_args(FilterXExpr *s, GPtrArray *args) return &self->super.super; } +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_otel_array_new(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_otel_kvlist_new(); +} + FILTERX_SIMPLE_FUNCTION(otel_resource, filterx_otel_resource_new_from_args); @@ -313,7 +325,7 @@ FILTERX_DEFINE_TYPE(otel_resource, FILTERX_TYPE_NAME(dict), .marshal = _marshal, .clone = _filterx_otel_resource_clone, .truthy = _truthy, - .list_factory = filterx_otel_array_new, - .dict_factory = filterx_otel_kvlist_new, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index d1a7751ec..e35bf9897 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -305,6 +305,18 @@ filterx_otel_scope_new_from_args(FilterXExpr *s, GPtrArray *args) return &self->super.super; } +static FilterXObject * +_list_factory(FilterXObject *self) +{ + return filterx_otel_array_new(); +} + +static FilterXObject * +_dict_factory(FilterXObject *self) +{ + return filterx_otel_kvlist_new(); +} + FILTERX_SIMPLE_FUNCTION(otel_scope, filterx_otel_scope_new_from_args); @@ -313,7 +325,7 @@ FILTERX_DEFINE_TYPE(otel_scope, FILTERX_TYPE_NAME(dict), .marshal = _marshal, .clone = _filterx_otel_scope_clone, .truthy = _truthy, - .list_factory = filterx_otel_array_new, - .dict_factory = filterx_otel_kvlist_new, + .list_factory = _list_factory, + .dict_factory = _dict_factory, .free_fn = _free, ); From b9122d72b01835a76c672c827fac76958dd895e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Wed, 2 Oct 2024 23:24:15 +0200 Subject: [PATCH 06/16] filterx: add FilterXRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deep-copy assignment (the `.something = `, `["something"] = `, `something =`, `something +=` operators) is an essential feature of the FilterX language, making it easy to trace and understand how values change. However, copy (especially deep-copy) is expensive, so this PR aims to fix the performance bottleneck by introducing copy-on-write for mutable objects. To achieve CoW semantics without introducing endless changes in the `filterx` folder, `FilterXRef` has been introduced. References are currently not part of the FilterX language (hopefully, they never will be). `FilterXRef` is used only internally to reference the same `FilterXObject` from multiple locations (variables, other types of assignments). For this reason, `FilterXRef` pretends to be a `FilterXObject`, but it's not a real FilterX type. `FilterXRef` transparently propagates operations (virtual function calls) to the referenced value, but it also makes sure that the value is copied on the first write operation in case multiple references exist. `FilterXRef` is created only for mutable objects, and only when they are needed: around assignment-like operators; this and its `FilterXObject` behavior was the key to avoid polluting all code paths with references. Signed-off-by: László Várady --- lib/filterx/CMakeLists.txt | 2 + lib/filterx/Makefile.am | 2 + lib/filterx/filterx-object.h | 2 + lib/filterx/filterx-ref.c | 270 +++++++++++++++++++++++++++++++++++ lib/filterx/filterx-ref.h | 63 ++++++++ 5 files changed, 339 insertions(+) create mode 100644 lib/filterx/filterx-ref.c create mode 100644 lib/filterx/filterx-ref.h diff --git a/lib/filterx/CMakeLists.txt b/lib/filterx/CMakeLists.txt index 39979c0c6..adaf2dbd5 100644 --- a/lib/filterx/CMakeLists.txt +++ b/lib/filterx/CMakeLists.txt @@ -33,6 +33,7 @@ set(FILTERX_HEADERS filterx/filterx-parser.h filterx/filterx-pipe.h filterx/filterx-private.h + filterx/filterx-ref.h filterx/filterx-scope.h filterx/filterx-variable.h filterx/filterx-weakrefs.h @@ -93,6 +94,7 @@ set(FILTERX_SOURCES filterx/filterx-parser.c filterx/filterx-pipe.c filterx/filterx-private.c + filterx/filterx-ref.c filterx/filterx-scope.c filterx/filterx-variable.c filterx/filterx-weakrefs.c diff --git a/lib/filterx/Makefile.am b/lib/filterx/Makefile.am index 8b57577df..374bee874 100644 --- a/lib/filterx/Makefile.am +++ b/lib/filterx/Makefile.am @@ -35,6 +35,7 @@ filterxinclude_HEADERS = \ lib/filterx/filterx-parser.h \ lib/filterx/filterx-pipe.h \ lib/filterx/filterx-private.h \ + lib/filterx/filterx-ref.h \ lib/filterx/filterx-scope.h \ lib/filterx/filterx-variable.h \ lib/filterx/filterx-weakrefs.h \ @@ -95,6 +96,7 @@ filterx_sources = \ lib/filterx/filterx-parser.c \ lib/filterx/filterx-pipe.c \ lib/filterx/filterx-private.c \ + lib/filterx/filterx-ref.c \ lib/filterx/filterx-scope.c \ lib/filterx/filterx-variable.c \ lib/filterx/filterx-weakrefs.c \ diff --git a/lib/filterx/filterx-object.h b/lib/filterx/filterx-object.h index 1539d7908..f7e5ea683 100644 --- a/lib/filterx/filterx-object.h +++ b/lib/filterx/filterx-object.h @@ -37,6 +37,7 @@ struct _FilterXType const gchar *name; gboolean is_mutable; + /* WARNING: adding a new method here requires an implementation in FilterXRef as well */ FilterXObject *(*unmarshal)(FilterXObject *self); gboolean (*marshal)(FilterXObject *self, GString *repr, LogMessageValueType *t); FilterXObject *(*clone)(FilterXObject *self); @@ -77,6 +78,7 @@ FILTERX_DECLARE_TYPE(object); struct _FilterXObject { GAtomicCounter ref_cnt; + GAtomicCounter fx_ref_cnt; /* NOTE: * diff --git a/lib/filterx/filterx-ref.c b/lib/filterx/filterx-ref.c new file mode 100644 index 000000000..083ae4c5f --- /dev/null +++ b/lib/filterx/filterx-ref.c @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2024 László Várady + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "filterx-ref.h" + +static FilterXObject * +_filterx_ref_clone(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + + return filterx_ref_new(filterx_object_ref(self->value)); +} + +static void +_filterx_ref_cow(FilterXRef *self) +{ + if (g_atomic_counter_get(&self->value->fx_ref_cnt) <= 1) + return; + + FilterXObject *cloned = filterx_object_clone(self->value); + + g_atomic_counter_dec_and_test(&self->value->fx_ref_cnt); + filterx_object_unref(self->value); + + self->value = cloned; + g_atomic_counter_inc(&self->value->fx_ref_cnt); +} + +FilterXObject * +filterx_ref_unwrap_ro(FilterXObject *s) +{ + if (!s || !filterx_object_is_type(s, &FILTERX_TYPE_NAME(ref))) + return s; + + FilterXRef *self = (FilterXRef *) s; + return self->value; +} + +FilterXObject * +filterx_ref_unwrap_rw(FilterXObject *s) +{ + if (!s || !filterx_object_is_type(s, &FILTERX_TYPE_NAME(ref))) + return s; + + FilterXRef *self = (FilterXRef *) s; + + _filterx_ref_cow(self); + + return self->value; +} + +/* mutator methods */ + +static gboolean +_filterx_ref_setattr(FilterXObject *s, FilterXObject *attr, FilterXObject **new_value) +{ + FilterXRef *self = (FilterXRef *) s; + + _filterx_ref_cow(self); + + return filterx_object_setattr(self->value, attr, new_value); +} + +static gboolean +_filterx_ref_set_subscript(FilterXObject *s, FilterXObject *key, FilterXObject **new_value) +{ + FilterXRef *self = (FilterXRef *) s; + + _filterx_ref_cow(self); + + return filterx_object_set_subscript(self->value, key, new_value); +} + +static gboolean +_filterx_ref_unset_key(FilterXObject *s, FilterXObject *key) +{ + FilterXRef *self = (FilterXRef *) s; + + _filterx_ref_cow(self); + + return filterx_object_unset_key(self->value, key); +} + +static void +_filterx_ref_free(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + + g_atomic_counter_dec_and_test(&self->value->fx_ref_cnt); + filterx_object_unref(self->value); + + filterx_object_free_method(s); +} + +void +_prohibit_readonly(FilterXObject *s) +{ + g_assert_not_reached(); +} + +/* readonly methods */ + +static FilterXObject * +_filterx_ref_unmarshal(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + FilterXObject *unmarshalled = filterx_object_unmarshal(self->value); + if (unmarshalled != self->value) + return unmarshalled; + + filterx_object_unref(self->value); + return filterx_object_ref(s); +} + +static gboolean +_filterx_ref_marshal(FilterXObject *s, GString *repr, LogMessageValueType *t) +{ + FilterXRef *self = (FilterXRef *) s; + + if (self->value->type->marshal) + return self->value->type->marshal(self->value, repr, t); + + return FALSE; +} + +static gboolean +_filterx_ref_map_to_json(FilterXObject *s, struct json_object **object, FilterXObject **assoc_object) +{ + FilterXRef *self = (FilterXRef *) s; + gboolean ok = filterx_object_map_to_json(self->value, object, assoc_object); + if (*assoc_object != self->value) + return ok; + + filterx_object_unref(self->value); + *assoc_object = filterx_object_ref(s); + return ok; +} + +static gboolean +_filterx_ref_truthy(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_truthy(self->value); +} + +static FilterXObject * +_filterx_ref_getattr(FilterXObject *s, FilterXObject *attr) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_getattr(self->value, attr); +} + +static FilterXObject * +_filterx_ref_get_subscript(FilterXObject *s, FilterXObject *key) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_get_subscript(self->value, key); +} + +static gboolean +_filterx_ref_is_key_set(FilterXObject *s, FilterXObject *key) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_is_key_set(self->value, key); +} + +static FilterXObject * +_filterx_ref_list_factory(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_create_list(self->value); +} + +static FilterXObject * +_filterx_ref_dict_factory(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_create_dict(self->value); +} + +static gboolean +_filterx_ref_repr(FilterXObject *s, GString *repr) +{ + FilterXRef *self = (FilterXRef *) s; + + if (self->value->type->repr) + return self->value->type->repr(self->value, repr); + + return FALSE; +} + +static gboolean +_filterx_ref_len(FilterXObject *s, guint64 *len) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_len(self->value, len); +} + +static FilterXObject * +_filterx_ref_add(FilterXObject *s, FilterXObject *object) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_add_object(self->value, object); +} + +FilterXObject * +filterx_ref_new(FilterXObject *value) +{ + if (!value || value->readonly || !_filterx_type_is_referenceable(value->type)) + return value; + + if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(ref))) + { + FilterXRef *absorbed_ref = (FilterXRef *) value; + value = filterx_object_ref(absorbed_ref->value); + filterx_object_unref(&absorbed_ref->super); + } + + FilterXRef *self = g_new0(FilterXRef, 1); + + filterx_object_init_instance(&self->super, &FILTERX_TYPE_NAME(ref)); + self->super.readonly = FALSE; + + self->value = value; + g_atomic_counter_inc(&self->value->fx_ref_cnt); + + return &self->super; +} + +FILTERX_DEFINE_TYPE(ref, FILTERX_TYPE_NAME(object), + .is_mutable = TRUE, + .unmarshal = _filterx_ref_unmarshal, + .marshal = _filterx_ref_marshal, + .clone = _filterx_ref_clone, + .map_to_json = _filterx_ref_map_to_json, + .truthy = _filterx_ref_truthy, + .getattr = _filterx_ref_getattr, + .setattr = _filterx_ref_setattr, + .get_subscript = _filterx_ref_get_subscript, + .set_subscript = _filterx_ref_set_subscript, + .is_key_set = _filterx_ref_is_key_set, + .unset_key = _filterx_ref_unset_key, + .list_factory = _filterx_ref_list_factory, + .dict_factory = _filterx_ref_dict_factory, + .repr = _filterx_ref_repr, + .len = _filterx_ref_len, + .add = _filterx_ref_add, + .make_readonly = _prohibit_readonly, + .free_fn = _filterx_ref_free, + ); diff --git a/lib/filterx/filterx-ref.h b/lib/filterx/filterx-ref.h new file mode 100644 index 000000000..9e68cfe72 --- /dev/null +++ b/lib/filterx/filterx-ref.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 László Várady + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef FILTERX_REF_H +#define FILTERX_REF_H + +#include "filterx/filterx-object.h" + +/* + * References are currently not part of the FilterX language (hopefully, they + * never will be). FilterXRef is used to reference the same FilterXObject from + * multiple locations (variables, other types of assignments) in order to + * implement copy-on-write. For this reason, FilterXRef pretends to be a + * FilterXObject, but it's not a real FilterX type. + * + * FilterXRef is a final class, do not inherit from it. + * + * The functionality behind FilterXRef and the locations where they are used are + * open for extension. + */ + +FILTERX_DECLARE_TYPE(ref); + +typedef struct _FilterXRef +{ + FilterXObject super; + FilterXObject *value; +} FilterXRef; + + +FilterXObject *filterx_ref_new(FilterXObject *value); + +/* Call them only where downcasting to a specific type is needed, the returned object should only be used locally. */ +FilterXObject *filterx_ref_unwrap_ro(FilterXObject *s); +FilterXObject *filterx_ref_unwrap_rw(FilterXObject *s); + +static inline gboolean +_filterx_type_is_referenceable(FilterXType *t) +{ + return t->is_mutable && t != &FILTERX_TYPE_NAME(ref); +} + +#endif From cc286b70aeca1184a4da94814d99bfc559399e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sun, 6 Oct 2024 15:45:28 +0200 Subject: [PATCH 07/16] filterx-json: eliminate unnecessary strlen() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/object-json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filterx/object-json.c b/lib/filterx/object-json.c index 79aae3c73..f7be6025e 100644 --- a/lib/filterx/object-json.c +++ b/lib/filterx/object-json.c @@ -103,7 +103,7 @@ filterx_json_convert_json_to_object(FilterXObject *root_obj, FilterXWeakRef *roo case json_type_int: return filterx_integer_new(json_object_get_int64(jso)); case json_type_string: - return filterx_string_new(json_object_get_string(jso), -1); + return filterx_string_new(json_object_get_string(jso), json_object_get_string_len(jso)); case json_type_array: return filterx_json_array_new_sub(json_object_get(jso), filterx_weakref_get(root_container) ? : filterx_object_ref(root_obj)); From 4dfcb96990e7eb5f1e45403e4fbcbe9e42c08faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 11 Oct 2024 15:52:22 +0200 Subject: [PATCH 08/16] filterx: move filterx_object_is_type() to a separate unit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function will be extended with FilterXRef-related functionality, which requires a separate unit to avoid circular dependency between FilterX objects and refs. Signed-off-by: László Várady --- lib/filterx/CMakeLists.txt | 1 + lib/filterx/Makefile.am | 1 + lib/filterx/expr-comparison.c | 1 + lib/filterx/expr-function.c | 1 + lib/filterx/expr-null-coalesce.c | 1 + lib/filterx/expr-plus-generator.c | 1 + lib/filterx/expr-regexp.c | 1 + lib/filterx/filterx-metrics-labels.c | 1 + lib/filterx/filterx-metrics.c | 1 + lib/filterx/filterx-object-istype.h | 44 +++++++++++++++++++ lib/filterx/filterx-object.h | 13 ------ lib/filterx/filterx-ref.c | 1 + lib/filterx/func-flatten.c | 1 + lib/filterx/func-istype.c | 1 + lib/filterx/func-str.c | 1 + lib/filterx/func-unset-empties.c | 1 + lib/filterx/object-datetime.c | 1 + lib/filterx/object-dict-interface.c | 1 + lib/filterx/object-extractor.c | 1 + lib/filterx/object-json-array.c | 1 + lib/filterx/object-json-object.c | 1 + lib/filterx/object-json.c | 1 + lib/filterx/object-list-interface.c | 1 + lib/filterx/object-primitive.c | 1 + lib/filterx/object-primitive.h | 1 + lib/filterx/object-string.c | 1 + lib/filterx/tests/test_builtin_functions.c | 1 + lib/filterx/tests/test_expr_compound.c | 1 + lib/filterx/tests/test_expr_condition.c | 1 + lib/filterx/tests/test_expr_function.c | 1 + lib/filterx/tests/test_expr_null_coalesce.c | 1 + lib/filterx/tests/test_expr_plus.c | 1 + lib/filterx/tests/test_expr_plus_generator.c | 1 + lib/filterx/tests/test_expr_regexp.c | 1 + lib/filterx/tests/test_filterx_expr.c | 1 + lib/filterx/tests/test_object_boolean.c | 1 + lib/filterx/tests/test_object_bytes.c | 1 + lib/filterx/tests/test_object_datetime.c | 1 + lib/filterx/tests/test_object_double.c | 1 + lib/filterx/tests/test_object_integer.c | 1 + lib/filterx/tests/test_object_json.c | 1 + lib/filterx/tests/test_object_protobuf.c | 1 + lib/filterx/tests/test_object_string.c | 1 + .../tests/test-filterx-function-parse-cef.c | 1 + .../tests/test-filterx-function-parse-leef.c | 1 + modules/csvparser/filterx-func-format-csv.c | 1 + modules/csvparser/filterx-func-parse-csv.c | 1 + .../tests/test_filterx_func_parse_csv.c | 1 + .../grpc/otel/filterx/object-otel-array.cpp | 1 + .../grpc/otel/filterx/object-otel-kvlist.cpp | 1 + .../otel/filterx/object-otel-logrecord.cpp | 1 + .../otel/filterx/object-otel-resource.cpp | 1 + .../grpc/otel/filterx/object-otel-scope.cpp | 1 + modules/grpc/otel/filterx/otel-field.cpp | 1 + modules/grpc/otel/filterx/protobuf-field.cpp | 1 + modules/grpc/otel/tests/test-otel-filterx.cpp | 1 + modules/json/filterx-cache-json-file.c | 1 + modules/json/filterx-format-json.c | 1 + modules/json/tests/test_filterx_format_json.c | 1 + modules/kvformat/filterx-func-format-kv.c | 1 + .../tests/test_filterx_func_parse_kv.c | 1 + .../xml/filterx-parse-windows-eventlog-xml.c | 1 + modules/xml/filterx-parse-xml.c | 1 + .../test_filterx_parse_windows_eventlog_xml.c | 1 + modules/xml/tests/test_filterx_parse_xml.c | 1 + 65 files changed, 107 insertions(+), 13 deletions(-) create mode 100644 lib/filterx/filterx-object-istype.h diff --git a/lib/filterx/CMakeLists.txt b/lib/filterx/CMakeLists.txt index adaf2dbd5..879ba4198 100644 --- a/lib/filterx/CMakeLists.txt +++ b/lib/filterx/CMakeLists.txt @@ -90,6 +90,7 @@ set(FILTERX_SOURCES filterx/filterx-globals.c filterx/filterx-metrics-labels.c filterx/filterx-metrics.c + filterx/filterx-object-istype.h filterx/filterx-object.c filterx/filterx-parser.c filterx/filterx-pipe.c diff --git a/lib/filterx/Makefile.am b/lib/filterx/Makefile.am index 374bee874..3089aae45 100644 --- a/lib/filterx/Makefile.am +++ b/lib/filterx/Makefile.am @@ -31,6 +31,7 @@ filterxinclude_HEADERS = \ lib/filterx/filterx-globals.h \ lib/filterx/filterx-metrics-labels.h \ lib/filterx/filterx-metrics.h \ + lib/filterx/filterx-object-istype.h \ lib/filterx/filterx-object.h \ lib/filterx/filterx-parser.h \ lib/filterx/filterx-pipe.h \ diff --git a/lib/filterx/expr-comparison.c b/lib/filterx/expr-comparison.c index 9bb3484ea..5e6d1bf5c 100644 --- a/lib/filterx/expr-comparison.c +++ b/lib/filterx/expr-comparison.c @@ -31,6 +31,7 @@ #include "filterx/object-json.h" #include "filterx/object-datetime.h" #include "filterx/object-message-value.h" +#include "filterx/filterx-object-istype.h" #include "object-primitive.h" #include "generic-number.h" #include "parse-number.h" diff --git a/lib/filterx/expr-function.c b/lib/filterx/expr-function.c index 1224e9eaf..e5a05adcc 100644 --- a/lib/filterx/expr-function.c +++ b/lib/filterx/expr-function.c @@ -31,6 +31,7 @@ #include "filterx/object-string.h" #include "filterx/object-primitive.h" #include "filterx/object-null.h" +#include "filterx/filterx-object-istype.h" #include "plugin.h" #include "cfg.h" #include "mainloop.h" diff --git a/lib/filterx/expr-null-coalesce.c b/lib/filterx/expr-null-coalesce.c index bdbe3c2e7..479130507 100644 --- a/lib/filterx/expr-null-coalesce.c +++ b/lib/filterx/expr-null-coalesce.c @@ -26,6 +26,7 @@ #include "filterx/object-null.h" #include "filterx/filterx-eval.h" #include "filterx/object-message-value.h" +#include "filterx/filterx-object-istype.h" typedef struct _FilterXNullCoalesce FilterXNullCoalesce; diff --git a/lib/filterx/expr-plus-generator.c b/lib/filterx/expr-plus-generator.c index 79c305a86..8919197d2 100644 --- a/lib/filterx/expr-plus-generator.c +++ b/lib/filterx/expr-plus-generator.c @@ -24,6 +24,7 @@ #include "expr-plus.h" #include "object-string.h" #include "filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "expr-generator.h" #include "object-list-interface.h" diff --git a/lib/filterx/expr-regexp.c b/lib/filterx/expr-regexp.c index 3a092ac7a..f18628a51 100644 --- a/lib/filterx/expr-regexp.c +++ b/lib/filterx/expr-regexp.c @@ -28,6 +28,7 @@ #include "filterx/object-list-interface.h" #include "filterx/object-dict-interface.h" #include "filterx/expr-function.h" +#include "filterx/filterx-object-istype.h" #include "compat/pcre.h" #include "scratch-buffers.h" diff --git a/lib/filterx/filterx-metrics-labels.c b/lib/filterx/filterx-metrics-labels.c index 42b2a7d9c..fd45bfbed 100644 --- a/lib/filterx/filterx-metrics-labels.c +++ b/lib/filterx/filterx-metrics-labels.c @@ -24,6 +24,7 @@ #include "filterx-metrics-labels.h" #include "filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "expr-literal.h" #include "expr-literal-generator.h" #include "object-string.h" diff --git a/lib/filterx/filterx-metrics.c b/lib/filterx/filterx-metrics.c index b64fcb0c1..c7bf1d27c 100644 --- a/lib/filterx/filterx-metrics.c +++ b/lib/filterx/filterx-metrics.c @@ -28,6 +28,7 @@ #include "object-extractor.h" #include "object-string.h" #include "filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "stats/stats-cluster-single.h" #include "stats/stats-registry.h" #include "metrics/dyn-metrics-cache.h" diff --git a/lib/filterx/filterx-object-istype.h b/lib/filterx/filterx-object-istype.h new file mode 100644 index 000000000..3b97c2f2f --- /dev/null +++ b/lib/filterx/filterx-object-istype.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2023 Balazs Scheidler + * Copyright (c) 2024 László Várady + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef FILTERX_OBJECT_ISTYPE_H +#define FILTERX_OBJECT_ISTYPE_H + +#include "syslog-ng.h" +#include "filterx/filterx-object.h" +#include "filterx/filterx-ref.h" + +static inline gboolean +filterx_object_is_type(FilterXObject *object, FilterXType *type) +{ + FilterXType *self_type = object->type; + while (self_type) + { + if (type == self_type) + return TRUE; + self_type = self_type->super_type; + } + return FALSE; +} diff --git a/lib/filterx/filterx-object.h b/lib/filterx/filterx-object.h index f7e5ea683..a94445334 100644 --- a/lib/filterx/filterx-object.h +++ b/lib/filterx/filterx-object.h @@ -104,19 +104,6 @@ void filterx_object_unfreeze_and_free(FilterXObject *self); void filterx_object_init_instance(FilterXObject *self, FilterXType *type); void filterx_object_free_method(FilterXObject *self); -static inline gboolean -filterx_object_is_type(FilterXObject *object, FilterXType *type) -{ - FilterXType *self_type = object->type; - while (self_type) - { - if (type == self_type) - return TRUE; - self_type = self_type->super_type; - } - return FALSE; -} - static inline void filterx_object_make_readonly(FilterXObject *self) { diff --git a/lib/filterx/filterx-ref.c b/lib/filterx/filterx-ref.c index 083ae4c5f..abf01ea48 100644 --- a/lib/filterx/filterx-ref.c +++ b/lib/filterx/filterx-ref.c @@ -22,6 +22,7 @@ */ #include "filterx-ref.h" +#include "filterx/filterx-object-istype.h" static FilterXObject * _filterx_ref_clone(FilterXObject *s) diff --git a/lib/filterx/func-flatten.c b/lib/filterx/func-flatten.c index e78954487..1cce79284 100644 --- a/lib/filterx/func-flatten.c +++ b/lib/filterx/func-flatten.c @@ -27,6 +27,7 @@ #include "filterx/object-primitive.h" #include "filterx/object-string.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #define FILTERX_FUNC_FLATTEN_USAGE "Usage: flatten(dict, separator=\".\")" diff --git a/lib/filterx/func-istype.c b/lib/filterx/func-istype.c index f06a7f06a..21a030303 100644 --- a/lib/filterx/func-istype.c +++ b/lib/filterx/func-istype.c @@ -28,6 +28,7 @@ #include "filterx/expr-literal.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-globals.h" +#include "filterx/filterx-object-istype.h" #define FILTERX_FUNC_ISTYPE_USAGE "Usage: istype(object, type_str)" diff --git a/lib/filterx/func-str.c b/lib/filterx/func-str.c index 7840cb3ca..8374fd5b1 100644 --- a/lib/filterx/func-str.c +++ b/lib/filterx/func-str.c @@ -31,6 +31,7 @@ #include "filterx/object-string.h" #include "filterx/object-message-value.h" #include "filterx/object-list-interface.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #define FILTERX_FUNC_STARTSWITH_USAGE "Usage: startswith(string, prefix, ignorecase=true)" \ diff --git a/lib/filterx/func-unset-empties.c b/lib/filterx/func-unset-empties.c index fff1affca..aaabf22a0 100644 --- a/lib/filterx/func-unset-empties.c +++ b/lib/filterx/func-unset-empties.c @@ -35,6 +35,7 @@ #include "filterx/func-flags.h" #include "filterx/expr-literal-generator.h" #include "filterx/expr-literal.h" +#include "filterx/filterx-object-istype.h" #define FILTERX_FUNC_UNSET_EMPTIES_USAGE "Usage: unset_empties(object, " \ FILTERX_FUNC_UNSET_EMPTIES_ARG_NAME_RECURSIVE"=bool, " \ diff --git a/lib/filterx/object-datetime.c b/lib/filterx/object-datetime.c index d7e26243d..f4f1884cf 100644 --- a/lib/filterx/object-datetime.c +++ b/lib/filterx/object-datetime.c @@ -37,6 +37,7 @@ #include "generic-number.h" #include "filterx/filterx-eval.h" #include "filterx-globals.h" +#include "filterx/filterx-object-istype.h" #include "compat/json.h" #define FILTERX_FUNC_STRPTIME_USAGE "Usage: strptime(time_str, format_str_1, ..., format_str_N)" diff --git a/lib/filterx/object-dict-interface.c b/lib/filterx/object-dict-interface.c index aa185400f..abceaf57b 100644 --- a/lib/filterx/object-dict-interface.c +++ b/lib/filterx/object-dict-interface.c @@ -25,6 +25,7 @@ #include "filterx/object-extractor.h" #include "filterx/object-string.h" #include "filterx/object-json.h" +#include "filterx/filterx-object-istype.h" #include "str-utils.h" gboolean diff --git a/lib/filterx/object-extractor.c b/lib/filterx/object-extractor.c index 13c5019b3..4ddaa12e3 100644 --- a/lib/filterx/object-extractor.c +++ b/lib/filterx/object-extractor.c @@ -28,6 +28,7 @@ #include "filterx/object-datetime.h" #include "filterx/object-null.h" #include "filterx/object-json.h" +#include "filterx/filterx-object-istype.h" gboolean filterx_object_extract_string_ref(FilterXObject *obj, const gchar **value, gsize *len) diff --git a/lib/filterx/object-json-array.c b/lib/filterx/object-json-array.c index b0c075ef2..bac4f90f1 100644 --- a/lib/filterx/object-json-array.c +++ b/lib/filterx/object-json-array.c @@ -30,6 +30,7 @@ #include "filterx/object-list-interface.h" #include "filterx/expr-function.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "logmsg/type-hinting.h" #include "str-repr/encode.h" diff --git a/lib/filterx/object-json-object.c b/lib/filterx/object-json-object.c index b224a229d..dc3743585 100644 --- a/lib/filterx/object-json-object.c +++ b/lib/filterx/object-json-object.c @@ -28,6 +28,7 @@ #include "filterx/object-string.h" #include "filterx/filterx-weakrefs.h" #include "filterx/object-dict-interface.h" +#include "filterx/filterx-object-istype.h" #include "syslog-ng.h" #include "str-utils.h" #include "logmsg/type-hinting.h" diff --git a/lib/filterx/object-json.c b/lib/filterx/object-json.c index f7be6025e..9f3b5ab57 100644 --- a/lib/filterx/object-json.c +++ b/lib/filterx/object-json.c @@ -29,6 +29,7 @@ #include "filterx/object-list-interface.h" #include "filterx/object-message-value.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scanner/list-scanner/list-scanner.h" #include "str-repr/encode.h" diff --git a/lib/filterx/object-list-interface.c b/lib/filterx/object-list-interface.c index 497ca0e99..fe28c0156 100644 --- a/lib/filterx/object-list-interface.c +++ b/lib/filterx/object-list-interface.c @@ -24,6 +24,7 @@ #include "filterx/object-list-interface.h" #include "filterx/object-primitive.h" #include "filterx/object-json.h" +#include "filterx/filterx-object-istype.h" FilterXObject * filterx_list_get_subscript(FilterXObject *s, gint64 index) diff --git a/lib/filterx/object-primitive.c b/lib/filterx/object-primitive.c index 569293e36..c0e703026 100644 --- a/lib/filterx/object-primitive.c +++ b/lib/filterx/object-primitive.c @@ -29,6 +29,7 @@ #include "plugin.h" #include "cfg.h" #include "filterx-globals.h" +#include "filterx/filterx-object-istype.h" #include "str-utils.h" #include "timeutils/misc.h" diff --git a/lib/filterx/object-primitive.h b/lib/filterx/object-primitive.h index e942b1a42..f68ff48ea 100644 --- a/lib/filterx/object-primitive.h +++ b/lib/filterx/object-primitive.h @@ -24,6 +24,7 @@ #define FILTERX_PRIMITIVE_H_INCLUDED #include "filterx/filterx-object.h" +#include "filterx/filterx-object-istype.h" #include "generic-number.h" FILTERX_DECLARE_TYPE(primitive); diff --git a/lib/filterx/object-string.c b/lib/filterx/object-string.c index 9f76ca529..23439fc87 100644 --- a/lib/filterx/object-string.c +++ b/lib/filterx/object-string.c @@ -25,6 +25,7 @@ #include "str-utils.h" #include "scratch-buffers.h" #include "filterx-globals.h" +#include "filterx/filterx-object-istype.h" #include "utf8utils.h" #include "str-format.h" #include "str-utils.h" diff --git a/lib/filterx/tests/test_builtin_functions.c b/lib/filterx/tests/test_builtin_functions.c index 8fa622c25..cdeb1f8e6 100644 --- a/lib/filterx/tests/test_builtin_functions.c +++ b/lib/filterx/tests/test_builtin_functions.c @@ -38,6 +38,7 @@ #include "filterx/expr-template.h" #include "filterx/expr-variable.h" #include "filterx/filterx-private.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_compound.c b/lib/filterx/tests/test_expr_compound.c index c999f3247..eb1f0669c 100644 --- a/lib/filterx/tests/test_expr_compound.c +++ b/lib/filterx/tests/test_expr_compound.c @@ -31,6 +31,7 @@ #include "filterx/expr-variable.h" #include "filterx/expr-assign.h" #include "filterx/expr-literal.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_condition.c b/lib/filterx/tests/test_expr_condition.c index 340a65424..26e3700e1 100644 --- a/lib/filterx/tests/test_expr_condition.c +++ b/lib/filterx/tests/test_expr_condition.c @@ -39,6 +39,7 @@ #include "filterx/object-datetime.h" #include "filterx/object-message-value.h" #include "filterx/filterx-private.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_function.c b/lib/filterx/tests/test_expr_function.c index 46315e3c5..d48c67284 100644 --- a/lib/filterx/tests/test_expr_function.c +++ b/lib/filterx/tests/test_expr_function.c @@ -39,6 +39,7 @@ #include "filterx/expr-variable.h" #include "filterx/expr-function.h" #include "filterx/filterx-private.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_null_coalesce.c b/lib/filterx/tests/test_expr_null_coalesce.c index 342889e77..19f5fcf0b 100644 --- a/lib/filterx/tests/test_expr_null_coalesce.c +++ b/lib/filterx/tests/test_expr_null_coalesce.c @@ -40,6 +40,7 @@ #include "filterx/filterx-eval.h" #include "filterx/func-len.h" #include "filterx/expr-function.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_plus.c b/lib/filterx/tests/test_expr_plus.c index d1be7619c..17cecb2aa 100644 --- a/lib/filterx/tests/test_expr_plus.c +++ b/lib/filterx/tests/test_expr_plus.c @@ -44,6 +44,7 @@ #include "filterx/object-json.h" #include "filterx/object-list-interface.h" #include "filterx/object-dict-interface.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_plus_generator.c b/lib/filterx/tests/test_expr_plus_generator.c index f5c560f8f..ff89937d8 100644 --- a/lib/filterx/tests/test_expr_plus_generator.c +++ b/lib/filterx/tests/test_expr_plus_generator.c @@ -46,6 +46,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/expr-generator.h" #include "filterx/expr-literal-generator.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_expr_regexp.c b/lib/filterx/tests/test_expr_regexp.c index f50c06eb9..b60009a6c 100644 --- a/lib/filterx/tests/test_expr_regexp.c +++ b/lib/filterx/tests/test_expr_regexp.c @@ -30,6 +30,7 @@ #include "filterx/object-primitive.h" #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" #include "compat/pcre.h" diff --git a/lib/filterx/tests/test_filterx_expr.c b/lib/filterx/tests/test_filterx_expr.c index beeaa14de..d45a6f5c1 100644 --- a/lib/filterx/tests/test_filterx_expr.c +++ b/lib/filterx/tests/test_filterx_expr.c @@ -40,6 +40,7 @@ #include "filterx/expr-getattr.h" #include "filterx/expr-set-subscript.h" #include "filterx/expr-get-subscript.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_boolean.c b/lib/filterx/tests/test_object_boolean.c index a45a791dd..110220892 100644 --- a/lib/filterx/tests/test_object_boolean.c +++ b/lib/filterx/tests/test_object_boolean.c @@ -26,6 +26,7 @@ #include "filterx/object-string.h" #include "filterx/object-null.h" #include "filterx/object-primitive.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_bytes.c b/lib/filterx/tests/test_object_bytes.c index 265cd537c..caf22217e 100644 --- a/lib/filterx/tests/test_object_bytes.c +++ b/lib/filterx/tests/test_object_bytes.c @@ -23,6 +23,7 @@ #include #include "filterx/object-string.h" #include "filterx/object-null.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_datetime.c b/lib/filterx/tests/test_object_datetime.c index 947091761..ed3472e89 100644 --- a/lib/filterx/tests/test_object_datetime.c +++ b/lib/filterx/tests/test_object_datetime.c @@ -34,6 +34,7 @@ #include "filterx/expr-function.h" #include "filterx/expr-get-subscript.h" #include "filterx/filterx-private.h" +#include "filterx/filterx-object-istype.h" Test(filterx_datetime, test_filterx_object_datetime_marshals_to_the_stored_values) diff --git a/lib/filterx/tests/test_object_double.c b/lib/filterx/tests/test_object_double.c index 828eba026..c64ae940a 100644 --- a/lib/filterx/tests/test_object_double.c +++ b/lib/filterx/tests/test_object_double.c @@ -27,6 +27,7 @@ #include "filterx/object-null.h" #include "filterx/object-primitive.h" #include "filterx/object-datetime.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_integer.c b/lib/filterx/tests/test_object_integer.c index 336ec6cad..880cf813b 100644 --- a/lib/filterx/tests/test_object_integer.c +++ b/lib/filterx/tests/test_object_integer.c @@ -27,6 +27,7 @@ #include "filterx/object-null.h" #include "filterx/object-primitive.h" #include "filterx/object-datetime.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_json.c b/lib/filterx/tests/test_object_json.c index acabffc51..84b070792 100644 --- a/lib/filterx/tests/test_object_json.c +++ b/lib/filterx/tests/test_object_json.c @@ -27,6 +27,7 @@ #include "filterx/object-string.h" #include "filterx/object-message-value.h" #include "filterx/expr-function.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_protobuf.c b/lib/filterx/tests/test_object_protobuf.c index 024c01fcf..b93cb1547 100644 --- a/lib/filterx/tests/test_object_protobuf.c +++ b/lib/filterx/tests/test_object_protobuf.c @@ -23,6 +23,7 @@ #include #include "filterx/object-string.h" #include "filterx/object-null.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/lib/filterx/tests/test_object_string.c b/lib/filterx/tests/test_object_string.c index 57cada90e..2b068cfda 100644 --- a/lib/filterx/tests/test_object_string.c +++ b/lib/filterx/tests/test_object_string.c @@ -25,6 +25,7 @@ #include "filterx/object-string.h" #include "filterx/object-null.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/modules/cef/tests/test-filterx-function-parse-cef.c b/modules/cef/tests/test-filterx-function-parse-cef.c index ceaa97601..5b3d2a6a2 100644 --- a/modules/cef/tests/test-filterx-function-parse-cef.c +++ b/modules/cef/tests/test-filterx-function-parse-cef.c @@ -37,6 +37,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/object-primitive.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" FilterXExpr * diff --git a/modules/cef/tests/test-filterx-function-parse-leef.c b/modules/cef/tests/test-filterx-function-parse-leef.c index e880f8b7f..0bf3cf774 100644 --- a/modules/cef/tests/test-filterx-function-parse-leef.c +++ b/modules/cef/tests/test-filterx-function-parse-leef.c @@ -37,6 +37,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/object-primitive.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" FilterXExpr * diff --git a/modules/csvparser/filterx-func-format-csv.c b/modules/csvparser/filterx-func-format-csv.c index 9d51f00a7..534584171 100644 --- a/modules/csvparser/filterx-func-format-csv.c +++ b/modules/csvparser/filterx-func-format-csv.c @@ -27,6 +27,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "utf8utils.h" diff --git a/modules/csvparser/filterx-func-parse-csv.c b/modules/csvparser/filterx-func-parse-csv.c index 582c3cecb..e7e68ffe6 100644 --- a/modules/csvparser/filterx-func-parse-csv.c +++ b/modules/csvparser/filterx-func-parse-csv.c @@ -36,6 +36,7 @@ #include "filterx/filterx-object.h" #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" +#include "filterx/filterx-object-istype.h" #include "scanner/csv-scanner/csv-scanner.h" #include "parser/parser-expr.h" diff --git a/modules/csvparser/tests/test_filterx_func_parse_csv.c b/modules/csvparser/tests/test_filterx_func_parse_csv.c index 32011f838..0b6bd4834 100644 --- a/modules/csvparser/tests/test_filterx_func_parse_csv.c +++ b/modules/csvparser/tests/test_filterx_func_parse_csv.c @@ -33,6 +33,7 @@ #include "filterx/object-list-interface.h" #include "scanner/csv-scanner/csv-scanner.h" #include "filterx/object-primitive.h" +#include "filterx/filterx-object-istype.h" static FilterXObject * _generate_string_list(const gchar *elts, ...) diff --git a/modules/grpc/otel/filterx/object-otel-array.cpp b/modules/grpc/otel/filterx/object-otel-array.cpp index f572bd15f..2a55a04ed 100644 --- a/modules/grpc/otel/filterx/object-otel-array.cpp +++ b/modules/grpc/otel/filterx/object-otel-array.cpp @@ -28,6 +28,7 @@ #include "filterx/object-primitive.h" #include "filterx/object-string.h" #include "filterx/object-message-value.h" +#include "filterx/filterx-object-istype.h" #include "compat/cpp-end.h" #include diff --git a/modules/grpc/otel/filterx/object-otel-kvlist.cpp b/modules/grpc/otel/filterx/object-otel-kvlist.cpp index 3f0644e84..5489ebac2 100644 --- a/modules/grpc/otel/filterx/object-otel-kvlist.cpp +++ b/modules/grpc/otel/filterx/object-otel-kvlist.cpp @@ -28,6 +28,7 @@ #include "filterx/object-string.h" #include "filterx/object-null.h" #include "filterx/object-message-value.h" +#include "filterx/filterx-object-istype.h" #include "compat/cpp-end.h" #include diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 19789bd40..f6fff7de3 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -30,6 +30,7 @@ #include "filterx/object-string.h" #include "filterx/object-datetime.h" #include "filterx/object-primitive.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "generic-number.h" diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 58ac31e05..8f5d1cc31 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -26,6 +26,7 @@ #include "compat/cpp-start.h" #include "filterx/object-extractor.h" #include "filterx/object-string.h" +#include "filterx/filterx-object-istype.h" #include "compat/cpp-end.h" #include diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index e35bf9897..ec421abd9 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -26,6 +26,7 @@ #include "compat/cpp-start.h" #include "filterx/object-extractor.h" #include "filterx/object-string.h" +#include "filterx/filterx-object-istype.h" #include "compat/cpp-end.h" #include diff --git a/modules/grpc/otel/filterx/otel-field.cpp b/modules/grpc/otel/filterx/otel-field.cpp index 7989318bf..9bd0f46aa 100644 --- a/modules/grpc/otel/filterx/otel-field.cpp +++ b/modules/grpc/otel/filterx/otel-field.cpp @@ -37,6 +37,7 @@ #include "filterx/object-message-value.h" #include "filterx/object-json.h" #include "filterx/object-null.h" +#include "filterx/filterx-object-istype.h" #include "compat/cpp-end.h" #include "opentelemetry/proto/logs/v1/logs.pb.h" diff --git a/modules/grpc/otel/filterx/protobuf-field.cpp b/modules/grpc/otel/filterx/protobuf-field.cpp index dbf65f1c1..4a3555e89 100644 --- a/modules/grpc/otel/filterx/protobuf-field.cpp +++ b/modules/grpc/otel/filterx/protobuf-field.cpp @@ -33,6 +33,7 @@ #include "generic-number.h" #include "filterx/object-json.h" #include "filterx/object-null.h" +#include "filterx/filterx-object-istype.h" #include "compat/cpp-end.h" #include diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 7fe15b58f..8cf5890e4 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -29,6 +29,7 @@ #include "compat/cpp-start.h" #include "filterx/object-string.h" #include "filterx/object-primitive.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "cfg.h" #include "compat/cpp-end.h" diff --git a/modules/json/filterx-cache-json-file.c b/modules/json/filterx-cache-json-file.c index 53a548b03..1d8e6b951 100644 --- a/modules/json/filterx-cache-json-file.c +++ b/modules/json/filterx-cache-json-file.c @@ -27,6 +27,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/expr-literal.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "compat/json.h" diff --git a/modules/json/filterx-format-json.c b/modules/json/filterx-format-json.c index 0e07bf4d5..7969e86fe 100644 --- a/modules/json/filterx-format-json.c +++ b/modules/json/filterx-format-json.c @@ -29,6 +29,7 @@ #include "filterx/object-json.h" #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "utf8utils.h" diff --git a/modules/json/tests/test_filterx_format_json.c b/modules/json/tests/test_filterx_format_json.c index a417ae79b..d0a9fd826 100644 --- a/modules/json/tests/test_filterx_format_json.c +++ b/modules/json/tests/test_filterx_format_json.c @@ -31,6 +31,7 @@ #include "filterx/object-json.h" #include "filterx/object-message-value.h" #include "filterx/object-list-interface.h" +#include "filterx/filterx-object-istype.h" #include "apphook.h" #include "scratch-buffers.h" diff --git a/modules/kvformat/filterx-func-format-kv.c b/modules/kvformat/filterx-func-format-kv.c index e5c4a1f2b..a7bbf61e6 100644 --- a/modules/kvformat/filterx-func-format-kv.c +++ b/modules/kvformat/filterx-func-format-kv.c @@ -27,6 +27,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "utf8utils.h" diff --git a/modules/kvformat/tests/test_filterx_func_parse_kv.c b/modules/kvformat/tests/test_filterx_func_parse_kv.c index 72b95ac02..d340f10f8 100644 --- a/modules/kvformat/tests/test_filterx_func_parse_kv.c +++ b/modules/kvformat/tests/test_filterx_func_parse_kv.c @@ -32,6 +32,7 @@ #include "filterx/expr-literal.h" #include "filterx/object-json.h" #include "filterx-func-parse-kv.h" +#include "filterx/filterx-object-istype.h" FilterXExpr * diff --git a/modules/xml/filterx-parse-windows-eventlog-xml.c b/modules/xml/filterx-parse-windows-eventlog-xml.c index 900948584..f76af6aff 100644 --- a/modules/xml/filterx-parse-windows-eventlog-xml.c +++ b/modules/xml/filterx-parse-windows-eventlog-xml.c @@ -24,6 +24,7 @@ #include "filterx-parse-windows-eventlog-xml.h" #include "filterx/object-string.h" #include "filterx/object-dict-interface.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" static void _set_error(GError **error, const gchar *format, ...) G_GNUC_PRINTF(2, 0); diff --git a/modules/xml/filterx-parse-xml.c b/modules/xml/filterx-parse-xml.c index c62b8b45e..88d3d7971 100644 --- a/modules/xml/filterx-parse-xml.c +++ b/modules/xml/filterx-parse-xml.c @@ -27,6 +27,7 @@ #include "filterx/object-list-interface.h" #include "filterx/object-dict-interface.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include diff --git a/modules/xml/tests/test_filterx_parse_windows_eventlog_xml.c b/modules/xml/tests/test_filterx_parse_windows_eventlog_xml.c index 53a37dbe5..0c34ba5a6 100644 --- a/modules/xml/tests/test_filterx_parse_windows_eventlog_xml.c +++ b/modules/xml/tests/test_filterx_parse_windows_eventlog_xml.c @@ -27,6 +27,7 @@ #include "filterx/object-string.h" #include "filterx/object-json.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "apphook.h" #include "cfg.h" diff --git a/modules/xml/tests/test_filterx_parse_xml.c b/modules/xml/tests/test_filterx_parse_xml.c index 9c477e702..a872da181 100644 --- a/modules/xml/tests/test_filterx_parse_xml.c +++ b/modules/xml/tests/test_filterx_parse_xml.c @@ -27,6 +27,7 @@ #include "filterx/object-string.h" #include "filterx/object-json.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-object-istype.h" #include "scratch-buffers.h" #include "apphook.h" #include "cfg.h" From efe9eb8317af2c7580aecec26f1af49b84223ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 11 Oct 2024 16:13:54 +0200 Subject: [PATCH 09/16] filterx-istype: add FilterXRef safety check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FilterXRef can mimic the functionality of its referenced value, but only through virtual methods. filterx_object_is_type() is usually followed by an implicit type cast, which is impossible to implement transparently (without the unwrap() call), so this safety check has been added. This is far from ideal, but doing downcasts is the original and real problem here. Signed-off-by: László Várady --- lib/filterx/filterx-object-istype.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/filterx/filterx-object-istype.h b/lib/filterx/filterx-object-istype.h index 3b97c2f2f..0f317ed98 100644 --- a/lib/filterx/filterx-object-istype.h +++ b/lib/filterx/filterx-object-istype.h @@ -31,7 +31,7 @@ #include "filterx/filterx-ref.h" static inline gboolean -filterx_object_is_type(FilterXObject *object, FilterXType *type) +_filterx_object_is_type(FilterXObject *object, FilterXType *type) { FilterXType *self_type = object->type; while (self_type) @@ -42,3 +42,14 @@ filterx_object_is_type(FilterXObject *object, FilterXType *type) } return FALSE; } + +static inline gboolean +filterx_object_is_type(FilterXObject *object, FilterXType *type) +{ +#if SYSLOG_NG_ENABLE_DEBUG + g_assert(!(_filterx_type_is_referenceable(type) && _filterx_object_is_type(object, &FILTERX_TYPE_NAME(ref))) + && "filterx_ref_unwrap() must be used before comparing to mutable types"); +#endif + + return _filterx_object_is_type(object, type); +} From 37bc73f699b1febae1b61e76c57cfbcae1bf53e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sun, 13 Oct 2024 23:45:55 +0200 Subject: [PATCH 10/16] filterx-istype: add filterx_assert_not_ref() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/filterx-object-istype.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/filterx/filterx-object-istype.h b/lib/filterx/filterx-object-istype.h index 0f317ed98..0ac9ff4e3 100644 --- a/lib/filterx/filterx-object-istype.h +++ b/lib/filterx/filterx-object-istype.h @@ -53,3 +53,15 @@ filterx_object_is_type(FilterXObject *object, FilterXType *type) return _filterx_object_is_type(object, type); } + +#if SYSLOG_NG_ENABLE_DEBUG +static inline void +filterx_assert_not_ref(FilterXObject *object) +{ + g_assert(!_filterx_object_is_type(object, &FILTERX_TYPE_NAME(ref))); +} +#else +#define filterx_assert_not_ref(o) +#endif + +#endif From 1bb933389825093f4fea80ea48d27996352f01ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sun, 13 Oct 2024 17:06:33 +0200 Subject: [PATCH 11/16] filterx: remove always-true condition from _add() of dict and list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "lhs_object" is "self". Signed-off-by: László Várady --- lib/filterx/object-dict-interface.c | 3 +-- lib/filterx/object-list-interface.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/filterx/object-dict-interface.c b/lib/filterx/object-dict-interface.c index abceaf57b..b677cbf38 100644 --- a/lib/filterx/object-dict-interface.c +++ b/lib/filterx/object-dict-interface.c @@ -215,8 +215,7 @@ filterx_dict_init_instance(FilterXDict *self, FilterXType *type) static FilterXObject * _add(FilterXObject *lhs_object, FilterXObject *rhs_object) { - if (!filterx_object_is_type(lhs_object, &FILTERX_TYPE_NAME(dict)) || - !filterx_object_is_type(rhs_object, &FILTERX_TYPE_NAME(dict))) + if (!filterx_object_is_type(rhs_object, &FILTERX_TYPE_NAME(dict))) return NULL; FilterXObject *cloned = filterx_object_clone(lhs_object); diff --git a/lib/filterx/object-list-interface.c b/lib/filterx/object-list-interface.c index fe28c0156..28c5ee6b2 100644 --- a/lib/filterx/object-list-interface.c +++ b/lib/filterx/object-list-interface.c @@ -302,8 +302,7 @@ filterx_list_init_instance(FilterXList *self, FilterXType *type) static FilterXObject * _add(FilterXObject *lhs_object, FilterXObject *rhs_object) { - if (!filterx_object_is_type(lhs_object, &FILTERX_TYPE_NAME(list)) || - !filterx_object_is_type(rhs_object, &FILTERX_TYPE_NAME(list))) + if (!filterx_object_is_type(rhs_object, &FILTERX_TYPE_NAME(list))) return NULL; FilterXObject *cloned = filterx_object_clone(lhs_object); From 9e7d312eea33e7f4f299fd7825c261d83b771878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Thu, 3 Oct 2024 00:27:55 +0200 Subject: [PATCH 12/16] filterx: create references where objects are assigned to names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `FilterXRef` is created only for mutable objects, and only when they are needed: around assignment-like operators; this and its `FilterXObject` behavior is the key to avoid polluting all code paths with references. Full CoW on subscript and attr operators (`.something`, `["something"]`) of JSON objects is currently not implemented. I've tried to make it work, but it requires a complete redesign of how we cache JSON subobjects (or alternatively, we need an internal representation for dicts and lists without using the json-c library). This will be implemented later. For now, such operators make unnecessary copies (CoW only partially works on them). Signed-off-by: László Várady --- lib/filterx/expr-assign.c | 9 +++++++++ lib/filterx/expr-literal-generator.c | 1 + lib/filterx/expr-set-subscript.c | 8 ++++++++ lib/filterx/expr-setattr.c | 8 ++++++++ 4 files changed, 26 insertions(+) diff --git a/lib/filterx/expr-assign.c b/lib/filterx/expr-assign.c index c0bda7fd3..04c20e4c4 100644 --- a/lib/filterx/expr-assign.c +++ b/lib/filterx/expr-assign.c @@ -22,6 +22,8 @@ */ #include "filterx/expr-assign.h" #include "filterx/object-primitive.h" +#include "filterx/filterx-ref.h" +#include "filterx/object-json.h" #include "scratch-buffers.h" static FilterXObject * @@ -34,6 +36,13 @@ _eval(FilterXExpr *s) if (!value) return NULL; + /* TODO: create ref unconditionally after implementing hierarchical CoW for JSON types + * (or after creating our own dict/list repr) */ + if (!value->weak_referenced) + { + value = filterx_ref_new(value); + } + if (!filterx_expr_assign(self->lhs, value)) { filterx_object_unref(value); diff --git a/lib/filterx/expr-literal-generator.c b/lib/filterx/expr-literal-generator.c index b970be48f..61eea5da5 100644 --- a/lib/filterx/expr-literal-generator.c +++ b/lib/filterx/expr-literal-generator.c @@ -23,6 +23,7 @@ #include "filterx/expr-literal-generator.h" #include "filterx/object-primitive.h" +#include "filterx/filterx-ref.h" struct FilterXLiteralGeneratorElem_ { diff --git a/lib/filterx/expr-set-subscript.c b/lib/filterx/expr-set-subscript.c index 9f6e7149e..3901d164c 100644 --- a/lib/filterx/expr-set-subscript.c +++ b/lib/filterx/expr-set-subscript.c @@ -23,6 +23,7 @@ #include "filterx/expr-set-subscript.h" #include "filterx/object-primitive.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" typedef struct _FilterXSetSubscript @@ -66,6 +67,13 @@ _eval(FilterXExpr *s) if (!new_value) goto exit; + /* TODO: create ref unconditionally after implementing hierarchical CoW for JSON types + * (or after creating our own dict/list repr) */ + if (!new_value->weak_referenced) + { + new_value = filterx_ref_new(new_value); + } + FilterXObject *cloned = filterx_object_clone(new_value); filterx_object_unref(new_value); diff --git a/lib/filterx/expr-setattr.c b/lib/filterx/expr-setattr.c index c88a2d479..bd5084292 100644 --- a/lib/filterx/expr-setattr.c +++ b/lib/filterx/expr-setattr.c @@ -24,6 +24,7 @@ #include "filterx/object-primitive.h" #include "filterx/object-string.h" #include "filterx/filterx-eval.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" typedef struct _FilterXSetAttr @@ -54,6 +55,13 @@ _eval(FilterXExpr *s) if (!new_value) goto exit; + /* TODO: create ref unconditionally after implementing hierarchical CoW for JSON types + * (or after creating our own dict/list repr) */ + if (!new_value->weak_referenced) + { + new_value = filterx_ref_new(new_value); + } + FilterXObject *cloned = filterx_object_clone(new_value); filterx_object_unref(new_value); From aea2b1039cd2931265cbf0715ccf1d072dd4752a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sun, 13 Oct 2024 23:42:43 +0200 Subject: [PATCH 13/16] filterx: unwrap references where downcasts are used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FilterXRef can mimic the functionality of its referenced value, but only through virtual methods. Where (implicit) downcasts are used, it is impossible to be transparent about references, so these unwrap methods have to be used. This is far from ideal, but doing downcasts based on is_type() is the original and real problem here. In such cases, `filterx_ref_unwrap()` must be called to extract the original object. The extracted object must only be used locally (short-term, without referencing or storing it anywhere). Signed-off-by: László Várady --- lib/filterx/expr-comparison.c | 17 +++++++--- lib/filterx/expr-plus-generator.c | 3 ++ lib/filterx/expr-regexp.c | 3 ++ lib/filterx/filterx-metrics-labels.c | 5 +-- lib/filterx/func-flatten.c | 13 ++++--- lib/filterx/func-istype.c | 5 ++- lib/filterx/func-str.c | 6 ++-- lib/filterx/func-unset-empties.c | 23 +++++++++---- lib/filterx/object-dict-interface.c | 3 ++ lib/filterx/object-json-array.c | 11 +++--- lib/filterx/object-json-object.c | 8 +++-- lib/filterx/object-json.c | 18 ++++++---- lib/filterx/object-list-interface.c | 4 +++ lib/filterx/tests/test_filterx_expr.c | 7 ++-- modules/csvparser/filterx-func-format-csv.c | 22 +++++++----- modules/csvparser/filterx-func-parse-csv.c | 17 ++++++---- .../grpc/otel/filterx/object-otel-array.cpp | 8 +++-- .../grpc/otel/filterx/object-otel-kvlist.cpp | 7 ++-- .../otel/filterx/object-otel-logrecord.cpp | 6 ++-- .../otel/filterx/object-otel-resource.cpp | 6 ++-- .../grpc/otel/filterx/object-otel-scope.cpp | 6 ++-- modules/grpc/otel/filterx/otel-field.cpp | 6 ++-- modules/grpc/otel/filterx/protobuf-field.cpp | 2 ++ modules/json/filterx-cache-json-file.c | 2 ++ modules/json/filterx-format-json.c | 10 +++--- modules/kvformat/filterx-func-format-kv.c | 19 ++++++----- .../xml/filterx-parse-windows-eventlog-xml.c | 15 +++++--- modules/xml/filterx-parse-xml.c | 34 ++++++++++++------- 28 files changed, 192 insertions(+), 94 deletions(-) diff --git a/lib/filterx/expr-comparison.c b/lib/filterx/expr-comparison.c index 5e6d1bf5c..2b4e5a226 100644 --- a/lib/filterx/expr-comparison.c +++ b/lib/filterx/expr-comparison.c @@ -32,6 +32,7 @@ #include "filterx/object-datetime.h" #include "filterx/object-message-value.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "object-primitive.h" #include "generic-number.h" #include "parse-number.h" @@ -141,6 +142,9 @@ _evaluate_as_num(FilterXObject *lhs, FilterXObject *rhs, gint operator) static gboolean _evaluate_type_aware(FilterXObject *lhs, FilterXObject *rhs, gint operator) { + filterx_assert_not_ref(lhs); + filterx_assert_not_ref(rhs); + if (lhs->type == rhs->type && (filterx_object_is_type(lhs, &FILTERX_TYPE_NAME(string)) || filterx_object_is_type(lhs, &FILTERX_TYPE_NAME(bytes)) || @@ -165,6 +169,9 @@ _evaluate_type_aware(FilterXObject *lhs, FilterXObject *rhs, gint operator) static gboolean _evaluate_type_and_value_based(FilterXObject *lhs, FilterXObject *rhs, gint operator) { + filterx_assert_not_ref(lhs); + filterx_assert_not_ref(rhs); + if (operator == FCMPX_EQ) { if (lhs->type != rhs->type) @@ -202,15 +209,17 @@ _eval(FilterXExpr *s) return NULL; } + FilterXObject *lhs = filterx_ref_unwrap_ro(lhs_object); + FilterXObject *rhs = filterx_ref_unwrap_ro(rhs_object); gboolean result = TRUE; if (compare_mode & FCMPX_TYPE_AWARE) - result = _evaluate_type_aware(lhs_object, rhs_object, operator); + result = _evaluate_type_aware(lhs, rhs, operator); else if (compare_mode & FCMPX_STRING_BASED) - result = _evaluate_as_string(lhs_object, rhs_object, operator); + result = _evaluate_as_string(lhs, rhs, operator); else if (compare_mode & FCMPX_NUM_BASED) - result = _evaluate_as_num(lhs_object, rhs_object, operator); + result = _evaluate_as_num(lhs, rhs, operator); else if (compare_mode & FCMPX_TYPE_AND_VALUE_BASED) - result = _evaluate_type_and_value_based(lhs_object, rhs_object, operator); + result = _evaluate_type_and_value_based(lhs, rhs, operator); else g_assert_not_reached(); diff --git a/lib/filterx/expr-plus-generator.c b/lib/filterx/expr-plus-generator.c index 8919197d2..f9fda257f 100644 --- a/lib/filterx/expr-plus-generator.c +++ b/lib/filterx/expr-plus-generator.c @@ -25,6 +25,7 @@ #include "object-string.h" #include "filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include "expr-generator.h" #include "object-list-interface.h" @@ -40,6 +41,8 @@ typedef struct FilterXOperatorPlusGenerator static gboolean _generate_obj(FilterXOperatorPlusGenerator *self, FilterXObject *obj, FilterXObject *fillable) { + fillable = filterx_ref_unwrap_rw(fillable); + if (filterx_object_is_type(fillable, &FILTERX_TYPE_NAME(list))) return filterx_list_merge(fillable, obj); diff --git a/lib/filterx/expr-regexp.c b/lib/filterx/expr-regexp.c index f18628a51..821956bca 100644 --- a/lib/filterx/expr-regexp.c +++ b/lib/filterx/expr-regexp.c @@ -29,6 +29,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/expr-function.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/pcre.h" #include "scratch-buffers.h" @@ -269,6 +270,8 @@ _store_matches_to_dict(pcre2_code_8 *pattern, const FilterXReMatchState *state, static gboolean _store_matches(pcre2_code_8 *pattern, const FilterXReMatchState *state, FilterXObject *fillable) { + fillable = filterx_ref_unwrap_rw(fillable); + if (filterx_object_is_type(fillable, &FILTERX_TYPE_NAME(list))) return _store_matches_to_list(pattern, state, fillable); diff --git a/lib/filterx/filterx-metrics-labels.c b/lib/filterx/filterx-metrics-labels.c index fd45bfbed..979f644cf 100644 --- a/lib/filterx/filterx-metrics-labels.c +++ b/lib/filterx/filterx-metrics-labels.c @@ -227,14 +227,15 @@ _format_expr_to_cache(FilterXExpr *expr, DynMetricsStore *cache) gboolean success = FALSE; - if (!filterx_object_is_type(obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *typed_obj = filterx_ref_unwrap_ro(obj); + if (!filterx_object_is_type(typed_obj, &FILTERX_TYPE_NAME(dict))) { filterx_eval_push_error_info("failed to format metrics labels, labels must be a dict", expr, g_strdup_printf("got %s instead", obj->type->name), TRUE); goto exit; } - success = filterx_dict_iter(obj, _format_dict_elem_to_cache, cache); + success = filterx_dict_iter(typed_obj, _format_dict_elem_to_cache, cache); if (!success) goto exit; diff --git a/lib/filterx/func-flatten.c b/lib/filterx/func-flatten.c index 1cce79284..bcda2ec8f 100644 --- a/lib/filterx/func-flatten.c +++ b/lib/filterx/func-flatten.c @@ -28,6 +28,7 @@ #include "filterx/object-string.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #define FILTERX_FUNC_FLATTEN_USAGE "Usage: flatten(dict, separator=\".\")" @@ -71,7 +72,8 @@ _collect_modifications_from_elem(FilterXObject *key, FilterXObject *value, gpoin GString *key_buffer = ((gpointer *) user_data)[3]; gboolean is_top_level = (gboolean) GPOINTER_TO_INT(((gpointer *) user_data)[4]); - if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(dict))) + FilterXObject *dict = filterx_ref_unwrap_ro(value); + if (filterx_object_is_type(dict, &FILTERX_TYPE_NAME(dict))) { if (is_top_level) *top_level_dict_keys = g_list_prepend(*top_level_dict_keys, filterx_object_ref(key)); @@ -85,7 +87,7 @@ _collect_modifications_from_elem(FilterXObject *key, FilterXObject *value, gpoin g_string_append(key_buffer, self->separator); gpointer inner_user_data[] = { self, flattened_kvs, NULL, key_buffer, GINT_TO_POINTER(FALSE)}; - gboolean result = filterx_dict_iter(value, _collect_modifications_from_elem, inner_user_data); + gboolean result = filterx_dict_iter(dict, _collect_modifications_from_elem, inner_user_data); g_string_truncate(key_buffer, orig_len); return result; @@ -185,12 +187,13 @@ _eval(FilterXExpr *s) { FilterXFunctionFlatten *self = (FilterXFunctionFlatten *) s; - FilterXObject *dict = filterx_expr_eval_typed(self->dict_expr); - if (!dict) + FilterXObject *obj = filterx_expr_eval_typed(self->dict_expr); + if (!obj) return NULL; gboolean result = FALSE; + FilterXObject *dict = filterx_ref_unwrap_rw(obj); if (!filterx_object_is_type(dict, &FILTERX_TYPE_NAME(dict))) { filterx_eval_push_error_info("object must be a dict", self->dict_expr, @@ -201,7 +204,7 @@ _eval(FilterXExpr *s) result = _flatten(self, dict); exit: - filterx_object_unref(dict); + filterx_object_unref(obj); return result ? filterx_boolean_new(TRUE) : NULL; } diff --git a/lib/filterx/func-istype.c b/lib/filterx/func-istype.c index 21a030303..fce1fc0d9 100644 --- a/lib/filterx/func-istype.c +++ b/lib/filterx/func-istype.c @@ -29,6 +29,7 @@ #include "filterx/filterx-eval.h" #include "filterx/filterx-globals.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #define FILTERX_FUNC_ISTYPE_USAGE "Usage: istype(object, type_str)" @@ -51,7 +52,9 @@ _eval(FilterXExpr *s) return NULL; } - gboolean result = filterx_object_is_type(object_expr, self->type); + FilterXObject *obj = filterx_ref_unwrap_ro(object_expr); + gboolean result = filterx_object_is_type(obj, self->type); + filterx_object_unref(object_expr); return filterx_boolean_new(result); } diff --git a/lib/filterx/func-str.c b/lib/filterx/func-str.c index 8374fd5b1..217473a76 100644 --- a/lib/filterx/func-str.c +++ b/lib/filterx/func-str.c @@ -247,7 +247,9 @@ _expr_affix_eval_needle(FilterXExprAffix *self) g_ptr_array_add(needles, str_value); return needles; } - if (filterx_object_is_type(needle_obj, &FILTERX_TYPE_NAME(list))) + + FilterXObject *list_needle = filterx_ref_unwrap_ro(needle_obj); + if (filterx_object_is_type(list_needle, &FILTERX_TYPE_NAME(list))) { guint64 len; filterx_object_len(needle_obj, &len); @@ -261,7 +263,7 @@ _expr_affix_eval_needle(FilterXExprAffix *self) for (guint64 i = 0; i < len; i++) { - FilterXObject *elem = filterx_list_get_subscript(needle_obj, i); + FilterXObject *elem = filterx_list_get_subscript(list_needle, i); GString *str_value = _obj_format(elem, self->ignore_case); filterx_object_unref(elem); diff --git a/lib/filterx/func-unset-empties.c b/lib/filterx/func-unset-empties.c index aaabf22a0..4806fd2bc 100644 --- a/lib/filterx/func-unset-empties.c +++ b/lib/filterx/func-unset-empties.c @@ -36,6 +36,7 @@ #include "filterx/expr-literal-generator.h" #include "filterx/expr-literal.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #define FILTERX_FUNC_UNSET_EMPTIES_USAGE "Usage: unset_empties(object, " \ FILTERX_FUNC_UNSET_EMPTIES_ARG_NAME_RECURSIVE"=bool, " \ @@ -105,6 +106,8 @@ static gboolean _should_unset_string(FilterXFunctionUnsetEmpties *self, FilterXO static gboolean _should_unset(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) { + filterx_assert_not_ref(obj); + if (check_flag(self->flags, FILTERX_FUNC_UNSET_EMPTIES_FLAG_REPLACE_NULL) && filterx_object_is_type(obj, &FILTERX_TYPE_NAME(null))) return TRUE; @@ -128,6 +131,7 @@ _add_key_to_unset_list_if_needed(FilterXObject *key, FilterXObject *value, gpoin FilterXFunctionUnsetEmpties *self = ((gpointer *) user_data)[0]; GList **keys_to_unset = ((gpointer *) user_data)[1]; + value = filterx_ref_unwrap_rw(value); if (check_flag(self->flags, FILTERX_FUNC_UNSET_EMPTIES_FLAG_RECURSIVE)) { if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(dict)) && !_process_dict(self, value)) @@ -189,22 +193,23 @@ _process_list(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) for (gint64 i = ((gint64) len) - 1; i >= 0; i--) { FilterXObject *elem = filterx_list_get_subscript(obj, i); + FilterXObject *elem_unwrapped = filterx_ref_unwrap_rw(elem); if (check_flag(self->flags, FILTERX_FUNC_UNSET_EMPTIES_FLAG_RECURSIVE)) { - if (filterx_object_is_type(elem, &FILTERX_TYPE_NAME(dict)) && !_process_dict(self, elem)) + if (filterx_object_is_type(elem_unwrapped, &FILTERX_TYPE_NAME(dict)) && !_process_dict(self, elem_unwrapped)) { filterx_object_unref(elem); return FALSE; } - if (filterx_object_is_type(elem, &FILTERX_TYPE_NAME(list)) && !_process_list(self, elem)) + if (filterx_object_is_type(elem_unwrapped, &FILTERX_TYPE_NAME(list)) && !_process_list(self, elem_unwrapped)) { filterx_object_unref(elem); return FALSE; } } - if (_should_unset(self, elem)) + if (_should_unset(self, elem_unwrapped)) { if (self->replacement) { @@ -248,11 +253,12 @@ _eval(FilterXExpr *s) return NULL; } - if (filterx_object_is_type(obj, &FILTERX_TYPE_NAME(dict))) - return _eval_on_dict(self, obj); + FilterXObject *obj_unwrapped = filterx_ref_unwrap_rw(obj); + if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(dict))) + return _eval_on_dict(self, obj_unwrapped); - if (filterx_object_is_type(obj, &FILTERX_TYPE_NAME(list))) - return _eval_on_list(self, obj); + if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(list))) + return _eval_on_list(self, obj_unwrapped); filterx_eval_push_error("Object must be dict or list. " FILTERX_FUNC_UNSET_EMPTIES_USAGE, s, obj); filterx_object_unref(obj); @@ -351,6 +357,9 @@ _handle_target_object(FilterXFunctionUnsetEmpties *self, FilterXObject *target, { g_assert(target); guint64 len; + + target = filterx_ref_unwrap_ro(target); + if (filterx_object_is_type(target, &FILTERX_TYPE_NAME(null))) set_flag(&self->flags, FILTERX_FUNC_UNSET_EMPTIES_FLAG_REPLACE_NULL, TRUE); else if (filterx_object_is_type(target, &FILTERX_TYPE_NAME(list))) diff --git a/lib/filterx/object-dict-interface.c b/lib/filterx/object-dict-interface.c index b677cbf38..57a857bb7 100644 --- a/lib/filterx/object-dict-interface.c +++ b/lib/filterx/object-dict-interface.c @@ -26,6 +26,7 @@ #include "filterx/object-string.h" #include "filterx/object-json.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "str-utils.h" gboolean @@ -54,6 +55,7 @@ filterx_dict_merge(FilterXObject *s, FilterXObject *other) { FilterXDict *self = (FilterXDict *) s; + other = filterx_ref_unwrap_ro(other); g_assert(filterx_object_is_type(other, &FILTERX_TYPE_NAME(dict))); return filterx_dict_iter(other, _add_elem_to_dict, self); } @@ -215,6 +217,7 @@ filterx_dict_init_instance(FilterXDict *self, FilterXType *type) static FilterXObject * _add(FilterXObject *lhs_object, FilterXObject *rhs_object) { + rhs_object = filterx_ref_unwrap_ro(rhs_object); if (!filterx_object_is_type(rhs_object, &FILTERX_TYPE_NAME(dict))) return NULL; diff --git a/lib/filterx/object-json-array.c b/lib/filterx/object-json-array.c index bac4f90f1..d2476c697 100644 --- a/lib/filterx/object-json-array.c +++ b/lib/filterx/object-json-array.c @@ -31,6 +31,7 @@ #include "filterx/expr-function.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "logmsg/type-hinting.h" #include "str-repr/encode.h" @@ -328,7 +329,8 @@ filterx_json_array_new_from_args(FilterXExpr *s, GPtrArray *args) FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(json_array))) + FilterXObject *json_arr = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(json_arr, &FILTERX_TYPE_NAME(json_array))) return filterx_object_ref(arg); struct json_object *jso; @@ -354,10 +356,11 @@ filterx_json_array_new_empty(void) const gchar * filterx_json_array_to_json_literal(FilterXObject *s) { - FilterXJsonArray *self = (FilterXJsonArray *) s; - + s = filterx_ref_unwrap_ro(s); if (!filterx_object_is_type(s, &FILTERX_TYPE_NAME(json_array))) return NULL; + + FilterXJsonArray *self = (FilterXJsonArray *) s; return _json_string(self); } @@ -365,11 +368,11 @@ filterx_json_array_to_json_literal(FilterXObject *s) struct json_object * filterx_json_array_get_value(FilterXObject *s) { + s = filterx_ref_unwrap_ro(s); if (!filterx_object_is_type(s, &FILTERX_TYPE_NAME(json_array))) return NULL; FilterXJsonArray *self = (FilterXJsonArray *) s; - return self->jso; } diff --git a/lib/filterx/object-json-object.c b/lib/filterx/object-json-object.c index dc3743585..d95ca476c 100644 --- a/lib/filterx/object-json-object.c +++ b/lib/filterx/object-json-object.c @@ -29,6 +29,7 @@ #include "filterx/filterx-weakrefs.h" #include "filterx/object-dict-interface.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "syslog-ng.h" #include "str-utils.h" #include "logmsg/type-hinting.h" @@ -293,10 +294,11 @@ filterx_json_object_new_empty(void) const gchar * filterx_json_object_to_json_literal(FilterXObject *s) { - FilterXJsonObject *self = (FilterXJsonObject *) s; - + s = filterx_ref_unwrap_ro(s); if (!filterx_object_is_type(s, &FILTERX_TYPE_NAME(json_object))) return NULL; + + FilterXJsonObject *self = (FilterXJsonObject *) s; return _json_string(self); } @@ -304,11 +306,11 @@ filterx_json_object_to_json_literal(FilterXObject *s) struct json_object * filterx_json_object_get_value(FilterXObject *s) { + s = filterx_ref_unwrap_ro(s); if (!filterx_object_is_type(s, &FILTERX_TYPE_NAME(json_object))) return NULL; FilterXJsonObject *self = (FilterXJsonObject *) s; - return self->jso; } diff --git a/lib/filterx/object-json.c b/lib/filterx/object-json.c index 9f3b5ab57..7fe7125e8 100644 --- a/lib/filterx/object-json.c +++ b/lib/filterx/object-json.c @@ -30,6 +30,7 @@ #include "filterx/object-message-value.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scanner/list-scanner/list-scanner.h" #include "str-repr/encode.h" @@ -198,14 +199,15 @@ filterx_json_new_from_args(FilterXExpr *s, GPtrArray *args) FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(json_array)) || - filterx_object_is_type(arg, &FILTERX_TYPE_NAME(json_object))) + FilterXObject *arg_unwrapped = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(arg_unwrapped, &FILTERX_TYPE_NAME(json_array)) || + filterx_object_is_type(arg_unwrapped, &FILTERX_TYPE_NAME(json_object))) return filterx_object_ref(arg); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + if (filterx_object_is_type(arg_unwrapped, &FILTERX_TYPE_NAME(dict))) { FilterXObject *self = filterx_json_object_new_empty(); - if (!filterx_dict_merge(self, arg)) + if (!filterx_dict_merge(self, arg_unwrapped)) { filterx_object_unref(self); return NULL; @@ -213,10 +215,10 @@ filterx_json_new_from_args(FilterXExpr *s, GPtrArray *args) return self; } - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(list))) + if (filterx_object_is_type(arg_unwrapped, &FILTERX_TYPE_NAME(list))) { FilterXObject *self = filterx_json_array_new_empty(); - if (!filterx_list_merge(self, arg)) + if (!filterx_list_merge(self, arg_unwrapped)) { filterx_object_unref(self); return NULL; @@ -234,7 +236,7 @@ filterx_json_new_from_args(FilterXExpr *s, GPtrArray *args) return filterx_json_new_from_repr(repr, repr_len); filterx_eval_push_error_info("Argument must be a json, a string or a syslog-ng list", s, - g_strdup_printf("got \"%s\" instead", arg->type->name), TRUE); + g_strdup_printf("got \"%s\" instead", arg_unwrapped->type->name), TRUE); return NULL; } @@ -253,6 +255,8 @@ filterx_json_new_from_object(struct json_object *jso) const gchar * filterx_json_to_json_literal(FilterXObject *s) { + s = filterx_ref_unwrap_ro(s); + if (filterx_object_is_type(s, &FILTERX_TYPE_NAME(json_object))) return filterx_json_object_to_json_literal(s); if (filterx_object_is_type(s, &FILTERX_TYPE_NAME(json_array))) diff --git a/lib/filterx/object-list-interface.c b/lib/filterx/object-list-interface.c index 28c5ee6b2..8d2825bc6 100644 --- a/lib/filterx/object-list-interface.c +++ b/lib/filterx/object-list-interface.c @@ -25,6 +25,7 @@ #include "filterx/object-primitive.h" #include "filterx/object-json.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" FilterXObject * filterx_list_get_subscript(FilterXObject *s, gint64 index) @@ -65,6 +66,7 @@ filterx_list_unset_index(FilterXObject *s, gint64 index) gboolean filterx_list_merge(FilterXObject *s, FilterXObject *other) { + other = filterx_ref_unwrap_ro(other); g_assert(filterx_object_is_type(other, &FILTERX_TYPE_NAME(list))); guint64 len; @@ -302,6 +304,8 @@ filterx_list_init_instance(FilterXList *self, FilterXType *type) static FilterXObject * _add(FilterXObject *lhs_object, FilterXObject *rhs_object) { + rhs_object = filterx_ref_unwrap_ro(rhs_object); + if (!filterx_object_is_type(rhs_object, &FILTERX_TYPE_NAME(list))) return NULL; diff --git a/lib/filterx/tests/test_filterx_expr.c b/lib/filterx/tests/test_filterx_expr.c index d45a6f5c1..299ed047e 100644 --- a/lib/filterx/tests/test_filterx_expr.c +++ b/lib/filterx/tests/test_filterx_expr.c @@ -41,6 +41,7 @@ #include "filterx/expr-set-subscript.h" #include "filterx/expr-get-subscript.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "apphook.h" #include "scratch-buffers.h" @@ -152,7 +153,7 @@ Test(filterx_expr, test_filterx_list_merge) FilterXObject *stored_inner_list = filterx_list_get_subscript(json_array, 2); cr_assert(stored_inner_list); - cr_assert(filterx_object_is_type(stored_inner_list, &FILTERX_TYPE_NAME(list))); + cr_assert(filterx_object_is_type(filterx_ref_unwrap_ro(stored_inner_list), &FILTERX_TYPE_NAME(list))); cr_assert(filterx_object_len(stored_inner_list, &len)); cr_assert_eq(len, 1); _assert_int_value_and_unref(filterx_list_get_subscript(stored_inner_list, 0), 1337); @@ -270,7 +271,7 @@ Test(filterx_expr, test_filterx_dict_merge) FilterXObject *stored_inner_dict = filterx_object_get_subscript(json, baz); cr_assert(stored_inner_dict); - cr_assert(filterx_object_is_type(stored_inner_dict, &FILTERX_TYPE_NAME(dict))); + cr_assert(filterx_object_is_type(filterx_ref_unwrap_ro(stored_inner_dict), &FILTERX_TYPE_NAME(dict))); cr_assert_eq(filterx_object_len(stored_inner_dict, &len), 1); _assert_int_value_and_unref(filterx_object_get_subscript(stored_inner_dict, foo), 1); filterx_object_unref(stored_inner_dict); @@ -287,7 +288,7 @@ Test(filterx_expr, test_filterx_dict_merge) stored_inner_dict = filterx_object_get_subscript(json, baz); cr_assert(stored_inner_dict); - cr_assert(filterx_object_is_type(stored_inner_dict, &FILTERX_TYPE_NAME(dict))); + cr_assert(filterx_object_is_type(filterx_ref_unwrap_ro(stored_inner_dict), &FILTERX_TYPE_NAME(dict))); cr_assert(filterx_object_len(stored_inner_dict, &len)); cr_assert_eq(len, 1); _assert_int_value_and_unref(filterx_object_get_subscript(stored_inner_dict, foo), 1); diff --git a/modules/csvparser/filterx-func-format-csv.c b/modules/csvparser/filterx-func-format-csv.c index 534584171..ac4ecd848 100644 --- a/modules/csvparser/filterx-func-format-csv.c +++ b/modules/csvparser/filterx-func-format-csv.c @@ -28,6 +28,7 @@ #include "filterx/object-list-interface.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include "utf8utils.h" @@ -55,11 +56,12 @@ _append_to_buffer(FilterXObject *key, FilterXObject *value, gpointer user_data) if (!value) value = self->default_value; - if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(dict)) || - filterx_object_is_type(value, &FILTERX_TYPE_NAME(list))) + FilterXObject *value_unwrapped = filterx_ref_unwrap_ro(value); + if (filterx_object_is_type(value_unwrapped, &FILTERX_TYPE_NAME(dict)) || + filterx_object_is_type(value_unwrapped, &FILTERX_TYPE_NAME(list))) { msg_debug("FilterX: format_csv(): skipping object, type not supported", - evt_tag_str("type", value->type->name)); + evt_tag_str("type", value_unwrapped->type->name)); return TRUE; } @@ -114,7 +116,8 @@ _handle_dict_input(FilterXFunctionFormatCSV *self, FilterXObject *csv_data, GStr if (self->columns) { FilterXObject *cols = filterx_expr_eval(self->columns); - if (!cols || !filterx_object_is_type(cols, &FILTERX_TYPE_NAME(list)) || !filterx_object_len(cols, &size)) + FilterXObject *cols_unwrapped = filterx_ref_unwrap_ro(cols); + if (!cols || !filterx_object_is_type(cols_unwrapped, &FILTERX_TYPE_NAME(list)) || !filterx_object_len(cols, &size)) { filterx_object_unref(cols); filterx_eval_push_error("Columns must represented as list. " FILTERX_FUNC_FORMAT_CSV_USAGE, &self->super.super, NULL); @@ -124,7 +127,7 @@ _handle_dict_input(FilterXFunctionFormatCSV *self, FilterXObject *csv_data, GStr gboolean success = TRUE; for (guint64 i = 0; i < size && success; i++) { - FilterXObject *col = filterx_list_get_subscript(cols, i); + FilterXObject *col = filterx_list_get_subscript(cols_unwrapped, i); FilterXObject *elt = filterx_object_get_subscript(csv_data, col); success = _append_to_buffer(col, elt, user_data); filterx_object_unref(col); @@ -149,10 +152,11 @@ _eval(FilterXExpr *s) gboolean success = FALSE; GString *formatted = scratch_buffers_alloc(); - if (filterx_object_is_type(csv_data, &FILTERX_TYPE_NAME(list))) - success = _handle_list_input(self, csv_data, formatted); - else if (filterx_object_is_type(csv_data, &FILTERX_TYPE_NAME(dict))) - success = _handle_dict_input(self, csv_data, formatted); + FilterXObject *csv_data_unwrapped = filterx_ref_unwrap_ro(csv_data); + if (filterx_object_is_type(csv_data_unwrapped, &FILTERX_TYPE_NAME(list))) + success = _handle_list_input(self, csv_data_unwrapped, formatted); + else if (filterx_object_is_type(csv_data_unwrapped, &FILTERX_TYPE_NAME(dict))) + success = _handle_dict_input(self, csv_data_unwrapped, formatted); else filterx_eval_push_error("input must be a dict or list. " FILTERX_FUNC_FORMAT_CSV_USAGE, s, csv_data); diff --git a/modules/csvparser/filterx-func-parse-csv.c b/modules/csvparser/filterx-func-parse-csv.c index e7e68ffe6..95049124f 100644 --- a/modules/csvparser/filterx-func-parse-csv.c +++ b/modules/csvparser/filterx-func-parse-csv.c @@ -37,6 +37,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scanner/csv-scanner/csv-scanner.h" #include "parser/parser-expr.h" @@ -70,10 +71,11 @@ _parse_list_argument(FilterXFunctionParseCSV *self, FilterXExpr *list_expr, GLis gboolean result = FALSE; if (!list_expr) return TRUE; - FilterXObject *list_obj = filterx_expr_eval(list_expr); - if (!list_obj) + FilterXObject *obj = filterx_expr_eval(list_expr); + if (!obj) return FALSE; + FilterXObject *list_obj = filterx_ref_unwrap_ro(obj); if (!filterx_object_is_type(list_obj, &FILTERX_TYPE_NAME(list))) { msg_error("list object argument must be a type of list.", @@ -99,7 +101,7 @@ _parse_list_argument(FilterXFunctionParseCSV *self, FilterXExpr *list_expr, GLis result = TRUE; exit: - filterx_object_unref(list_obj); + filterx_object_unref(obj); return result; } @@ -145,15 +147,16 @@ _maybe_init_columns(FilterXFunctionParseCSV *self, FilterXObject **columns, guin if (!*columns) return FALSE; - if (!filterx_object_is_type(*columns, &FILTERX_TYPE_NAME(list))) + FilterXObject *cols_unwrapped = filterx_ref_unwrap_ro(*columns); + if (!filterx_object_is_type(cols_unwrapped, &FILTERX_TYPE_NAME(list))) { msg_error("list object argument must be a type of list.", - evt_tag_str("current_type", (*columns)->type->name), + evt_tag_str("current_type", cols_unwrapped->type->name), evt_tag_str("argument_name", FILTERX_FUNC_PARSE_CSV_ARG_NAME_COLUMNS)); return FALSE; } - if (!filterx_object_len(*columns, num_of_columns)) + if (!filterx_object_len(cols_unwrapped, num_of_columns)) return FALSE; return TRUE; @@ -197,6 +200,8 @@ _fill_array_element(CSVScanner *scanner, FilterXObject *result) static gboolean _validate_fillable(FilterXFunctionParseCSV *self, FilterXObject *fillable) { + fillable = filterx_ref_unwrap_ro(fillable); + if (_are_column_names_set(self) && !filterx_object_is_type(fillable, &FILTERX_TYPE_NAME(dict))) { filterx_eval_push_error_info("fillable must be dict", &self->super.super.super, diff --git a/modules/grpc/otel/filterx/object-otel-array.cpp b/modules/grpc/otel/filterx/object-otel-array.cpp index 2a55a04ed..333f40f21 100644 --- a/modules/grpc/otel/filterx/object-otel-array.cpp +++ b/modules/grpc/otel/filterx/object-otel-array.cpp @@ -29,6 +29,7 @@ #include "filterx/object-string.h" #include "filterx/object-message-value.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/cpp-end.h" #include @@ -262,10 +263,12 @@ filterx_otel_array_new_from_args(FilterXExpr *s, GPtrArray *args) else if (args->len == 1) { FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(list))) + + FilterXObject *list_arg = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(list_arg, &FILTERX_TYPE_NAME(list))) { self->cpp = new Array(self); - if (!filterx_list_merge(&self->super.super, arg)) + if (!filterx_list_merge(&self->super.super, list_arg)) throw std::runtime_error("Failed to merge list"); } else @@ -363,6 +366,7 @@ bool OtelArrayField::FilterXObjectSetter(google::protobuf::Message *message, ProtoReflectors reflectors, FilterXObject *object, FilterXObject **assoc_object) { + object = filterx_ref_unwrap_rw(object); if (!filterx_object_is_type(object, &FILTERX_TYPE_NAME(otel_array))) { if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(list))) diff --git a/modules/grpc/otel/filterx/object-otel-kvlist.cpp b/modules/grpc/otel/filterx/object-otel-kvlist.cpp index 5489ebac2..179414395 100644 --- a/modules/grpc/otel/filterx/object-otel-kvlist.cpp +++ b/modules/grpc/otel/filterx/object-otel-kvlist.cpp @@ -29,6 +29,7 @@ #include "filterx/object-null.h" #include "filterx/object-message-value.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/cpp-end.h" #include @@ -377,10 +378,11 @@ filterx_otel_kvlist_new_from_args(FilterXExpr *s, GPtrArray *args) else if (args->len == 1) { FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + FilterXObject *dict_arg = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(dict_arg, &FILTERX_TYPE_NAME(dict))) { self->cpp = new KVList(self); - if (!filterx_dict_merge(&self->super.super, arg)) + if (!filterx_dict_merge(&self->super.super, dict_arg)) throw std::runtime_error("Failed to merge dict"); } else @@ -507,6 +509,7 @@ bool OtelKVListField::FilterXObjectSetter(google::protobuf::Message *message, ProtoReflectors reflectors, FilterXObject *object, FilterXObject **assoc_object) { + object = filterx_ref_unwrap_rw(object); if (!filterx_object_is_type(object, &FILTERX_TYPE_NAME(otel_kvlist))) { if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(dict))) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index f6fff7de3..596ea515f 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -31,6 +31,7 @@ #include "filterx/object-datetime.h" #include "filterx/object-primitive.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include "generic-number.h" @@ -294,10 +295,11 @@ filterx_otel_logrecord_new_from_args(FilterXExpr *s, GPtrArray *args) else if (args->len == 1) { FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + FilterXObject *dict_arg = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(dict_arg, &FILTERX_TYPE_NAME(dict))) { self->cpp = new LogRecord(self); - if (!filterx_dict_merge(&self->super.super, arg)) + if (!filterx_dict_merge(&self->super.super, dict_arg)) throw std::runtime_error("Failed to merge dict"); } else diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 8f5d1cc31..475b32e31 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -27,6 +27,7 @@ #include "filterx/object-extractor.h" #include "filterx/object-string.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/cpp-end.h" #include @@ -280,10 +281,11 @@ filterx_otel_resource_new_from_args(FilterXExpr *s, GPtrArray *args) else if (args->len == 1) { FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + FilterXObject *dict_arg = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(dict_arg, &FILTERX_TYPE_NAME(dict))) { self->cpp = new Resource(self); - if (!filterx_dict_merge(&self->super.super, arg)) + if (!filterx_dict_merge(&self->super.super, dict_arg)) throw std::runtime_error("Failed to merge dict"); } else diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index ec421abd9..ead2c40c1 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -27,6 +27,7 @@ #include "filterx/object-extractor.h" #include "filterx/object-string.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/cpp-end.h" #include @@ -280,10 +281,11 @@ filterx_otel_scope_new_from_args(FilterXExpr *s, GPtrArray *args) else if (args->len == 1) { FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); - if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + FilterXObject *dict_arg = filterx_ref_unwrap_ro(arg); + if (filterx_object_is_type(dict_arg, &FILTERX_TYPE_NAME(dict))) { self->cpp = new Scope(self); - if (!filterx_dict_merge(&self->super.super, arg)) + if (!filterx_dict_merge(&self->super.super, dict_arg)) throw std::runtime_error("Failed to merge dict"); } else diff --git a/modules/grpc/otel/filterx/otel-field.cpp b/modules/grpc/otel/filterx/otel-field.cpp index 9bd0f46aa..ad84ac77e 100644 --- a/modules/grpc/otel/filterx/otel-field.cpp +++ b/modules/grpc/otel/filterx/otel-field.cpp @@ -38,6 +38,7 @@ #include "filterx/object-json.h" #include "filterx/object-null.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/cpp-end.h" #include "opentelemetry/proto/logs/v1/logs.pb.h" @@ -177,6 +178,7 @@ AnyField::FilterXObjectDirectSetter(AnyValue *anyValue, FilterXObject *object, F ProtobufField *converter = nullptr; const char *typeFieldName; + FilterXObject *object_unwrapped = filterx_ref_unwrap_ro(object); if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(boolean)) || (filterx_object_is_type(object, &FILTERX_TYPE_NAME(message_value)) && filterx_message_value_get_type(object) == LM_VT_BOOLEAN)) @@ -219,14 +221,14 @@ AnyField::FilterXObjectDirectSetter(AnyValue *anyValue, FilterXObject *object, F converter = protobuf_converter_by_type(FieldDescriptor::TYPE_BYTES); typeFieldName = "bytes_value"; } - else if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(dict)) || + else if (filterx_object_is_type(object_unwrapped, &FILTERX_TYPE_NAME(dict)) || (filterx_object_is_type(object, &FILTERX_TYPE_NAME(message_value)) && filterx_message_value_get_type(object) == LM_VT_JSON)) { converter = &filterx::otel_kvlist_converter; typeFieldName = "kvlist_value"; } - else if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(list)) || + else if (filterx_object_is_type(object_unwrapped, &FILTERX_TYPE_NAME(list)) || (filterx_object_is_type(object, &FILTERX_TYPE_NAME(message_value)) && filterx_message_value_get_type(object) == LM_VT_LIST)) { diff --git a/modules/grpc/otel/filterx/protobuf-field.cpp b/modules/grpc/otel/filterx/protobuf-field.cpp index 4a3555e89..b59f314d8 100644 --- a/modules/grpc/otel/filterx/protobuf-field.cpp +++ b/modules/grpc/otel/filterx/protobuf-field.cpp @@ -34,6 +34,7 @@ #include "filterx/object-json.h" #include "filterx/object-null.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "compat/cpp-end.h" #include @@ -224,6 +225,7 @@ class StringField : public ProtobufField goto success; } + object = filterx_ref_unwrap_ro(object); if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(json_object)) || filterx_object_is_type(object, &FILTERX_TYPE_NAME(json_array))) { diff --git a/modules/json/filterx-cache-json-file.c b/modules/json/filterx-cache-json-file.c index 1d8e6b951..2a69d566c 100644 --- a/modules/json/filterx-cache-json-file.c +++ b/modules/json/filterx-cache-json-file.c @@ -28,6 +28,7 @@ #include "filterx/expr-literal.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include "compat/json.h" @@ -189,6 +190,7 @@ _deep_freeze(FilterXFuntionCacheJsonFile *self, FilterXObject *object) if (filterx_object_freeze(object)) g_ptr_array_add(self->frozen_objects, object); + object = filterx_ref_unwrap_ro(object); if (filterx_object_is_type(object, &FILTERX_TYPE_NAME(json_object))) _deep_freeze_dict(self, object); diff --git a/modules/json/filterx-format-json.c b/modules/json/filterx-format-json.c index 7969e86fe..79051e561 100644 --- a/modules/json/filterx-format-json.c +++ b/modules/json/filterx-format-json.c @@ -30,6 +30,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/object-list-interface.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include "utf8utils.h" @@ -266,11 +267,12 @@ _format_and_append_value(FilterXObject *value, GString *result) if (filterx_object_extract_string_ref(value, &str, &str_len)) return _format_and_append_string(str, str_len, result); - if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(dict))) - return _format_and_append_dict(value, result); + FilterXObject *value_unwrapped = filterx_ref_unwrap_ro(value); + if (filterx_object_is_type(value_unwrapped, &FILTERX_TYPE_NAME(dict))) + return _format_and_append_dict(value_unwrapped, result); - if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(list))) - return _format_and_append_list(value, result); + if (filterx_object_is_type(value_unwrapped, &FILTERX_TYPE_NAME(list))) + return _format_and_append_list(value_unwrapped, result); /* FIXME: handle datetime based on object-datetime.c:_convert_unix_time_to_string() */ diff --git a/modules/kvformat/filterx-func-format-kv.c b/modules/kvformat/filterx-func-format-kv.c index a7bbf61e6..478d2ef1a 100644 --- a/modules/kvformat/filterx-func-format-kv.c +++ b/modules/kvformat/filterx-func-format-kv.c @@ -28,6 +28,7 @@ #include "filterx/object-list-interface.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include "utf8utils.h" @@ -48,11 +49,12 @@ _append_kv_to_buffer(FilterXObject *key, FilterXObject *value, gpointer user_dat FilterXFunctionFormatKV *self = ((gpointer *) user_data)[0]; GString *buffer = ((gpointer *) user_data)[1]; - if (filterx_object_is_type(value, &FILTERX_TYPE_NAME(dict)) || - filterx_object_is_type(value, &FILTERX_TYPE_NAME(list))) + FilterXObject *value_unwrapped = filterx_ref_unwrap_ro(value); + if (filterx_object_is_type(value_unwrapped, &FILTERX_TYPE_NAME(dict)) || + filterx_object_is_type(value_unwrapped, &FILTERX_TYPE_NAME(list))) { msg_debug("FilterX: format_kv(): skipping object, type not supported", - evt_tag_str("type", value->type->name)); + evt_tag_str("type", value_unwrapped->type->name)); return TRUE; } @@ -91,17 +93,18 @@ _eval(FilterXExpr *s) { FilterXFunctionFormatKV *self = (FilterXFunctionFormatKV *) s; - FilterXObject *kvs = filterx_expr_eval_typed(self->kvs); - if (!kvs) + FilterXObject *obj = filterx_expr_eval_typed(self->kvs); + if (!obj) { filterx_eval_push_error("Failed to evaluate kvs_dict. " FILTERX_FUNC_FORMAT_KV_USAGE, s, NULL); return NULL; } + FilterXObject *kvs = filterx_ref_unwrap_ro(obj); if (!filterx_object_is_type(kvs, &FILTERX_TYPE_NAME(dict))) { - filterx_eval_push_error("kvs_dict must be a dict. " FILTERX_FUNC_FORMAT_KV_USAGE, s, kvs); - filterx_object_unref(kvs); + filterx_eval_push_error("kvs_dict must be a dict. " FILTERX_FUNC_FORMAT_KV_USAGE, s, obj); + filterx_object_unref(obj); return NULL; } @@ -109,7 +112,7 @@ _eval(FilterXExpr *s) gpointer user_data[] = { self, formatted }; gboolean success = filterx_dict_iter(kvs, _append_kv_to_buffer, user_data); - filterx_object_unref(kvs); + filterx_object_unref(obj); return success ? filterx_string_new(formatted->str, formatted->len) : NULL; } diff --git a/modules/xml/filterx-parse-windows-eventlog-xml.c b/modules/xml/filterx-parse-windows-eventlog-xml.c index f76af6aff..3d9b675eb 100644 --- a/modules/xml/filterx-parse-windows-eventlog-xml.c +++ b/modules/xml/filterx-parse-windows-eventlog-xml.c @@ -25,6 +25,7 @@ #include "filterx/object-string.h" #include "filterx/object-dict-interface.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" static void _set_error(GError **error, const gchar *format, ...) G_GNUC_PRINTF(2, 0); @@ -77,7 +78,8 @@ _convert_to_dict(GMarkupParseContext *context, XmlElemContext *elem_context, GEr if (!dict_obj) goto exit; - if (!filterx_object_is_type(elem_context->parent_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *parent_unwrapped = filterx_ref_unwrap_ro(elem_context->parent_obj); + if (!filterx_object_is_type(parent_unwrapped, &FILTERX_TYPE_NAME(dict))) { _set_error(error, "failed to convert EventData string to dict, parent must be a dict"); goto exit; @@ -119,10 +121,11 @@ _prepare_elem(const gchar *new_elem_name, XmlElemContext *last_elem_context, Xml } existing_obj = filterx_object_get_subscript(new_elem_context->parent_obj, new_elem_key); - if (!filterx_object_is_type(existing_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *existing_obj_unwrapped = filterx_ref_unwrap_ro(existing_obj); + if (!filterx_object_is_type(existing_obj_unwrapped, &FILTERX_TYPE_NAME(dict))) { _set_error(error, "failed to prepare dict for named param, parent must be dict, got \"%s\"", - existing_obj->type->name); + existing_obj_unwrapped->type->name); goto exit; } @@ -284,7 +287,8 @@ _start_elem(FilterXGeneratorFunctionParseXml *s, return; } - if (!filterx_object_is_type(last_elem_context->current_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *current_obj = filterx_ref_unwrap_ro(last_elem_context->current_obj); + if (!filterx_object_is_type(current_obj, &FILTERX_TYPE_NAME(dict))) { if (!_convert_to_dict(context, last_elem_context, error)) return; @@ -318,7 +322,8 @@ _text(FilterXGeneratorFunctionParseXml *s, FilterXParseWEVTState *state = (FilterXParseWEVTState *) st; XmlElemContext *elem_context = xml_elem_context_stack_peek_last(state->super.xml_elem_context_stack); - if (!filterx_object_is_type(elem_context->current_obj, &FILTERX_TYPE_NAME(dict)) || + FilterXObject *current_obj = filterx_ref_unwrap_ro(elem_context->current_obj); + if (!filterx_object_is_type(current_obj, &FILTERX_TYPE_NAME(dict)) || !state->has_named_data) { filterx_parse_xml_text_method(s, context, text, text_len, st, error); diff --git a/modules/xml/filterx-parse-xml.c b/modules/xml/filterx-parse-xml.c index 88d3d7971..56db99cca 100644 --- a/modules/xml/filterx-parse-xml.c +++ b/modules/xml/filterx-parse-xml.c @@ -28,6 +28,7 @@ #include "filterx/object-dict-interface.h" #include "filterx/filterx-eval.h" #include "filterx/filterx-object-istype.h" +#include "filterx/filterx-ref.h" #include "scratch-buffers.h" #include @@ -186,6 +187,7 @@ _store_first_elem(XmlElemContext *new_elem_context, FilterXObject *new_elem_key, static gboolean _is_obj_storing_a_single_elem(FilterXObject *obj) { + obj = filterx_ref_unwrap_ro(obj); return filterx_object_is_type(obj, &FILTERX_TYPE_NAME(string)) || filterx_object_is_type(obj, &FILTERX_TYPE_NAME(dict)); } @@ -219,6 +221,7 @@ _store_second_elem(XmlElemContext *new_elem_context, FilterXObject **existing_ob static gboolean _is_obj_storing_multiple_elems(FilterXObject *obj) { + obj = filterx_ref_unwrap_ro(obj); return filterx_object_is_type(obj, &FILTERX_TYPE_NAME(list)); } @@ -239,7 +242,8 @@ static gboolean _prepare_elem(const gchar *new_elem_name, XmlElemContext *last_elem_context, gboolean has_attrs, XmlElemContext *new_elem_context, GError **error) { - g_assert(filterx_object_is_type(last_elem_context->current_obj, &FILTERX_TYPE_NAME(dict))); + FilterXObject *current_obj = filterx_ref_unwrap_ro(last_elem_context->current_obj); + g_assert(filterx_object_is_type(current_obj, &FILTERX_TYPE_NAME(dict))); const gchar *new_elem_repr; FilterXObject *new_elem_obj = _create_object_for_new_elem(last_elem_context->current_obj, has_attrs, &new_elem_repr); @@ -295,8 +299,9 @@ _collect_attrs(const gchar *element_name, XmlElemContext *elem_context, const gchar **attribute_names, const gchar **attribute_values, GError **error) { + FilterXObject *current_obj = filterx_ref_unwrap_ro(elem_context->current_obj); /* Ensured by _prepare_elem() and _create_object_for_new_elem(). */ - g_assert(filterx_object_is_type(elem_context->current_obj, &FILTERX_TYPE_NAME(dict))); + g_assert(filterx_object_is_type(current_obj, &FILTERX_TYPE_NAME(dict))); ScratchBuffersMarker marker; GString *attr_key = scratch_buffers_alloc_and_mark(&marker); @@ -356,16 +361,17 @@ _convert_to_dict(GMarkupParseContext *context, XmlElemContext *elem_context, GEr } } - if (filterx_object_is_type(elem_context->parent_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *parent_obj = filterx_ref_unwrap_rw(elem_context->parent_obj); + if (filterx_object_is_type(parent_obj, &FILTERX_TYPE_NAME(dict))) { - if (!filterx_object_set_subscript(elem_context->parent_obj, key, &dict_obj)) + if (!filterx_object_set_subscript(parent_obj, key, &dict_obj)) _set_error(error, "failed to replace leaf node object with: \"%s\"={}", parent_elem_name); goto exit; } - if (filterx_object_is_type(elem_context->parent_obj, &FILTERX_TYPE_NAME(list))) + if (filterx_object_is_type(parent_obj, &FILTERX_TYPE_NAME(list))) { - if (!filterx_list_set_subscript(elem_context->parent_obj, -1, &dict_obj)) + if (!filterx_list_set_subscript(parent_obj, -1, &dict_obj)) _set_error(error, "failed to replace leaf node object with: {}"); goto exit; } @@ -389,7 +395,8 @@ filterx_parse_xml_start_elem_method(FilterXGeneratorFunctionParseXml *self, { XmlElemContext *last_elem_context = xml_elem_context_stack_peek_last(state->xml_elem_context_stack); - if (!filterx_object_is_type(last_elem_context->current_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *current_obj = filterx_ref_unwrap_ro(last_elem_context->current_obj); + if (!filterx_object_is_type(current_obj, &FILTERX_TYPE_NAME(dict))) { /* * We need the last node to be a dict, so we can start a new inner element in it. @@ -442,10 +449,11 @@ _replace_string_text(XmlElemContext *elem_context, const gchar *element_name, co { FilterXObject *text_obj = filterx_string_new(text, text_len); - if (filterx_object_is_type(elem_context->parent_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *parent_obj = filterx_ref_unwrap_rw(elem_context->parent_obj); + if (filterx_object_is_type(parent_obj, &FILTERX_TYPE_NAME(dict))) { FilterXObject *key = filterx_string_new(element_name, -1); - gboolean result = filterx_object_set_subscript(elem_context->parent_obj, key, &text_obj); + gboolean result = filterx_object_set_subscript(parent_obj, key, &text_obj); filterx_object_unref(key); if (!result) @@ -456,9 +464,9 @@ _replace_string_text(XmlElemContext *elem_context, const gchar *element_name, co goto success; } - if (filterx_object_is_type(elem_context->parent_obj, &FILTERX_TYPE_NAME(list))) + if (filterx_object_is_type(parent_obj, &FILTERX_TYPE_NAME(list))) { - if (!filterx_list_set_subscript(elem_context->parent_obj, -1, &text_obj)) + if (!filterx_list_set_subscript(parent_obj, -1, &text_obj)) { _set_error(error, "failed to add text to list: \"%s\"", text); goto fail; @@ -553,7 +561,8 @@ filterx_parse_xml_text_method(FilterXGeneratorFunctionParseXml *self, goto exit; } - if (filterx_object_is_type(elem_context->current_obj, &FILTERX_TYPE_NAME(dict))) + FilterXObject *current_obj = filterx_ref_unwrap_ro(elem_context->current_obj); + if (filterx_object_is_type(current_obj, &FILTERX_TYPE_NAME(dict))) { _add_text_to_dict(elem_context, stripped_text, stripped_text_len, error); goto exit; @@ -568,6 +577,7 @@ filterx_parse_xml_text_method(FilterXGeneratorFunctionParseXml *self, static gboolean _validate_fillable(FilterXGeneratorFunctionParseXml *self, FilterXObject *fillable) { + fillable = filterx_ref_unwrap_ro(fillable); if (!filterx_object_is_type(fillable, &FILTERX_TYPE_NAME(dict))) { filterx_eval_push_error_info("fillable must be dict", &self->super.super.super, From 31dbe98551d764c77ad855c156bbff12cd2df248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 18 Oct 2024 17:33:56 +0200 Subject: [PATCH 14/16] filterx: propagate modified_in_place through refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/filterx-object.h | 24 +++++++++++++++++++++++- lib/filterx/filterx-ref.c | 18 +++++++++++++++++- lib/filterx/filterx-scope.c | 4 ++-- lib/filterx/object-json-array.c | 12 ++++++------ lib/filterx/object-json-object.c | 8 ++++---- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/lib/filterx/filterx-object.h b/lib/filterx/filterx-object.h index a94445334..4ba32e902 100644 --- a/lib/filterx/filterx-object.h +++ b/lib/filterx/filterx-object.h @@ -55,6 +55,8 @@ struct _FilterXType gboolean (*len)(FilterXObject *self, guint64 *len); FilterXObject *(*add)(FilterXObject *self, FilterXObject *object); void (*make_readonly)(FilterXObject *self); + gboolean (*is_modified_in_place)(FilterXObject *self); + void (*set_modified_in_place)(FilterXObject *self, gboolean modified); void (*free_fn)(FilterXObject *self); }; @@ -83,7 +85,9 @@ struct _FilterXObject /* NOTE: * * modified_in_place -- set to TRUE in case the value in this - * FilterXObject was changed + * FilterXObject was changed. + * don't use it directly, use + * filterx_object_{is,set}_modified_in_place() * readonly -- marks the object as unmodifiable, * propagates to the inner elements lazily * @@ -296,4 +300,22 @@ filterx_object_add_object(FilterXObject *self, FilterXObject *object) return self->type->add(self, object); } +static inline gboolean +filterx_object_is_modified_in_place(FilterXObject *self) +{ + if (G_UNLIKELY(self->type->is_modified_in_place)) + return self->type->is_modified_in_place(self); + + return self->modified_in_place; +} + +static inline void +filterx_object_set_modified_in_place(FilterXObject *self, gboolean modified) +{ + if (G_UNLIKELY(self->type->set_modified_in_place)) + return self->type->set_modified_in_place(self, modified); + + self->modified_in_place = modified; +} + #endif diff --git a/lib/filterx/filterx-ref.c b/lib/filterx/filterx-ref.c index abf01ea48..08f7b8052 100644 --- a/lib/filterx/filterx-ref.c +++ b/lib/filterx/filterx-ref.c @@ -113,12 +113,26 @@ _filterx_ref_free(FilterXObject *s) filterx_object_free_method(s); } -void +static void _prohibit_readonly(FilterXObject *s) { g_assert_not_reached(); } +static gboolean +_is_modified_in_place(FilterXObject *s) +{ + FilterXRef *self = (FilterXRef *) s; + return filterx_object_is_modified_in_place(self->value); +} + +static void +_set_modified_in_place(FilterXObject *s, gboolean modified) +{ + FilterXRef *self = (FilterXRef *) s; + filterx_object_set_modified_in_place(self->value, modified); +} + /* readonly methods */ static FilterXObject * @@ -267,5 +281,7 @@ FILTERX_DEFINE_TYPE(ref, FILTERX_TYPE_NAME(object), .len = _filterx_ref_len, .add = _filterx_ref_add, .make_readonly = _prohibit_readonly, + .is_modified_in_place = _is_modified_in_place, + .set_modified_in_place = _set_modified_in_place, .free_fn = _filterx_ref_free, ); diff --git a/lib/filterx/filterx-scope.c b/lib/filterx/filterx-scope.c index a57708f57..b5580d3d3 100644 --- a/lib/filterx/filterx-scope.c +++ b/lib/filterx/filterx-scope.c @@ -254,7 +254,7 @@ filterx_scope_sync(FilterXScope *self, LogMessage *msg) log_msg_unset_value(msg, filterx_variable_get_nv_handle(v)); filterx_variable_unassign(v); } - else if (filterx_variable_is_assigned(v) || v->value->modified_in_place) + else if (filterx_variable_is_assigned(v) || filterx_object_is_modified_in_place(v->value)) { LogMessageValueType t; @@ -265,7 +265,7 @@ filterx_scope_sync(FilterXScope *self, LogMessage *msg) if (!filterx_object_marshal(v->value, buffer, &t)) g_assert_not_reached(); log_msg_set_value_with_type(msg, filterx_variable_get_nv_handle(v), buffer->str, buffer->len, t); - v->value->modified_in_place = FALSE; + filterx_object_set_modified_in_place(v->value, FALSE); filterx_variable_unassign(v); } else diff --git a/lib/filterx/object-json-array.c b/lib/filterx/object-json-array.c index d2476c697..3596461c1 100644 --- a/lib/filterx/object-json-array.c +++ b/lib/filterx/object-json-array.c @@ -167,11 +167,11 @@ _append(FilterXList *s, FilterXObject **new_value) return FALSE; } - self->super.super.modified_in_place = TRUE; + filterx_object_set_modified_in_place(&self->super.super, TRUE); FilterXObject *root_container = filterx_weakref_get(&self->root_container); if (root_container) { - root_container->modified_in_place = TRUE; + filterx_object_set_modified_in_place(root_container, TRUE); filterx_object_unref(root_container); } @@ -203,11 +203,11 @@ _set_subscript(FilterXList *s, guint64 index, FilterXObject **new_value) return FALSE; } - self->super.super.modified_in_place = TRUE; + filterx_object_set_modified_in_place(&self->super.super, TRUE); FilterXObject *root_container = filterx_weakref_get(&self->root_container); if (root_container) { - root_container->modified_in_place = TRUE; + filterx_object_set_modified_in_place(root_container, TRUE); filterx_object_unref(root_container); } @@ -228,11 +228,11 @@ _unset_index(FilterXList *s, guint64 index) if (json_object_array_del_idx(self->jso, index, 1) != 0) return FALSE; - self->super.super.modified_in_place = TRUE; + filterx_object_set_modified_in_place(&self->super.super, TRUE); FilterXObject *root_container = filterx_weakref_get(&self->root_container); if (root_container) { - root_container->modified_in_place = TRUE; + filterx_object_set_modified_in_place(root_container, TRUE); filterx_object_unref(root_container); } diff --git a/lib/filterx/object-json-object.c b/lib/filterx/object-json-object.c index d95ca476c..59837566e 100644 --- a/lib/filterx/object-json-object.c +++ b/lib/filterx/object-json-object.c @@ -149,11 +149,11 @@ _set_subscript(FilterXDict *s, FilterXObject *key, FilterXObject **new_value) return FALSE; } - self->super.super.modified_in_place = TRUE; + filterx_object_set_modified_in_place(&self->super.super, TRUE); FilterXObject *root_container = filterx_weakref_get(&self->root_container); if (root_container) { - root_container->modified_in_place = TRUE; + filterx_object_set_modified_in_place(root_container, TRUE); filterx_object_unref(root_container); } @@ -177,11 +177,11 @@ _unset_key(FilterXDict *s, FilterXObject *key) json_object_object_del(self->jso, key_str); - self->super.super.modified_in_place = TRUE; + filterx_object_set_modified_in_place(&self->super.super, TRUE); FilterXObject *root_container = filterx_weakref_get(&self->root_container); if (root_container) { - root_container->modified_in_place = TRUE; + filterx_object_set_modified_in_place(root_container, TRUE); filterx_object_unref(root_container); } From 9dcb3299637cce94fb74f7ac268b2b2d6aaba961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sun, 20 Oct 2024 12:43:34 +0200 Subject: [PATCH 15/16] filterx-unset-empties: fix use-after-free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/func-unset-empties.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/lib/filterx/func-unset-empties.c b/lib/filterx/func-unset-empties.c index 4806fd2bc..2a8e1321d 100644 --- a/lib/filterx/func-unset-empties.c +++ b/lib/filterx/func-unset-empties.c @@ -173,15 +173,6 @@ _process_dict(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) return success; } -/* Takes reference of obj. */ -static FilterXObject * -_eval_on_dict(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) -{ - gboolean success = _process_dict(self, obj); - filterx_object_unref(obj); - return success ? filterx_boolean_new(TRUE) : NULL; -} - static gboolean _process_list(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) { @@ -232,15 +223,6 @@ _process_list(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) return TRUE; } -/* Takes reference of obj. */ -static FilterXObject * -_eval_on_list(FilterXFunctionUnsetEmpties *self, FilterXObject *obj) -{ - gboolean success = _process_list(self, obj); - filterx_object_unref(obj); - return success ? filterx_boolean_new(TRUE) : NULL; -} - static FilterXObject * _eval(FilterXExpr *s) { @@ -255,10 +237,18 @@ _eval(FilterXExpr *s) FilterXObject *obj_unwrapped = filterx_ref_unwrap_rw(obj); if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(dict))) - return _eval_on_dict(self, obj_unwrapped); + { + gboolean success = _process_dict(self, obj_unwrapped); + filterx_object_unref(obj); + return success ? filterx_boolean_new(TRUE) : NULL; + } if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(list))) - return _eval_on_list(self, obj_unwrapped); + { + gboolean success = _process_list(self, obj_unwrapped); + filterx_object_unref(obj); + return success ? filterx_boolean_new(TRUE) : NULL; + } filterx_eval_push_error("Object must be dict or list. " FILTERX_FUNC_UNSET_EMPTIES_USAGE, s, obj); filterx_object_unref(obj); From 699c32c52b3c7b09ba3d0e8d651a89a1b74222bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sun, 20 Oct 2024 12:49:12 +0200 Subject: [PATCH 16/16] filterx-unset-empties: refactor eval() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- lib/filterx/func-unset-empties.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/filterx/func-unset-empties.c b/lib/filterx/func-unset-empties.c index 2a8e1321d..0ad388e3d 100644 --- a/lib/filterx/func-unset-empties.c +++ b/lib/filterx/func-unset-empties.c @@ -235,24 +235,17 @@ _eval(FilterXExpr *s) return NULL; } + gboolean success = FALSE; FilterXObject *obj_unwrapped = filterx_ref_unwrap_rw(obj); if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(dict))) - { - gboolean success = _process_dict(self, obj_unwrapped); - filterx_object_unref(obj); - return success ? filterx_boolean_new(TRUE) : NULL; - } - - if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(list))) - { - gboolean success = _process_list(self, obj_unwrapped); - filterx_object_unref(obj); - return success ? filterx_boolean_new(TRUE) : NULL; - } + success = _process_dict(self, obj_unwrapped); + else if (filterx_object_is_type(obj_unwrapped, &FILTERX_TYPE_NAME(list))) + success = _process_list(self, obj_unwrapped); + else + filterx_eval_push_error("Object must be dict or list. " FILTERX_FUNC_UNSET_EMPTIES_USAGE, s, obj); - filterx_eval_push_error("Object must be dict or list. " FILTERX_FUNC_UNSET_EMPTIES_USAGE, s, obj); filterx_object_unref(obj); - return NULL; + return success ? filterx_boolean_new(TRUE) : NULL; } static void