save axoflow#5

lib/scanner/kv-scanner/kv-scanner.c

@@ -264,8 +264,6 @@ _decode_value(KVScanner *self)
     .match_delimiter_data = self,
     .delimiter_chars = { ' ', self->pair_separator[0], self->stop_char },
   };
-  if (self->cef_mode)
-    options.unquoted_escaped_chars = "=";
 
   self->value_was_quoted = _is_quoted(input);
   if (str_repr_decode_with_options(self->value, input, &end, &options))

lib/scanner/kv-scanner/kv-scanner.h

@@ -46,7 +46,6 @@ struct _KVScanner
   const gchar *pair_separator;
   gsize pair_separator_len;
   gchar stop_char;
-  gboolean cef_mode;
 
   KVTransformValueFunc transform_value;
   KVExtractAnnotationFunc extract_annotation;
@@ -125,12 +124,6 @@ kv_scanner_set_stop_character(KVScanner *self, gchar stop_char)
   self->stop_char = stop_char;
 }
 
-static inline void
-kv_scanner_set_cef_mode(KVScanner *self, gboolean enable)
-{
-  self->cef_mode = enable;
-}
-
 gboolean kv_scanner_scan_next(KVScanner *self);
 
 #endif

lib/str-repr/decode.c

@@ -36,7 +36,6 @@ enum
   KV_UNQUOTED_CHARACTERS,
   KV_FINISH_SUCCESS,
   KV_FINISH_FAILURE,
-  KV_UNQUOTED_BACKSLASH,
 };
 
 static void
@@ -155,27 +154,10 @@ _process_quoted_string_characters(StrReprDecodeState *state)
 static gint
 _process_backslash_escaped_character_in_strings(StrReprDecodeState *state)
 {
-  // if (state->options->unquoted_escaped_chars && strchr(state->options->unquoted_escaped_chars, *state->cur) != NULL)
-  // {
-  //   g_string_append_c(state->value, *state->cur);
-  //   return KV_QUOTE_STRING;
-  // }
   _decode_backslash_escape(state->value, state->quote_char, *state->cur);
   return KV_QUOTE_STRING;
 }
 
-static gint
-_process_backslash_escaped_character_in_unquoted_strings(StrReprDecodeState *state)
-{
-  if (state->options->unquoted_escaped_chars && strchr(state->options->unquoted_escaped_chars, *state->cur) != NULL)
-  {
-    g_string_append_c(state->value, *state->cur);
-    return KV_UNQUOTED_CHARACTERS;
-  }
-  _decode_backslash_escape(state->value, state->quote_char, *state->cur);
-  return KV_UNQUOTED_CHARACTERS;
-}
-
 static gint
 _process_delimiter_characters_after_a_quoted_string(StrReprDecodeState *state)
 {
@@ -197,8 +179,6 @@ _process_unquoted_characters(StrReprDecodeState *state)
 {
   if (_match_and_skip_delimiter(state))
     return KV_FINISH_SUCCESS;
-  else if (state->options->unquoted_escaped_chars && (*state->cur == '\\'))
-    return KV_UNQUOTED_BACKSLASH;
 
   g_string_append_c(state->value, *state->cur);
   return KV_UNQUOTED_CHARACTERS;
@@ -240,9 +220,6 @@ _decode(StrReprDecodeState *state)
         case KV_UNQUOTED_CHARACTERS:
           quote_state = _process_unquoted_characters(state);
           break;
-        case KV_UNQUOTED_BACKSLASH:
-          quote_state = _process_backslash_escaped_character_in_unquoted_strings(state);
-          break;
         default:
           break;
         }

lib/str-repr/decode.h

@@ -33,7 +33,6 @@ typedef struct _StrReprDecodeOptions
   MatchDelimiterFunc match_delimiter;
   gpointer match_delimiter_data;
   gchar delimiter_chars[3];
-  gchar *unquoted_escaped_chars;
 } StrReprDecodeOptions;
 
 gboolean str_repr_decode(GString *value, const gchar *input, const gchar **end);

modules/cef/CMakeLists.txt

@@ -2,11 +2,13 @@ set(CEF_SOURCES
     format-cef-extension.c
     format-cef-extension.h
     cef-plugin.c
-    parser.c
-    parser.h
-    parser-cfg.h
+    event-format-parser.c
+    event-format-parser.h
+    event-format-parser-cfg.h
     filterx-func-parse-cef.c
     filterx-func-parse-cef.h
+    filterx-func-parse-leef.c
+    filterx-func-parse-leef.h
 )
 
 add_module(

modules/cef/Makefile.am

@@ -5,11 +5,13 @@ EXTRA_DIST += modules/cef/CMakeLists.txt
 modules_cef_libcef_la_SOURCES	=	\
 	modules/cef/format-cef-extension.c	\
 	modules/cef/format-cef-extension.h	\
-	modules/cef/parser-cfg.h	\
-	modules/cef/parser.c	\
-	modules/cef/parser.h	\
+	modules/cef/event-format-parser-cfg.h	\
+	modules/cef/event-format-parser.c	\
+	modules/cef/event-format-parser.h	\
 	modules/cef/filterx-func-parse-cef.c	\
 	modules/cef/filterx-func-parse-cef.h	\
+	modules/cef/filterx-func-parse-leef.c	\
+	modules/cef/filterx-func-parse-leef.h	\
 	modules/cef/cef-plugin.c
 
 modules_cef_libcef_la_CFLAGS	=	\

modules/cef/cef-plugin.c

@@ -23,12 +23,14 @@
 #include "plugin.h"
 #include "plugin-types.h"
 #include "filterx-func-parse-cef.h"
+#include "filterx-func-parse-leef.h"
 #include "filterx/expr-function.h"
 
 static Plugin cef_plugins[] =
 {
   TEMPLATE_FUNCTION_PLUGIN(tf_cef, "format-cef-extension"),
   FILTERX_GENERATOR_FUNCTION_PLUGIN(parse_cef),
+  FILTERX_GENERATOR_FUNCTION_PLUGIN(parse_leef),
 };
 
 gboolean

modules/cef/parser-cfg.h → modules/cef/event-format-parser-cfg.h

@@ -21,28 +21,44 @@
  *
  */
 
-#ifndef CEF_PARSER_CFG_H_INCLUDED
-#define CEF_PARSER_CFG_H_INCLUDED
+#ifndef EVENT_FORMAT_PARSER_CFG_H_INCLUDED
+#define EVENT_FORMAT_PARSER_CFG_H_INCLUDED
 
 #include "filterx/filterx-object.h"
 
-typedef struct _FilterXFunctionParser FilterXFunctionParser;
+typedef enum _EventFormatLogType {
+    EVENFORMAT_LOGTYPE_CEF,
+    EVENFORMAT_LOGTYPE_LEEF,
+    EVENFORMAT_LOGTYPE_COUNT,
+} EventFormatLogType;
 
-typedef FilterXObject *(*FieldParser)(FilterXFunctionParser *parser, const gchar *value, GError **error, gpointer user_data);
+static gchar *EventFormatLogTypeSignatures[] = {
+    "CEF",
+    "LEEF",
+};
+
+typedef struct _FilterXFunctionEventFormatParser FilterXFunctionEventFormatParser;
+
+typedef FilterXObject *(*FieldParser)(FilterXFunctionEventFormatParser *parser, const gchar *value, GError **error, gpointer user_data);
 
 typedef struct _Field {
     const gchar *name;
     FieldParser field_parser;
 } Field;
 
 typedef struct _Config {
-    const gchar *signature; // CEF or LEEF
+    EventFormatLogType log_type;
     const gchar *delimiters;
     size_t num_fields;
     gchar extensions_value_separator;
     const gchar *extensions_pair_separator;
-    Field *fields; // field names and handlers
+    Field *fields;
 } Config;
 
+static inline gchar* signature_for_logtype(EventFormatLogType log_type)
+{
+    g_assert(log_type < EVENFORMAT_LOGTYPE_COUNT);
+    return EventFormatLogTypeSignatures[log_type];
+}
 
 #endif

modules/cef/parser.c → modules/cef/event-format-parser.c

@@ -22,8 +22,8 @@
  *
  */
 
-#include "parser.h"
 #include <string.h>
+#include "event-format-parser.h"
 #include "filterx-func-parse-cef.h"
 #include "filterx/object-string.h"
 #include "filterx/object-primitive.h"
@@ -41,36 +41,35 @@
 #include "filterx/object-string.h"
 
 Field
-field(FilterXFunctionParser *self, int index)
+field(FilterXFunctionEventFormatParser *self, int index)
 {
   g_assert(index >= 0 && index < self->config.num_fields);
   return self->config.fields[index];
 }
 
 static FilterXObject *
-parse_default(FilterXFunctionParser *self, const gchar *value, GError **error, gpointer user_data)
+parse_default(FilterXFunctionEventFormatParser *self, const gchar *value, GError **error, gpointer user_data)
 {
   return filterx_string_new(value, -1);
 }
 
 FilterXObject *
-parse_version(FilterXFunctionParser *self, const gchar *value, GError **error, gpointer user_data)
+parse_version(FilterXFunctionEventFormatParser *self, const gchar *value, GError **error, gpointer user_data)
 {
-  const gchar *log_signature = self->config.signature;
+  const gchar *log_signature = signature_for_logtype(self->config.log_type);
   gchar *colon_pos = strchr(value, ':');
   if (!colon_pos || colon_pos == value)
   {
     // TODO: fix error type, id, message
     g_set_error(error, FILTERX_FUNCTION_ERROR, 444,
-                FX_CEF_PARSER_ERR_NO_LOG_SING, log_signature);
+                EVENT_FORMAT_PARSER_ERR_NO_LOG_SING, log_signature);
     return FALSE;
   }
-
   if (!strncmp(value, log_signature, colon_pos - value) == 0)
   {
     // TODO: fix error type, id, message
     g_set_error(error, FILTERX_FUNCTION_ERROR, 444,
-                FX_CEF_PARSER_ERR_LOG_SIGN_DIFFERS, value, log_signature);
+                EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS, value, log_signature);
     return FALSE;
   }
   return filterx_string_new(++colon_pos, -1);
@@ -91,15 +90,32 @@ _set_dict_value(FilterXObject *out,
   return ok;
 }
 
+gboolean unescape_value_separators(KVScanner *self)
+{
+  gchar escaped_separator[3] = {'\\', self->value_separator, '\0'};
+
+  const gchar *start = self->value->str;
+  const gchar *pos = start;
+
+  while ((pos = g_strstr_len(start, self->value->len - (pos - start), escaped_separator)) != NULL) {
+    g_string_append_len(self->decoded_value, start, pos - start);
+    g_string_append_c(self->decoded_value, self->value_separator);
+    start = pos + 2;
+  }
+  g_string_append(self->decoded_value, start);
+  return TRUE;
+}
+
+
 FilterXObject *
-parse_extensions(FilterXFunctionParser *self, const gchar *input, GError **error, gpointer user_data)
+parse_extensions(FilterXFunctionEventFormatParser *self, const gchar *input, GError **error, gpointer user_data)
 {
   FilterXObject *fillable = (FilterXObject*)user_data;
   FilterXObject *output = filterx_object_create_dict(fillable);
 
   KVScanner kv_scanner;
   kv_scanner_init(&kv_scanner, self->config.extensions_value_separator, self->config.extensions_pair_separator, FALSE);
-  kv_scanner_set_cef_mode(&kv_scanner, TRUE);
+  kv_scanner_set_transform_value(&kv_scanner, unescape_value_separators);
   kv_scanner_input(&kv_scanner, input);
   while (kv_scanner_scan_next(&kv_scanner))
     {
@@ -118,7 +134,7 @@ parse_extensions(FilterXFunctionParser *self, const gchar *input, GError **error
 }
 
 static inline gboolean
-_fill_object_col(FilterXFunctionParser *self, gint64 index, const gchar *input, FilterXObject *fillable, GError **error)
+_fill_object_col(FilterXFunctionEventFormatParser *self, gint64 index, const gchar *input, FilterXObject *fillable, GError **error)
 {
   Field f = field(self, index);
   FilterXObject *key = filterx_string_new(f.name, -1);
@@ -139,18 +155,18 @@ _fill_object_col(FilterXFunctionParser *self, gint64 index, const gchar *input,
 }
 
 gboolean
-parse(FilterXFunctionParser *self, const gchar *log, FilterXObject *fillable, GError **error)
+parse(FilterXFunctionEventFormatParser *self, const gchar *log, FilterXObject *fillable, GError **error)
 {
   gboolean ok = FALSE;
-  size_t nfields = self->config.num_fields;
+  gsize num_fields = self->config.num_fields;
 
   CSVScanner csv_scanner;
   csv_scanner_init(&csv_scanner, &self->csv_opts, log);
 
   guint64 i = 0;
   while (csv_scanner_scan_next(&csv_scanner))
     {
-        if (i >= nfields)
+        if (i >= num_fields)
           break;
 
         const gchar *input = csv_scanner_get_current_value(&csv_scanner);
@@ -166,7 +182,7 @@ parse(FilterXFunctionParser *self, const gchar *log, FilterXObject *fillable, GE
   if (i < self->csv_opts.expected_columns)
     {
       g_set_error(error, FILTERX_FUNCTION_ERROR, 444,
-                FX_CEF_PARSER_ERR_MISSING_COLUMNS, i, self->config.num_fields);
+                EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS, i, self->config.num_fields);
     }
 
 
@@ -179,7 +195,7 @@ parse(FilterXFunctionParser *self, const gchar *log, FilterXObject *fillable, GE
 static gboolean
 _generate(FilterXExprGenerator *s, FilterXObject *fillable)
 {
-  FilterXFunctionParser *self = (FilterXFunctionParser *) s;
+  FilterXFunctionEventFormatParser *self = (FilterXFunctionEventFormatParser *) s;
   gboolean ok = FALSE;
 
   FilterXObject *obj = filterx_expr_eval(self->msg);
@@ -212,7 +228,7 @@ _generate(FilterXExprGenerator *s, FilterXObject *fillable)
 static void
 _free(FilterXExpr *s)
 {
-  FilterXFunctionParser *self = (FilterXFunctionParser *) s;
+  FilterXFunctionEventFormatParser *self = (FilterXFunctionEventFormatParser *) s;
   filterx_expr_unref(self->msg);
   csv_scanner_options_clean(&self->csv_opts);
   filterx_generator_function_free_method(&self->super);
@@ -234,7 +250,7 @@ _extract_msg_expr(FilterXFunctionArgs *args, GError **error)
 }
 
 static gboolean
-_extract_args(FilterXFunctionParser *self, FilterXFunctionArgs *args, GError **error)
+_extract_args(FilterXFunctionEventFormatParser *self, FilterXFunctionArgs *args, GError **error)
 {
   gsize args_len = filterx_function_args_len(args);
   if (args_len != 1)
@@ -258,7 +274,7 @@ _create_container(FilterXExprGenerator *s, FilterXExpr *fillable_parent)
 }
 
 gboolean
-filterx_function_parser_init_instance(FilterXFunctionParser *self, const gchar *fn_name, FilterXFunctionArgs *args, GError **error)
+filterx_function_parser_init_instance(FilterXFunctionEventFormatParser *self, const gchar *fn_name, FilterXFunctionArgs *args, GError **error)
 {
   filterx_generator_function_init_instance(&self->super, fn_name);
   self->super.super.generate = _generate;
@@ -272,7 +288,7 @@ filterx_function_parser_init_instance(FilterXFunctionParser *self, const gchar *
 }
 
 void
-filterx_function_parser_set_config(FilterXFunctionParser *self, Config *cfg)
+filterx_function_parser_set_config(FilterXFunctionEventFormatParser *self, Config *cfg)
 {
   g_assert(cfg);
   self->config = *cfg;
@@ -281,13 +297,12 @@ filterx_function_parser_set_config(FilterXFunctionParser *self, Config *cfg)
   csv_scanner_options_set_dialect(&self->csv_opts, CSV_SCANNER_ESCAPE_UNQUOTED_DELIMITER);
   csv_scanner_options_set_expected_columns(&self->csv_opts, cfg->num_fields);
   self->csv_opts.flags |= CSV_SCANNER_GREEDY;
-  // self->csv_opts.flags |= CSV_SCANNER_STRIP_WHITESPACE;
 }
 
 FilterXExpr *
 filterx_function_parse_new(FilterXFunctionArgs *args, GError **error)
 {
-  FilterXFunctionParser *self = g_new0(FilterXFunctionParser, 1);
+  FilterXFunctionEventFormatParser *self = g_new0(FilterXFunctionEventFormatParser, 1);
   if (!filterx_function_parser_init_instance(self, "parser", args, error))
     goto error;