Skip to content

Commit

Permalink
Merge pull request #181 from bshifter/slash-string-fix
Browse files Browse the repository at this point in the history 
lexer fix for slash strings
  • Loading branch information
@alltilla
alltilla authored Jul 2, 2024
2 parents cb19b84 + b27acd3 commit 3a04ea2
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 23 deletions.
50 changes: 27 additions & 23 deletions lib/cfg-lex.l
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,28 +367,32 @@ filterx_word [^ \#'"/\(\)\{\}\[\]\\;\r\n\t,|\.@:]
<INITIAL,filterx>. { return (unsigned char) yytext[0]; }
/* continuation line within a string: just move the location and skip the newline character as if it was never there */
<string,qstring,slash_string>\\\r?\n { _cfg_lex_extend_token_location_to_next_line(yyextra); }
<string>\\a { g_string_append_c(yyextra->string_buffer, 7); }
<string>\\n { g_string_append_c(yyextra->string_buffer, 10); }
<string>\\r { g_string_append_c(yyextra->string_buffer, 13); }
<string>\\t { g_string_append_c(yyextra->string_buffer, 9); }
<string>\\v { g_string_append_c(yyextra->string_buffer, 11); }
<string>\\x{xdigit}{1,2} { g_string_append_c(yyextra->string_buffer, strtol(yytext+2, NULL, 16)); }
<string>\\o{odigit}{1,3} { g_string_append_c(yyextra->string_buffer, strtol(yytext+2, NULL, 8)); }
<string>\\[^anrtv] { g_string_append_c(yyextra->string_buffer, yytext[1]); }
<string>\" {
yy_pop_state(yyscanner);
yylval->cptr = strdup(yyextra->string_buffer->str);
return LL_STRING;
}
<slash_string>\\\/ { g_string_append_c(yyextra->string_buffer, '/'); }
<slash_string>[^/\r\n]+ { g_string_append(yyextra->string_buffer, yytext); }
<slash_string>\/ {
yy_pop_state(yyscanner);
yylval->cptr = strdup(yyextra->string_buffer->str);
return LL_STRING;
}
<string>[^"\\\r\n]+ { g_string_append(yyextra->string_buffer, yytext); }
<string,qstring,slash_string>\\\r?\n { _cfg_lex_extend_token_location_to_next_line(yyextra); }
<string,slash_string>\\a { g_string_append_c(yyextra->string_buffer, 7); }
<string,slash_string>\\n { g_string_append_c(yyextra->string_buffer, 10); }
<string,slash_string>\\r { g_string_append_c(yyextra->string_buffer, 13); }
<string,slash_string>\\t { g_string_append_c(yyextra->string_buffer, 9); }
<string,slash_string>\\v { g_string_append_c(yyextra->string_buffer, 11); }
<string,slash_string>\\x{xdigit}{1,2} { g_string_append_c(yyextra->string_buffer, strtol(yytext+2, NULL, 16)); }
<string,slash_string>\\o{odigit}{1,3} { g_string_append_c(yyextra->string_buffer, strtol(yytext+2, NULL, 8)); }
<string>\\[^anrtv] { g_string_append_c(yyextra->string_buffer, yytext[1]); }
<string>[^"\\\r\n]+ { g_string_append(yyextra->string_buffer, yytext); }
<string>\" {
yy_pop_state(yyscanner);
yylval->cptr = strdup(yyextra->string_buffer->str);
return LL_STRING;
}
<slash_string>\\b { g_string_append_c(yyextra->string_buffer, 8); }
<slash_string>\\f { g_string_append_c(yyextra->string_buffer, 12); }
<slash_string>\\\\ { g_string_append_c(yyextra->string_buffer, '\\'); }
<slash_string>\\\/ { g_string_append_c(yyextra->string_buffer, '/'); }
<slash_string>\\[^anrtvbfox] { g_string_append_c(yyextra->string_buffer, yytext[1]); }
<slash_string>[^\/\\\r\n]+ { g_string_append(yyextra->string_buffer, yytext); }
<slash_string>\/ {
yy_pop_state(yyscanner);
yylval->cptr = strdup(yyextra->string_buffer->str);
return LL_STRING;
}
<qstring>[^'\r\n]+ { g_string_append(yyextra->string_buffer, yytext); }
<qstring>\' {
yy_pop_state(yyscanner);
Expand Down Expand Up @@ -421,7 +425,7 @@ filterx_word [^ \#'"/\(\)\{\}\[\]\\;\r\n\t,|\.@:]
* - the rule below gets matched and the newline is included in the string
*/
<string,qstring>\r?\n {
<string,qstring,slash_string>\r?\n {
g_string_append(yyextra->string_buffer, yytext);
_cfg_lex_extend_token_location_to_next_line(yyextra);
}
Expand Down
57 changes: 57 additions & 0 deletions tests/light/functional_tests/filterx/test_filterx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1680,3 +1680,60 @@ def = "default";
assert file_true.get_stats()["processed"] == 1
assert "processed" not in file_false.get_stats()
assert file_true.read_log() == '{"a":"default","b":"2","c":"2","d":"3","e":"1"}\n'


def test_slash_string_features(config, syslog_ng):
cfg = r"""
$MSG = json();
$MSG.base = /foo bar/;
$MSG.line_break = /foo
bar/;
$MSG.joint_lines = /foo \
bar/;
## escaped characters
$MSG.escaped_backslash = /foo\\bar/;
$MSG.escaped_slash = /foo\/bar/;
$MSG.non_escaped_single_quotes = /foo'bar/;
$MSG.non_escaped_double_quotes = /foo"bar/;
## special characters
$MSG.new_line = /foo\nbar/;
$MSG.tab = /foo\tbar/;
$MSG.carrige_return = /foo\rbar/;
$MSG.vertical_tab = /foo\vbar/;
$MSG.form_feed = /foo\fbar/;
$MSG.backspace = /foo\bbar/;
$MSG.alert = /foo\abar/;
##
$MSG.hexadecimal_value = /foo\x40bar/;
$MSG.octal_value = /foo\o100bar/;
"""

(file_true, file_false) = create_config(
config, cfg,
)
syslog_ng.start(config)

assert file_true.get_stats()["processed"] == 1
assert "processed" not in file_false.get_stats()
exp = (
r"""{"base":"foo bar","""
r""""line_break":"foo\nbar","""
r""""joint_lines":"foo bar","""
r""""escaped_backslash":"foo\\bar","""
r""""escaped_slash":"foo\/bar","""
r""""non_escaped_single_quotes":"foo'bar","""
r""""non_escaped_double_quotes":"foo\"bar","""
r""""new_line":"foo\nbar","""
r""""tab":"foo\tbar","""
r""""carrige_return":"foo\rbar","""
r""""vertical_tab":"foo\u000bbar","""
r""""form_feed":"foo\fbar","""
r""""backspace":"foo\bbar","""
r""""alert":"foo\u0007bar","""
r""""hexadecimal_value":"foo@bar","""
r""""octal_value":"foo@bar"}""" + "\n"
)
res = file_true.read_log()
assert res == exp

0 comments on commit 3a04ea2

Please sign in to comment.