From 99f3284e6ead1b88e2cd709b3b47a7e681791829 Mon Sep 17 00:00:00 2001 From: yashmathne Date: Sun, 18 Sep 2022 19:24:29 +0530 Subject: [PATCH] csv-parser: add parsing to log message matches Instead of always having to come up with names for the columns we parse, this patch makes it possible to parse into the "matches" array, e.g. $1, $2, $3... To do this, simply omit the previously mandatory columns() option to csv-parser(). @version: current log { source { tcp(port(2000) flags(no-parse)); }; parser { csv-parser(delimiters(',') dialect(escape-backslash)); }; destination { stdout(template("$ISODATE $*\n")); }; }; This makes the parsed content also available as a syslog-ng list, as the "$*" macro can be used to produce a list out of the parsed values. Signed-off-by: yashmathne Signed-off-by: Balazs Scheidler --- modules/csvparser/csvparser.c | 26 +++++++++++++++---- modules/csvparser/csvparser.h | 1 + modules/csvparser/tests/Makefile.am | 12 ++++++--- .../tests/test_csvparser_from_config.c | 11 ++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/modules/csvparser/csvparser.c b/modules/csvparser/csvparser.c index 54bd8aa0fd3..71ce63e59a7 100644 --- a/modules/csvparser/csvparser.c +++ b/modules/csvparser/csvparser.c @@ -151,13 +151,29 @@ csv_parser_process(LogParser *s, LogMessage **pmsg, const LogPathOptions *path_o g_string_assign(key_scratch, self->prefix); key_formatter_t _key_formatter = dispatch_key_formatter(self->prefix); + gint match_index = 1; + while (csv_scanner_scan_next(&scanner)) { - - log_msg_set_value_by_name(msg, - _key_formatter(key_scratch, csv_scanner_get_current_name(&scanner), self->prefix_len), - csv_scanner_get_current_value(&scanner), - csv_scanner_get_current_value_len(&scanner)); + const gchar *current_name = csv_scanner_get_current_name(&scanner); + + if (current_name) + { + log_msg_set_value_by_name(msg, + _key_formatter(key_scratch, csv_scanner_get_current_name(&scanner), self->prefix_len), + csv_scanner_get_current_value(&scanner), + csv_scanner_get_current_value_len(&scanner)); + } + else + { + if (match_index == 1) + log_msg_unset_match(msg, 0); + log_msg_set_match_with_type(msg, + match_index, csv_scanner_get_current_value(&scanner), + csv_scanner_get_current_value_len(&scanner), + LM_VT_STRING); + } + match_index++; } gboolean result = TRUE; diff --git a/modules/csvparser/csvparser.h b/modules/csvparser/csvparser.h index eeeb78cb65e..eaa2296b36d 100644 --- a/modules/csvparser/csvparser.h +++ b/modules/csvparser/csvparser.h @@ -31,6 +31,7 @@ CSVScannerOptions *csv_parser_get_scanner_options(LogParser *s); gboolean csv_parser_set_flags(LogParser *s, guint32 flags); void csv_parser_set_drop_invalid(LogParser *s, gboolean drop_invalid); void csv_parser_set_prefix(LogParser *s, const gchar *prefix); +void csv_parser_set_list_name(LogParser *s, const gchar *list_name); LogParser *csv_parser_new(GlobalConfig *cfg); guint32 csv_parser_lookup_flag(const gchar *flag); diff --git a/modules/csvparser/tests/Makefile.am b/modules/csvparser/tests/Makefile.am index e1f8e599582..4ea2f75e657 100644 --- a/modules/csvparser/tests/Makefile.am +++ b/modules/csvparser/tests/Makefile.am @@ -1,12 +1,15 @@ modules_csvparser_tests_TESTS = \ modules/csvparser/tests/test_csvparser \ modules/csvparser/tests/test_csvparser_from_config \ - modules/csvparser/tests/test_csvparser_perf + modules/csvparser/tests/test_csvparser_perf check_PROGRAMS += \ - ${modules_csvparser_tests_TESTS} + ${modules_csvparser_tests_TESTS} -EXTRA_DIST += modules/csvparser/tests/CMakeLists.txt +EXTRA_DIST += modules/csvparser/tests/CMakeLists.txt \ + modules/basicfuncs/list-funcs.c \ + modules/basicfuncs/tf-template.c \ + modules/basicfuncs/tests/CMakeLists.txt modules_csvparser_tests_test_csvparser_CFLAGS = \ $(TEST_CFLAGS) -I$(top_srcdir)/modules/csvparser @@ -18,7 +21,8 @@ modules_csvparser_tests_test_csvparser_from_config_CFLAGS = \ $(TEST_CFLAGS) -I$(top_srcdir)/modules/csvparser modules_csvparser_tests_test_csvparser_from_config_LDADD = \ $(TEST_LDADD) \ - -dlpreopen $(top_builddir)/modules/csvparser/libcsvparser.la + $(PREOPEN_SYSLOGFORMAT) $(PREOPEN_BASICFUNCS) \ + -dlpreopen $(top_builddir)/modules/csvparser/libcsvparser.la modules_csvparser_tests_test_csvparser_perf_CFLAGS = \ $(TEST_CFLAGS) -I$(top_srcdir)/modules/csvparser diff --git a/modules/csvparser/tests/test_csvparser_from_config.c b/modules/csvparser/tests/test_csvparser_from_config.c index ff2af136077..a021a705534 100644 --- a/modules/csvparser/tests/test_csvparser_from_config.c +++ b/modules/csvparser/tests/test_csvparser_from_config.c @@ -94,6 +94,17 @@ Test(parser, condition_success) assert_log_message_value_by_name(msg, "c", "baz"); } +Test(parser, test_index_macros) +{ + LogParser *test_parser = create_parser_rule("csv-parser();"); + + invoke_parser_rule(test_parser, msg); + assert_log_message_value_by_name(msg, "1", "foo"); + assert_log_message_value_by_name(msg, "2", "bar"); + assert_log_message_value_by_name(msg, "3", "baz"); + assert_template_format_msg("$*", "foo,bar,baz", msg); +} + void setup(void) {