csv-parser: add parsing to log message matches

Instead of always having to come up with names for the columns we parse,
this patch makes it possible to parse into the "matches" array, e.g.
$1, $2, $3...

To do this, simply omit the previously mandatory columns() option to
csv-parser().

	@Version: current

	log {
		source { tcp(port(2000) flags(no-parse)); };

		parser { csv-parser(delimiters(',')  dialect(escape-backslash)); };
		destination { stdout(template("$ISODATE $*\n")); };
	};

This makes the parsed content also available as a syslog-ng list, as
the "$*" macro can be used to produce a list out of the parsed values.

Signed-off-by: yashmathne <yash6866@gmail.com>
Signed-off-by: Balazs Scheidler <balazs.scheidler@axoflow.com>

modules/csvparser/csvparser.c

@@ -151,13 +151,29 @@ csv_parser_process(LogParser *s, LogMessage **pmsg, const LogPathOptions *path_o
     g_string_assign(key_scratch, self->prefix);
 
   key_formatter_t _key_formatter = dispatch_key_formatter(self->prefix);
+  gint match_index = 1;
+
   while (csv_scanner_scan_next(&scanner))
     {
-
-      log_msg_set_value_by_name(msg,
-                                _key_formatter(key_scratch, csv_scanner_get_current_name(&scanner), self->prefix_len),
-                                csv_scanner_get_current_value(&scanner),
-                                csv_scanner_get_current_value_len(&scanner));
+      const gchar *current_name = csv_scanner_get_current_name(&scanner);
+
+      if (current_name)
+        {
+          log_msg_set_value_by_name(msg,
+                                    _key_formatter(key_scratch, csv_scanner_get_current_name(&scanner), self->prefix_len),
+                                    csv_scanner_get_current_value(&scanner),
+                                    csv_scanner_get_current_value_len(&scanner));
+        }
+      else
+        {
+          if (match_index == 1)
+            log_msg_unset_match(msg, 0);
+          log_msg_set_match_with_type(msg,
+                                      match_index, csv_scanner_get_current_value(&scanner),
+                                      csv_scanner_get_current_value_len(&scanner),
+                                      LM_VT_STRING);
+        }
+      match_index++;
     }
 
   gboolean result = TRUE;

modules/csvparser/csvparser.h

@@ -31,6 +31,7 @@ CSVScannerOptions *csv_parser_get_scanner_options(LogParser *s);
 gboolean csv_parser_set_flags(LogParser *s, guint32 flags);
 void csv_parser_set_drop_invalid(LogParser *s, gboolean drop_invalid);
 void csv_parser_set_prefix(LogParser *s, const gchar *prefix);
+void csv_parser_set_list_name(LogParser *s, const gchar *list_name);
 LogParser *csv_parser_new(GlobalConfig *cfg);
 
 guint32 csv_parser_lookup_flag(const gchar *flag);

modules/csvparser/tests/Makefile.am

@@ -1,12 +1,15 @@
 modules_csvparser_tests_TESTS			=	\
 	modules/csvparser/tests/test_csvparser		\
 	modules/csvparser/tests/test_csvparser_from_config \
-	modules/csvparser/tests/test_csvparser_perf
+	modules/csvparser/tests/test_csvparser_perf 
 
 check_PROGRAMS					+=	\
-	${modules_csvparser_tests_TESTS}
+	${modules_csvparser_tests_TESTS} 
 
-EXTRA_DIST += modules/csvparser/tests/CMakeLists.txt
+EXTRA_DIST += modules/csvparser/tests/CMakeLists.txt \
+	modules/basicfuncs/list-funcs.c			\
+	modules/basicfuncs/tf-template.c		\
+	modules/basicfuncs/tests/CMakeLists.txt 
 
 modules_csvparser_tests_test_csvparser_CFLAGS	=	\
 	$(TEST_CFLAGS) -I$(top_srcdir)/modules/csvparser
@@ -18,7 +21,8 @@ modules_csvparser_tests_test_csvparser_from_config_CFLAGS	=	\
 	$(TEST_CFLAGS) -I$(top_srcdir)/modules/csvparser
 modules_csvparser_tests_test_csvparser_from_config_LDADD	=	\
 	$(TEST_LDADD)					\
-	-dlpreopen $(top_builddir)/modules/csvparser/libcsvparser.la
+	$(PREOPEN_SYSLOGFORMAT) $(PREOPEN_BASICFUNCS) \
+	-dlpreopen $(top_builddir)/modules/csvparser/libcsvparser.la 
 
 modules_csvparser_tests_test_csvparser_perf_CFLAGS	=	\
 	$(TEST_CFLAGS) -I$(top_srcdir)/modules/csvparser

modules/csvparser/tests/test_csvparser_from_config.c

@@ -94,6 +94,17 @@ Test(parser, condition_success)
   assert_log_message_value_by_name(msg, "c", "baz");
 }
 
+Test(parser, test_index_macros)
+{
+  LogParser *test_parser = create_parser_rule("csv-parser();");
+
+  invoke_parser_rule(test_parser, msg);
+  assert_log_message_value_by_name(msg, "1", "foo");
+  assert_log_message_value_by_name(msg, "2", "bar");
+  assert_log_message_value_by_name(msg, "3", "baz");
+  assert_template_format_msg("$*", "foo,bar,baz", msg);
+}
+
 void
 setup(void)
 {