Merge pull request #50 from logzio/json_markers

Added option to send a one level json format marker
logzio · Aug 29, 2018 · dd35469 · dd35469
2 parents bc8d486 + c64f5fe
commit dd35469
Show file tree

Hide file tree

Showing 3 changed files with 97 additions and 34 deletions.
diff --git a/README.md b/README.md
@@ -50,7 +50,8 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender)
 | **debug**       | *false*                                    | Print some debug messages to stdout to help to diagnose issues |
 | **line**       | *false*                                    | Print the line of code that generated this log  |
 | **compressRequests**       | *false*                                    | Boolean. `true` if logs are compressed in gzip format before sending. `false` if logs are sent uncompressed. |
-| **format** | *text*  | Optional. `json` if the logged message is to be parsed as a JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text.
+| **format** | *text*  | Optional. `json` if the logged message is to be parsed as a one level JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text.|
+| **markersFormat** | *text*  | Optional. `json` if the marker is to be parsed as a one level JSON (in such a way that each JSON node will be a field in logz.io), `text` if the marker is to be treated as plain text, or ignore if the marker is to be ignored and not included in the log.
 
 ### Code Example
 ```java
@@ -123,6 +124,8 @@ Will send a log to Logz.io that looks like this:
 ```
 
 ### Release notes
+ - 1.0.19 - 1.0.20
+   - added support for one level json format log message and marker. 
  - 1.0.18
    - added `compressRequests` parameter to enable gzip compression of the logs before they are sent.
    - added option to inject system property value into additionalFields, logzioUrl and token.

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -15,11 +15,14 @@
 import java.io.File;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 
 public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
@@ -35,8 +38,9 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private static final String EXCEPTION = "exception";
     private static final String FORMAT_TEXT = "text";
     private static final String FORMAT_JSON = "json";
+    private static final String FORMAT_IGNORE = "ignore";
 
-    private static final Set<String> reservedFields =  new HashSet<>(Arrays.asList(new String[] {TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION}));
+    private static final Set<String> reservedFields =  new HashSet<>(Arrays.asList(TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION));
 
     private LogzioSender logzioSender;
     private ThrowableProxyConverter throwableProxyConverter;
@@ -58,6 +62,7 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private boolean compressRequests = false;
     private int gcPersistedQueueFilesIntervalSeconds = 30;
     private String format = FORMAT_TEXT;
+    private String markersFormat = FORMAT_TEXT;
 
     public LogzioLogbackAppender() {
         super();
@@ -71,6 +76,14 @@ public void setFormat(String format) {
         this.format = format;
     }
 
+    public String getMarkersFormat() {
+        return markersFormat;
+    }
+
+    public void setMarkersFormat(String markersFormat) {
+        this.markersFormat = markersFormat;
+    }
+
     public void setToken(String logzioToken) {
         this.logzioToken = getValueFromSystemEnvironmentIfNeeded(logzioToken);
     }
@@ -208,7 +221,7 @@ public void start() {
         }
         throwableProxyConverter = new ThrowableProxyConverter();
         lineOfCallerConverter = new LineOfCallerConverter();
-        throwableProxyConverter.setOptionList(Arrays.asList("full"));
+        throwableProxyConverter.setOptionList(Collections.singletonList("full"));
         throwableProxyConverter.start();
         super.start();
     }
@@ -234,16 +247,10 @@ private String getValueFromSystemEnvironmentIfNeeded(String value) {
     }
 
     private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) {
-        JsonObject logMessage;
+        JsonObject logMessage = new JsonObject();
 
         if (format.equals(FORMAT_JSON)) {
-            try {
-                JsonElement jsonElement = gson.fromJson(loggingEvent.getFormattedMessage(), JsonElement.class);
-                logMessage = jsonElement.getAsJsonObject();
-            } catch (Exception e) {
-                logMessage = new JsonObject();
-                logMessage.addProperty(MESSAGE, loggingEvent.getFormattedMessage());
-            }
+            addJsonObject(logMessage, loggingEvent.getFormattedMessage(), MESSAGE);
         } else {
             logMessage = new JsonObject();
             logMessage.addProperty(MESSAGE, loggingEvent.getFormattedMessage());
@@ -254,11 +261,20 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) {
             loggingEvent.getMDCPropertyMap().forEach(logMessage::addProperty);
         }
 
-        logMessage.addProperty(TIMESTAMP, new Date(loggingEvent.getTimeStamp()).toInstant().toString());
+        logMessage.addProperty(TIMESTAMP, Instant.ofEpochMilli(loggingEvent.getTimeStamp()).toString());
         logMessage.addProperty(LOGLEVEL,loggingEvent.getLevel().levelStr);
 
         if (loggingEvent.getMarker() != null) {
-            logMessage.addProperty(MARKER, loggingEvent.getMarker().toString());
+            switch (markersFormat) {
+                case FORMAT_IGNORE:
+                    break;
+                case FORMAT_JSON:
+                    addJsonObject(logMessage, loggingEvent.getMarker().toString(), MARKER);
+                    break;
+                default:
+                    logMessage.addProperty(MARKER, loggingEvent.getMarker().toString());
+                    break;
+            }
         }
 
         logMessage.addProperty(LOGGER, loggingEvent.getLoggerName());
@@ -278,6 +294,20 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) {
         return logMessage;
     }
 
+    private void addJsonObject(JsonObject logMessage, String jsonData, String keyOnFail) {
+        try {
+            JsonElement jsonElement = gson.fromJson(jsonData, JsonElement.class);
+            Set<Entry<String, JsonElement>> jsonSet = jsonElement.getAsJsonObject().entrySet();
+            for (Entry<String, JsonElement> entry : jsonSet) {
+                logMessage.add(entry.getKey(), entry.getValue());
+            }
+
+        } catch (Exception e) {
+            //on fail just add as a string
+            logMessage.addProperty(keyOnFail, jsonData);
+        }
+    }
+
     @Override
     protected void append(ILoggingEvent loggingEvent) {
         if (!loggingEvent.getLoggerName().contains("io.logz.sender")) {

diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -6,7 +6,11 @@
 import io.logz.sender.com.google.gson.Gson;
 import io.logz.test.MockLogzioBulkListener;
 import org.junit.Test;
-import org.slf4j.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
 
 import java.net.InetAddress;
 import java.util.HashMap;
@@ -22,9 +26,9 @@ public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest {
 
     @Test
     public void validateJsonMessage(){
-        String token = "validatingAdditionalFields";
-        String type = "willTryWithOrWithoutEnvironmentVariables";
-        String loggerName = "additionalLogger";
+        String token = "validateJsonMessageToken";
+        String type = "validateJsonMessageType";
+        String loggerName = "validateJsonMessageLogger";
         int drainTimeout = 1;
         String messageText = "message test";
 
@@ -52,7 +56,7 @@ public void validateJsonMessage(){
     }
 
     @Test
-    public void simpleAppending() throws Exception {
+    public void simpleAppending() {
         String token = "aBcDeFgHiJkLmNoPqRsT";
         String type = "awesomeType";
         String loggerName = "simpleAppending";
@@ -72,7 +76,7 @@ public void simpleAppending() throws Exception {
     }
 
     @Test
-    public void simpleGzipAppending() throws Exception {
+    public void simpleGzipAppending() {
         String token = "aBcDeFgHiJkLmNoPqRsTGzIp";
         String type = "awesomeGzipType";
         String loggerName = "simpleGzipAppending";
@@ -92,7 +96,7 @@ public void simpleGzipAppending() throws Exception {
     }
 
     @Test
-    public void validateAdditionalFields() throws Exception {
+    public void validateAdditionalFields() {
         String token = "validatingAdditionalFields";
         String type = "willTryWithOrWithoutEnvironmentVariables";
         String loggerName = "additionalLogger";
@@ -120,7 +124,7 @@ public void existingHostname() throws Exception {
         String type = "withOrWithoutHostnamr";
         String loggerName = "runningOutOfIdeasHere";
         int drainTimeout = 1;
-        String message1 = "Hostname log - " +  random(5);
+        String message1 = "Hostname log - " + random(5);
 
         Logger testLogger = createLogger(token, type, loggerName, drainTimeout, true, false, null);
         testLogger.info(message1);
@@ -137,12 +141,12 @@ public void existingHostname() throws Exception {
     }
 
     @Test
-    public void existingLine() throws Exception {
+    public void existingLine() {
         String token = "checkingLine";
         String type = "withLineType";
         String loggerName = "test";
         int drainTimeout = 1;
-        String message1 = "Hostname log - " +  random(5);
+        String message1 = "Hostname log - " + random(5);
 
         Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, true, null);
         testLogger.info(message1);
@@ -159,7 +163,7 @@ public void existingLine() throws Exception {
 
     @SuppressWarnings("ConstantConditions")
     @Test
-    public void sendException() throws Exception {
+    public void sendException() {
         String token = "checkingExceptions";
         String type = "badType";
         String loggerName = "exceptionProducer";
@@ -190,12 +194,12 @@ public void sendException() throws Exception {
     }
 
     @Test
-    public void testMDC() throws Exception {
+    public void testMDC() {
         String token = "mdcTokensAreTheBest";
         String type = "mdcType";
         String loggerName = "mdcTesting";
         int drainTimeout = 1;
-        String message1 = "Simple log line - "+random(5);
+        String message1 = "Simple log line - " + random(5);
         String mdcKey = "mdc-key";
         String mdcValue = "mdc-value";
 
@@ -216,20 +220,18 @@ public void testMDC() throws Exception {
     }
 
     @Test
-    public void testMarker() throws Exception {
+    public void testMarker() {
         String token = "markerToken";
         String type = "markerType";
         String loggerName = "markerTesting";
         String markerKey = "marker";
         String markerTestValue = "MyMarker";
         int drainTimeout = 1;
         String message1 = "Simple log line - "+random(5);
-
         Marker marker = MarkerFactory.getMarker(markerTestValue);
-
         Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
-        testLogger.info(marker, message1);
 
+        testLogger.info(marker, message1);
         sleepSeconds(2 * drainTimeout);
 
         mockListener.assertNumberOfReceivedMsgs(1);
@@ -239,15 +241,43 @@ public void testMarker() throws Exception {
     }
 
     @Test
-    public void testContextReset() throws Exception {
+    public void testJsonMarkers() {
+        String token = "markersJsonToken";
+        String type = "markersJsonType";
+        String loggerName = "markersJsonTesting";
+        String markerKey = "marker";
+        int drainTimeout = 1;
+        String message1 = "Simple log line - " + random(5);
+
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        LogzioLogbackAppender logzioLogbackAppender =
+                (LogzioLogbackAppender)((ch.qos.logback.classic.Logger)testLogger).getAppender("LogzioLogbackAppender");
+        logzioLogbackAppender.setMarkersFormat("json");
+
+        Map markerMap = new HashMap();
+        markerMap.put("customkey_str", "value1");
+        markerMap.put("projectid_int", 5);
+        testLogger.info(MarkerFactory.getMarker(markerMap.toString()), message1);
+
+        sleepSeconds(2 * drainTimeout);
+
+        mockListener.assertNumberOfReceivedMsgs(1);
+        MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
+        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
+        assertThat(logRequest.getStringFieldOrNull("customkey_str")).isEqualTo("value1");
+        assertThat(logRequest.getStringFieldOrNull("projectid_int")).isEqualTo("5");
+    }
+
+    @Test
+    public void testContextReset() {
         logger.info("context.reset() is called when logback loads a new logback.xml in-flight");
         String token = "testingContextReset";
         String type = "contextResetType";
         String loggerName = "ContextResetLogger";
         int drainTimeout = 1;
 
-        String message1 = "Before Reset Line - "+random(5);
-        String message2 = "After Reset Line - "+random(5);
+        String message1 = "Before Reset Line - " + random(5);
+        String message2 = "After Reset Line - " + random(5);
 
         Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
 
@@ -293,7 +323,7 @@ public void testTokenAndLogzioUrlFromSystemEnvironment() {
         mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
     }
 
-    public void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map<String, String> additionalFields) {
+    private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map<String, String> additionalFields) {
         additionalFields.forEach((field, value) -> {
             String fieldValueInLog = logRequest.getStringFieldOrNull(field);
             assertThat(fieldValueInLog)