From d0b76ead32e4b3cf1ee4c9f16f179570c0ba0523 Mon Sep 17 00:00:00 2001 From: idohalevi Date: Tue, 28 Aug 2018 14:53:10 +0300 Subject: [PATCH 1/8] adding support for json markers --- .../logz/logback/LogzioLogbackAppender.java | 51 +++++++++++++++---- .../logback/LogzioLogbackAppenderTest.java | 32 +++++++----- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java index 4d86098..e1fb1d0 100644 --- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java +++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java @@ -16,10 +16,12 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; public class LogzioLogbackAppender extends UnsynchronizedAppenderBase { @@ -35,8 +37,9 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase reservedFields = new HashSet<>(Arrays.asList(new String[] {TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION})); + private static final Set reservedFields = new HashSet<>(Arrays.asList(TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION)); private LogzioSender logzioSender; private ThrowableProxyConverter throwableProxyConverter; @@ -58,6 +61,7 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase> jsonSet = jsonElement.getAsJsonObject().entrySet(); + for (Entry entry : jsonSet) { + logMessage.add(entry.getKey(), entry.getValue()); + } + + } catch (Exception e) { + //on fail just add as a string + logMessage.addProperty(MESSAGE, jsonData); + } + } + @Override protected void append(ILoggingEvent loggingEvent) { if (!loggingEvent.getLoggerName().contains("io.logz.sender")) { diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java index 46b341a..5d95e62 100644 --- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java +++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java @@ -6,7 +6,11 @@ import io.logz.sender.com.google.gson.Gson; import io.logz.test.MockLogzioBulkListener; import org.junit.Test; -import org.slf4j.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; import java.net.InetAddress; import java.util.HashMap; @@ -52,7 +56,7 @@ public void validateJsonMessage(){ } @Test - public void simpleAppending() throws Exception { + public void simpleAppending() { String token = "aBcDeFgHiJkLmNoPqRsT"; String type = "awesomeType"; String loggerName = "simpleAppending"; @@ -72,7 +76,7 @@ public void simpleAppending() throws Exception { } @Test - public void simpleGzipAppending() throws Exception { + public void simpleGzipAppending() { String token = "aBcDeFgHiJkLmNoPqRsTGzIp"; String type = "awesomeGzipType"; String loggerName = "simpleGzipAppending"; @@ -92,7 +96,7 @@ public void simpleGzipAppending() throws Exception { } @Test - public void validateAdditionalFields() throws Exception { + public void validateAdditionalFields() { String token = "validatingAdditionalFields"; String type = "willTryWithOrWithoutEnvironmentVariables"; String loggerName = "additionalLogger"; @@ -137,7 +141,7 @@ public void existingHostname() throws Exception { } @Test - public void existingLine() throws Exception { + public void existingLine() { String token = "checkingLine"; String type = "withLineType"; String loggerName = "test"; @@ -159,7 +163,7 @@ public void existingLine() throws Exception { @SuppressWarnings("ConstantConditions") @Test - public void sendException() throws Exception { + public void sendException() { String token = "checkingExceptions"; String type = "badType"; String loggerName = "exceptionProducer"; @@ -190,7 +194,7 @@ public void sendException() throws Exception { } @Test - public void testMDC() throws Exception { + public void testMDC() { String token = "mdcTokensAreTheBest"; String type = "mdcType"; String loggerName = "mdcTesting"; @@ -216,7 +220,7 @@ public void testMDC() throws Exception { } @Test - public void testMarker() throws Exception { + public void testMarker() { String token = "markerToken"; String type = "markerType"; String loggerName = "markerTesting"; @@ -224,12 +228,14 @@ public void testMarker() throws Exception { String markerTestValue = "MyMarker"; int drainTimeout = 1; String message1 = "Simple log line - "+random(5); - Marker marker = MarkerFactory.getMarker(markerTestValue); - Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null); - testLogger.info(marker, message1); +// Map logstashMarker = new HashMap(); +// logstashMarker.put("customkey_str", "value1"); +// logstashMarker.put("projectid_int", 5); +// testLogger.info(Markers.appendEntries(logstashMarker) , message1); + testLogger.info(marker, message1); sleepSeconds(2 * drainTimeout); mockListener.assertNumberOfReceivedMsgs(1); @@ -239,7 +245,7 @@ public void testMarker() throws Exception { } @Test - public void testContextReset() throws Exception { + public void testContextReset() { logger.info("context.reset() is called when logback loads a new logback.xml in-flight"); String token = "testingContextReset"; String type = "contextResetType"; @@ -293,7 +299,7 @@ public void testTokenAndLogzioUrlFromSystemEnvironment() { mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr); } - public void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map additionalFields) { + private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map additionalFields) { additionalFields.forEach((field, value) -> { String fieldValueInLog = logRequest.getStringFieldOrNull(field); assertThat(fieldValueInLog) From 90cbdbee26d24b27ca082cb92d1379e468e81024 Mon Sep 17 00:00:00 2001 From: idohalevi Date: Tue, 28 Aug 2018 15:27:03 +0300 Subject: [PATCH 2/8] adding test for json markers --- pom.xml | 5 +++ .../logz/logback/LogzioLogbackAppender.java | 4 +-- .../logback/LogzioLogbackAppenderTest.java | 33 ++++++++++++++++--- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 292b37d..2dff75a 100644 --- a/pom.xml +++ b/pom.xml @@ -181,5 +181,10 @@ 9.3.9.M1 test + + net.logstash.logback + logstash-logback-encoder + 5.2 + \ No newline at end of file diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java index e1fb1d0..57a48cd 100644 --- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java +++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java @@ -75,11 +75,11 @@ public void setFormat(String format) { this.format = format; } - public String getMarkersFormatFormat() { + public String getMarkersFormat() { return markersFormat; } - public void setMarkersFormatFormat(String markersFormat) { + public void setMarkersFormat(String markersFormat) { this.markersFormat = markersFormat; } diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java index 5d95e62..d8453e5 100644 --- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java +++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java @@ -5,6 +5,7 @@ import ch.qos.logback.classic.LoggerContext; import io.logz.sender.com.google.gson.Gson; import io.logz.test.MockLogzioBulkListener; +import net.logstash.logback.marker.Markers; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -230,10 +231,6 @@ public void testMarker() { String message1 = "Simple log line - "+random(5); Marker marker = MarkerFactory.getMarker(markerTestValue); Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null); -// Map logstashMarker = new HashMap(); -// logstashMarker.put("customkey_str", "value1"); -// logstashMarker.put("projectid_int", 5); -// testLogger.info(Markers.appendEntries(logstashMarker) , message1); testLogger.info(marker, message1); sleepSeconds(2 * drainTimeout); @@ -244,6 +241,34 @@ public void testMarker() { assertThat(logRequest.getStringFieldOrNull(markerKey)).isEqualTo(markerTestValue); } + @Test + public void testJsonMarkers() { + String token = "markersJsonToken"; + String type = "markersJsonType"; + String loggerName = "markersJsonTesting"; + String markerKey = "marker"; + int drainTimeout = 1; + String message1 = "Simple log line - "+random(5); + + Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null); + LogzioLogbackAppender logzioLogbackAppender = + (LogzioLogbackAppender)((ch.qos.logback.classic.Logger)testLogger).getAppender("LogzioLogbackAppender"); + logzioLogbackAppender.setMarkersFormat("json"); + + Map logstashMarker = new HashMap(); + logstashMarker.put("customkey_str", "value1"); + logstashMarker.put("projectid_int", 5); + testLogger.info(Markers.appendEntries(logstashMarker) , message1); + + sleepSeconds(2 * drainTimeout); + + mockListener.assertNumberOfReceivedMsgs(1); + MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1); + mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr); + assertThat(logRequest.getStringFieldOrNull("customkey_str")).isEqualTo("value1"); + assertThat(logRequest.getStringFieldOrNull("projectid_int")).isEqualTo("5"); + } + @Test public void testContextReset() { logger.info("context.reset() is called when logback loads a new logback.xml in-flight"); From b80629e73473ae9327d39affa5bc88ee047b98cb Mon Sep 17 00:00:00 2001 From: idohalevi Date: Tue, 28 Aug 2018 15:49:28 +0300 Subject: [PATCH 3/8] changed naming of test var --- .../java/io/logz/logback/LogzioLogbackAppenderTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java index d8453e5..90b3f3a 100644 --- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java +++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java @@ -27,9 +27,9 @@ public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest { @Test public void validateJsonMessage(){ - String token = "validatingAdditionalFields"; - String type = "willTryWithOrWithoutEnvironmentVariables"; - String loggerName = "additionalLogger"; + String token = "validateJsonMessageToken"; + String type = "validateJsonMessageType"; + String loggerName = "validateJsonMessageLogger"; int drainTimeout = 1; String messageText = "message test"; From 3968b6e8a05945b830ae956207b984b2038059e0 Mon Sep 17 00:00:00 2001 From: idohalevi Date: Wed, 29 Aug 2018 09:57:21 +0300 Subject: [PATCH 4/8] update readme with marker option --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 900d372..97cbe2c 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender) | **debug** | *false* | Print some debug messages to stdout to help to diagnose issues | | **line** | *false* | Print the line of code that generated this log | | **compressRequests** | *false* | Boolean. `true` if logs are compressed in gzip format before sending. `false` if logs are sent uncompressed. | -| **format** | *text* | Optional. `json` if the logged message is to be parsed as a JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text. +| **format** | *text* | Optional. `json` if the logged message is to be parsed as a JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text.| +| **markersFormat** | *text* | Optional. `json` if the marker is to be parsed as a JSON (in such a way that each JSON node will be a field in logz.io), `text` if the marker is to be treated as plain text, or ignore if the marker is to be ignored and not included in the log. ### Code Example ```java From 2972816b26ced8f51f033b9332346bda361756db Mon Sep 17 00:00:00 2001 From: idohalevi Date: Wed, 29 Aug 2018 10:00:49 +0300 Subject: [PATCH 5/8] adding key option in case of failing to parse the json --- src/main/java/io/logz/logback/LogzioLogbackAppender.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java index 57a48cd..544e198 100644 --- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java +++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java @@ -249,7 +249,7 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) { JsonObject logMessage = new JsonObject(); if (format.equals(FORMAT_JSON)) { - addJsonObject(logMessage, loggingEvent.getFormattedMessage()); + addJsonObject(logMessage, loggingEvent.getFormattedMessage(), MESSAGE); } else { logMessage = new JsonObject(); logMessage.addProperty(MESSAGE, loggingEvent.getFormattedMessage()); @@ -268,7 +268,7 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) { case FORMAT_IGNORE: break; case FORMAT_JSON: - addJsonObject(logMessage, loggingEvent.getMarker().toString()); + addJsonObject(logMessage, loggingEvent.getMarker().toString(), MARKER); break; default: logMessage.addProperty(MARKER, loggingEvent.getMarker().toString()); @@ -293,7 +293,7 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) { return logMessage; } - private void addJsonObject(JsonObject logMessage, String jsonData) { + private void addJsonObject(JsonObject logMessage, String jsonData, String keyOnFail) { try { JsonElement jsonElement = gson.fromJson(jsonData, JsonElement.class); Set> jsonSet = jsonElement.getAsJsonObject().entrySet(); @@ -303,7 +303,7 @@ private void addJsonObject(JsonObject logMessage, String jsonData) { } catch (Exception e) { //on fail just add as a string - logMessage.addProperty(MESSAGE, jsonData); + logMessage.addProperty(keyOnFail, jsonData); } } From 7d9b952b12473cb5d20fb5b5d9f24d6bdccb7f95 Mon Sep 17 00:00:00 2001 From: idohalevi Date: Wed, 29 Aug 2018 10:45:22 +0300 Subject: [PATCH 6/8] removing dependency and updating json markers test --- pom.xml | 5 ----- .../logback/LogzioLogbackAppenderTest.java | 21 +++++++++---------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index 2dff75a..292b37d 100644 --- a/pom.xml +++ b/pom.xml @@ -181,10 +181,5 @@ 9.3.9.M1 test - - net.logstash.logback - logstash-logback-encoder - 5.2 - \ No newline at end of file diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java index 90b3f3a..edecc48 100644 --- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java +++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java @@ -5,7 +5,6 @@ import ch.qos.logback.classic.LoggerContext; import io.logz.sender.com.google.gson.Gson; import io.logz.test.MockLogzioBulkListener; -import net.logstash.logback.marker.Markers; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,7 +124,7 @@ public void existingHostname() throws Exception { String type = "withOrWithoutHostnamr"; String loggerName = "runningOutOfIdeasHere"; int drainTimeout = 1; - String message1 = "Hostname log - " + random(5); + String message1 = "Hostname log - " + random(5); Logger testLogger = createLogger(token, type, loggerName, drainTimeout, true, false, null); testLogger.info(message1); @@ -147,7 +146,7 @@ public void existingLine() { String type = "withLineType"; String loggerName = "test"; int drainTimeout = 1; - String message1 = "Hostname log - " + random(5); + String message1 = "Hostname log - " + random(5); Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, true, null); testLogger.info(message1); @@ -200,7 +199,7 @@ public void testMDC() { String type = "mdcType"; String loggerName = "mdcTesting"; int drainTimeout = 1; - String message1 = "Simple log line - "+random(5); + String message1 = "Simple log line - " + random(5); String mdcKey = "mdc-key"; String mdcValue = "mdc-value"; @@ -248,17 +247,17 @@ public void testJsonMarkers() { String loggerName = "markersJsonTesting"; String markerKey = "marker"; int drainTimeout = 1; - String message1 = "Simple log line - "+random(5); + String message1 = "Simple log line - " + random(5); Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null); LogzioLogbackAppender logzioLogbackAppender = (LogzioLogbackAppender)((ch.qos.logback.classic.Logger)testLogger).getAppender("LogzioLogbackAppender"); logzioLogbackAppender.setMarkersFormat("json"); - Map logstashMarker = new HashMap(); - logstashMarker.put("customkey_str", "value1"); - logstashMarker.put("projectid_int", 5); - testLogger.info(Markers.appendEntries(logstashMarker) , message1); + Map markerMap = new HashMap(); + markerMap.put("customkey_str", "value1"); + markerMap.put("projectid_int", 5); + testLogger.info(MarkerFactory.getMarker(markerMap.toString()), message1); sleepSeconds(2 * drainTimeout); @@ -277,8 +276,8 @@ public void testContextReset() { String loggerName = "ContextResetLogger"; int drainTimeout = 1; - String message1 = "Before Reset Line - "+random(5); - String message2 = "After Reset Line - "+random(5); + String message1 = "Before Reset Line - " + random(5); + String message2 = "After Reset Line - " + random(5); Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null); From 30be8e8e45e46f9c24965659f3675319e4d87ff6 Mon Sep 17 00:00:00 2001 From: idohalevi Date: Wed, 29 Aug 2018 10:48:14 +0300 Subject: [PATCH 7/8] change @timestamp to use ofEpochMilli --- src/main/java/io/logz/logback/LogzioLogbackAppender.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java index 544e198..65dcb48 100644 --- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java +++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java @@ -15,6 +15,7 @@ import java.io.File; import java.net.InetAddress; import java.net.UnknownHostException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -260,7 +261,7 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) { loggingEvent.getMDCPropertyMap().forEach(logMessage::addProperty); } - logMessage.addProperty(TIMESTAMP, new Date(loggingEvent.getTimeStamp()).toInstant().toString()); + logMessage.addProperty(TIMESTAMP, Instant.ofEpochMilli(loggingEvent.getTimeStamp()).toString()); logMessage.addProperty(LOGLEVEL,loggingEvent.getLevel().levelStr); if (loggingEvent.getMarker() != null) { From c64f5fee05b091da076f83f8282aee91803f73f7 Mon Sep 17 00:00:00 2001 From: idohalevi Date: Wed, 29 Aug 2018 11:20:56 +0300 Subject: [PATCH 8/8] updating release notes --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 97cbe2c..ee1d696 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,8 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender) | **debug** | *false* | Print some debug messages to stdout to help to diagnose issues | | **line** | *false* | Print the line of code that generated this log | | **compressRequests** | *false* | Boolean. `true` if logs are compressed in gzip format before sending. `false` if logs are sent uncompressed. | -| **format** | *text* | Optional. `json` if the logged message is to be parsed as a JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text.| -| **markersFormat** | *text* | Optional. `json` if the marker is to be parsed as a JSON (in such a way that each JSON node will be a field in logz.io), `text` if the marker is to be treated as plain text, or ignore if the marker is to be ignored and not included in the log. +| **format** | *text* | Optional. `json` if the logged message is to be parsed as a one level JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text.| +| **markersFormat** | *text* | Optional. `json` if the marker is to be parsed as a one level JSON (in such a way that each JSON node will be a field in logz.io), `text` if the marker is to be treated as plain text, or ignore if the marker is to be ignored and not included in the log. ### Code Example ```java @@ -124,6 +124,8 @@ Will send a log to Logz.io that looks like this: ``` ### Release notes + - 1.0.19 - 1.0.20 + - added support for one level json format log message and marker. - 1.0.18 - added `compressRequests` parameter to enable gzip compression of the logs before they are sent. - added option to inject system property value into additionalFields, logzioUrl and token.