Skip to content

Commit

Permalink
Re-Configurable loggers (#736)
Browse files Browse the repository at this point in the history 
* LogFilter types

* LogFilter, LogFormat, LoggerNameExtractor, LogGroup - sealed and custom types, configuration - equal

* example with  http api for updates 

* ReconfigurableLogger

* LoggerLayers

* ConfigurableLoggerApp

* examples and docs updates

* benchmarks

* LoggerFactory typo fix
  • Loading branch information
@justcoon
justcoon authored Jan 18, 2024
1 parent 9b5b852 commit dca943d
Show file tree
Hide file tree
Showing 51 changed files with 2,501 additions and 1,205 deletions.
85 changes: 50 additions & 35 deletions benchmarks/src/main/scala/zio/logging/FilterBenchmarks.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class FilterBenchmarks {
val runtime = Runtime.default

val unfilteredLogging: ZLayer[Any, Nothing, Unit] =
Runtime.removeDefaultLoggers >>> consoleLogger(ConsoleLoggerConfig(LogFormat.default, LogFilter.acceptAll))
Runtime.removeDefaultLoggers >>> makeSystemOutLogger(LogFormat.default.toLogger).install

val handWrittenFilteredLogging: ZLayer[Any, Nothing, Unit] = {
val loggerNameGroup: LogGroup[Any, String] = LoggerNameExtractor.loggerNameAnnotationOrTrace.toLogGroup()
Expand All @@ -37,40 +37,45 @@ class FilterBenchmarks {
}
}
)
Runtime.removeDefaultLoggers >>> consoleLogger(ConsoleLoggerConfig(LogFormat.default, filter))
Runtime.removeDefaultLoggers >>> makeSystemOutLogger(LogFormat.default.toLogger)
.filter(filter)
.install
}

val filterConfig: LogFilter.LogLevelByNameConfig = LogFilter.LogLevelByNameConfig(
LogLevel.Debug,
"a.b.c" -> LogLevel.Info,
"a.b.d" -> LogLevel.Warning,
"e" -> LogLevel.Info,
"f.g" -> LogLevel.Error,
"f" -> LogLevel.Info
)

val filterByLogLevelAndNameLogging: ZLayer[Any, Nothing, Unit] =
Runtime.removeDefaultLoggers >>> consoleLogger(
ConsoleLoggerConfig(
LogFormat.default,
LogFilter.logLevelByName(
LogLevel.Debug,
"a.b.c" -> LogLevel.Info,
"a.b.d" -> LogLevel.Warning,
"e" -> LogLevel.Info,
"f.g" -> LogLevel.Error,
"f" -> LogLevel.Info
)
)
)
Runtime.removeDefaultLoggers >>> makeSystemOutLogger(LogFormat.default.toLogger)
.filter(filterConfig.toFilter)
.install

val cachedFilterByLogLevelAndNameLogging: ZLayer[Any, Nothing, Unit] =
Runtime.removeDefaultLoggers >>> consoleLogger(
ConsoleLoggerConfig(
LogFormat.default,
LogFilter
.logLevelByName(
LogLevel.Debug,
"a.b.c" -> LogLevel.Info,
"a.b.d" -> LogLevel.Warning,
"e" -> LogLevel.Info,
"f.g" -> LogLevel.Error,
"f" -> LogLevel.Info
)
.cached
Runtime.removeDefaultLoggers >>> makeSystemOutLogger(LogFormat.default.toLogger)
.filter(filterConfig.toFilter.cached)
.install

val reconfigurableFilterByLogLevelAndNameLogging: ZLayer[Any, Nothing, Unit] =
Runtime.removeDefaultLoggers >>> ReconfigurableLogger
.make[Any, Nothing, String, Any, ConsoleLoggerConfig](
ZIO.succeed(ConsoleLoggerConfig(LogFormat.default, filterConfig)),
(config, _) => makeSystemOutLogger(config.format.toLogger).filter(config.toFilter)
)
)
.installUnscoped[Any]

val reconfigurableCachedFilterByLogLevelAndNameLogging: ZLayer[Any, Nothing, Unit] =
Runtime.removeDefaultLoggers >>> ReconfigurableLogger
.make[Any, Nothing, String, Any, ConsoleLoggerConfig](
ZIO.succeed(ConsoleLoggerConfig(LogFormat.default, filterConfig)),
(config, _) => makeSystemOutLogger(config.format.toLogger).filter(config.toFilter.cached)
)
.installUnscoped[Any]

val names: List[String] = List(
"a",
Expand Down Expand Up @@ -108,15 +113,17 @@ class FilterBenchmarks {
}

/**
* 2022/10/28 Initial results
* 2023/12/26 Initial results
*
* jmh:run -i 3 -wi 3 -f1 -t1 .*FilterBenchmarks.*
*
* Benchmark Mode Cnt Score Error Units
* FilterBenchmarks.cachedFilterByLogLevelAndNameLog thrpt 3 16623.054 ± 15855.331 ops/s
* FilterBenchmarks.filterByLogLevelAndNameLog thrpt 3 18048.598 ± 3868.976 ops/s
* FilterBenchmarks.handWrittenFilterLog thrpt 3 16352.488 ± 2316.372 ops/s
* FilterBenchmarks.noFilteringLog thrpt 3 15104.002 ± 3857.108 ops/s
* Benchmark Mode Cnt Score Error Units
* FilterBenchmarks.cachedFilterByLogLevelAndNameLog thrpt 3 14795.547 ± 1372.850 ops/s
* FilterBenchmarks.filterByLogLevelAndNameLog thrpt 3 15093.994 ± 1230.494 ops/s
* FilterBenchmarks.handWrittenFilterLog thrpt 3 13157.888 ± 10193.287 ops/s
* FilterBenchmarks.noFilteringLog thrpt 3 11043.746 ± 230.514 ops/s
* FilterBenchmarks.reconfigurableCachedFilterByLogLevelAndNameLog thrpt 3 7532.412 ± 415.760 ops/s
* FilterBenchmarks.reconfigurableFilterByLogLevelAndNameLog thrpt 3 7482.096 ± 628.534 ops/s
*/

@Benchmark
Expand All @@ -135,4 +142,12 @@ class FilterBenchmarks {
def cachedFilterByLogLevelAndNameLog(): Unit =
testLoggingWith(cachedFilterByLogLevelAndNameLogging)

@Benchmark
def reconfigurableFilterByLogLevelAndNameLog(): Unit =
testLoggingWith(reconfigurableFilterByLogLevelAndNameLogging)

@Benchmark
def reconfigurableCachedFilterByLogLevelAndNameLog(): Unit =
testLoggingWith(reconfigurableCachedFilterByLogLevelAndNameLogging)

}
8 changes: 5 additions & 3 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ lazy val core = crossProject(JSPlatform, JVMPlatform)
.settings(enableZIO(enableStreaming = true))
.settings(
libraryDependencies ++= Seq(
"dev.zio" %%% "zio-parser" % zioParser
"dev.zio" %%% "zio-parser" % zioParser,
"dev.zio" %%% "zio-prelude" % zioPrelude
)
)
.jvmSettings(
Expand Down Expand Up @@ -198,8 +199,9 @@ lazy val examplesCore = project
.settings(
publish / skip := true,
libraryDependencies ++= Seq(
"dev.zio" %% "zio-metrics-connectors" % zioMetricsConnectorsVersion,
"dev.zio" %% "zio-config-typesafe" % zioConfig
"dev.zio" %% "zio-metrics-connectors-prometheus" % zioMetricsConnectorsVersion,
"dev.zio" %% "zio-http" % zioHttp,
"dev.zio" %% "zio-config-typesafe" % zioConfig
)
)

Expand Down
25 changes: 25 additions & 0 deletions core/jvm/src/test/scala/zio/logging/ConsoleLoggerConfigSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,31 @@ object ConsoleLoggerConfigSpec extends ZIOSpecDefault {
assertTrue(true)
}
},
test("equals config with same sources") {

// "%highlight{%timestamp{yyyy-MM-dd'T'HH:mm:ssZ} %fixed{7}{%level} [%fiberId] %name:%line %message %cause}" //FIXME

val logFormat =
"%highlight{%timestamp %fixed{7}{%level} [%fiberId] %name:%line %message %cause}"

val configProvider: ConfigProvider = ConfigProvider.fromMap(
Map(
"logger/format" -> logFormat,
"logger/filter/rootLevel" -> LogLevel.Info.label,
"logger/filter/mappings/zio.logging.example.LivePingService" -> LogLevel.Debug.label
),
"/"
)

import zio.prelude._
for {
c1 <- configProvider.load(ConsoleLoggerConfig.config.nested("logger"))
c2 <- configProvider.load(ConsoleLoggerConfig.config.nested("logger"))
} yield assertTrue(
c1.format == c2.format,
c1 === c2
)
},
test("fail on invalid filter config") {
val logFormat =
"%highlight{%timestamp{yyyy-MM-dd'T'HH:mm:ssZ} %fixed{7}{%level} [%fiberId] %name:%line %message %cause}"
Expand Down
27 changes: 27 additions & 0 deletions core/jvm/src/test/scala/zio/logging/FileLoggerConfigSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,33 @@ object FileLoggerConfigSpec extends ZIOSpecDefault {
assertTrue(loadedConfig.bufferedIOSize.isEmpty)
}
},
test("equals config with same sources") {

val logFormat =
"%highlight{%timestamp %fixed{7}{%level} [%fiberId] %name:%line %message %cause}"

val configProvider: ConfigProvider = ConfigProvider.fromMap(
Map(
"logger/format" -> logFormat,
"logger/path" -> "file:///tmp/test.log",
"logger/autoFlushBatchSize" -> "2",
"logger/bufferedIOSize" -> "4096",
"logger/rollingPolicy/type" -> "TimeBasedRollingPolicy",
"logger/filter/rootLevel" -> LogLevel.Info.label,
"logger/filter/mappings/zio.logging.example.LivePingService" -> LogLevel.Debug.label
),
"/"
)

import zio.prelude._
for {
c1 <- configProvider.load(ConsoleLoggerConfig.config.nested("logger"))
c2 <- configProvider.load(ConsoleLoggerConfig.config.nested("logger"))
} yield assertTrue(
c1.format == c2.format,
c1 === c2
)
},
test("fail on invalid charset and filter config") {
val logFormat =
"%highlight{%timestamp{yyyy-MM-dd'T'HH:mm:ssZ} %fixed{7}{%level} [%fiberId] %name:%line %message %cause}"
Expand Down
139 changes: 138 additions & 1 deletion core/jvm/src/test/scala/zio/logging/LogFilterSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,21 @@ package zio.logging
import zio.logging.test.TestService
import zio.test.ZTestLogger.LogEntry
import zio.test._
import zio.{ Cause, Chunk, Config, ConfigProvider, FiberId, FiberRefs, LogLevel, LogSpan, Runtime, Trace, ZIO, ZLogger }
import zio.{
Cause,
Chunk,
Config,
ConfigProvider,
FiberId,
FiberRefs,
LogLevel,
LogSpan,
Runtime,
Trace,
ZIO,
ZIOAspect,
ZLogger
}

object LogFilterSpec extends ZIOSpecDefault {

Expand Down Expand Up @@ -399,6 +413,129 @@ object LogFilterSpec extends ZIOSpecDefault {
"**" -> LogLevel.Info
)
)
},
test("log filters by log level and name from same configuration should be equal") {

val configProvider = ConfigProvider.fromMap(
Map(
"logger/rootLevel" -> LogLevel.Debug.label,
"logger/mappings/a" -> LogLevel.Info.label,
"logger/mappings/a.b.c" -> LogLevel.Warning.label,
"logger/mappings/e.f" -> LogLevel.Error.label
),
"/"
)

import zio.prelude._

for {
f1 <- configProvider
.load(LogFilter.LogLevelByNameConfig.config.nested("logger"))
.map(LogFilter.logLevelByName)
f2 <- configProvider
.load(LogFilter.LogLevelByNameConfig.config.nested("logger"))
.map(LogFilter.logLevelByName)
f3 <- configProvider
.load(LogFilter.LogLevelByNameConfig.config.nested("logger"))
.map(LogFilter.logLevelByName)
.map(_.cached)
f4 <- configProvider
.load(LogFilter.LogLevelByNameConfig.config.nested("logger"))
.map(LogFilter.logLevelByName)
.map(_.cached)
} yield assertTrue(f1 === f2, f3 === f4)
},
test("and") {

val filter = LogFilter.causeNonEmpty.and(LogFilter.logLevel(LogLevel.Info))

def testFilter(level: LogLevel, cause: Cause[_], expected: Boolean) =
assertTrue(
filter(
Trace.empty,
FiberId.None,
level,
() => "",
cause,
FiberRefs.empty,
List.empty,
Map.empty
) == expected
)

testFilter(LogLevel.Info, Cause.fail("fail"), true) && testFilter(
LogLevel.Info,
Cause.empty,
false
) && testFilter(LogLevel.Debug, Cause.fail("fail"), false)
},
test("or") {

val filter = LogFilter.causeNonEmpty.or(LogFilter.logLevel(LogLevel.Info))

def testFilter(level: LogLevel, cause: Cause[_], expected: Boolean) =
assertTrue(
filter(
Trace.empty,
FiberId.None,
level,
() => "",
cause,
FiberRefs.empty,
List.empty,
Map.empty
) == expected
)

testFilter(LogLevel.Info, Cause.fail("fail"), true) && testFilter(LogLevel.Info, Cause.empty, true) && testFilter(
LogLevel.Debug,
Cause.fail("fail"),
true
) && testFilter(LogLevel.Debug, Cause.empty, false)
},
test("not") {

val filter = LogFilter.causeNonEmpty.and(LogFilter.logLevel(LogLevel.Info)).not

def testFilter(level: LogLevel, cause: Cause[_], expected: Boolean) =
assertTrue(
filter(
Trace.empty,
FiberId.None,
level,
() => "",
cause,
FiberRefs.empty,
List.empty,
Map.empty
) == expected
)

testFilter(LogLevel.Info, Cause.fail("fail"), false) && testFilter(
LogLevel.Info,
Cause.empty,
true
) && testFilter(LogLevel.Debug, Cause.fail("fail"), true)
},
test("cached") {
val logOutputRef = new java.util.concurrent.atomic.AtomicReference[Chunk[LogEntry]](Chunk.empty)

val filter = LogFilter
.logLevelByGroup(
LogLevel.Info,
LogGroup.loggerName,
"zio.logger1" -> LogLevel.Debug,
"zio.logging.test" -> LogLevel.Warning
)
.cached
.asInstanceOf[LogFilter.CachedFilter[String]]

(for {
_ <- ZIO.logDebug("debug") @@ ZIOAspect.annotated(loggerNameAnnotationKey, "zio.logger1")
res1 = filter.cache.get(List("zio", "logger1") -> LogLevel.Debug)
_ <- ZIO.logDebug("debug") @@ ZIOAspect.annotated(loggerNameAnnotationKey, "zio.logger2")
res2 = filter.cache.get(List("zio", "logger2") -> LogLevel.Debug)
} yield assertTrue(res1 == true, res2 == false)).provideLayer(testLogger(logOutputRef, filter))
}
)
}
Loading

0 comments on commit dca943d

Please sign in to comment.