Support new multiline core introduced in fluentbit 1.8 (#884)

Currently some container runtimes like containerd & cri can write longer log messages separated into multiple lines in the log file. A special flag is used to indicate if the message is partial (additional lines will follow) or full. Currently you can handle this by using Parser_Firstline & Parser_N fields but with new multiline core support introduced in fluentbit 1.8 this can be achieved by only setting multiline.parser to the appropriate value and fluentbit handles this automatically. Multiple parser can be added which allow different node types to be scraped by the same fluentbit configuration. See more on https://docs.fluentbit.io/manual/pipeline/inputs/tail#multiline-support. Signed-off-by: David Mohar <d.mohar@sportradar.com>
kube-logging · Nov 26, 2021 · abb834e · abb834e
1 parent 27b6439
commit abb834e
Show file tree

Hide file tree

Showing 15 changed files with 51 additions and 12 deletions.
diff --git a/charts/logging-operator-logging/README.md b/charts/logging-operator-logging/README.md
@@ -22,7 +22,7 @@ The following tables lists the configurable parameters of the logging-operator-l
 | `fluentbit.enabled`                                 | Install fluent-bit                                                       | true                                                       |
 | `fluentbit.namespace`                               | Specified fluentbit installation namespace                               | same as operator namespace                                 |
 | `fluentbit.tolerations`                             | Tolerations for fluentbit daemonset                                      | none                                                       |
-| `fluentbit.image.tag`                               | Fluentbit container image tag                                            | `1.8.8`                                                    |
+| `fluentbit.image.tag`                               | Fluentbit container image tag                                            | `1.8.9`                                                    |
 | `fluentbit.image.repository`                        | Fluentbit container image repository                                     | `fluent/fluent-bit`                                        |
 | `fluentbit.image.pullPolicy`                        | Fluentbit container pull policy                                          | `IfNotPresent`                                             |
 | `fluentbit.podPriorityClassName`                    | Priority class name for fluentbit pods                                   | none                                                       |

diff --git a/charts/logging-operator-logging/values.yaml b/charts/logging-operator-logging/values.yaml
@@ -29,7 +29,7 @@ nodeAgents: {}
 #          template:
 #            spec:
 #              containers:
-#              - image: banzaicloud/fluentbit:1.8.8
+#              - image: banzaicloud/fluentbit:1.8.9
 #                name: fluent-bit
 #      tls:
 #        enabled: false

diff --git a/charts/logging-operator/README.md b/charts/logging-operator/README.md
@@ -112,7 +112,7 @@ The following tables lists the configurable parameters of the logging-operator-l
 | `tls.sharedKey`                                     | Shared key between nodes (fluentd-fluentbit)           | [autogenerated]                |
 | `fluentbit.enabled`                                 | Install fluent-bit                                     | true                           |
 | `fluentbit.namespace`                               | Specified fluentbit installation namespace             | same as operator namespace     |
-| `fluentbit.image.tag`                               | Fluentbit container image tag                          | `1.8.8`                        |
+| `fluentbit.image.tag`                               | Fluentbit container image tag                          | `1.8.9`                        |
 | `fluentbit.image.repository`                        | Fluentbit container image repository                   | `fluent/fluent-bit`            |
 | `fluentbit.image.pullPolicy`                        | Fluentbit container pull policy                        | `IfNotPresent`                 |
 | `fluentd.enabled`                                   | Install fluentd                                        | true                           |

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml
@@ -1483,6 +1483,10 @@ spec:
                         type: string
                       Tag_Regex:
                         type: string
+                      multiline.parser:
+                        items:
+                          type: string
+                        type: array
                       storage.type:
                         type: string
                     type: object
@@ -7874,6 +7878,10 @@ spec:
                               type: string
                             Tag_Regex:
                               type: string
+                            multiline.parser:
+                              items:
+                                type: string
+                              type: array
                             storage.type:
                               type: string
                           type: object

diff --git a/config/crd/bases/logging.banzaicloud.io_loggings.yaml b/config/crd/bases/logging.banzaicloud.io_loggings.yaml
@@ -1483,6 +1483,10 @@ spec:
                         type: string
                       Tag_Regex:
                         type: string
+                      multiline.parser:
+                        items:
+                          type: string
+                        type: array
                       storage.type:
                         type: string
                     type: object
@@ -7874,6 +7878,10 @@ spec:
                               type: string
                             Tag_Regex:
                               type: string
+                            multiline.parser:
+                              items:
+                                type: string
+                              type: array
                             storage.type:
                               type: string
                           type: object

diff --git a/config/samples/logging_v1alpha2_logging_debug.yaml b/config/samples/logging_v1alpha2_logging_debug.yaml
@@ -8,7 +8,7 @@ spec:
     disablePvc: true
   fluentbit:
     image:
-      tag: 1.8.8-debug
+      tag: 1.8.9-debug
       repository: fluent/fluent-bit
       pullPolicy: IfNotPresent
   controlNamespace: default
diff --git a/pkg/resources/fluentbit/config.go b/pkg/resources/fluentbit/config.go
@@ -48,6 +48,9 @@ var fluentBitConfigTemplate = `
     Parse_{{ $id}} {{$v}}
     {{- end }}
     {{- end }}
+    {{- if .Input.MultilineParser }}
+    multiline.parser: {{- range $i, $v := .Input.MultilineParser }}{{ if $i }},{{ end}} {{ $v }}{{ end }}
+    {{- end }}
 
 {{- if not .DisableKubernetesFilter }}
 [FILTER]

diff --git a/pkg/resources/fluentbit/configsecret.go b/pkg/resources/fluentbit/configsecret.go
@@ -31,8 +31,9 @@ import (
 )
 
 type fluentbitInputConfig struct {
-	Values  map[string]string
-	ParserN []string
+	Values          map[string]string
+	ParserN         []string
+	MultilineParser []string
 }
 
 type upstreamNode struct {
@@ -123,6 +124,10 @@ func (r *Reconciler) configSecret() (runtime.Object, reconciler.DesiredState, er
 		fluentbitInput.ParserN = r.Logging.Spec.FluentbitSpec.InputTail.ParserN
 		inputTail.ParserN = nil
 	}
+	if len(inputTail.MultilineParser) > 0 {
+		fluentbitInput.MultilineParser = r.Logging.Spec.FluentbitSpec.InputTail.MultilineParser
+		inputTail.MultilineParser = nil
+	}
 	fluentbitInputValues, err := mapper.StringsMap(inputTail)
 	if err != nil {
 		return nil, reconciler.StatePresent, errors.WrapIf(err, "failed to map container tailer config for fluentbit")

diff --git a/pkg/resources/nodeagent/config.go b/pkg/resources/nodeagent/config.go
@@ -43,6 +43,9 @@ var fluentBitConfigTemplate = `
     Parse_{{ $id}} {{$v}}
     {{- end }}
     {{- end }}
+    {{- if .Input.MultilineParser }}
+    multiline.parser: {{- range $i, $v := .Input.MultilineParser }}{{ if $i }},{{ end}} {{ $v }}{{ end }}
+    {{- end }}
 
 {{- if not .DisableKubernetesFilter }}
 [FILTER]

diff --git a/pkg/resources/nodeagent/configsecret.go b/pkg/resources/nodeagent/configsecret.go
@@ -31,8 +31,9 @@ import (
 )
 
 type fluentbitInputConfig struct {
-	Values  map[string]string
-	ParserN []string
+	Values          map[string]string
+	ParserN         []string
+	MultilineParser []string
 }
 
 type upstreamNode struct {
@@ -123,6 +124,10 @@ func (n *nodeAgentInstance) configSecret() (runtime.Object, reconciler.DesiredSt
 		fluentbitInput.ParserN = n.nodeAgent.FluentbitSpec.InputTail.ParserN
 		inputTail.ParserN = nil
 	}
+	if len(inputTail.MultilineParser) > 0 {
+		fluentbitInput.MultilineParser = n.nodeAgent.FluentbitSpec.InputTail.MultilineParser
+		inputTail.MultilineParser = nil
+	}
 	fluentbitInputValues, err := mapper.StringsMap(inputTail)
 	if err != nil {
 		return nil, reconciler.StatePresent, errors.WrapIf(err, "failed to map container tailer config for fluentbit")

diff --git a/pkg/resources/nodeagent/nodeagent.go b/pkg/resources/nodeagent/nodeagent.go
@@ -61,7 +61,7 @@ func NodeAgentFluentbitDefaults(userDefined **v1beta1.NodeAgent) (*v1beta1.NodeA
 							Containers: []v1.Container{
 								{
 									Name:            containerName,
-									Image:           "fluent/fluent-bit:1.8.8",
+									Image:           "fluent/fluent-bit:1.8.9",
 									Command:         []string{"/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/conf_operator/fluent-bit.conf"},
 									ImagePullPolicy: v1.PullIfNotPresent,
 									Resources: v1.ResourceRequirements{

diff --git a/pkg/sdk/api/v1beta1/fluentbit_types.go b/pkg/sdk/api/v1beta1/fluentbit_types.go
@@ -183,6 +183,8 @@ type InputTail struct {
 	DockerModeParser string `json:"Docker_Mode_Parser,omitempty"`
 	//Wait period time in seconds to flush queued unfinished split lines. (default:4)
 	DockerModeFlush string `json:"Docker_Mode_Flush,omitempty"`
+	// Specify one or multiple parser definitions to apply to the content. Part of the new Multiline Core support in 1.8 (default: "")
+	MultilineParser []string `json:"multiline.parser,omitempty"`
 }
 
 // FilterKubernetes Fluent Bit Kubernetes Filter allows to enrich your log files with Kubernetes metadata.

diff --git a/pkg/sdk/api/v1beta1/logging_types.go b/pkg/sdk/api/v1beta1/logging_types.go
@@ -114,7 +114,7 @@ type DefaultFlowSpec struct {
 
 const (
 	DefaultFluentbitImageRepository         = "fluent/fluent-bit"
-	DefaultFluentbitImageTag                = "1.8.8"
+	DefaultFluentbitImageTag                = "1.8.9"
 	DefaultFluentdImageRepository           = "ghcr.io/banzaicloud/fluentd"
 	DefaultFluentdImageTag                  = "v1.13.3-alpine-11"
 	DefaultFluentdBufferStorageVolumeName   = "fluentd-buffer"

diff --git a/pkg/sdk/api/v1beta1/zz_generated.deepcopy.go b/pkg/sdk/api/v1beta1/zz_generated.deepcopy.go
diff --git a/pkg/sdk/static/gen/crds/generated.go b/pkg/sdk/static/gen/crds/generated.go