Loki Extra labels and Labels fix (#212)

* - Extra Label merge
kube-logging · Nov 6, 2019 · c689b53 · c689b53
1 parent 0a4465e
commit c689b53
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 9 deletions.
diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml
@@ -2366,12 +2366,17 @@ spec:
                 extra_labels:
                   additionalProperties:
                     type: string
-                  description: Set of labels to include with every Loki stream.
+                  description: Set of extra labels to include with every Loki stream.
                   type: object
                 extract_kubernetes_labels:
                   description: 'Extract kubernetes labels as loki labels (default:
                     false)'
                   type: boolean
+                labels:
+                  additionalProperties:
+                    type: string
+                  description: Set of labels to include with every Loki stream.
+                  type: object
                 line_format:
                   description: 'Format to use when flattening the record to a log
                     line: json, key_value (default: key_value)'

diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml
@@ -2362,12 +2362,17 @@ spec:
                 extra_labels:
                   additionalProperties:
                     type: string
-                  description: Set of labels to include with every Loki stream.
+                  description: Set of extra labels to include with every Loki stream.
                   type: object
                 extract_kubernetes_labels:
                   description: 'Extract kubernetes labels as loki labels (default:
                     false)'
                   type: boolean
+                labels:
+                  additionalProperties:
+                    type: string
+                  description: Set of labels to include with every Loki stream.
+                  type: object
                 line_format:
                   description: 'Format to use when flattening the record to a log
                     line: json, key_value (default: key_value)'

diff --git a/docs/plugins/outputs/loki.md b/docs/plugins/outputs/loki.md
@@ -23,7 +23,8 @@ More info at https://github.com/banzaicloud/fluent-plugin-kubernetes-loki
 | username | *secret.Secret | No | - | Specify a username if the Loki server requires authentication.<br>[Secret](./secret.md)<br> |
 | password | *secret.Secret | No | - | Specify password if the Loki server requires authentication.<br>[Secret](./secret.md)<br> |
 | tenant | string | No | - | Loki is a multi-tenant log storage platform and all requests sent must include a tenant.<br> |
-| extra_labels | Label | No | - | Set of labels to include with every Loki stream.<br> |
+| labels | Label | No | - | Set of labels to include with every Loki stream.<br> |
+| extra_labels | map[string]string | No | - | Set of extra labels to include with every Loki stream.<br> |
 | line_format | string | No | json | Format to use when flattening the record to a log line: json, key_value (default: key_value)<br> |
 | extract_kubernetes_labels | bool | No |  false | Extract kubernetes labels as loki labels <br> |
 | remove_keys | []string | No |  [] | Comma separated list of needless record keys to remove <br> |

diff --git a/pkg/model/output/loki.go b/pkg/model/output/loki.go
@@ -51,7 +51,9 @@ type LokiOutput struct {
 	// Loki is a multi-tenant log storage platform and all requests sent must include a tenant.
 	Tenant string `json:"tenant,omitempty"`
 	// Set of labels to include with every Loki stream.
-	ExtraLabels Label `json:"extra_labels,omitempty"`
+	Labels Label `json:"labels,omitempty"`
+	// Set of extra labels to include with every Loki stream.
+	ExtraLabels map[string]string `json:"extra_labels,omitempty"`
 	// Format to use when flattening the record to a log line: json, key_value (default: key_value)
 	LineFormat string `json:"line_format,omitempty" plugin:"default:json"`
 	// Extract kubernetes labels as loki labels (default: false)
@@ -78,6 +80,13 @@ func (r Label) ToDirective(secretLoader secret.SecretLoader, id string) (types.D
 	}
 	return directive, nil
 }
+
+func (r Label) merge(input Label) {
+	for k, v := range input {
+		r[k] = v
+	}
+}
+
 func (l *LokiOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
 	pluginType := "loki"
 	pluginID := id + "_" + pluginType
@@ -90,14 +99,18 @@ func (l *LokiOutput) ToDirective(secretLoader secret.SecretLoader, id string) (t
 		},
 	}
 	if l.ConfigureKubernetesLabels {
-		l.ExtraLabels = Label{
+		if l.Labels == nil {
+			l.Labels = Label{}
+		}
+		l.Labels.merge(Label{
 			"namespace":    `$.kubernetes.namespace_name`,
 			"pod":          `$.kubernetes.pod_name`,
 			"container_id": `$.kubernetes.docker_id`,
 			"container":    `$.kubernetes.container_name`,
 			"pod_id":       `$.kubernetes.pod_id`,
 			"host":         `$.kubernetes.host`,
-		}
+		})
+
 		if l.RemoveKeys != nil {
 			if !util.Contains(l.RemoveKeys, "kubernetes") {
 				l.RemoveKeys = append(l.RemoveKeys, "kubernetes")
@@ -114,8 +127,8 @@ func (l *LokiOutput) ToDirective(secretLoader secret.SecretLoader, id string) (t
 	} else {
 		loki.Params = params
 	}
-	if l.ExtraLabels != nil {
-		if meta, err := l.ExtraLabels.ToDirective(secretLoader, ""); err != nil {
+	if l.Labels != nil {
+		if meta, err := l.Labels.ToDirective(secretLoader, ""); err != nil {
 			return nil, err
 		} else {
 			loki.SubDirectives = append(loki.SubDirectives, meta)

diff --git a/pkg/model/output/loki_test.go b/pkg/model/output/loki_test.go
@@ -26,6 +26,10 @@ func TestLoki(t *testing.T) {
 	CONFIG := []byte(`
 url: http://loki:3100
 configure_kubernetes_labels: true
+labels:
+  name: $.name
+extra_labels:
+  testing: "testing"
 buffer:
   timekey: 1m
   timekey_wait: 30s
@@ -35,6 +39,7 @@ buffer:
   <match **>
     @type loki
     @id test_loki
+    extra_labels {"testing":"testing"}
     extract_kubernetes_labels true
     line_format json
     remove_keys ["kubernetes"]
@@ -43,6 +48,7 @@ buffer:
       container $.kubernetes.container_name
       container_id $.kubernetes.docker_id
       host $.kubernetes.host
+      name $.name
       namespace $.kubernetes.namespace_name
       pod $.kubernetes.pod_name
       pod_id $.kubernetes.pod_id

diff --git a/pkg/model/output/zz_generated.deepcopy.go b/pkg/model/output/zz_generated.deepcopy.go
diff --git a/pkg/model/types/stringmaps.go b/pkg/model/types/stringmaps.go
@@ -176,6 +176,25 @@ func (s *StructToStringMapper) fillMap(value reflect.Value, out map[string]strin
 					}
 				}
 			}
+		case reflect.Map:
+			if mapStringString, ok := v.Interface().(map[string]string); ok {
+				if len(mapStringString) > 0 {
+					b, err := json.Marshal(mapStringString)
+					if err != nil {
+						multierror = errors.Combine(multierror, errors.Errorf("can't marshal field: %q value: %q as json", name, mapStringString), err)
+					}
+					out[name] = string(b)
+				} else {
+					if ok, def := pluginTagOpts.ValueForPrefix("default:"); ok {
+						validate := map[string]string{}
+						if err := json.Unmarshal([]byte(def), &validate); err != nil {
+							multierror = errors.Combine(multierror, errors.Errorf("can't marshal field: %q value: %q as json", name, def), err)
+						}
+						out[name] = def
+					}
+				}
+			}
+
 		}
 	}
 	return multierror