diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml index 60de402e8..56405b5c2 100644 --- a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml @@ -183,8 +183,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -352,8 +353,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -867,8 +869,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -1429,8 +1432,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -1697,8 +1701,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -1928,8 +1933,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -2265,8 +2271,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -2588,8 +2595,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml index 2f7884294..c4b41aa40 100644 --- a/config/crd/bases/logging.banzaicloud.io_outputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_outputs.yaml @@ -183,8 +183,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -352,8 +353,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -863,8 +865,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -1425,8 +1428,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -1693,8 +1697,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -1924,8 +1929,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -2261,8 +2267,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin @@ -2584,8 +2591,9 @@ spec: to accept newly incoming chunk' type: string path: - description: The path where buffer chunks are stored. The '*' - is replaced with random characters. This parameter is required. + description: 'The path where buffer chunks are stored. The ''*'' + is replaced with random characters. It''s highly recommended + to leave this default. (default: operator generated)' type: string queue_limit_length: description: The queue length limitation of this buffer plugin diff --git a/docs/plugins/outputs/buffer.md b/docs/plugins/outputs/buffer.md index 3141d57e3..96815ae79 100644 --- a/docs/plugins/outputs/buffer.md +++ b/docs/plugins/outputs/buffer.md @@ -3,7 +3,7 @@ |---|---|---|---|---| | type | string | No | - | Fluentd core bundles memory and file plugins. 3rd party plugins are also available when installed.
| | tags | string | No | tag,time | When tag is specified as buffer chunk key, output plugin writes events into chunks separately per tags.
| -| path | string | No | /buffers/default.*.buffer | The path where buffer chunks are stored. The '*' is replaced with random characters. This parameter is required.
| +| path | string | No | operator generated | The path where buffer chunks are stored. The '*' is replaced with random characters. It's highly recommended to leave this default.
| | chunk_limit_size | string | No | - | The max size of each chunks: events will be written into chunks until the size of chunks become this size
| | chunk_limit_records | int | No | - | The max number of events that each chunks can store in it
| | total_limit_size | string | No | - | The size limitation of this buffer plugin instance. Once the total size of stored buffer reached this threshold, all append operations will fail with error (and data will be lost)
| diff --git a/pkg/model/filter/dedot.go b/pkg/model/filter/dedot.go index 88aff9cff..c98b96e09 100644 --- a/pkg/model/filter/dedot.go +++ b/pkg/model/filter/dedot.go @@ -41,6 +41,6 @@ func (c *DedotFilterConfig) ToDirective(secretLoader secret.SecretLoader, id str Type: pluginType, Directive: "filter", Tag: "**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, c, secretLoader) } diff --git a/pkg/model/filter/geoip.go b/pkg/model/filter/geoip.go index 08306694a..98ca62c01 100644 --- a/pkg/model/filter/geoip.go +++ b/pkg/model/filter/geoip.go @@ -68,7 +68,7 @@ func (g *GeoIP) ToDirective(secretLoader secret.SecretLoader, id string) (types. Type: pluginType, Directive: "filter", Tag: "**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(g); err != nil { diff --git a/pkg/model/filter/parser.go b/pkg/model/filter/parser.go index 0d907fb26..fb11782b5 100644 --- a/pkg/model/filter/parser.go +++ b/pkg/model/filter/parser.go @@ -82,7 +82,7 @@ func (p *ParserConfig) ToDirective(secretLoader secret.SecretLoader, id string) Type: pluginType, Directive: "filter", Tag: "**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(p); err != nil { diff --git a/pkg/model/filter/record_transformer.go b/pkg/model/filter/record_transformer.go index 143acebb4..33f7e24c7 100644 --- a/pkg/model/filter/record_transformer.go +++ b/pkg/model/filter/record_transformer.go @@ -59,7 +59,7 @@ func (r *RecordTransformer) ToDirective(secretLoader secret.SecretLoader, id str Type: pluginType, Directive: "filter", Tag: "**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(r); err != nil { diff --git a/pkg/model/filter/stdout.go b/pkg/model/filter/stdout.go index 179fe57df..db0f98232 100644 --- a/pkg/model/filter/stdout.go +++ b/pkg/model/filter/stdout.go @@ -34,6 +34,6 @@ func (c *StdOutFilterConfig) ToDirective(secretLoader secret.SecretLoader, id st Type: pluginType, Directive: "filter", Tag: "**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, c, secretLoader) } diff --git a/pkg/model/filter/tagnormaliser.go b/pkg/model/filter/tagnormaliser.go index 58949f5d6..2668d4325 100644 --- a/pkg/model/filter/tagnormaliser.go +++ b/pkg/model/filter/tagnormaliser.go @@ -47,6 +47,6 @@ func (t *TagNormaliser) ToDirective(secretLoader secret.SecretLoader, id string) Type: pluginType, Directive: "match", Tag: "kubernetes.**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, t, secretLoader) } diff --git a/pkg/model/input/forward.go b/pkg/model/input/forward.go index d2201638b..10f415c2d 100644 --- a/pkg/model/input/forward.go +++ b/pkg/model/input/forward.go @@ -35,11 +35,12 @@ func NewForwardInputConfig() *ForwardInputConfig { func (f *ForwardInputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "forward" + pluginID := id + "_" + pluginType forward := &types.GenericDirective{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "source", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(f); err != nil { diff --git a/pkg/model/input/tail.go b/pkg/model/input/tail.go index fb329df36..60b7a00ea 100644 --- a/pkg/model/input/tail.go +++ b/pkg/model/input/tail.go @@ -31,9 +31,10 @@ func NewTailInputConfig(path string) *TailInputConfig { func (c *TailInputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "tail" + pluginID := id + "_" + pluginType return types.NewFlatDirective(types.PluginMeta{ Type: pluginType, Directive: "source", - Id: id + "-" + pluginType, + Id: pluginID, }, c, secretLoader) } diff --git a/pkg/model/output/azurestore.go b/pkg/model/output/azurestore.go index 60ed895cf..84daa8b22 100644 --- a/pkg/model/output/azurestore.go +++ b/pkg/model/output/azurestore.go @@ -53,12 +53,13 @@ type AzureStorage struct { func (a *AzureStorage) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "azurestorage" + pluginID := id + "_" + pluginType azure := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(a); err != nil { @@ -67,7 +68,7 @@ func (a *AzureStorage) ToDirective(secretLoader secret.SecretLoader, id string) azure.Params = params } if a.Buffer != nil { - if buffer, err := a.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := a.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { azure.SubDirectives = append(azure.SubDirectives, buffer) diff --git a/pkg/model/output/buffer.go b/pkg/model/output/buffer.go index c7861a1cd..00ab6c6b3 100644 --- a/pkg/model/output/buffer.go +++ b/pkg/model/output/buffer.go @@ -15,6 +15,8 @@ package output import ( + "fmt" + "github.com/banzaicloud/logging-operator/pkg/model/secret" "github.com/banzaicloud/logging-operator/pkg/model/types" ) @@ -26,8 +28,8 @@ type Buffer struct { Type string `json:"type,omitempty"` // When tag is specified as buffer chunk key, output plugin writes events into chunks separately per tags. (default: tag,time) Tags string `json:"tags,omitempty"` - // The path where buffer chunks are stored. The '*' is replaced with random characters. This parameter is required. - Path string `json:"path,omitempty" plugin:"default:/buffers/default.*.buffer"` + // The path where buffer chunks are stored. The '*' is replaced with random characters. It's highly recommended to leave this default. (default: operator generated) + Path string `json:"path,omitempty"` // The max size of each chunks: events will be written into chunks until the size of chunks become this size ChunkLimitSize string `json:"chunk_limit_size,omitempty"` // The max number of events that each chunks can store in it @@ -106,6 +108,9 @@ func (b *Buffer) ToDirective(secretLoader secret.SecretLoader, id string) (types } else { metadata.Tag = "tag,time" } + if b.Path == "" { + b.Path = fmt.Sprintf("/buffers/%s.*.buffer", id) + } b.Tags = "" return types.NewFlatDirective(metadata, b, secretLoader) diff --git a/pkg/model/output/elasticsearch.go b/pkg/model/output/elasticsearch.go index ed469d207..2162b6cd3 100644 --- a/pkg/model/output/elasticsearch.go +++ b/pkg/model/output/elasticsearch.go @@ -194,12 +194,13 @@ type ElasticsearchOutput struct { func (e *ElasticsearchOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "elasticsearch" + pluginID := id + "_" + pluginType elasticsearch := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(e); err != nil { @@ -208,7 +209,7 @@ func (e *ElasticsearchOutput) ToDirective(secretLoader secret.SecretLoader, id s elasticsearch.Params = params } if e.Buffer != nil { - if buffer, err := e.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := e.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { elasticsearch.SubDirectives = append(elasticsearch.SubDirectives, buffer) diff --git a/pkg/model/output/file.go b/pkg/model/output/file.go index 77794588b..537724a91 100644 --- a/pkg/model/output/file.go +++ b/pkg/model/output/file.go @@ -27,10 +27,11 @@ type FileOutputConfig struct { func (c *FileOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "file" + pluginID := id + "_" + pluginType return types.NewFlatDirective(types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, c, secretLoader) } diff --git a/pkg/model/output/forward.go b/pkg/model/output/forward.go index e2c09788a..2c35e059a 100644 --- a/pkg/model/output/forward.go +++ b/pkg/model/output/forward.go @@ -91,12 +91,13 @@ type ForwardOutput struct { func (f *ForwardOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "forward" + pluginID := id + "_" + pluginType forward := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(f); err != nil { @@ -105,7 +106,7 @@ func (f *ForwardOutput) ToDirective(secretLoader secret.SecretLoader, id string) forward.Params = params } if f.Buffer != nil { - if buffer, err := f.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := f.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { forward.SubDirectives = append(forward.SubDirectives, buffer) diff --git a/pkg/model/output/gcs.go b/pkg/model/output/gcs.go index c6436760c..4a869ea67 100644 --- a/pkg/model/output/gcs.go +++ b/pkg/model/output/gcs.go @@ -67,12 +67,13 @@ type GCSOutput struct { func (g *GCSOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "gcs" + pluginID := id + "_" + pluginType gcs := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(g); err != nil { @@ -81,7 +82,7 @@ func (g *GCSOutput) ToDirective(secretLoader secret.SecretLoader, id string) (ty gcs.Params = params } if g.Buffer != nil { - if buffer, err := g.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := g.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { gcs.SubDirectives = append(gcs.SubDirectives, buffer) diff --git a/pkg/model/output/kafka.go b/pkg/model/output/kafka.go index 1755fb1aa..5c7b314e2 100644 --- a/pkg/model/output/kafka.go +++ b/pkg/model/output/kafka.go @@ -94,12 +94,13 @@ type KafkaOutputConfig struct { func (e *KafkaOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "kafka2" + pluginID := id + "_" + pluginType kafka := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(e); err != nil { @@ -108,7 +109,7 @@ func (e *KafkaOutputConfig) ToDirective(secretLoader secret.SecretLoader, id str kafka.Params = params } if e.Buffer != nil { - if buffer, err := e.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := e.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { kafka.SubDirectives = append(kafka.SubDirectives, buffer) diff --git a/pkg/model/output/loki.go b/pkg/model/output/loki.go index 87224a86a..dc8fa3696 100644 --- a/pkg/model/output/loki.go +++ b/pkg/model/output/loki.go @@ -80,12 +80,13 @@ func (r Label) ToDirective(secretLoader secret.SecretLoader, id string) (types.D } func (l *LokiOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "loki" + pluginID := id + "_" + pluginType loki := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if l.ConfigureKubernetesLabels { @@ -121,7 +122,7 @@ func (l *LokiOutput) ToDirective(secretLoader secret.SecretLoader, id string) (t } } if l.Buffer != nil { - if buffer, err := l.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := l.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { loki.SubDirectives = append(loki.SubDirectives, buffer) diff --git a/pkg/model/output/loki_test.go b/pkg/model/output/loki_test.go index 8e75af540..1c097320b 100644 --- a/pkg/model/output/loki_test.go +++ b/pkg/model/output/loki_test.go @@ -34,7 +34,7 @@ buffer: expected := ` @type loki - @id test-loki + @id test_loki extract_kubernetes_labels true line_format json remove_keys ["kubernetes"] @@ -49,7 +49,7 @@ buffer: @type file - path /buffers/default.*.buffer + path /buffers/test_loki.*.buffer retry_forever true timekey 1m timekey_use_utc true diff --git a/pkg/model/output/null.go b/pkg/model/output/null.go index a77d6cd97..7431c5bed 100644 --- a/pkg/model/output/null.go +++ b/pkg/model/output/null.go @@ -30,10 +30,11 @@ func NewNullOutputConfig() *NullOutputConfig { func (c *NullOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "null" + pluginID := id + "_" + pluginType return types.NewFlatDirective(types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, c, secretLoader) } diff --git a/pkg/model/output/oss.go b/pkg/model/output/oss.go index fe0811a69..09186957e 100644 --- a/pkg/model/output/oss.go +++ b/pkg/model/output/oss.go @@ -85,12 +85,13 @@ type OSSOutput struct { func (o *OSSOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "oss" + pluginID := id + "_" + pluginType oss := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(o); err != nil { @@ -99,7 +100,7 @@ func (o *OSSOutput) ToDirective(secretLoader secret.SecretLoader, id string) (ty oss.Params = params } if o.Buffer != nil { - if buffer, err := o.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := o.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { oss.SubDirectives = append(oss.SubDirectives, buffer) diff --git a/pkg/model/output/s3.go b/pkg/model/output/s3.go index 53e8bb615..b7f69ec90 100644 --- a/pkg/model/output/s3.go +++ b/pkg/model/output/s3.go @@ -182,12 +182,13 @@ type S3SharedCredentials struct { func (c *S3OutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "s3" + pluginID := id + "_" + pluginType s3 := &types.OutputPlugin{ PluginMeta: types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, } if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(c); err != nil { @@ -196,7 +197,7 @@ func (c *S3OutputConfig) ToDirective(secretLoader secret.SecretLoader, id string s3.Params = params } if c.Buffer != nil { - if buffer, err := c.Buffer.ToDirective(secretLoader, ""); err != nil { + if buffer, err := c.Buffer.ToDirective(secretLoader, pluginID); err != nil { return nil, err } else { s3.SubDirectives = append(s3.SubDirectives, buffer) diff --git a/pkg/model/output/sumologic.go b/pkg/model/output/sumologic.go index 69980b618..a6b5ad9d6 100644 --- a/pkg/model/output/sumologic.go +++ b/pkg/model/output/sumologic.go @@ -61,10 +61,11 @@ type SumologicOutput struct { func (s *SumologicOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { pluginType := "sumologic" + pluginID := id + "_" + pluginType return types.NewFlatDirective(types.PluginMeta{ Type: pluginType, Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: pluginID, }, s, secretLoader) } diff --git a/pkg/model/render/fluent_test.go b/pkg/model/render/fluent_test.go index 0b573e806..14f1be6c2 100644 --- a/pkg/model/render/fluent_test.go +++ b/pkg/model/render/fluent_test.go @@ -144,7 +144,7 @@ func TestRenderDirective(t *testing.T) { expected: heredoc.Doc(` @type tail - @id test-tail + @id test_tail path /path/to/input `, ), @@ -155,7 +155,7 @@ func TestRenderDirective(t *testing.T) { expected: heredoc.Doc(` @type stdout - @id test-stdout + @id test_stdout `, ), }, @@ -165,7 +165,7 @@ func TestRenderDirective(t *testing.T) { expected: heredoc.Doc(` @type null - @id test-null + @id test_null `, ), }, @@ -207,7 +207,7 @@ func TestRenderDirective(t *testing.T) { expected: heredoc.Doc(` @type label_router - @id test-label_router + @id test_label_router @label @d41d8cd98f00b204e9800998ecf8427e @@ -223,7 +223,7 @@ func TestRenderDirective(t *testing.T) { expected: heredoc.Doc(` @type label_router - @id test-label_router + @id test_label_router @label @098f6bcd4621d373cade4e832627b4f6 namespace test @@ -240,7 +240,7 @@ func TestRenderDirective(t *testing.T) { expected: heredoc.Doc(` @type label_router - @id test-label_router + @id test_label_router @label @092f5fa58e4f619d739f5b65f2ed38bc labels a:b,c:d @@ -310,12 +310,12 @@ func TestMultipleOutput(t *testing.T) { expected := ` @type tail - @id test-tail + @id test_tail path input.log @type label_router - @id test-label_router + @id test_label_router @label @901f778f9602a78e8fd702c1973d8d8d labels key1:val1,key2:val2 @@ -325,17 +325,17 @@ func TestMultipleOutput(t *testing.T) { ` @@ -384,12 +384,12 @@ func TestRenderFullFluentConfig(t *testing.T) { expected := ` @type tail - @id test-tail + @id test_tail path input.log @type label_router - @id test-label_router + @id test_label_router @label @901f778f9602a78e8fd702c1973d8d8d labels key1:val1,key2:val2 @@ -399,11 +399,11 @@ func TestRenderFullFluentConfig(t *testing.T) { ` @@ -434,7 +434,7 @@ func TestRenderS3(t *testing.T) { }, }, expected: ` @type s3 - @id test-s3 + @id test_s3 path /var/buffer s3_bucket test_bucket @@ -456,7 +456,7 @@ func TestRenderS3(t *testing.T) { InstanceProfileCredentials: &output.S3InstanceProfileCredentials{}, }, expected: ` @type s3 - @id test-s3 + @id test_s3 path /var/buffer s3_bucket test_bucket @@ -473,7 +473,7 @@ func TestRenderS3(t *testing.T) { }, }, expected: ` @type s3 - @id test-s3 + @id test_s3 path /var/buffer s3_bucket test_bucket @@ -551,12 +551,12 @@ func ValidateRenderS3(t *testing.T, s3Config output.S3OutputConfig, expected str expected = fmt.Sprintf(` @type tail - @id test-tail + @id test_tail path input.log @type label_router - @id test-label_router + @id test_label_router @label @901f778f9602a78e8fd702c1973d8d8d labels key1:val1,key2:val2 diff --git a/pkg/model/render/json_test.go b/pkg/model/render/json_test.go index 1b157c469..ceda7f35f 100644 --- a/pkg/model/render/json_test.go +++ b/pkg/model/render/json_test.go @@ -77,7 +77,7 @@ func TestJsonRender(t *testing.T) { expected := `{ "input": { "type": "tail", - "id": "test-tail", + "id": "test_tail", "directive": "source", "params": { "path": "input.log" @@ -85,7 +85,7 @@ func TestJsonRender(t *testing.T) { }, "router": { "type": "label_router", - "id": "test-label_router", + "id": "test_label_router", "directive": "match", "tag": "**", "routes": [ @@ -107,7 +107,7 @@ func TestJsonRender(t *testing.T) { "filters": [ { "type": "stdout", - "id": "test-stdout", + "id": "test_stdout", "directive": "filter", "tag": "**" } @@ -115,7 +115,7 @@ func TestJsonRender(t *testing.T) { "outputs": [ { "type": "null", - "id": "test-null", + "id": "test_null", "directive": "match", "tag": "**" } diff --git a/pkg/model/render/test_helper.go b/pkg/model/render/test_helper.go index 4258594c7..a278e606b 100644 --- a/pkg/model/render/test_helper.go +++ b/pkg/model/render/test_helper.go @@ -66,12 +66,12 @@ func NewOutputPluginTest(t *testing.T, plugin plugins.DirectiveConverter) *Plugi Prefix: ` @type tail - @id test-tail + @id test_tail path input.log @type label_router - @id test-label_router + @id test_label_router @label @a42fd8d29c181fcf9887280c4a51bd1e namespace ns-test diff --git a/pkg/model/types/router.go b/pkg/model/types/router.go index bb9066c40..abca676eb 100644 --- a/pkg/model/types/router.go +++ b/pkg/model/types/router.go @@ -88,7 +88,7 @@ func NewRouter(id string) *Router { Type: "label_router", Directive: "match", Tag: "**", - Id: id + "-" + pluginType, + Id: id + "_" + pluginType, }, } } diff --git a/pkg/resources/model/system.go b/pkg/resources/model/system.go index e5470c4de..1e1bacd16 100644 --- a/pkg/resources/model/system.go +++ b/pkg/resources/model/system.go @@ -118,7 +118,8 @@ FindOutputForAllRefs: for _, output := range l.Outputs { // only an output from the same namespace can be used with a matching name if output.Namespace == namespace && outputRef == output.Name { - plugin, err := plugins.CreateOutput(output.Spec, output.Name, secret.NewSecretLoader(l.client, output.Namespace, fluentd.OutputSecretPath, l.Secrets)) + outputId := namespace + "_" + flowCr.Name + "_" + output.Name + plugin, err := plugins.CreateOutput(output.Spec, outputId, secret.NewSecretLoader(l.client, output.Namespace, fluentd.OutputSecretPath, l.Secrets)) if err != nil { multierr = errors.Combine(multierr, errors.WrapIff(err, "failed to create configured output %s", outputRef)) continue FindOutputForAllRefs @@ -130,7 +131,8 @@ FindOutputForAllRefs: } for _, clusterOutput := range l.ClusterOutputs { if outputRef == clusterOutput.Name { - plugin, err := plugins.CreateOutput(clusterOutput.Spec.OutputSpec, clusterOutput.Name, secret.NewSecretLoader(l.client, clusterOutput.Namespace, fluentd.OutputSecretPath, l.Secrets)) + outputId := "_" + flowCr.Name + "_" + clusterOutput.Name + plugin, err := plugins.CreateOutput(clusterOutput.Spec.OutputSpec, outputId, secret.NewSecretLoader(l.client, clusterOutput.Namespace, fluentd.OutputSecretPath, l.Secrets)) if err != nil { multierr = errors.Combine(multierr, errors.WrapIff(err, "failed to create configured output %s", outputRef)) continue FindOutputForAllRefs