Skip to content

Commit

Permalink
Fix plugin IDs and Buffer path default (#193)
Browse files Browse the repository at this point in the history 
Fix plugin IDs and buffer Path
  • Loading branch information
@tarokkk
tarokkk authored Oct 25, 2019
1 parent 60fe9fb commit 58157ed
Show file tree
Hide file tree
Showing 29 changed files with 129 additions and 93 deletions.
40 changes: 24 additions & 16 deletions config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -352,8 +353,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -867,8 +869,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -1429,8 +1432,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -1697,8 +1701,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -1928,8 +1933,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -2265,8 +2271,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -2588,8 +2595,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down
40 changes: 24 additions & 16 deletions config/crd/bases/logging.banzaicloud.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -352,8 +353,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -863,8 +865,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -1425,8 +1428,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -1693,8 +1697,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -1924,8 +1929,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -2261,8 +2267,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down Expand Up @@ -2584,8 +2591,9 @@ spec:
to accept newly incoming chunk'
type: string
path:
description: The path where buffer chunks are stored. The '*'
is replaced with random characters. This parameter is required.
description: 'The path where buffer chunks are stored. The ''*''
is replaced with random characters. It''s highly recommended
to leave this default. (default: operator generated)'
type: string
queue_limit_length:
description: The queue length limitation of this buffer plugin
Expand Down
2 changes: 1 addition & 1 deletion docs/plugins/outputs/buffer.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
|---|---|---|---|---|
| type | string | No | - | Fluentd core bundles memory and file plugins. 3rd party plugins are also available when installed.<br> |
| tags | string | No | tag,time | When tag is specified as buffer chunk key, output plugin writes events into chunks separately per tags. <br> |
| path | string | No | /buffers/default.*.buffer | The path where buffer chunks are stored. The '*' is replaced with random characters. This parameter is required.<br> |
| path | string | No | operator generated | The path where buffer chunks are stored. The '*' is replaced with random characters. It's highly recommended to leave this default. <br> |
| chunk_limit_size | string | No | - | The max size of each chunks: events will be written into chunks until the size of chunks become this size<br> |
| chunk_limit_records | int | No | - | The max number of events that each chunks can store in it<br> |
| total_limit_size | string | No | - | The size limitation of this buffer plugin instance. Once the total size of stored buffer reached this threshold, all append operations will fail with error (and data will be lost)<br> |
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/filter/dedot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,6 @@ func (c *DedotFilterConfig) ToDirective(secretLoader secret.SecretLoader, id str
Type: pluginType,
Directive: "filter",
Tag: "**",
Id: id + "-" + pluginType,
Id: id + "_" + pluginType,
}, c, secretLoader)
}
2 changes: 1 addition & 1 deletion pkg/model/filter/geoip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func (g *GeoIP) ToDirective(secretLoader secret.SecretLoader, id string) (types.
Type: pluginType,
Directive: "filter",
Tag: "**",
Id: id + "-" + pluginType,
Id: id + "_" + pluginType,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(g); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/filter/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func (p *ParserConfig) ToDirective(secretLoader secret.SecretLoader, id string)
Type: pluginType,
Directive: "filter",
Tag: "**",
Id: id + "-" + pluginType,
Id: id + "_" + pluginType,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(p); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/filter/record_transformer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (r *RecordTransformer) ToDirective(secretLoader secret.SecretLoader, id str
Type: pluginType,
Directive: "filter",
Tag: "**",
Id: id + "-" + pluginType,
Id: id + "_" + pluginType,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(r); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/filter/stdout.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,6 @@ func (c *StdOutFilterConfig) ToDirective(secretLoader secret.SecretLoader, id st
Type: pluginType,
Directive: "filter",
Tag: "**",
Id: id + "-" + pluginType,
Id: id + "_" + pluginType,
}, c, secretLoader)
}
2 changes: 1 addition & 1 deletion pkg/model/filter/tagnormaliser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,6 @@ func (t *TagNormaliser) ToDirective(secretLoader secret.SecretLoader, id string)
Type: pluginType,
Directive: "match",
Tag: "kubernetes.**",
Id: id + "-" + pluginType,
Id: id + "_" + pluginType,
}, t, secretLoader)
}
3 changes: 2 additions & 1 deletion pkg/model/input/forward.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,12 @@ func NewForwardInputConfig() *ForwardInputConfig {

func (f *ForwardInputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
pluginType := "forward"
pluginID := id + "_" + pluginType
forward := &types.GenericDirective{
PluginMeta: types.PluginMeta{
Type: pluginType,
Directive: "source",
Id: id + "-" + pluginType,
Id: pluginID,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(f); err != nil {
Expand Down
3 changes: 2 additions & 1 deletion pkg/model/input/tail.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@ func NewTailInputConfig(path string) *TailInputConfig {

func (c *TailInputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
pluginType := "tail"
pluginID := id + "_" + pluginType
return types.NewFlatDirective(types.PluginMeta{
Type: pluginType,
Directive: "source",
Id: id + "-" + pluginType,
Id: pluginID,
}, c, secretLoader)
}
5 changes: 3 additions & 2 deletions pkg/model/output/azurestore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,13 @@ type AzureStorage struct {

func (a *AzureStorage) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
pluginType := "azurestorage"
pluginID := id + "_" + pluginType
azure := &types.OutputPlugin{
PluginMeta: types.PluginMeta{
Type: pluginType,
Directive: "match",
Tag: "**",
Id: id + "-" + pluginType,
Id: pluginID,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(a); err != nil {
Expand All @@ -67,7 +68,7 @@ func (a *AzureStorage) ToDirective(secretLoader secret.SecretLoader, id string)
azure.Params = params
}
if a.Buffer != nil {
if buffer, err := a.Buffer.ToDirective(secretLoader, ""); err != nil {
if buffer, err := a.Buffer.ToDirective(secretLoader, pluginID); err != nil {
return nil, err
} else {
azure.SubDirectives = append(azure.SubDirectives, buffer)
Expand Down
9 changes: 7 additions & 2 deletions pkg/model/output/buffer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package output

import (
"fmt"

"github.com/banzaicloud/logging-operator/pkg/model/secret"
"github.com/banzaicloud/logging-operator/pkg/model/types"
)
Expand All @@ -26,8 +28,8 @@ type Buffer struct {
Type string `json:"type,omitempty"`
// When tag is specified as buffer chunk key, output plugin writes events into chunks separately per tags. (default: tag,time)
Tags string `json:"tags,omitempty"`
// The path where buffer chunks are stored. The '*' is replaced with random characters. This parameter is required.
Path string `json:"path,omitempty" plugin:"default:/buffers/default.*.buffer"`
// The path where buffer chunks are stored. The '*' is replaced with random characters. It's highly recommended to leave this default. (default: operator generated)
Path string `json:"path,omitempty"`
// The max size of each chunks: events will be written into chunks until the size of chunks become this size
ChunkLimitSize string `json:"chunk_limit_size,omitempty"`
// The max number of events that each chunks can store in it
Expand Down Expand Up @@ -106,6 +108,9 @@ func (b *Buffer) ToDirective(secretLoader secret.SecretLoader, id string) (types
} else {
metadata.Tag = "tag,time"
}
if b.Path == "" {
b.Path = fmt.Sprintf("/buffers/%s.*.buffer", id)
}

b.Tags = ""
return types.NewFlatDirective(metadata, b, secretLoader)
Expand Down
5 changes: 3 additions & 2 deletions pkg/model/output/elasticsearch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,12 +194,13 @@ type ElasticsearchOutput struct {

func (e *ElasticsearchOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
pluginType := "elasticsearch"
pluginID := id + "_" + pluginType
elasticsearch := &types.OutputPlugin{
PluginMeta: types.PluginMeta{
Type: pluginType,
Directive: "match",
Tag: "**",
Id: id + "-" + pluginType,
Id: pluginID,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(e); err != nil {
Expand All @@ -208,7 +209,7 @@ func (e *ElasticsearchOutput) ToDirective(secretLoader secret.SecretLoader, id s
elasticsearch.Params = params
}
if e.Buffer != nil {
if buffer, err := e.Buffer.ToDirective(secretLoader, ""); err != nil {
if buffer, err := e.Buffer.ToDirective(secretLoader, pluginID); err != nil {
return nil, err
} else {
elasticsearch.SubDirectives = append(elasticsearch.SubDirectives, buffer)
Expand Down
3 changes: 2 additions & 1 deletion pkg/model/output/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ type FileOutputConfig struct {

func (c *FileOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
pluginType := "file"
pluginID := id + "_" + pluginType
return types.NewFlatDirective(types.PluginMeta{
Type: pluginType,
Directive: "match",
Tag: "**",
Id: id + "-" + pluginType,
Id: pluginID,
}, c, secretLoader)
}
5 changes: 3 additions & 2 deletions pkg/model/output/forward.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,13 @@ type ForwardOutput struct {

func (f *ForwardOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
pluginType := "forward"
pluginID := id + "_" + pluginType
forward := &types.OutputPlugin{
PluginMeta: types.PluginMeta{
Type: pluginType,
Directive: "match",
Tag: "**",
Id: id + "-" + pluginType,
Id: pluginID,
},
}
if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(f); err != nil {
Expand All @@ -105,7 +106,7 @@ func (f *ForwardOutput) ToDirective(secretLoader secret.SecretLoader, id string)
forward.Params = params
}
if f.Buffer != nil {
if buffer, err := f.Buffer.ToDirective(secretLoader, ""); err != nil {
if buffer, err := f.Buffer.ToDirective(secretLoader, pluginID); err != nil {
return nil, err
} else {
forward.SubDirectives = append(forward.SubDirectives, buffer)
Expand Down
Loading

0 comments on commit 58157ed

Please sign in to comment.