diff --git a/docs/plugins/plugins.json b/docs/plugins/plugins.json index cf8fc3ef..b4489ab3 100644 --- a/docs/plugins/plugins.json +++ b/docs/plugins/plugins.json @@ -164,6 +164,14 @@ "glob", "path" ] + }, + { + "name": "yaml", + "type": "data-source", + "arguments": [ + "glob", + "path" + ] } ] }, diff --git a/internal/builtin/data_yaml.go b/internal/builtin/data_yaml.go new file mode 100644 index 00000000..4a01adac --- /dev/null +++ b/internal/builtin/data_yaml.go @@ -0,0 +1,198 @@ +package builtin + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/plugin" + "github.com/blackstork-io/fabric/plugin/dataspec" + "github.com/blackstork-io/fabric/plugin/plugindata" +) + +type yamlData struct { + data plugindata.Data +} + +func makeYAMLDataSource() *plugin.DataSource { + return &plugin.DataSource{ + DataFunc: fetchYAMLData, + Args: &dataspec.RootSpec{ + Attrs: []*dataspec.AttrSpec{ + { + Name: "glob", + Type: cty.String, + ExampleVal: cty.StringVal("path/to/file*.yaml"), + Doc: `A glob pattern to select YAML files to read`, + }, + { + Name: "path", + Type: cty.String, + ExampleVal: cty.StringVal("path/to/file.yaml"), + Doc: `A file path to a YAML file to read`, + }, + }, + }, + Doc: ` + Loads YAML files with the names that match provided ` + "`glob`" + ` pattern or a single file from provided ` + "`path`" + `value. + + Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set. + + When ` + "`path`" + ` argument is specified, the data source returns only the content of a file. + When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example: + + ` + "```yaml" + ` + [ + { + "file_path": "path/file-a.yaml", + "file_name": "file-a.yaml", + "content": { + "foo": "bar" + } + }, + { + "file_path": "path/file-b.yaml", + "file_name": "file-b.yaml", + "content": [ + {"x": "y"} + ] + } + ] + ` + "```", + } +} + +func fetchYAMLData(ctx context.Context, params *plugin.RetrieveDataParams) (plugindata.Data, diagnostics.Diag) { + glob := params.Args.GetAttrVal("glob") + path := params.Args.GetAttrVal("path") + + if !path.IsNull() && path.AsString() != "" { + slog.Debug("Reading a file from a path", "path", path.AsString()) + data, err := readAndDecodeYAMLFile(path.AsString()) + if err != nil { + slog.Error( + "Error while reading a YAML file", + slog.String("path", path.AsString()), + slog.Any("error", err), + ) + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to read the file", + Detail: err.Error(), + }} + } + return data, nil + } else if !glob.IsNull() && glob.AsString() != "" { + slog.Debug("Reading the files that match the glob pattern", "glob", glob.AsString()) + data, err := readYAMLFiles(ctx, glob.AsString()) + if err != nil { + slog.Error( + "Error while reading the YAML files", + slog.String("glob", glob.AsString()), + slog.Any("error", err), + ) + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to read the files", + Detail: err.Error(), + }} + } + return data, nil + } + slog.Error("Either \"glob\" value or \"path\" value must be provided") + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Failed to parse provided arguments", + Detail: "Either \"glob\" value or \"path\" value must be provided", + }} +} + +func readAndDecodeYAMLFile(path string) (plugindata.Data, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var content yamlData + err = yaml.Unmarshal(yamlFile, &content) + if err != nil { + return nil, err + } + return content.data, nil +} + +func readYAMLFiles(ctx context.Context, pattern string) (plugindata.List, error) { + paths, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + result := make(plugindata.List, 0, len(paths)) + for _, path := range paths { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + content, err := readAndDecodeYAMLFile(path) + if err != nil { + return result, err + } + result = append(result, plugindata.Map{ + "file_path": plugindata.String(path), + "file_name": plugindata.String(filepath.Base(path)), + "content": content, + }) + } + } + return result, nil +} + +func (d yamlData) toData(v any) (res plugindata.Data, err error) { + switch v := v.(type) { + case nil: + return nil, nil + case int: + return plugindata.Number(v), nil + case float64: + return plugindata.Number(v), nil + case string: + return plugindata.String(v), nil + case bool: + return plugindata.Bool(v), nil + case map[string]any: + m := make(plugindata.Map) + for k, v := range v { + m[k], err = d.toData(v) + if err != nil { + return nil, err + } + } + return m, nil + case []any: + l := make(plugindata.List, len(v)) + for i, v := range v { + l[i], err = d.toData(v) + if err != nil { + return nil, err + } + } + return l, nil + default: + return nil, fmt.Errorf("can't convert type %T into `plugindata.Data`", v) + } +} + +func (d *yamlData) UnmarshalYAML(node *yaml.Node) (err error) { + var result any + if err := node.Decode(&result); err != nil { + return err + } + d.data, err = d.toData(result) + return err +} diff --git a/internal/builtin/data_yaml_test.go b/internal/builtin/data_yaml_test.go new file mode 100644 index 00000000..13474c2b --- /dev/null +++ b/internal/builtin/data_yaml_test.go @@ -0,0 +1,177 @@ +package builtin + +import ( + "context" + "log/slog" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/zclconf/go-cty/cty" + + "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/diagnostics/diagtest" + "github.com/blackstork-io/fabric/plugin" + "github.com/blackstork-io/fabric/plugin/plugindata" + "github.com/blackstork-io/fabric/plugin/plugintest" +) + +func Test_makeYAMLDataSchema(t *testing.T) { + schema := makeYAMLDataSource() + assert.Nil(t, schema.Config) + assert.NotNil(t, schema.Args) + assert.NotNil(t, schema.DataFunc) +} + +func Test_fetchYAMLData(t *testing.T) { + slog.SetLogLoggerLevel(slog.LevelDebug) + + tt := []struct { + name string + glob string + path string + expectedData plugindata.Data + expectedDiags diagtest.Asserts + }{ + { + name: "invalid_yaml_file", + glob: filepath.Join("testdata", "yaml", "invalid.*"), + expectedDiags: diagtest.Asserts{{ + diagtest.IsError, + diagtest.SummaryEquals("Failed to read the files"), + diagtest.DetailContains("yaml: line 2: could not find expected ':'"), + }}, + }, + { + name: "invalid_yaml_file_with_path", + path: filepath.Join("testdata", "yaml", "invalid.txt"), + expectedDiags: diagtest.Asserts{{ + diagtest.IsError, + diagtest.SummaryEquals("Failed to read the file"), + diagtest.DetailContains("yaml: line 2: could not find expected ':'"), + }}, + }, + { + name: "no_params", + expectedDiags: diagtest.Asserts{{ + diagtest.IsError, + diagtest.SummaryEquals("Failed to parse provided arguments"), + diagtest.DetailEquals("Either \"glob\" value or \"path\" value must be provided"), + }}, + }, + { + name: "no_glob_matches", + glob: filepath.Join("testdata", "yaml", "unknown_dir", "*.yaml"), + expectedData: plugindata.List{}, + }, + { + name: "no_path_match", + path: filepath.Join("testdata", "yaml", "unknown_dir", "does-not-exist.yaml"), + expectedDiags: diagtest.Asserts{{ + diagtest.IsError, + diagtest.SummaryEquals("Failed to read the file"), + diagtest.DetailContains("open", "does-not-exist.yaml"), + }}, + }, + { + name: "load_one_file_with_path", + path: filepath.Join("testdata", "yaml", "a.yaml"), + expectedData: plugindata.Map{ + "property_for": plugindata.String("a.yaml"), + }, + }, + { + name: "glob_matches_one_file", + glob: filepath.Join("testdata", "yaml", "a.yaml"), + expectedData: plugindata.List{ + plugindata.Map{ + "file_name": plugindata.String("a.yaml"), + "file_path": plugindata.String(filepath.Join("testdata", "yaml", "a.yaml")), + "content": plugindata.Map{ + "property_for": plugindata.String("a.yaml"), + }, + }, + }, + }, + { + name: "glob_matches_multiple_files", + glob: filepath.Join("testdata", "yaml", "dir", "*.yaml"), + expectedData: plugindata.List{ + plugindata.Map{ + "file_name": plugindata.String("b.yaml"), + "file_path": plugindata.String(filepath.Join("testdata", "yaml", "dir", "b.yaml")), + "content": plugindata.List{ + plugindata.Map{ + "id": plugindata.Number(1), + "property_for": plugindata.String("dir/b.yaml"), + }, + plugindata.Map{ + "id": plugindata.Number(2), + "property_for": plugindata.String("dir/b.yaml"), + }, + }, + }, + plugindata.Map{ + "file_name": plugindata.String("c.yaml"), + "file_path": plugindata.String(filepath.Join("testdata", "yaml", "dir", "c.yaml")), + "content": plugindata.List{ + plugindata.Map{ + "id": plugindata.Number(3), + "property_for": plugindata.String("dir/c.yaml"), + }, + plugindata.Map{ + "id": plugindata.Number(4), + "property_for": plugindata.String("dir/c.yaml"), + }, + }, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + p := &plugin.Schema{ + DataSources: plugin.DataSources{ + "yaml": makeYAMLDataSource(), + }, + } + + args := plugintest.NewTestDecoder(t, p.DataSources["yaml"].Args) + if tc.path != "" { + args.SetAttr("path", cty.StringVal(tc.path)) + } + if tc.glob != "" { + args.SetAttr("glob", cty.StringVal(tc.glob)) + } + + argVal, fm, diags := args.DecodeDiagFiles() + + var ( + data plugindata.Data + diag diagnostics.Diag + ) + if !diags.HasErrors() { + ctx := context.Background() + data, diag = p.RetrieveData(ctx, "yaml", &plugin.RetrieveDataParams{Args: argVal}) + + slog.Info("WHAT1", "data", data) + slog.Info("WHAT2", "diag", diag) + + + diags.Extend(diag) + } + assert.Equal(t, tc.expectedData, data) + tc.expectedDiags.AssertMatch(t, diags, fm) + }) + } +} + +func Test_readYAMLFilesCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + data, err := readYAMLFiles(ctx, filepath.Join("testdata", "yaml", "a.yaml")) + assert.Nil(t, data) + assert.Error(t, context.Canceled, err) +} + diff --git a/internal/builtin/plugin.go b/internal/builtin/plugin.go index ca343994..b401bc48 100644 --- a/internal/builtin/plugin.go +++ b/internal/builtin/plugin.go @@ -19,6 +19,7 @@ func Plugin(version string, logger *slog.Logger, tracer trace.Tracer) *plugin.Sc "txt": makeTXTDataSource(), "rss": makeRSSDataSource(), "json": makeJSONDataSource(), + "yaml": makeYAMLDataSource(), "http": makeHTTPDataSource(version), }, ContentProviders: plugin.ContentProviders{ diff --git a/internal/builtin/testdata/yaml/a.yaml b/internal/builtin/testdata/yaml/a.yaml new file mode 100644 index 00000000..399fbf6e --- /dev/null +++ b/internal/builtin/testdata/yaml/a.yaml @@ -0,0 +1 @@ +property_for: a.yaml diff --git a/internal/builtin/testdata/yaml/dir/b.yaml b/internal/builtin/testdata/yaml/dir/b.yaml new file mode 100644 index 00000000..dd321cd2 --- /dev/null +++ b/internal/builtin/testdata/yaml/dir/b.yaml @@ -0,0 +1,4 @@ +- id: 1 + property_for: dir/b.yaml +- id: 2 + property_for: dir/b.yaml diff --git a/internal/builtin/testdata/yaml/dir/c.yaml b/internal/builtin/testdata/yaml/dir/c.yaml new file mode 100644 index 00000000..7c8c1fee --- /dev/null +++ b/internal/builtin/testdata/yaml/dir/c.yaml @@ -0,0 +1,5 @@ +--- +- id: 3 + property_for: dir/c.yaml +- id: 4 + property_for: dir/c.yaml diff --git a/internal/builtin/testdata/yaml/invalid.txt b/internal/builtin/testdata/yaml/invalid.txt new file mode 100644 index 00000000..e7acd046 --- /dev/null +++ b/internal/builtin/testdata/yaml/invalid.txt @@ -0,0 +1,2 @@ +foo: +bar