YAML data source

docs/plugins/plugins.json

@@ -164,6 +164,14 @@
           "glob",
           "path"
         ]
+      },
+      {
+        "name": "yaml",
+        "type": "data-source",
+        "arguments": [
+          "glob",
+          "path"
+        ]
       }
     ]
   },

internal/builtin/data_yaml.go

@@ -0,0 +1,198 @@
+package builtin
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/blackstork-io/fabric/pkg/diagnostics"
+	"github.com/blackstork-io/fabric/plugin"
+	"github.com/blackstork-io/fabric/plugin/dataspec"
+	"github.com/blackstork-io/fabric/plugin/plugindata"
+)
+
+type yamlData struct {
+	data plugindata.Data
+}
+
+func makeYAMLDataSource() *plugin.DataSource {
+	return &plugin.DataSource{
+		DataFunc: fetchYAMLData,
+		Args: &dataspec.RootSpec{
+			Attrs: []*dataspec.AttrSpec{
+				{
+					Name:       "glob",
+					Type:       cty.String,
+					ExampleVal: cty.StringVal("path/to/file*.yaml"),
+					Doc:        `A glob pattern to select YAML files to read`,
+				},
+				{
+					Name:       "path",
+					Type:       cty.String,
+					ExampleVal: cty.StringVal("path/to/file.yaml"),
+					Doc:        `A file path to a YAML file to read`,
+				},
+			},
+		},
+		Doc: `
+		Loads YAML files with the names that match provided ` + "`glob`" + ` pattern or a single file from provided ` + "`path`" + `value.
+
+		Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set.
+
+		When ` + "`path`" + ` argument is specified, the data source returns only the content of a file.
+		When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example:
+
+		` + "```yaml" + `
+		[
+		  {
+			"file_path": "path/file-a.yaml",
+			"file_name": "file-a.yaml",
+			"content": {
+			  "foo": "bar"
+			}
+		  },
+		  {
+			"file_path": "path/file-b.yaml",
+			"file_name": "file-b.yaml",
+			"content": [
+			  {"x": "y"}
+			]
+		  }
+		]
+		` + "```",
+	}
+}
+
+func fetchYAMLData(ctx context.Context, params *plugin.RetrieveDataParams) (plugindata.Data, diagnostics.Diag) {
+	glob := params.Args.GetAttrVal("glob")
+	path := params.Args.GetAttrVal("path")
+
+	if !path.IsNull() && path.AsString() != "" {
+		slog.Debug("Reading a file from a path", "path", path.AsString())
+		data, err := readAndDecodeYAMLFile(path.AsString())
+		if err != nil {
+			slog.Error(
+				"Error while reading a YAML file",
+				slog.String("path", path.AsString()),
+				slog.Any("error", err),
+			)
+			return nil, diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to read the file",
+				Detail:   err.Error(),
+			}}
+		}
+		return data, nil
+	} else if !glob.IsNull() && glob.AsString() != "" {
+		slog.Debug("Reading the files that match the glob pattern", "glob", glob.AsString())
+		data, err := readYAMLFiles(ctx, glob.AsString())
+		if err != nil {
+			slog.Error(
+				"Error while reading the YAML files",
+				slog.String("glob", glob.AsString()),
+				slog.Any("error", err),
+			)
+			return nil, diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to read the files",
+				Detail:   err.Error(),
+			}}
+		}
+		return data, nil
+	}
+	slog.Error("Either \"glob\" value or \"path\" value must be provided")
+	return nil, diagnostics.Diag{{
+		Severity: hcl.DiagError,
+		Summary:  "Failed to parse provided arguments",
+		Detail:   "Either \"glob\" value or \"path\" value must be provided",
+	}}
+}
+
+func readAndDecodeYAMLFile(path string) (plugindata.Data, error) {
+	yamlFile, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var content yamlData
+	err = yaml.Unmarshal(yamlFile, &content)
+	if err != nil {
+		return nil, err
+	}
+	return content.data, nil
+}
+
+func readYAMLFiles(ctx context.Context, pattern string) (plugindata.List, error) {
+	paths, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+	result := make(plugindata.List, 0, len(paths))
+	for _, path := range paths {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+			content, err := readAndDecodeYAMLFile(path)
+			if err != nil {
+				return result, err
+			}
+			result = append(result, plugindata.Map{
+				"file_path": plugindata.String(path),
+				"file_name": plugindata.String(filepath.Base(path)),
+				"content":   content,
+			})
+		}
+	}
+	return result, nil
+}
+
+func (d yamlData) toData(v any) (res plugindata.Data, err error) {
+	switch v := v.(type) {
+	case nil:
+		return nil, nil
+	case int:
+		return plugindata.Number(v), nil
+	case float64:
+		return plugindata.Number(v), nil
+	case string:
+		return plugindata.String(v), nil
+	case bool:
+		return plugindata.Bool(v), nil
+	case map[string]any:
+		m := make(plugindata.Map)
+		for k, v := range v {
+			m[k], err = d.toData(v)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return m, nil
+	case []any:
+		l := make(plugindata.List, len(v))
+		for i, v := range v {
+			l[i], err = d.toData(v)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return l, nil
+	default:
+		return nil, fmt.Errorf("can't convert type %T into `plugindata.Data`", v)
+	}
+}
+
+func (d *yamlData) UnmarshalYAML(node *yaml.Node) (err error) {
+	var result any
+	if err := node.Decode(&result); err != nil {
+		return err
+	}
+	d.data, err = d.toData(result)
+	return err
+}

internal/builtin/data_yaml_test.go

@@ -0,0 +1,177 @@
+package builtin
+
+import (
+	"context"
+	"log/slog"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/blackstork-io/fabric/pkg/diagnostics"
+	"github.com/blackstork-io/fabric/pkg/diagnostics/diagtest"
+	"github.com/blackstork-io/fabric/plugin"
+	"github.com/blackstork-io/fabric/plugin/plugindata"
+	"github.com/blackstork-io/fabric/plugin/plugintest"
+)
+
+func Test_makeYAMLDataSchema(t *testing.T) {
+	schema := makeYAMLDataSource()
+	assert.Nil(t, schema.Config)
+	assert.NotNil(t, schema.Args)
+	assert.NotNil(t, schema.DataFunc)
+}
+
+func Test_fetchYAMLData(t *testing.T) {
+	slog.SetLogLoggerLevel(slog.LevelDebug)
+
+	tt := []struct {
+		name          string
+		glob          string
+		path          string
+		expectedData  plugindata.Data
+		expectedDiags diagtest.Asserts
+	}{
+		{
+			name: "invalid_yaml_file",
+			glob: filepath.Join("testdata", "yaml", "invalid.*"),
+			expectedDiags: diagtest.Asserts{{
+				diagtest.IsError,
+				diagtest.SummaryEquals("Failed to read the files"),
+				diagtest.DetailContains("yaml: line 2: could not find expected ':'"),
+			}},
+		},
+		{
+			name: "invalid_yaml_file_with_path",
+			path: filepath.Join("testdata", "yaml", "invalid.txt"),
+			expectedDiags: diagtest.Asserts{{
+				diagtest.IsError,
+				diagtest.SummaryEquals("Failed to read the file"),
+				diagtest.DetailContains("yaml: line 2: could not find expected ':'"),
+			}},
+		},
+		{
+			name: "no_params",
+			expectedDiags: diagtest.Asserts{{
+				diagtest.IsError,
+				diagtest.SummaryEquals("Failed to parse provided arguments"),
+				diagtest.DetailEquals("Either \"glob\" value or \"path\" value must be provided"),
+			}},
+		},
+		{
+			name:         "no_glob_matches",
+			glob:         filepath.Join("testdata", "yaml", "unknown_dir", "*.yaml"),
+			expectedData: plugindata.List{},
+		},
+		{
+			name: "no_path_match",
+			path: filepath.Join("testdata", "yaml", "unknown_dir", "does-not-exist.yaml"),
+			expectedDiags: diagtest.Asserts{{
+				diagtest.IsError,
+				diagtest.SummaryEquals("Failed to read the file"),
+				diagtest.DetailContains("open", "does-not-exist.yaml"),
+			}},
+		},
+		{
+			name: "load_one_file_with_path",
+			path: filepath.Join("testdata", "yaml", "a.yaml"),
+			expectedData: plugindata.Map{
+				"property_for": plugindata.String("a.yaml"),
+			},
+		},
+		{
+			name: "glob_matches_one_file",
+			glob: filepath.Join("testdata", "yaml", "a.yaml"),
+			expectedData: plugindata.List{
+				plugindata.Map{
+					"file_name": plugindata.String("a.yaml"),
+					"file_path": plugindata.String(filepath.Join("testdata", "yaml", "a.yaml")),
+					"content": plugindata.Map{
+						"property_for": plugindata.String("a.yaml"),
+					},
+				},
+			},
+		},
+		{
+			name: "glob_matches_multiple_files",
+			glob: filepath.Join("testdata", "yaml", "dir", "*.yaml"),
+			expectedData: plugindata.List{
+				plugindata.Map{
+					"file_name": plugindata.String("b.yaml"),
+					"file_path": plugindata.String(filepath.Join("testdata", "yaml", "dir", "b.yaml")),
+					"content": plugindata.List{
+						plugindata.Map{
+							"id":           plugindata.Number(1),
+							"property_for": plugindata.String("dir/b.yaml"),
+						},
+						plugindata.Map{
+							"id":           plugindata.Number(2),
+							"property_for": plugindata.String("dir/b.yaml"),
+						},
+					},
+				},
+				plugindata.Map{
+					"file_name": plugindata.String("c.yaml"),
+					"file_path": plugindata.String(filepath.Join("testdata", "yaml", "dir", "c.yaml")),
+					"content": plugindata.List{
+						plugindata.Map{
+							"id":           plugindata.Number(3),
+							"property_for": plugindata.String("dir/c.yaml"),
+						},
+						plugindata.Map{
+							"id":           plugindata.Number(4),
+							"property_for": plugindata.String("dir/c.yaml"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			p := &plugin.Schema{
+				DataSources: plugin.DataSources{
+					"yaml": makeYAMLDataSource(),
+				},
+			}
+
+			args := plugintest.NewTestDecoder(t, p.DataSources["yaml"].Args)
+			if tc.path != "" {
+				args.SetAttr("path", cty.StringVal(tc.path))
+			}
+			if tc.glob != "" {
+				args.SetAttr("glob", cty.StringVal(tc.glob))
+			}
+
+			argVal, fm, diags := args.DecodeDiagFiles()
+
+			var (
+				data plugindata.Data
+				diag diagnostics.Diag
+			)
+			if !diags.HasErrors() {
+				ctx := context.Background()
+				data, diag = p.RetrieveData(ctx, "yaml", &plugin.RetrieveDataParams{Args: argVal})
+
+				slog.Info("WHAT1", "data", data)
+				slog.Info("WHAT2", "diag", diag)
+
+
+				diags.Extend(diag)
+			}
+			assert.Equal(t, tc.expectedData, data)
+			tc.expectedDiags.AssertMatch(t, diags, fm)
+		})
+	}
+}
+
+func Test_readYAMLFilesCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	data, err := readYAMLFiles(ctx, filepath.Join("testdata", "yaml", "a.yaml"))
+	assert.Nil(t, data)
+	assert.Error(t, context.Canceled, err)
+}
+

internal/builtin/plugin.go

@@ -19,6 +19,7 @@ func Plugin(version string, logger *slog.Logger, tracer trace.Tracer) *plugin.Sc
 			"txt":  makeTXTDataSource(),
 			"rss":  makeRSSDataSource(),
 			"json": makeJSONDataSource(),
+			"yaml": makeYAMLDataSource(),
 			"http": makeHTTPDataSource(version),
 		},
 		ContentProviders: plugin.ContentProviders{