From 1d1276c1c70db0317caa4dce000a5d3d77dbd5d7 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 18 Oct 2023 15:57:02 +0800 Subject: [PATCH] add parse package --- load_mod/load_mod.go | 34 +- misc/load_pipeline.go | 134 ++-- misc/workspace_profile_loader.go | 2 +- parse/connection.go | 166 +++++ parse/context_functions.go | 125 ++++ parse/decode.go | 422 ++++++----- parse/decode_args.go | 46 +- parse/decode_body.go | 65 +- parse/decode_children.go | 4 +- parse/decode_options.go | 5 +- parse/decode_result.go | 5 +- parse/dependency_test.go | 64 ++ parse/installed_mod.go | 11 + parse/limiter.go | 20 + parse/mod.go | 82 +-- parse/mod_dependency_config.go | 1 - parse/mod_parse_context.go | 96 +-- parse/mod_parse_context_blocks.go | 5 +- parse/parse_context.go | 50 +- parse/parser.go | 28 +- parse/plugin.go | 50 ++ parse/query_invocation_test.go | 127 ++++ parse/references.go | 7 +- parse/schema.go | 130 ++-- parse/unresolved_block.go | 14 +- parse/validate.go | 61 +- parse/workspace_profile.go | 13 +- parse/workspace_profile_parse_context.go | 8 +- parse_v/decode.go | 801 +++++++++++++++++++++ parse_v/decode_args.go | 356 +++++++++ parse_v/decode_body.go | 301 ++++++++ parse_v/decode_children.go | 94 +++ parse_v/decode_options.go | 59 ++ parse_v/decode_result.go | 59 ++ parse_v/metadata.go | 39 + parse_v/mod.go | 219 ++++++ parse_v/mod_dependency_config.go | 27 + parse_v/mod_parse_context.go | 744 +++++++++++++++++++ parse_v/mod_parse_context_blocks.go | 94 +++ parse_v/parse_context.go | 241 +++++++ parse_v/parser.go | 203 ++++++ {parse => parse_v}/pipeline_decode.go | 2 +- parse_v/query_invocation.go | 185 +++++ parse_v/references.go | 38 + parse_v/schema.go | 344 +++++++++ parse_v/unresolved_block.go | 26 + parse_v/validate.go | 133 ++++ parse_v/workspace_profile.go | 208 ++++++ parse_v/workspace_profile_parse_context.go | 64 ++ workspace/workspace.go | 1 - 50 files changed, 5311 insertions(+), 702 deletions(-) create mode 100644 parse/connection.go create mode 100644 parse/context_functions.go create mode 100644 parse/dependency_test.go create mode 100644 parse/installed_mod.go create mode 100644 parse/limiter.go create mode 100644 parse/plugin.go create mode 100644 parse/query_invocation_test.go create mode 100644 parse_v/decode.go create mode 100644 parse_v/decode_args.go create mode 100644 parse_v/decode_body.go create mode 100644 parse_v/decode_children.go create mode 100644 parse_v/decode_options.go create mode 100644 parse_v/decode_result.go create mode 100644 parse_v/metadata.go create mode 100644 parse_v/mod.go create mode 100644 parse_v/mod_dependency_config.go create mode 100644 parse_v/mod_parse_context.go create mode 100644 parse_v/mod_parse_context_blocks.go create mode 100644 parse_v/parse_context.go create mode 100644 parse_v/parser.go rename {parse => parse_v}/pipeline_decode.go (99%) create mode 100644 parse_v/query_invocation.go create mode 100644 parse_v/references.go create mode 100644 parse_v/schema.go create mode 100644 parse_v/unresolved_block.go create mode 100644 parse_v/validate.go create mode 100644 parse_v/workspace_profile.go create mode 100644 parse_v/workspace_profile_parse_context.go diff --git a/load_mod/load_mod.go b/load_mod/load_mod.go index 6b0fb7c6..44e8eb0a 100644 --- a/load_mod/load_mod.go +++ b/load_mod/load_mod.go @@ -1,33 +1,33 @@ package load_mod import ( + "context" "fmt" + "github.com/turbot/pipe-fittings/parse" "log" "os" "path/filepath" "strings" - "github.com/hashicorp/hcl/v2" filehelpers "github.com/turbot/go-kit/files" "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/constants" "github.com/turbot/pipe-fittings/error_helpers" "github.com/turbot/pipe-fittings/filepaths" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/parse" "github.com/turbot/pipe-fittings/perr" "github.com/turbot/pipe-fittings/versionmap" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" ) -func LoadModWithFileName(modPath string, modFile string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings *error_helpers.ErrorAndWarnings) { +func LoadModWithFileName(ctx context.Context, modPath string, modFile string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings *error_helpers.ErrorAndWarnings) { defer func() { if r := recover(); r != nil { errorsAndWarnings = error_helpers.NewErrorsAndWarning(helpers.ToError(r)) } }() - mod, loadModResult := loadModDefinition(modPath, modFile, parseCtx) + mod, loadModResult := loadModDefinition(ctx, modPath, parseCtx) if loadModResult.Error != nil { return nil, loadModResult } @@ -68,7 +68,7 @@ func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mo } }() - return LoadModWithFileName(modPath, filepaths.PipesComponentModsFileName, parseCtx) + return LoadModWithFileName(context.Background(), modPath, filepaths.PipesComponentModsFileName, parseCtx) } func ModFileExists(modPath, modFile string) bool { @@ -91,22 +91,18 @@ func ModFileExists(modPath, modFile string) bool { return false } -func loadModDefinition(modPath string, modFile string, parseCtx *parse.ModParseContext) (*modconfig.Mod, *error_helpers.ErrorAndWarnings) { - var mod *modconfig.Mod - errorsAndWarnings := &error_helpers.ErrorAndWarnings{} - - if parseCtx.ShouldCreateCreateTransientLocalMod() && !ModFileExists(modPath, modFile) { - mod = modconfig.NewMod("local", modPath, hcl.Range{}) - return mod, errorsAndWarnings - } - +func loadModDefinition(ctx context.Context, modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings *error_helpers.ErrorAndWarnings) { + errorsAndWarnings = &error_helpers.ErrorAndWarnings{} // verify the mod folder exists - modFileFound := ModFileExists(modPath, modFile) + _, err := os.Stat(modPath) + if os.IsNotExist(err) { + return nil, error_helpers.NewErrorsAndWarning(fmt.Errorf("mod folder %s does not exist", modPath)) + } - if modFileFound { + if parse.ModfileExists(modPath) { // load the mod definition to get the dependencies var res *parse.DecodeResult - mod, res = parse.ParseModDefinitionWithFileName(modPath, modFile, parseCtx.EvalCtx) + mod, res = parse.ParseModDefinition(modPath, parseCtx.EvalCtx) errorsAndWarnings = error_helpers.DiagsToErrorsAndWarnings("mod load failed", res.Diags) if res.Diags.HasErrors() { return nil, errorsAndWarnings @@ -114,7 +110,7 @@ func loadModDefinition(modPath string, modFile string, parseCtx *parse.ModParseC } else { // so there is no mod file - should we create a default? if !parseCtx.ShouldCreateDefaultMod() { - errorsAndWarnings.Error = perr.BadRequestWithMessage(fmt.Sprintf("mod folder does not contain a mod resource definition '%s'", modPath)) + errorsAndWarnings.Error = fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath) // ShouldCreateDefaultMod flag NOT set - fail return nil, errorsAndWarnings } @@ -218,7 +214,7 @@ func loadModResources(mod *modconfig.Mod, parseCtx *parse.ModParseContext) (*mod } // parse all hcl files (NOTE - this reads the CurrentMod out of ParseContext and adds to it) - mod, errAndWarnings := parse.ParseMod(fileData, pseudoResources, parseCtx) + mod, errAndWarnings := parse.ParseMod(context.Background(), fileData, pseudoResources, parseCtx) return mod, errAndWarnings } diff --git a/misc/load_pipeline.go b/misc/load_pipeline.go index 3d76a52c..f5f2a4fb 100644 --- a/misc/load_pipeline.go +++ b/misc/load_pipeline.go @@ -3,15 +3,7 @@ package misc import ( "context" "fmt" - "github.com/turbot/pipe-fittings/load_mod" - "os" - "path/filepath" - - filehelpers "github.com/turbot/go-kit/files" - "github.com/turbot/pipe-fittings/filepaths" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/parse" - "github.com/turbot/pipe-fittings/perr" ) // ToError formats the supplied value as an error (or just returns it if already an error) @@ -31,66 +23,68 @@ func ToError(val interface{}) error { // We can potentially remove this function, but we have to refactor all our test cases func LoadPipelines(ctx context.Context, configPath string) (map[string]*modconfig.Pipeline, map[string]*modconfig.Trigger, error) { - var modDir string - var fileName string - var modFileNameToLoad string - - // Get information about the path - info, err := os.Stat(configPath) - if err != nil { - if os.IsNotExist(err) { - return map[string]*modconfig.Pipeline{}, map[string]*modconfig.Trigger{}, nil - } - return nil, nil, err - } - - // Check if it's a regular file - if info.Mode().IsRegular() { - fileName = filepath.Base(configPath) - modDir = filepath.Dir(configPath) - - // TODO: this is a hack (ish) to let the existing automated test to pass - if filepath.Ext(fileName) == ".fp" { - modFileNameToLoad = "ignore.sp" - } else { - modFileNameToLoad = fileName - } - } else if info.IsDir() { // Check if it's a directory - - defaultModSp := filepath.Join(configPath, filepaths.PipesComponentModsFileName) - - _, err := os.Stat(defaultModSp) - if err == nil { - // default mod.hcl exist - fileName = filepaths.PipesComponentModsFileName - modDir = configPath - } else { - fileName = "*.fp" - modDir = configPath - } - modFileNameToLoad = fileName - } else { - return nil, nil, perr.BadRequestWithMessage("invalid path") - } - - parseCtx := parse.NewModParseContext( - ctx, - nil, - modDir, - parse.CreateTransientLocalMod, - &filehelpers.ListOptions{ - Flags: filehelpers.Files | filehelpers.Recursive, - Include: []string{"**/" + fileName}, - }) - - mod, errorsAndWarnings := load_mod.LoadModWithFileName(modDir, modFileNameToLoad, parseCtx) - - var pipelines map[string]*modconfig.Pipeline - var triggers map[string]*modconfig.Trigger - - if mod != nil && mod.ResourceMaps != nil { - pipelines = mod.ResourceMaps.Pipelines - triggers = mod.ResourceMaps.Triggers - } - return pipelines, triggers, errorsAndWarnings.Error + // TODO KAI FIX ME + //var modDir string + //var fileName string + //var modFileNameToLoad string + // + //// Get information about the path + //info, err := os.Stat(configPath) + //if err != nil { + // if os.IsNotExist(err) { + // return map[string]*modconfig.Pipeline{}, map[string]*modconfig.Trigger{}, nil + // } + // return nil, nil, err + //} + // + //// Check if it's a regular file + //if info.Mode().IsRegular() { + // fileName = filepath.Base(configPath) + // modDir = filepath.Dir(configPath) + // + // // TODO: this is a hack (ish) to let the existing automated test to pass + // if filepath.Ext(fileName) == ".fp" { + // modFileNameToLoad = "ignore.sp" + // } else { + // modFileNameToLoad = fileName + // } + //} else if info.IsDir() { // Check if it's a directory + // + // defaultModSp := filepath.Join(configPath, filepaths.PipesComponentModsFileName) + // + // _, err := os.Stat(defaultModSp) + // if err == nil { + // // default mod.hcl exist + // fileName = filepaths.PipesComponentModsFileName + // modDir = configPath + // } else { + // fileName = "*.fp" + // modDir = configPath + // } + // modFileNameToLoad = fileName + //} else { + // return nil, nil, perr.BadRequestWithMessage("invalid path") + //} + // + //parseCtx := parse.NewModParseContext( + // ctx, + // nil, + // modDir, + // parse_v.CreateTransientLocalMod, + // &filehelpers.ListOptions{ + // Flags: filehelpers.Files | filehelpers.Recursive, + // Include: []string{"**/" + fileName}, + // }) + // + //mod, errorsAndWarnings := load_mod.LoadModWithFileName(context.Background(), modDir, modFileNameToLoad, parseCtx) + // + //var pipelines map[string]*modconfig.Pipeline + //var triggers map[string]*modconfig.Trigger + // + //if mod != nil && mod.ResourceMaps != nil { + // pipelines = mod.ResourceMaps.Pipelines + // triggers = mod.ResourceMaps.Triggers + //} + //return pipelines, triggers, errorsAndWarnings.Error + return nil, nil, nil } diff --git a/misc/workspace_profile_loader.go b/misc/workspace_profile_loader.go index db473ea5..3da935cc 100644 --- a/misc/workspace_profile_loader.go +++ b/misc/workspace_profile_loader.go @@ -71,7 +71,7 @@ func (l *WorkspaceProfileLoader) get(name string) (*modconfig.WorkspaceProfile, func (l *WorkspaceProfileLoader) load(runCtx context.Context) (map[string]*modconfig.WorkspaceProfile, error) { // get all the config files in the directory - return parse.LoadWorkspaceProfiles(runCtx, l.workspaceProfilePath) + return parse.LoadWorkspaceProfiles(l.workspaceProfilePath) } /* diff --git a/parse/connection.go b/parse/connection.go new file mode 100644 index 00000000..5f0db4e0 --- /dev/null +++ b/parse/connection.go @@ -0,0 +1,166 @@ +package parse + +import ( + "fmt" + "github.com/turbot/go-kit/hcl_helpers" + "log" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/zclconf/go-cty/cty" + "golang.org/x/exp/maps" +) + +func DecodeConnection(block *hcl.Block) (*modconfig.Connection, hcl.Diagnostics) { + connectionContent, rest, diags := block.Body.PartialContent(ConnectionBlockSchema) + if diags.HasErrors() { + return nil, diags + } + + connection := modconfig.NewConnection(block) + + // decode the plugin property + // NOTE: this mutates connection to set PluginAlias and possible PluginInstance + diags = decodeConnectionPluginProperty(connectionContent, connection) + if diags.HasErrors() { + return nil, diags + } + + if connectionContent.Attributes["type"] != nil { + var connectionType string + diags = gohcl.DecodeExpression(connectionContent.Attributes["type"].Expr, nil, &connectionType) + if diags.HasErrors() { + return nil, diags + } + connection.Type = connectionType + } + if connectionContent.Attributes["import_schema"] != nil { + var importSchema string + diags = gohcl.DecodeExpression(connectionContent.Attributes["import_schema"].Expr, nil, &importSchema) + if diags.HasErrors() { + return nil, diags + } + connection.ImportSchema = importSchema + } + if connectionContent.Attributes["connections"] != nil { + var connections []string + diags = gohcl.DecodeExpression(connectionContent.Attributes["connections"].Expr, nil, &connections) + if diags.HasErrors() { + return nil, diags + } + connection.ConnectionNames = connections + } + + // check for nested options + for _, connectionBlock := range connectionContent.Blocks { + switch connectionBlock.Type { + case "options": + // if we already found settings, fail + opts, moreDiags := DecodeOptions(connectionBlock) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + break + } + moreDiags = connection.SetOptions(opts, connectionBlock) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + } + + // TODO: remove in 0.22 [https://github.com/turbot/steampipe/issues/3251] + if connection.Options != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: fmt.Sprintf("%s in %s have been deprecated and will be removed in subsequent versions of steampipe", constants.Bold("'connection' options"), constants.Bold("'connection' blocks")), + Subject: hcl_helpers.BlockRangePointer(connectionBlock), + }) + } + + default: + // this can never happen + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("connections do not support '%s' blocks", block.Type), + Subject: hcl_helpers.BlockRangePointer(connectionBlock), + }) + } + } + + // tactical - update when support for options blocks is removed + // this needs updating to use a single block check + // at present we do not support blocks for plugin specific connection config + // so any blocks present in 'rest' are an error + if hclBody, ok := rest.(*hclsyntax.Body); ok { + for _, b := range hclBody.Blocks { + if b.Type != "options" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("connections do not support '%s' blocks", b.Type), + Subject: hcl_helpers.HclSyntaxBlockRangePointer(b), + }) + } + } + } + + // convert the remaining config to a hcl string to pass to the plugin + config, moreDiags := hcl_helpers.HclBodyToHclString(rest, connectionContent) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + } else { + connection.Config = config + } + + return connection, diags +} + +func decodeConnectionPluginProperty(connectionContent *hcl.BodyContent, connection *modconfig.Connection) hcl.Diagnostics { + var pluginName string + evalCtx := &hcl.EvalContext{Variables: make(map[string]cty.Value)} + + diags := gohcl.DecodeExpression(connectionContent.Attributes["plugin"].Expr, evalCtx, &pluginName) + res := newDecodeResult() + res.handleDecodeDiags(diags) + if res.Diags.HasErrors() { + return res.Diags + } + if len(res.Depends) > 0 { + log.Printf("[INFO] decodeConnectionPluginProperty plugin property is HCL reference") + // if this is a plugin reference, extract the plugin instance + pluginInstance, ok := getPluginInstanceFromDependency(maps.Values(res.Depends)) + if !ok { + log.Printf("[INFO] failed to resolve plugin property") + // return the original diagnostics + return diags + } + + // so we have resolved a reference to a plugin config + // we will validate that this block exists later in initializePlugins + // set PluginInstance ONLY + // (the PluginInstance property being set means that we will raise the correct error if we fail to resolve the plugin block) + connection.PluginInstance = &pluginInstance + return nil + } + + // NOTE: plugin property is set in initializePlugins + connection.PluginAlias = pluginName + + return nil +} + +func getPluginInstanceFromDependency(dependencies []*modconfig.ResourceDependency) (string, bool) { + if len(dependencies) != 1 { + return "", false + } + if len(dependencies[0].Traversals) != 1 { + return "", false + } + traversalString := hcl_helpers.TraversalAsString(dependencies[0].Traversals[0]) + split := strings.Split(traversalString, ".") + if len(split) != 2 || split[0] != "plugin" { + return "", false + } + return split[1], true +} diff --git a/parse/context_functions.go b/parse/context_functions.go new file mode 100644 index 00000000..55a0402f --- /dev/null +++ b/parse/context_functions.go @@ -0,0 +1,125 @@ +package parse + +import ( + "github.com/hashicorp/hcl/v2/ext/tryfunc" + "github.com/turbot/terraform-components/lang/funcs" + ctyyaml "github.com/zclconf/go-cty-yaml" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" + "github.com/zclconf/go-cty/cty/function/stdlib" +) + +// from `github.com/turbot/terraform-components/lang/functions.go` + +// ContextFunctions returns the set of functions that should be used to when evaluating expressions +func ContextFunctions(workspaceDir string) map[string]function.Function { + + ctxFuncs := map[string]function.Function{ + "abs": stdlib.AbsoluteFunc, + "abspath": funcs.AbsPathFunc, + "basename": funcs.BasenameFunc, + "base64decode": funcs.Base64DecodeFunc, + "base64encode": funcs.Base64EncodeFunc, + "base64gzip": funcs.Base64GzipFunc, + "base64sha256": funcs.Base64Sha256Func, + "base64sha512": funcs.Base64Sha512Func, + "bcrypt": funcs.BcryptFunc, + "can": tryfunc.CanFunc, + "ceil": stdlib.CeilFunc, + "chomp": stdlib.ChompFunc, + "cidrhost": funcs.CidrHostFunc, + "cidrnetmask": funcs.CidrNetmaskFunc, + "cidrsubnet": funcs.CidrSubnetFunc, + "cidrsubnets": funcs.CidrSubnetsFunc, + "coalesce": funcs.CoalesceFunc, + "coalescelist": stdlib.CoalesceListFunc, + "compact": stdlib.CompactFunc, + "concat": stdlib.ConcatFunc, + "contains": stdlib.ContainsFunc, + "csvdecode": stdlib.CSVDecodeFunc, + "dirname": funcs.DirnameFunc, + "distinct": stdlib.DistinctFunc, + "element": stdlib.ElementFunc, + "chunklist": stdlib.ChunklistFunc, + "file": funcs.MakeFileFunc(workspaceDir, false), + "fileexists": funcs.MakeFileExistsFunc(workspaceDir), + "fileset": funcs.MakeFileSetFunc(workspaceDir), + "filebase64": funcs.MakeFileFunc(workspaceDir, true), + "filebase64sha256": funcs.MakeFileBase64Sha256Func(workspaceDir), + "filebase64sha512": funcs.MakeFileBase64Sha512Func(workspaceDir), + "filemd5": funcs.MakeFileMd5Func(workspaceDir), + "filesha1": funcs.MakeFileSha1Func(workspaceDir), + "filesha256": funcs.MakeFileSha256Func(workspaceDir), + "filesha512": funcs.MakeFileSha512Func(workspaceDir), + "flatten": stdlib.FlattenFunc, + "floor": stdlib.FloorFunc, + "format": stdlib.FormatFunc, + "formatdate": stdlib.FormatDateFunc, + "formatlist": stdlib.FormatListFunc, + "indent": stdlib.IndentFunc, + "index": funcs.IndexFunc, // stdlib.IndexFunc is not compatible + "join": stdlib.JoinFunc, + "jsondecode": stdlib.JSONDecodeFunc, + "jsonencode": stdlib.JSONEncodeFunc, + "keys": stdlib.KeysFunc, + "length": funcs.LengthFunc, + "list": funcs.ListFunc, + "log": stdlib.LogFunc, + "lookup": funcs.LookupFunc, + "lower": stdlib.LowerFunc, + "map": funcs.MapFunc, + "matchkeys": funcs.MatchkeysFunc, + "max": stdlib.MaxFunc, + "md5": funcs.Md5Func, + "merge": stdlib.MergeFunc, + "min": stdlib.MinFunc, + "parseint": stdlib.ParseIntFunc, + "pathexpand": funcs.PathExpandFunc, + "pow": stdlib.PowFunc, + "range": stdlib.RangeFunc, + "regex": stdlib.RegexFunc, + "regexall": stdlib.RegexAllFunc, + "replace": funcs.ReplaceFunc, + "reverse": stdlib.ReverseListFunc, + "rsadecrypt": funcs.RsaDecryptFunc, + "setintersection": stdlib.SetIntersectionFunc, + "setproduct": stdlib.SetProductFunc, + "setsubtract": stdlib.SetSubtractFunc, + "setunion": stdlib.SetUnionFunc, + "sha1": funcs.Sha1Func, + "sha256": funcs.Sha256Func, + "sha512": funcs.Sha512Func, + "signum": stdlib.SignumFunc, + "slice": stdlib.SliceFunc, + "sort": stdlib.SortFunc, + "split": stdlib.SplitFunc, + "strrev": stdlib.ReverseFunc, + "substr": stdlib.SubstrFunc, + "sum": funcs.SumFunc, + "timestamp": funcs.TimestampFunc, + "timeadd": stdlib.TimeAddFunc, + "title": stdlib.TitleFunc, + "tostring": funcs.MakeToFunc(cty.String), + "tonumber": funcs.MakeToFunc(cty.Number), + "tobool": funcs.MakeToFunc(cty.Bool), + "toset": funcs.MakeToFunc(cty.Set(cty.DynamicPseudoType)), + "tolist": funcs.MakeToFunc(cty.List(cty.DynamicPseudoType)), + "tomap": funcs.MakeToFunc(cty.Map(cty.DynamicPseudoType)), + "transpose": funcs.TransposeFunc, + "trim": stdlib.TrimFunc, + "trimprefix": stdlib.TrimPrefixFunc, + "trimspace": stdlib.TrimSpaceFunc, + "trimsuffix": stdlib.TrimSuffixFunc, + "try": tryfunc.TryFunc, + "upper": stdlib.UpperFunc, + "urlencode": funcs.URLEncodeFunc, + "uuid": funcs.UUIDFunc, + "uuidv5": funcs.UUIDV5Func, + "values": stdlib.ValuesFunc, + "yamldecode": ctyyaml.YAMLDecodeFunc, + "yamlencode": ctyyaml.YAMLEncodeFunc, + "zipmap": stdlib.ZipmapFunc, + } + + return ctxFuncs +} diff --git a/parse/decode.go b/parse/decode.go index 781df663..f1645e54 100644 --- a/parse/decode.go +++ b/parse/decode.go @@ -2,14 +2,13 @@ package parse import ( "fmt" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/modconfig/var_config" - "github.com/turbot/pipe-fittings/schema" ) // A consistent detail message for all "not a valid identifier" diagnostics. @@ -40,7 +39,7 @@ func decode(parseCtx *ModParseContext) hcl.Diagnostics { parseCtx.ClearDependencies() for _, block := range blocks { - if block.Type == schema.BlockTypeLocals { + if block.Type == modconfig.BlockTypeLocals { resources, res := decodeLocalsBlock(block, parseCtx) if !res.Success() { diags = append(diags, res.Diags...) @@ -74,21 +73,19 @@ func addResourceToMod(resource modconfig.HclResource, block *hcl.Block, parseCtx } func shouldAddToMod(resource modconfig.HclResource, block *hcl.Block, parseCtx *ModParseContext) bool { - return true - // TODO: commented out due to dashboard - // switch resource.(type) { - // // do not add mods, withs - // case *modconfig.Mod, *modconfig.DashboardWith: - // return false - - // case *modconfig.DashboardCategory, *modconfig.DashboardInput: - // // if this is a dashboard category or dashboard input, only add top level blocks - // // this is to allow nested categories/inputs to have the same name as top level categories - // // (nested inputs are added by Dashboard.InitInputs) - // return parseCtx.IsTopLevelBlock(block) - // default: - // return true - // } + switch resource.(type) { + // do not add mods, withs + case *modconfig.Mod, *modconfig.DashboardWith: + return false + + case *modconfig.DashboardCategory, *modconfig.DashboardInput: + // if this is a dashboard category or dashboard input, only add top level blocks + // this is to allow nested categories/inputs to have the same name as top level categories + // (nested inputs are added by Dashboard.InitInputs) + return parseCtx.IsTopLevelBlock(block) + default: + return true + } } // special case decode logic for locals @@ -146,40 +143,32 @@ func decodeBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclReso // now do the actual decode switch { - case helpers.StringSliceContains(schema.NodeAndEdgeProviderBlocks, block.Type): + case helpers.StringSliceContains(modconfig.NodeAndEdgeProviderBlocks, block.Type): resource, res = decodeNodeAndEdgeProvider(block, parseCtx) - case helpers.StringSliceContains(schema.QueryProviderBlocks, block.Type): + case helpers.StringSliceContains(modconfig.QueryProviderBlocks, block.Type): resource, res = decodeQueryProvider(block, parseCtx) default: switch block.Type { - case schema.BlockTypeMod: + case modconfig.BlockTypeMod: // decodeMode has slightly different args as this code is shared with ParseModDefinition resource, res = decodeMod(block, parseCtx.EvalCtx, parseCtx.CurrentMod) - // TODO: commented out: dashboard - // case schema.BlockTypeDashboard: - // resource, res = decodeDashboard(block, parseCtx) - // case schema.BlockTypeContainer: - // resource, res = decodeDashboardContainer(block, parseCtx) - case schema.BlockTypeVariable: + case modconfig.BlockTypeDashboard: + resource, res = decodeDashboard(block, parseCtx) + case modconfig.BlockTypeContainer: + resource, res = decodeDashboardContainer(block, parseCtx) + case modconfig.BlockTypeVariable: resource, res = decodeVariable(block, parseCtx) - case schema.BlockTypeBenchmark: + case modconfig.BlockTypeBenchmark: resource, res = decodeBenchmark(block, parseCtx) - case schema.BlockTypePipeline: - resource, res = decodePipeline(parseCtx.CurrentMod, block, parseCtx) - case schema.BlockTypeTrigger: - resource, res = decodeTrigger(parseCtx.CurrentMod, block, parseCtx) default: // all other blocks are treated the same: resource, res = decodeResource(block, parseCtx) } } - // Note that an interface value that holds a nil concrete value is itself non-nil. - if !helpers.IsNil(resource) { - // handle the result - // - if there are dependencies, add to run context - handleModDecodeResult(resource, res, block, parseCtx) - } + // handle the result + // - if there are dependencies, add to run context + handleModDecodeResult(resource, res, block, parseCtx) return resource, res } @@ -218,25 +207,25 @@ func resourceForBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.Hc factoryFuncs := map[string]func(*hcl.Block, *modconfig.Mod, string) modconfig.HclResource{ // for block type mod, just use the current mod - schema.BlockTypeMod: func(*hcl.Block, *modconfig.Mod, string) modconfig.HclResource { return mod }, - schema.BlockTypeQuery: modconfig.NewQuery, - // schema.BlockTypeControl: modconfig.NewControl, - // schema.BlockTypeBenchmark: modconfig.NewBenchmark, - // schema.BlockTypeDashboard: modconfig.NewDashboard, - // schema.BlockTypeContainer: modconfig.NewDashboardContainer, - // schema.BlockTypeChart: modconfig.NewDashboardChart, - // schema.BlockTypeCard: modconfig.NewDashboardCard, - // schema.BlockTypeFlow: modconfig.NewDashboardFlow, - // schema.BlockTypeGraph: modconfig.NewDashboardGraph, - // schema.BlockTypeHierarchy: modconfig.NewDashboardHierarchy, - // schema.BlockTypeImage: modconfig.NewDashboardImage, - // schema.BlockTypeInput: modconfig.NewDashboardInput, - // schema.BlockTypeTable: modconfig.NewDashboardTable, - // schema.BlockTypeText: modconfig.NewDashboardText, - // schema.BlockTypeNode: modconfig.NewDashboardNode, - // schema.BlockTypeEdge: modconfig.NewDashboardEdge, - // schema.BlockTypeCategory: modconfig.NewDashboardCategory, - // schema.BlockTypeWith: modconfig.NewDashboardWith, + modconfig.BlockTypeMod: func(*hcl.Block, *modconfig.Mod, string) modconfig.HclResource { return mod }, + modconfig.BlockTypeQuery: modconfig.NewQuery, + modconfig.BlockTypeControl: modconfig.NewControl, + modconfig.BlockTypeBenchmark: modconfig.NewBenchmark, + modconfig.BlockTypeDashboard: modconfig.NewDashboard, + modconfig.BlockTypeContainer: modconfig.NewDashboardContainer, + modconfig.BlockTypeChart: modconfig.NewDashboardChart, + modconfig.BlockTypeCard: modconfig.NewDashboardCard, + modconfig.BlockTypeFlow: modconfig.NewDashboardFlow, + modconfig.BlockTypeGraph: modconfig.NewDashboardGraph, + modconfig.BlockTypeHierarchy: modconfig.NewDashboardHierarchy, + modconfig.BlockTypeImage: modconfig.NewDashboardImage, + modconfig.BlockTypeInput: modconfig.NewDashboardInput, + modconfig.BlockTypeTable: modconfig.NewDashboardTable, + modconfig.BlockTypeText: modconfig.NewDashboardText, + modconfig.BlockTypeNode: modconfig.NewDashboardNode, + modconfig.BlockTypeEdge: modconfig.NewDashboardEdge, + modconfig.BlockTypeCategory: modconfig.NewDashboardCategory, + modconfig.BlockTypeWith: modconfig.NewDashboardWith, } factoryFunc, ok := factoryFuncs[block.Type] @@ -244,7 +233,7 @@ func resourceForBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.Hc return nil, hcl.Diagnostics{&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("resourceForBlock called for unsupported block type %s", block.Type), - Subject: &block.DefRange, + Subject: hcl_helpers.BlockRangePointer(block), }, } } @@ -303,9 +292,6 @@ func decodeVariable(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Var func decodeQueryProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.QueryProvider, *DecodeResult) { res := newDecodeResult() - - // TODO [node_reuse] need raise errors for invalid properties https://github.com/turbot/steampipe/issues/2923 - // get shell resource resource, diags := resourceForBlock(block, parseCtx) res.handleDecodeDiags(diags) @@ -338,7 +324,7 @@ func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resour panic(fmt.Sprintf("block type %s not convertible to a QueryProvider", block.Type)) } - if attr, exists := content.Attributes[schema.AttributeTypeArgs]; exists { + if attr, exists := content.Attributes[modconfig.AttributeArgs]; exists { args, runtimeDependencies, diags := decodeArgs(attr.AsHCLAttribute(), parseCtx.EvalCtx, queryProvider) if diags.HasErrors() { // handle dependencies @@ -353,7 +339,7 @@ func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resour for _, b := range content.Blocks { block = b.AsHCLBlock() switch block.Type { - case schema.BlockTypeParam: + case modconfig.BlockTypeParam: paramDef, runtimeDependencies, moreDiags := decodeParam(block, parseCtx) if !moreDiags.HasErrors() { params = append(params, paramDef) @@ -373,8 +359,6 @@ func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resour func decodeNodeAndEdgeProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) { res := newDecodeResult() - // TODO [node_reuse] need raise errors for invalid properties https://github.com/turbot/steampipe/issues/2923 - // get shell resource resource, diags := resourceForBlock(block, parseCtx) res.handleDecodeDiags(diags) @@ -419,21 +403,20 @@ func decodeNodeAndEdgeProviderBlocks(content *hclsyntax.Body, nodeAndEdgeProvide for _, b := range content.Blocks { block := b.AsHCLBlock() switch block.Type { - // TODO: commented out: dashboard - // case schema.BlockTypeCategory: - // // decode block - // category, blockRes := decodeBlock(block, parseCtx) - // res.Merge(blockRes) - // if !blockRes.Success() { - // continue - // } + case modconfig.BlockTypeCategory: + // decode block + category, blockRes := decodeBlock(block, parseCtx) + res.Merge(blockRes) + if !blockRes.Success() { + continue + } - // // add the category to the nodeAndEdgeProvider - // res.addDiags(nodeAndEdgeProvider.AddCategory(category.(*modconfig.DashboardCategory))) + // add the category to the nodeAndEdgeProvider + res.addDiags(nodeAndEdgeProvider.AddCategory(category.(*modconfig.DashboardCategory))) - // DO NOT add the category to the mod + // DO NOT add the category to the mod - case schema.BlockTypeNode, schema.BlockTypeEdge: + case modconfig.BlockTypeNode, modconfig.BlockTypeEdge: child, childRes := decodeQueryProvider(block, parseCtx) // TACTICAL if child has any runtime dependencies, claim them @@ -450,152 +433,150 @@ func decodeNodeAndEdgeProviderBlocks(content *hclsyntax.Body, nodeAndEdgeProvide moreDiags := nodeAndEdgeProvider.AddChild(child) res.addDiags(moreDiags) } - // TODO: commented out: dashboard - // case schema.BlockTypeWith: - // with, withRes := decodeBlock(block, parseCtx) - // res.Merge(withRes) - // if res.Success() { - // moreDiags := nodeAndEdgeProvider.AddWith(with.(*modconfig.DashboardWith)) - // res.addDiags(moreDiags) - // } + case modconfig.BlockTypeWith: + with, withRes := decodeBlock(block, parseCtx) + res.Merge(withRes) + if res.Success() { + moreDiags := nodeAndEdgeProvider.AddWith(with.(*modconfig.DashboardWith)) + res.addDiags(moreDiags) + } + } + + } + + return res +} + +func decodeDashboard(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Dashboard, *DecodeResult) { + res := newDecodeResult() + dashboard := modconfig.NewDashboard(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.Dashboard) + + // do a partial decode using an empty schema - use to pull out all body content in the remain block + _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) + body := r.(*hclsyntax.Body) + res.handleDecodeDiags(diags) + + // decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded + diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, dashboard) + // handle any resulting diags, which may specify dependencies + res.handleDecodeDiags(diags) + + if dashboard.Base != nil && len(dashboard.Base.ChildNames) > 0 { + supportedChildren := []string{modconfig.BlockTypeContainer, modconfig.BlockTypeChart, modconfig.BlockTypeControl, modconfig.BlockTypeCard, modconfig.BlockTypeFlow, modconfig.BlockTypeGraph, modconfig.BlockTypeHierarchy, modconfig.BlockTypeImage, modconfig.BlockTypeInput, modconfig.BlockTypeTable, modconfig.BlockTypeText} + // TACTICAL: we should be passing in the block for the Base resource - but this is only used for diags + // and we do not expect to get any (as this function has already succeeded when the base was originally parsed) + children, _ := resolveChildrenFromNames(dashboard.Base.ChildNames, block, supportedChildren, parseCtx) + dashboard.Base.SetChildren(children) + } + if !res.Success() { + return dashboard, res + } + + // now decode child blocks + if len(body.Blocks) > 0 { + blocksRes := decodeDashboardBlocks(body, dashboard, parseCtx) + res.Merge(blocksRes) + } + + return dashboard, res +} + +func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboard, parseCtx *ModParseContext) *DecodeResult { + var res = newDecodeResult() + // set dashboard as parent on the run context - this is used when generating names for anonymous blocks + parseCtx.PushParent(dashboard) + defer func() { + parseCtx.PopParent() + }() + + for _, b := range content.Blocks { + block := b.AsHCLBlock() + + // decode block + resource, blockRes := decodeBlock(block, parseCtx) + res.Merge(blockRes) + if !blockRes.Success() { + continue + } + + // we expect either inputs or child report nodes + // add the resource to the mod + res.addDiags(addResourceToMod(resource, block, parseCtx)) + // add to the dashboard children + // (we expect this cast to always succeed) + if child, ok := resource.(modconfig.ModTreeItem); ok { + dashboard.AddChild(child) } } + moreDiags := dashboard.InitInputs() + res.addDiags(moreDiags) + return res } -// TODO: commented out: dashboard -// func decodeDashboard(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Dashboard, *DecodeResult) { -// res := newDecodeResult() -// dashboard := modconfig.NewDashboard(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.Dashboard) - -// // do a partial decode using an empty schema - use to pull out all body content in the remain block -// _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) -// body := r.(*hclsyntax.Body) -// res.handleDecodeDiags(diags) - -// // decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded -// diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, dashboard) -// // handle any resulting diags, which may specify dependencies -// res.handleDecodeDiags(diags) - -// if dashboard.Base != nil && len(dashboard.Base.ChildNames) > 0 { -// supportedChildren := []string{schema.BlockTypeContainer, schema.BlockTypeChart, schema.BlockTypeControl, schema.BlockTypeCard, schema.BlockTypeFlow, schema.BlockTypeGraph, schema.BlockTypeHierarchy, schema.BlockTypeImage, schema.BlockTypeInput, schema.BlockTypeTable, schema.BlockTypeText} -// // TACTICAL: we should be passing in the block for the Base resource - but this is only used for diags -// // and we do not expect to get any (as this function has already succeeded when the base was originally parsed) -// children, _ := resolveChildrenFromNames(dashboard.Base.ChildNames, block, supportedChildren, parseCtx) -// dashboard.Base.SetChildren(children) -// } -// if !res.Success() { -// return dashboard, res -// } - -// // now decode child blocks -// if len(body.Blocks) > 0 { -// blocksRes := decodeDashboardBlocks(body, dashboard, parseCtx) -// res.Merge(blocksRes) -// } - -// return dashboard, res -// } - -// func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboard, parseCtx *ModParseContext) *DecodeResult { -// var res = newDecodeResult() -// // set dashboard as parent on the run context - this is used when generating names for anonymous blocks -// parseCtx.PushParent(dashboard) -// defer func() { -// parseCtx.PopParent() -// }() - -// for _, b := range content.Blocks { -// block := b.AsHCLBlock() - -// // decode block -// resource, blockRes := decodeBlock(block, parseCtx) -// res.Merge(blockRes) -// if !blockRes.Success() { -// continue -// } - -// // we expect either inputs or child report nodes -// // add the resource to the mod -// res.addDiags(addResourceToMod(resource, block, parseCtx)) -// // add to the dashboard children -// // (we expect this cast to always succeed) -// if child, ok := resource.(modconfig.ModTreeItem); ok { -// dashboard.AddChild(child) -// } - -// } - -// moreDiags := dashboard.InitInputs() -// res.addDiags(moreDiags) - -// return res -// } - -// func decodeDashboardContainer(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.DashboardContainer, *DecodeResult) { -// res := newDecodeResult() -// container := modconfig.NewDashboardContainer(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.DashboardContainer) - -// // do a partial decode using an empty schema - use to pull out all body content in the remain block -// _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) -// body := r.(*hclsyntax.Body) -// res.handleDecodeDiags(diags) -// if !res.Success() { -// return nil, res -// } - -// // decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded -// diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, container) -// // handle any resulting diags, which may specify dependencies -// res.handleDecodeDiags(diags) - -// // now decode child blocks -// if len(body.Blocks) > 0 { -// blocksRes := decodeDashboardContainerBlocks(body, container, parseCtx) -// res.Merge(blocksRes) -// } - -// return container, res -// } - -// func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer *modconfig.DashboardContainer, parseCtx *ModParseContext) *DecodeResult { -// var res = newDecodeResult() - -// // set container as parent on the run context - this is used when generating names for anonymous blocks -// parseCtx.PushParent(dashboardContainer) -// defer func() { -// parseCtx.PopParent() -// }() - -// for _, b := range content.Blocks { -// block := b.AsHCLBlock() -// resource, blockRes := decodeBlock(block, parseCtx) -// res.Merge(blockRes) -// if !blockRes.Success() { -// continue -// } - -// // special handling for inputs -// if b.Type == schema.BlockTypeInput { -// input := resource.(*modconfig.DashboardInput) -// dashboardContainer.Inputs = append(dashboardContainer.Inputs, input) -// dashboardContainer.AddChild(input) -// // the input will be added to the mod by the parent dashboard - -// } else { -// // for all other children, add to mod and children -// res.addDiags(addResourceToMod(resource, block, parseCtx)) -// if child, ok := resource.(modconfig.ModTreeItem); ok { -// dashboardContainer.AddChild(child) -// } -// } -// } - -// return res -// } +func decodeDashboardContainer(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.DashboardContainer, *DecodeResult) { + res := newDecodeResult() + container := modconfig.NewDashboardContainer(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.DashboardContainer) + + // do a partial decode using an empty schema - use to pull out all body content in the remain block + _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) + body := r.(*hclsyntax.Body) + res.handleDecodeDiags(diags) + if !res.Success() { + return nil, res + } + + // decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded + diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, container) + // handle any resulting diags, which may specify dependencies + res.handleDecodeDiags(diags) + + // now decode child blocks + if len(body.Blocks) > 0 { + blocksRes := decodeDashboardContainerBlocks(body, container, parseCtx) + res.Merge(blocksRes) + } + + return container, res +} + +func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer *modconfig.DashboardContainer, parseCtx *ModParseContext) *DecodeResult { + var res = newDecodeResult() + + // set container as parent on the run context - this is used when generating names for anonymous blocks + parseCtx.PushParent(dashboardContainer) + defer func() { + parseCtx.PopParent() + }() + + for _, b := range content.Blocks { + block := b.AsHCLBlock() + resource, blockRes := decodeBlock(block, parseCtx) + res.Merge(blockRes) + if !blockRes.Success() { + continue + } + + // special handling for inputs + if b.Type == modconfig.BlockTypeInput { + input := resource.(*modconfig.DashboardInput) + dashboardContainer.Inputs = append(dashboardContainer.Inputs, input) + dashboardContainer.AddChild(input) + // the input will be added to the mod by the parent dashboard + + } else { + // for all other children, add to mod and children + res.addDiags(addResourceToMod(resource, block, parseCtx)) + if child, ok := resource.(modconfig.ModTreeItem); ok { + dashboardContainer.AddChild(child) + } + } + } + + return res +} func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Benchmark, *DecodeResult) { res := newDecodeResult() @@ -626,7 +607,7 @@ func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Be // now add children if res.Success() { - supportedChildren := []string{schema.BlockTypeBenchmark, schema.BlockTypeControl} + supportedChildren := []string{modconfig.BlockTypeBenchmark, modconfig.BlockTypeControl} children, diags := resolveChildrenFromNames(benchmark.ChildNames.StringList(), block, supportedChildren, parseCtx) res.handleDecodeDiags(diags) @@ -638,7 +619,7 @@ func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Be diags = decodeProperty(content, "base", &benchmark.Base, parseCtx.EvalCtx) res.handleDecodeDiags(diags) if benchmark.Base != nil && len(benchmark.Base.ChildNames) > 0 { - supportedChildren := []string{schema.BlockTypeBenchmark, schema.BlockTypeControl} + supportedChildren := []string{modconfig.BlockTypeBenchmark, modconfig.BlockTypeControl} // TACTICAL: we should be passing in the block for the Base resource - but this is only used for diags // and we do not expect to get any (as this function has already succeeded when the base was originally parsed) children, _ := resolveChildrenFromNames(benchmark.Base.ChildNameStrings, block, supportedChildren, parseCtx) @@ -698,8 +679,7 @@ func handleModDecodeResult(resource modconfig.HclResource, res *DecodeResult, bl // if resource supports metadata, save it if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok { - body := block.Body.(*hclsyntax.Body) - moreDiags = addResourceMetadata(resourceWithMetadata, body.SrcRange, parseCtx) + moreDiags = addResourceMetadata(resourceWithMetadata, resource.GetHclResourceImpl().DeclRange, parseCtx) res.addDiags(moreDiags) } } @@ -794,7 +774,7 @@ func validateHcl(blockType string, body *hclsyntax.Body, schema *hcl.BodySchema) func isDeprecated(attribute *hclsyntax.Attribute, blockType string) bool { switch attribute.Name { case "search_path", "search_path_prefix": - return blockType == schema.BlockTypeQuery || blockType == schema.BlockTypeControl + return blockType == modconfig.BlockTypeQuery || blockType == modconfig.BlockTypeControl default: return false } diff --git a/parse/decode_args.go b/parse/decode_args.go index 2728e5e3..5fea3333 100644 --- a/parse/decode_args.go +++ b/parse/decode_args.go @@ -2,14 +2,13 @@ package parse import ( "fmt" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/go-kit/helpers" - "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/go-kit/type_conversion" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/schema" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" ) @@ -76,7 +75,7 @@ func ctyTupleToArgArray(attr *hcl.Attribute, val cty.Value) ([]any, []*modconfig for idx, v := range values { // if the value is unknown, this is a runtime dependency if !v.IsKnown() { - runtimeDependency, err := identifyRuntimeDependenciesFromArray(attr, idx, schema.AttributeTypeArgs) + runtimeDependency, err := identifyRuntimeDependenciesFromArray(attr, idx, modconfig.AttributeArgs) if err != nil { return nil, nil, err } @@ -84,7 +83,7 @@ func ctyTupleToArgArray(attr *hcl.Attribute, val cty.Value) ([]any, []*modconfig runtimeDependencies = append(runtimeDependencies, runtimeDependency) } else { // decode the value into a go type - val, err := hclhelpers.CtyToGo(v) + val, err := type_conversion.CtyToGo(v) if err != nil { err := fmt.Errorf("invalid value provided for arg #%d: %v", idx, err) return nil, nil, err @@ -111,20 +110,20 @@ func ctyObjectToArgMap(attr *hcl.Attribute, val cty.Value, evalCtx *hcl.EvalCont // if the value is unknown, this is a runtime dependency if !v.IsKnown() { - runtimeDependency, err := identifyRuntimeDependenciesFromObject(attr, key, schema.AttributeTypeArgs, evalCtx) + runtimeDependency, err := identifyRuntimeDependenciesFromObject(attr, key, modconfig.AttributeArgs, evalCtx) if err != nil { return nil, nil, err } runtimeDependencies = append(runtimeDependencies, runtimeDependency) } else if getWrappedUnknownVal(v) { - runtimeDependency, err := identifyRuntimeDependenciesFromObject(attr, key, schema.AttributeTypeArgs, evalCtx) + runtimeDependency, err := identifyRuntimeDependenciesFromObject(attr, key, modconfig.AttributeArgs, evalCtx) if err != nil { return nil, nil, err } runtimeDependencies = append(runtimeDependencies, runtimeDependency) } else { // decode the value into a go type - val, err := hclhelpers.CtyToGo(v) + val, err := type_conversion.CtyToGo(v) if err != nil { err := fmt.Errorf("invalid value provided for param '%s': %v", key, err) return nil, nil, err @@ -184,7 +183,7 @@ func getRuntimeDepFromExpression(expr hcl.Expression, targetProperty, parentProp return nil, err } - if propertyPath.ItemType == schema.BlockTypeInput { + if propertyPath.ItemType == modconfig.BlockTypeInput { // tactical: validate input dependency if err := validateInputRuntimeDependency(propertyPath); err != nil { return nil, err @@ -207,14 +206,14 @@ dep_loop: for { switch e := expr.(type) { case *hclsyntax.ScopeTraversalExpr: - propertyPathStr = hclhelpers.TraversalAsString(e.Traversal) + propertyPathStr = hcl_helpers.TraversalAsString(e.Traversal) break dep_loop case *hclsyntax.SplatExpr: - root := hclhelpers.TraversalAsString(e.Source.(*hclsyntax.ScopeTraversalExpr).Traversal) + root := hcl_helpers.TraversalAsString(e.Source.(*hclsyntax.ScopeTraversalExpr).Traversal) var suffix string // if there is a property path, add it if each, ok := e.Each.(*hclsyntax.RelativeTraversalExpr); ok { - suffix = fmt.Sprintf(".%s", hclhelpers.TraversalAsString(each.Traversal)) + suffix = fmt.Sprintf(".%s", hcl_helpers.TraversalAsString(each.Traversal)) } propertyPathStr = fmt.Sprintf("%s.*%s", root, suffix) break dep_loop @@ -256,7 +255,7 @@ func identifyRuntimeDependenciesFromArray(attr *hcl.Attribute, idx int, parentPr return nil, err } // tactical: validate input dependency - if propertyPath.ItemType == schema.BlockTypeInput { + if propertyPath.ItemType == modconfig.BlockTypeInput { if err := validateInputRuntimeDependency(propertyPath); err != nil { return nil, err } @@ -278,11 +277,9 @@ func identifyRuntimeDependenciesFromArray(attr *hcl.Attribute, idx int, parentPr // TODO - include this with the main runtime dependency validation, when it is rewritten https://github.com/turbot/steampipe/issues/2925 func validateInputRuntimeDependency(propertyPath *modconfig.ParsedPropertyPath) error { // input references must be of form self.input..value - - // TODO: commented out: dashboard - // if propertyPath.Scope != modconfig.RuntimeDependencyDashboardScope { - // return fmt.Errorf("could not resolve runtime dependency resource %s", propertyPath.Original) - // } + if propertyPath.Scope != modconfig.RuntimeDependencyDashboardScope { + return fmt.Errorf("could not resolve runtime dependency resource %s", propertyPath.Original) + } return nil } @@ -299,16 +296,7 @@ func decodeParam(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.ParamD defaultValue, deps, moreDiags := decodeParamDefault(attr, parseCtx, def.UnqualifiedName) diags = append(diags, moreDiags...) if !helpers.IsNil(defaultValue) { - err := def.SetDefault(defaultValue) - if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "invalid default config for " + def.UnqualifiedName, - Detail: err.Error(), - Subject: &attr.Range, - }) - return nil, nil, diags - } + def.SetDefault(defaultValue) } runtimeDependencies = deps } @@ -320,7 +308,7 @@ func decodeParamDefault(attr *hcl.Attribute, parseCtx *ModParseContext, paramNam if v.IsKnown() { // convert the raw default into a string representation - val, err := hclhelpers.CtyToGo(v) + val, err := type_conversion.CtyToGo(v) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, diff --git a/parse/decode_body.go b/parse/decode_body.go index dc4e973e..3f065fa7 100644 --- a/parse/decode_body.go +++ b/parse/decode_body.go @@ -1,16 +1,14 @@ package parse import ( - "reflect" - "strings" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/go-kit/helpers" - "github.com/turbot/pipe-fittings/hclhelpers" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/schema" + "reflect" + "strings" ) func decodeHclBody(body hcl.Body, evalCtx *hcl.EvalContext, resourceProvider modconfig.ResourceMapsProvider, resource modconfig.HclResource) (diags hcl.Diagnostics) { @@ -100,29 +98,29 @@ func getResourceSchema(resource modconfig.HclResource, nestedStructs []any) *hcl // special cases for manually parsed attributes and blocks switch resource.BlockType() { - case schema.BlockTypeMod: - res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: schema.BlockTypeRequire}) - case schema.BlockTypeDashboard, schema.BlockTypeContainer: + case modconfig.BlockTypeMod: + res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: modconfig.BlockTypeRequire}) + case modconfig.BlockTypeDashboard, modconfig.BlockTypeContainer: res.Blocks = append(res.Blocks, - hcl.BlockHeaderSchema{Type: schema.BlockTypeControl}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeBenchmark}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeCard}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeChart}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeContainer}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeFlow}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeGraph}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeHierarchy}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeImage}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeInput}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeTable}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeText}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeWith}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeControl}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeBenchmark}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeCard}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeChart}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeContainer}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeFlow}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeGraph}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeHierarchy}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeImage}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeInput}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeTable}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeText}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeWith}, ) - case schema.BlockTypeQuery: + case modconfig.BlockTypeQuery: // remove `Query` from attributes var querySchema = &hcl.BodySchema{} for _, a := range res.Attributes { - if a.Name != schema.AttributeQuery { + if a.Name != modconfig.AttributeQuery { querySchema.Attributes = append(querySchema.Attributes, a) } } @@ -130,22 +128,21 @@ func getResourceSchema(resource modconfig.HclResource, nestedStructs []any) *hcl } if _, ok := resource.(modconfig.QueryProvider); ok { - res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: schema.BlockTypeParam}) + res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: modconfig.BlockTypeParam}) // if this is NOT query, add args - if resource.BlockType() != schema.BlockTypeQuery { - res.Attributes = append(res.Attributes, hcl.AttributeSchema{Name: schema.AttributeTypeArgs}) + if resource.BlockType() != modconfig.BlockTypeQuery { + res.Attributes = append(res.Attributes, hcl.AttributeSchema{Name: modconfig.AttributeArgs}) } } if _, ok := resource.(modconfig.NodeAndEdgeProvider); ok { res.Blocks = append(res.Blocks, - hcl.BlockHeaderSchema{Type: schema.BlockTypeCategory}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeNode}, - hcl.BlockHeaderSchema{Type: schema.BlockTypeEdge}) + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeCategory}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeNode}, + hcl.BlockHeaderSchema{Type: modconfig.BlockTypeEdge}) + } + if _, ok := resource.(modconfig.WithProvider); ok { + res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: modconfig.BlockTypeWith}) } - // TODO: dashbaord related (WithProvider) - // if _, ok := resource.(modconfig.WithProvider); ok { - // res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: schema.BlockTypeWith}) - // } return res } @@ -212,7 +209,7 @@ func resolveReferences(body hcl.Body, resourceMapsProvider modconfig.ResourceMap if _, ok := v.(modconfig.HclResource); ok { if hclVal, ok := attributes[hclAttribute]; ok { if scopeTraversal, ok := hclVal.Expr.(*hclsyntax.ScopeTraversalExpr); ok { - path := hclhelpers.TraversalAsString(scopeTraversal.Traversal) + path := hcl_helpers.TraversalAsString(scopeTraversal.Traversal) if parsedName, err := modconfig.ParseResourceName(path); err == nil { if r, ok := resourceMapsProvider.GetResource(parsedName); ok { f := rv.FieldByName(field.Name) diff --git a/parse/decode_children.go b/parse/decode_children.go index 9d2aa4af..516de58a 100644 --- a/parse/decode_children.go +++ b/parse/decode_children.go @@ -2,8 +2,8 @@ package parse import ( "fmt" - "github.com/hashicorp/hcl/v2" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/modconfig" ) @@ -69,7 +69,7 @@ func checkForDuplicateChildren(names []string, block *hcl.Block) hcl.Diagnostics diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("'%s.%s' has duplicate child name '%s'", block.Type, block.Labels[0], n), - Subject: &block.DefRange}) + Subject: hcl_helpers.BlockRangePointer(block)}) } nameMap[n] = nameCount + 1 } diff --git a/parse/decode_options.go b/parse/decode_options.go index 4c8bd334..b5579580 100644 --- a/parse/decode_options.go +++ b/parse/decode_options.go @@ -2,9 +2,9 @@ package parse import ( "fmt" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/pipe-fittings/options" ) @@ -21,7 +21,7 @@ func DecodeOptions(block *hcl.Block, overrides ...BlockMappingOverride) (options diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("Unexpected options type '%s'", block.Labels[0]), - Subject: &block.DefRange, + Subject: hcl_helpers.BlockRangePointer(block), }) return nil, diags } @@ -45,6 +45,7 @@ func defaultOptionsBlockMapping() OptionsBlockMapping { options.QueryBlock: &options.Query{}, options.CheckBlock: &options.Check{}, options.DashboardBlock: &options.GlobalDashboard{}, + options.PluginBlock: &options.Plugin{}, } return mapping } diff --git a/parse/decode_result.go b/parse/decode_result.go index 60c70360..99a49777 100644 --- a/parse/decode_result.go +++ b/parse/decode_result.go @@ -1,9 +1,8 @@ package parse import ( - "slices" - "github.com/hashicorp/hcl/v2" + "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/modconfig" ) @@ -48,7 +47,7 @@ func (p *DecodeResult) handleDecodeDiags(diags hcl.Diagnostics) { // determine whether the diag is a dependency error, and if so, return a dependency object func diagsToDependency(diag *hcl.Diagnostic) *modconfig.ResourceDependency { - if slices.Contains[[]string, string](missingVariableErrors, diag.Summary) { + if helpers.StringSliceContains(missingVariableErrors, diag.Summary) { return &modconfig.ResourceDependency{Range: diag.Expression.Range(), Traversals: diag.Expression.Variables()} } return nil diff --git a/parse/dependency_test.go b/parse/dependency_test.go new file mode 100644 index 00000000..d67f944e --- /dev/null +++ b/parse/dependency_test.go @@ -0,0 +1,64 @@ +package parse + +import ( + "reflect" + "testing" + + "github.com/turbot/go-kit/helpers" +) + +type dependencyTreeTest struct { + input [][]string + expected []string +} + +var testCasesDependencyTree = map[string]dependencyTreeTest{ + "no overlap": { + input: [][]string{{"a", "b", "c"}, {"d", "e", "f"}}, + expected: []string{"a", "b", "c", "d", "e", "f"}, + }, + "overlap": { + input: [][]string{{"a", "b", "c"}, {"b", "c"}}, + expected: []string{"a", "b", "c"}, + }, + "multiple overlaps": { + input: [][]string{{"a", "b", "c"}, {"b", "c"}, {"d", "e", "f", "g", "h", "i"}, {"g", "h", "i"}, {"h", "i"}}, + expected: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i"}, + }, +} + +func TestDependencyTree(t *testing.T) { + + for name, test := range testCasesDependencyTree { + + res := combineDependencyOrders(test.input) + if !reflect.DeepEqual(res, test.expected) { + t.Errorf("Test %s FAILED. Expected %v, got %v", name, test.expected, res) + } + } +} + +func combineDependencyOrders(deps [][]string) []string { + + // we assume every dep is unique + // for each dep, if first element exists in any other dep, then it cannot be the longest + // first dedupe + var longestDeps []string + for i, d1 := range deps { + longest := true + for j, d2 := range deps { + if i == j { + continue + } + if helpers.StringSliceContains(d2, d1[0]) { + longest = false + continue + } + } + if longest { + longestDeps = append(longestDeps, d1...) + } + } + + return longestDeps +} diff --git a/parse/installed_mod.go b/parse/installed_mod.go new file mode 100644 index 00000000..974e8070 --- /dev/null +++ b/parse/installed_mod.go @@ -0,0 +1,11 @@ +package parse + +import ( + "github.com/Masterminds/semver/v3" + "github.com/turbot/pipe-fittings/modconfig" +) + +type InstalledMod struct { + Mod *modconfig.Mod + Version *semver.Version +} diff --git a/parse/limiter.go b/parse/limiter.go new file mode 100644 index 00000000..d7a3c8da --- /dev/null +++ b/parse/limiter.go @@ -0,0 +1,20 @@ +package parse + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/turbot/pipe-fittings/modconfig" +) + +func DecodeLimiter(block *hcl.Block) (*modconfig.RateLimiter, hcl.Diagnostics) { + var limiter = &modconfig.RateLimiter{ + // populate name from label + Name: block.Labels[0], + } + diags := gohcl.DecodeBody(block.Body, nil, limiter) + if !diags.HasErrors() { + limiter.OnDecoded(block) + } + + return limiter, diags +} diff --git a/parse/mod.go b/parse/mod.go index 8026aa0c..2d18f4b3 100644 --- a/parse/mod.go +++ b/parse/mod.go @@ -1,21 +1,17 @@ package parse import ( + "context" "fmt" + "github.com/turbot/go-kit/hcl_helpers" "log" "os" "path" - "path/filepath" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/turbot/pipe-fittings/error_helpers" "github.com/turbot/pipe-fittings/filepaths" - "github.com/turbot/pipe-fittings/funcs" - "github.com/turbot/pipe-fittings/hclhelpers" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/perr" - "github.com/turbot/pipe-fittings/schema" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/zclconf/go-cty/cty" ) @@ -27,7 +23,7 @@ func LoadModfile(modPath string) (*modconfig.Mod, error) { // build an eval context just containing functions evalCtx := &hcl.EvalContext{ - Functions: funcs.ContextFunctions(modPath), + Functions: ContextFunctions(modPath), Variables: make(map[string]cty.Value), } @@ -39,31 +35,18 @@ func LoadModfile(modPath string) (*modconfig.Mod, error) { return mod, nil } -func ParseModDefinitionWithFileName(modPath string, modFileName string, evalCtx *hcl.EvalContext) (*modconfig.Mod, *DecodeResult) { +// ParseModDefinition parses the modfile only +// it is expected the calling code will have verified the existence of the modfile by calling ModfileExists +// this is called before parsing the workspace to, for example, identify dependency mods +func ParseModDefinition(modPath string, evalCtx *hcl.EvalContext) (*modconfig.Mod, *DecodeResult) { res := newDecodeResult() // if there is no mod at this location, return error - modFilePath := filepath.Join(modPath, modFileName) - - modFileFound := true + modFilePath := filepaths.ModFilePath(modPath) if _, err := os.Stat(modFilePath); os.IsNotExist(err) { - modFileFound = false - for _, file := range filepaths.PipesComponentValidModFiles { - modFilePath = filepath.Join(modPath, file) - if _, err := os.Stat(modFilePath); os.IsNotExist(err) { - - continue - } else { - modFileFound = true - break - } - } - } - - if !modFileFound { res.Diags = append(res.Diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "no mod file found in " + modPath, + Summary: fmt.Sprintf("no mod file found in %s", modPath), }) return nil, res } @@ -86,18 +69,15 @@ func ParseModDefinitionWithFileName(modPath string, modFileName string, evalCtx return nil, res } - block := hclhelpers.GetFirstBlockOfType(workspaceContent.Blocks, schema.BlockTypeMod) + block := hcl_helpers.GetFirstBlockOfType(workspaceContent.Blocks, modconfig.BlockTypeMod) if block == nil { res.Diags = append(res.Diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: fmt.Sprintf("no mod definition found in %s", modPath), + Summary: fmt.Sprintf("failed to parse mod definition file: no mod definition found in %s", fmt.Sprintf("%s/mod.sp", modPath)), }) return nil, res } - var defRange = block.DefRange - if hclBody, ok := block.Body.(*hclsyntax.Body); ok { - defRange = hclBody.SrcRange - } + var defRange = hcl_helpers.BlockRange(block) mod := modconfig.NewMod(block.Labels[0], path.Dir(modFilePath), defRange) // set modFilePath mod.SetFilePath(modFilePath) @@ -116,18 +96,9 @@ func ParseModDefinitionWithFileName(modPath string, modFileName string, evalCtx return mod, res } -// ParseModDefinition parses the modfile only -// it is expected the calling code will have verified the existence of the modfile by calling ModfileExists -// this is called before parsing the workspace to, for example, identify dependency mods -// -// This function only parse the "mod" block, and does not parse any resources in the mod file -func ParseModDefinition(modPath string, evalCtx *hcl.EvalContext) (*modconfig.Mod, *DecodeResult) { - return ParseModDefinitionWithFileName(modPath, filepaths.PipesComponentModsFileName, evalCtx) -} - // ParseMod parses all source hcl files for the mod path and associated resources, and returns the mod object // NOTE: the mod definition has already been parsed (or a default created) and is in opts.RunCtx.RootMod -func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableResource, parseCtx *ModParseContext) (*modconfig.Mod, *error_helpers.ErrorAndWarnings) { +func ParseMod(ctx context.Context, fileData map[string][]byte, pseudoResources []modconfig.MappableResource, parseCtx *ModParseContext) (*modconfig.Mod, *error_helpers.ErrorAndWarnings) { body, diags := ParseHclFiles(fileData) if diags.HasErrors() { return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod source files", diags)) @@ -158,33 +129,25 @@ func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableRe } } + // collect warnings as we parse + var res = &error_helpers.ErrorAndWarnings{} + // add pseudo resources to the mod - addPseudoResourcesToMod(pseudoResources, hclResources, mod) + errorsAndWarnings := addPseudoResourcesToMod(pseudoResources, hclResources, mod) + + // merge the warnings generated while adding pseudoresources + res.Merge(errorsAndWarnings) // add the parsed content to the run context parseCtx.SetDecodeContent(content, fileData) // add the mod to the run context // - this it to ensure all pseudo resources get added and build the eval context with the variables we just added - - // ! This is the place where the child mods (dependent mods) resources are "pulled up" into this current evaluation - // ! context. - // ! - // ! Step through the code to find the place where the child mod resources are added to the "referencesValue" - // ! - // ! Note that this resource MUST implement ModItem interface, otherwise it will look "flat", i.e. it will be added - // ! to the current mod - // ! - // ! There's also a bug where we test for ModTreeItem, we added a new interface ModItem for resources that are mod - // ! resources but not necessarily need to be in the mod tree - // ! + // - it also adds the top level resources of the any dependency mods if diags = parseCtx.AddModResources(mod); diags.HasErrors() { return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to add mod to run context", diags)) } - // collect warnings as we parse - var res = &error_helpers.ErrorAndWarnings{} - // we may need to decode more than once as we gather dependencies as we go // continue decoding as long as the number of unresolved blocks decreases prevUnresolvedBlocks := 0 @@ -205,8 +168,7 @@ func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableRe // if the number of unresolved blocks has NOT reduced, fail if prevUnresolvedBlocks != 0 && unresolvedBlocks >= prevUnresolvedBlocks { str := parseCtx.FormatDependencies() - msg := fmt.Sprintf("Failed to resolve dependencies after %d passes. Unresolved blocks:\n%s", attempts+1, str) - return nil, error_helpers.NewErrorsAndWarning(perr.BadRequestWithTypeAndMessage(perr.ErrorCodeDependencyFailure, msg)) + return nil, error_helpers.NewErrorsAndWarning(fmt.Errorf("failed to resolve dependencies for mod '%s' after %d attempts\nDependencies:\n%s", mod.FullName, attempts+1, str)) } // update prevUnresolvedBlocks prevUnresolvedBlocks = unresolvedBlocks diff --git a/parse/mod_dependency_config.go b/parse/mod_dependency_config.go index dd3010e5..97093668 100644 --- a/parse/mod_dependency_config.go +++ b/parse/mod_dependency_config.go @@ -2,7 +2,6 @@ package parse import ( "fmt" - "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/versionmap" ) diff --git a/parse/mod_parse_context.go b/parse/mod_parse_context.go index 798a75d5..79b776b0 100644 --- a/parse/mod_parse_context.go +++ b/parse/mod_parse_context.go @@ -1,8 +1,8 @@ package parse import ( - "context" "fmt" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/terraform-components/terraform" "strings" @@ -10,11 +10,8 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" filehelpers "github.com/turbot/go-kit/files" "github.com/turbot/go-kit/helpers" - "github.com/turbot/pipe-fittings/hclhelpers" "github.com/turbot/pipe-fittings/inputvars" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/perr" - "github.com/turbot/pipe-fittings/schema" "github.com/turbot/pipe-fittings/utils" "github.com/turbot/pipe-fittings/versionmap" "github.com/zclconf/go-cty/cty" @@ -27,7 +24,6 @@ type ParseModFlag uint32 const ( CreateDefaultMod ParseModFlag = 1 << iota CreatePseudoResources - CreateTransientLocalMod ) /* @@ -44,10 +40,6 @@ type ReferenceTypeValueMap map[string]map[string]cty.Value type ModParseContext struct { ParseContext - - // PipelineHcls map[string]*modconfig.Pipeline - TriggerHcls map[string]*modconfig.Trigger - // the mod which is currently being parsed CurrentMod *modconfig.Mod // the workspace lock data @@ -85,17 +77,10 @@ type ModParseContext struct { DependencyConfig *ModDependencyConfig } -func NewModParseContext(runContext context.Context, workspaceLock *versionmap.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *ModParseContext { - - parseContext := NewParseContext(runContext, rootEvalPath) +func NewModParseContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *ModParseContext { + parseContext := NewParseContext(rootEvalPath) c := &ModParseContext{ - ParseContext: parseContext, - - // TODO: fix this issue - // TODO: temporary mapping until we sort out merging Flowpipe and Steampipe - // PipelineHcls: make(map[string]*modconfig.Pipeline), - TriggerHcls: make(map[string]*modconfig.Trigger), - + ParseContext: parseContext, Flags: flags, WorkspaceLock: workspaceLock, ListOptions: listOptions, @@ -118,7 +103,6 @@ func NewModParseContext(runContext context.Context, workspaceLock *versionmap.Wo func NewChildModParseContext(parent *ModParseContext, modVersion *versionmap.ResolvedVersionConstraint, rootEvalPath string) *ModParseContext { // create a child run context child := NewModParseContext( - parent.RunCtx, parent.WorkspaceLock, rootEvalPath, parent.Flags, @@ -146,9 +130,7 @@ func (m *ModParseContext) EnsureWorkspaceLock(mod *modconfig.Mod) error { // if the mod has dependencies, there must a workspace lock object in the run context // (mod MUST be the workspace mod, not a dependency, as we would hit this error as soon as we parse it) if mod.HasDependentMods() && (m.WorkspaceLock.Empty() || m.WorkspaceLock.Incomplete()) { - // logger := fplog.Logger(m.RunCtx) - // logger.Error("mod has dependencies but no workspace lock file found", "mod", mod.Name(), "m.HasDependentMods()", mod.HasDependentMods(), "m.WorkspaceLock.Empty()", m.WorkspaceLock.Empty(), "m.WorkspaceLock.Incomplete()", m.WorkspaceLock.Incomplete()) - return perr.BadRequestWithTypeAndMessage(perr.ErrorCodeDependencyFailure, "not all dependencies are installed - run 'steampipe mod install'") + return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'") } return nil @@ -237,8 +219,8 @@ func (m *ModParseContext) AddModResources(mod *modconfig.Mod) hcl.Diagnostics { // do not add variables (as they have already been added) // if the resource is for a dependency mod, do not add locals shouldAdd := func(item modconfig.HclResource) bool { - if item.BlockType() == schema.BlockTypeVariable || - item.BlockType() == schema.BlockTypeLocals && item.(modconfig.ModItem).GetMod().ShortName != m.CurrentMod.ShortName { + if item.BlockType() == modconfig.BlockTypeVariable || + item.BlockType() == modconfig.BlockTypeLocals && item.(modconfig.ModTreeItem).GetMod().ShortName != m.CurrentMod.ShortName { return false } return true @@ -253,15 +235,7 @@ func (m *ModParseContext) AddModResources(mod *modconfig.Mod) hcl.Diagnostics { // continue walking return true, nil } - err := mod.WalkResources(resourceFunc) - if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "error walking mod resources", - Detail: err.Error(), - }) - return diags - } + mod.WalkResources(resourceFunc) // rebuild the eval context m.buildEvalContext() @@ -295,10 +269,6 @@ func (m *ModParseContext) ShouldCreateDefaultMod() bool { return m.Flags&CreateDefaultMod == CreateDefaultMod } -func (m *ModParseContext) ShouldCreateCreateTransientLocalMod() bool { - return m.Flags&CreateTransientLocalMod == CreateTransientLocalMod -} - // CreatePseudoResources returns whether the flag is set to create pseudo resources func (m *ModParseContext) CreatePseudoResources() bool { return m.Flags&CreatePseudoResources == CreatePseudoResources @@ -364,7 +334,6 @@ func (m *ModParseContext) buildEvalContext() { // now for each mod add all the values for mod, modMap := range m.referenceValues { - // TODO: this code is from steampipe, looks like there's a special treatment if the mod is named "local"? if mod == "local" { for k, v := range modMap { referenceValues[k] = cty.ObjectVal(v) @@ -375,7 +344,6 @@ func (m *ModParseContext) buildEvalContext() { // mod map is map[string]map[string]cty.Value // for each element (i.e. map[string]cty.Value) convert to cty object refTypeMap := make(map[string]cty.Value) - // TODO: this code is from steampipe, looks like there's a special treatment if the mod is named "local"? if mod == "local" { for k, v := range modMap { referenceValues[k] = cty.ObjectVal(v) @@ -390,7 +358,7 @@ func (m *ModParseContext) buildEvalContext() { } // rebuild the eval context - m.ParseContext.BuildEvalContext(referenceValues) + m.ParseContext.buildEvalContext(referenceValues) } // store the resource as a cty value in the reference valuemap @@ -500,7 +468,7 @@ func (m *ModParseContext) addReferenceValue(resource modconfig.HclResource, valu // most resources will have a mod property - use this if available var mod *modconfig.Mod - if modTreeItem, ok := resource.(modconfig.ModItem); ok { + if modTreeItem, ok := resource.(modconfig.ModTreeItem); ok { mod = modTreeItem.GetMod() } // fall back to current mod @@ -678,7 +646,7 @@ func (m *ModParseContext) getErrorStringForUnresolvedArg(parsedVarName *modconfi return "", fmt.Errorf("failed to get args details for %s", parsedVarName.ToResourceName()) } // ok we have the expression - build the error string - exprString := hclhelpers.TraversalAsString(expr.Traversal) + exprString := hcl_helpers.TraversalAsString(expr.Traversal) r := expr.Range() sourceRange := fmt.Sprintf("%s:%d", r.Filename, r.Start.Line) res := fmt.Sprintf("\"%s = %s\" (%s %s)", @@ -695,50 +663,10 @@ func (m *ModParseContext) getErrorStringForUnresolvedArg(parsedVarName *modconfi func (m *ModParseContext) getModRequireBlock() *hclsyntax.Block { for _, b := range m.CurrentMod.ResourceWithMetadataBaseRemain.(*hclsyntax.Body).Blocks { - if b.Type == schema.BlockTypeRequire { + if b.Type == modconfig.BlockTypeRequire { return b } } return nil } - -// TODO: transition period -// AddPipeline stores this resource as a variable to be added to the eval context. It alse -func (m *ModParseContext) AddPipeline(pipelineHcl *modconfig.Pipeline) hcl.Diagnostics { - - // Split and get the last part for pipeline name - // pipelineFullName := pipelineHcl.Name() - // parts := strings.Split(pipelineFullName, ".") - // pipelineNameOnly := parts[len(parts)-1] - - // m.PipelineHcls[pipelineNameOnly] = pipelineHcl - - diags := m.addReferenceValue(pipelineHcl, pipelineHcl.AsCtyValue()) - if diags.HasErrors() { - return diags - } - - // remove this resource from unparsed blocks - delete(m.UnresolvedBlocks, pipelineHcl.Name()) - - m.buildEvalContext() - return nil -} - -func (m *ModParseContext) AddTrigger(trigger *modconfig.Trigger) hcl.Diagnostics { - - // Split and get the last part for pipeline name - parts := strings.Split(trigger.Name(), ".") - triggerNameOnly := parts[len(parts)-1] - - m.TriggerHcls[triggerNameOnly] = trigger - - // remove this resource from unparsed blocks - delete(m.UnresolvedBlocks, trigger.Name()) - - m.buildEvalContext() - return nil -} - -// TODO: transition period diff --git a/parse/mod_parse_context_blocks.go b/parse/mod_parse_context_blocks.go index b956a7f4..231fcad4 100644 --- a/parse/mod_parse_context_blocks.go +++ b/parse/mod_parse_context_blocks.go @@ -2,6 +2,7 @@ package parse import ( "fmt" + "github.com/turbot/go-kit/hcl_helpers" "strings" "github.com/hashicorp/hcl/v2" @@ -67,7 +68,7 @@ func (m *ModParseContext) cacheBlockName(block *hcl.Block, shortName string) { } func (m *ModParseContext) blockHash(block *hcl.Block) string { - return helpers.GetMD5Hash(block.DefRange.String()) + return helpers.GetMD5Hash(hcl_helpers.BlockRange(block).String()) } // getUniqueName returns a name unique within the scope of this execution tree @@ -85,7 +86,7 @@ func (m *ModParseContext) getUniqueName(blockType string, parent string) string childCount++ } } - sanitisedParentName := strings.ReplaceAll(parent, ".", "_") + sanitisedParentName := strings.Replace(parent, ".", "_", -1) return fmt.Sprintf("%s_anonymous_%s_%d", sanitisedParentName, blockType, childCount) } diff --git a/parse/parse_context.go b/parse/parse_context.go index 161e503a..9479fea2 100644 --- a/parse/parse_context.go +++ b/parse/parse_context.go @@ -1,30 +1,22 @@ package parse import ( - "context" "fmt" - "strings" - "github.com/hashicorp/hcl/v2" "github.com/stevenle/topsort" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/go-kit/helpers" - "github.com/turbot/pipe-fittings/funcs" - "github.com/turbot/pipe-fittings/hclhelpers" "github.com/turbot/pipe-fittings/modconfig" "github.com/zclconf/go-cty/cty" + "strings" ) type ParseContext struct { - // This is the running application context - RunCtx context.Context UnresolvedBlocks map[string]*unresolvedBlock FileData map[string][]byte - // the eval context used to decode references in HCL EvalCtx *hcl.EvalContext - Diags hcl.Diagnostics - RootEvalPath string // if set, only decode these blocks @@ -36,11 +28,10 @@ type ParseContext struct { blocks hcl.Blocks } -func NewParseContext(runContext context.Context, rootEvalPath string) ParseContext { +func NewParseContext(rootEvalPath string) ParseContext { c := ParseContext{ UnresolvedBlocks: make(map[string]*unresolvedBlock), RootEvalPath: rootEvalPath, - RunCtx: runContext, } // add root node - this will depend on all other nodes c.dependencyGraph = c.newDependencyGraph() @@ -63,19 +54,8 @@ func (r *ParseContext) ClearDependencies() { // 2) add dependencies to our tree of dependencies func (r *ParseContext) AddDependencies(block *hcl.Block, name string, dependencies map[string]*modconfig.ResourceDependency) hcl.Diagnostics { var diags hcl.Diagnostics - - if r.UnresolvedBlocks[name] != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: fmt.Sprintf("duplicate unresolved block name '%s'", name), - Detail: fmt.Sprintf("block '%s' already exists. This could mean that there are unresolved duplicate resources,", name), - Subject: &block.DefRange, - }) - return diags - } - // store unresolved block - r.UnresolvedBlocks[name] = &unresolvedBlock{Name: name, Block: block, Dependencies: dependencies} + r.UnresolvedBlocks[name] = newUnresolvedBlock(block, name, dependencies) // store dependency in tree - d if !r.dependencyGraph.ContainsNode(name) { @@ -86,23 +66,19 @@ func (r *ParseContext) AddDependencies(block *hcl.Block, name string, dependenci diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "failed to add root dependency to graph", - Detail: err.Error(), - Subject: &block.DefRange, - }) + Detail: err.Error()}) } for _, dep := range dependencies { // each dependency object may have multiple traversals for _, t := range dep.Traversals { - parsedPropertyPath, err := modconfig.ParseResourcePropertyPath(hclhelpers.TraversalAsString(t)) + parsedPropertyPath, err := modconfig.ParseResourcePropertyPath(hcl_helpers.TraversalAsString(t)) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "failed to parse dependency", - Detail: err.Error(), - Subject: &block.DefRange, - }) + Detail: err.Error()}) continue } @@ -116,9 +92,7 @@ func (r *ParseContext) AddDependencies(block *hcl.Block, name string, dependenci diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "failed to add dependency to graph", - Detail: err.Error(), - Subject: &block.DefRange, - }) + Detail: err.Error()}) } } } @@ -145,10 +119,10 @@ func (r *ParseContext) BlocksToDecode() (hcl.Blocks, error) { // depOrder is all the blocks required to resolve dependencies. // if this one is unparsed, added to list block, ok := r.UnresolvedBlocks[name] - if ok && !blocksMap[block.Block.DefRange.String()] { + if ok && !blocksMap[block.DeclRange.String()] && ok { blocksToDecode = append(blocksToDecode, block.Block) // add to map - blocksMap[block.Block.DefRange.String()] = true + blocksMap[block.DeclRange.String()] = true } } return blocksToDecode, nil @@ -230,12 +204,12 @@ func (r *ParseContext) getDependencyOrder() ([]string, error) { } // eval functions -func (r *ParseContext) BuildEvalContext(variables map[string]cty.Value) { +func (r *ParseContext) buildEvalContext(variables map[string]cty.Value) { // create evaluation context r.EvalCtx = &hcl.EvalContext{ Variables: variables, // use the mod path as the file root for functions - Functions: funcs.ContextFunctions(r.RootEvalPath), + Functions: ContextFunctions(r.RootEvalPath), } } diff --git a/parse/parser.go b/parse/parser.go index 1ac24616..72ffc825 100644 --- a/parse/parser.go +++ b/parse/parser.go @@ -12,10 +12,8 @@ import ( "github.com/hashicorp/hcl/v2/hclparse" "github.com/hashicorp/hcl/v2/json" "github.com/turbot/pipe-fittings/constants" - "github.com/turbot/pipe-fittings/filepaths" + "github.com/turbot/pipe-fittings/error_helpers" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/schema" - "github.com/turbot/pipe-fittings/utils" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "sigs.k8s.io/yaml" ) @@ -85,7 +83,7 @@ func buildOrderedFileNameList(fileData map[string][]byte) []string { // ModfileExists returns whether a mod file exists at the specified path func ModfileExists(modPath string) bool { - modFilePath := filepath.Join(modPath, filepaths.PipesComponentModsFileName) + modFilePath := filepath.Join(modPath, "mod.sp") if _, err := os.Stat(modFilePath); os.IsNotExist(err) { return false } @@ -129,24 +127,22 @@ func parseYamlFile(filename string) (*hcl.File, hcl.Diagnostics) { return json.Parse(jsonData, filename) } -func addPseudoResourcesToMod(pseudoResources []modconfig.MappableResource, hclResources map[string]bool, mod *modconfig.Mod) { - var duplicates []string +func addPseudoResourcesToMod(pseudoResources []modconfig.MappableResource, hclResources map[string]bool, mod *modconfig.Mod) *error_helpers.ErrorAndWarnings { + res := error_helpers.EmptyErrorsAndWarning() for _, r := range pseudoResources { // is there a hcl resource with the same name as this pseudo resource - it takes precedence name := r.GetUnqualifiedName() if _, ok := hclResources[name]; ok { - duplicates = append(duplicates, r.GetDeclRange().Filename) + res.AddWarning(fmt.Sprintf("%s ignored as hcl resources of same name is already defined", r.GetDeclRange().Filename)) + log.Printf("[TRACE] %s ignored as hcl resources of same name is already defined", r.GetDeclRange().Filename) continue } // add pseudo resource to mod - mod.AddResource(r.(modconfig.HclResource)) //nolint:errcheck // TODO: handle error + mod.AddResource(r.(modconfig.HclResource)) // add to map of existing resources hclResources[name] = true } - numDupes := len(duplicates) - if numDupes > 0 { - log.Printf("[TRACE] %d %s not converted into resources as hcl resources of same name are defined: %v", numDupes, utils.Pluralize("file", numDupes), duplicates) - } + return res } // get names of all resources defined in hcl which may also be created as pseudo resources @@ -157,7 +153,7 @@ func loadMappableResourceNames(content *hcl.BodyContent) (map[string]bool, error for _, block := range content.Blocks { // if this is a mod, build a shell mod struct (with just the name populated) switch block.Type { - case schema.BlockTypeQuery: + case modconfig.BlockTypeQuery: // for any mappable resource, store the resource name name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) hclResources[name] = true @@ -185,15 +181,15 @@ func ParseModResourceNames(fileData map[string][]byte) (*modconfig.WorkspaceReso // if this is a mod, build a shell mod struct (with just the name populated) switch block.Type { - case schema.BlockTypeQuery: + case modconfig.BlockTypeQuery: // for any mappable resource, store the resource name name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) resources.Query[name] = true - case schema.BlockTypeControl: + case modconfig.BlockTypeControl: // for any mappable resource, store the resource name name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) resources.Control[name] = true - case schema.BlockTypeBenchmark: + case modconfig.BlockTypeBenchmark: // for any mappable resource, store the resource name name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) resources.Benchmark[name] = true diff --git a/parse/plugin.go b/parse/plugin.go new file mode 100644 index 00000000..049ab7ff --- /dev/null +++ b/parse/plugin.go @@ -0,0 +1,50 @@ +package parse + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" + + "github.com/turbot/pipe-fittings/modconfig" +) + +func DecodePlugin(block *hcl.Block) (*modconfig.Plugin, hcl.Diagnostics) { + // manually decode child limiter blocks + content, rest, diags := block.Body.PartialContent(PluginBlockSchema) + if diags.HasErrors() { + return nil, diags + } + body := rest.(*hclsyntax.Body) + + // decode attributes using 'rest' (these are automativally parsed so are not in schema) + var plugin = &modconfig.Plugin{ + // default source and name to label + Instance: block.Labels[0], + Alias: block.Labels[0], + } + moreDiags := gohcl.DecodeBody(body, nil, plugin) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + return nil, diags + } + + // decode limiter blocks using 'content' + for _, block := range content.Blocks { + switch block.Type { + // only block defined in schema + case modconfig.BlockTypeRateLimiter: + limiter, moreDiags := DecodeLimiter(block) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + limiter.SetPlugin(plugin) + plugin.Limiters = append(plugin.Limiters, limiter) + } + } + if !diags.HasErrors() { + plugin.OnDecoded(block) + } + + return plugin, diags +} diff --git a/parse/query_invocation_test.go b/parse/query_invocation_test.go new file mode 100644 index 00000000..84ca990c --- /dev/null +++ b/parse/query_invocation_test.go @@ -0,0 +1,127 @@ +package parse + +import ( + "fmt" + "testing" + + "github.com/turbot/pipe-fittings/utils" + + "github.com/turbot/pipe-fittings/modconfig" +) + +// NOTE: all query arg values must be JSON representations +type parseQueryInvocationTest struct { + input string + expected parseQueryInvocationResult +} + +type parseQueryInvocationResult struct { + queryName string + args *modconfig.QueryArgs +} + +var emptyParams = modconfig.NewQueryArgs() +var testCasesParseQueryInvocation = map[string]parseQueryInvocationTest{ + "no brackets": { + input: `query.q1`, + expected: parseQueryInvocationResult{"query.q1", emptyParams}, + }, + "no params": { + input: `query.q1()`, + expected: parseQueryInvocationResult{"query.q1", emptyParams}, + }, + "invalid params 1": { + input: `query.q1(foo)`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{}, + }, + }, + "invalid params 4": { + input: `query.q1("foo", "bar"])`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + + args: &modconfig.QueryArgs{}, + }, + }, + + "single positional param": { + input: `query.q1("foo")`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgList: []*string{utils.ToStringPointer("foo")}}, + }, + }, + "single positional param extra spaces": { + input: `query.q1("foo" ) `, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgList: []*string{utils.ToStringPointer("foo")}}, + }, + }, + "multiple positional params": { + input: `query.q1("foo", "bar", "foo-bar")`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgList: []*string{utils.ToStringPointer("foo"), utils.ToStringPointer("bar"), utils.ToStringPointer("foo-bar")}}, + }, + }, + "multiple positional params extra spaces": { + input: `query.q1("foo", "bar", "foo-bar" )`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgList: []*string{utils.ToStringPointer("foo"), utils.ToStringPointer("bar"), utils.ToStringPointer("foo-bar")}}, + }, + }, + "single named param": { + input: `query.q1(p1 => "foo")`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgMap: map[string]string{"p1": "foo"}}, + }, + }, + "single named param extra spaces": { + input: `query.q1( p1 => "foo" ) `, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgMap: map[string]string{"p1": "foo"}}, + }, + }, + "multiple named params": { + input: `query.q1(p1 => "foo", p2 => "bar")`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgMap: map[string]string{"p1": "foo", "p2": "bar"}}, + }, + }, + "multiple named params extra spaces": { + input: ` query.q1 ( p1 => "foo" , p2 => "bar" ) `, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgMap: map[string]string{"p1": "foo", "p2": "bar"}}, + }, + }, + "named param with dot in value": { + input: `query.q1(p1 => "foo.bar")`, + expected: parseQueryInvocationResult{ + queryName: `query.q1`, + args: &modconfig.QueryArgs{ArgMap: map[string]string{"p1": "foo.bar"}}, + }, + }, +} + +func TestParseQueryInvocation(t *testing.T) { + for name, test := range testCasesParseQueryInvocation { + queryName, args, _ := ParseQueryInvocation(test.input) + + if queryName != test.expected.queryName || !test.expected.args.Equals(args) { + fmt.Printf("") + t.Errorf("Test: '%s'' FAILED : expected:\nquery: %s params: %s\n\ngot:\nquery: %s params: %s", + name, + test.expected.queryName, + test.expected.args, + queryName, args) + } + } +} diff --git a/parse/references.go b/parse/references.go index ee513d62..d52c462e 100644 --- a/parse/references.go +++ b/parse/references.go @@ -3,9 +3,8 @@ package parse import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/go-kit/hcl_helpers" "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/schema" ) // AddReferences populates the 'References' resource field, used for the introspection tables @@ -18,8 +17,8 @@ func AddReferences(resource modconfig.HclResource, block *hcl.Block, parseCtx *M var diags hcl.Diagnostics for _, attr := range block.Body.(*hclsyntax.Body).Attributes { for _, v := range attr.Expr.Variables() { - for _, referenceBlockType := range schema.ReferenceBlocks { - if referenceString, ok := hclhelpers.ResourceNameFromTraversal(referenceBlockType, v); ok { + for _, referenceBlockType := range modconfig.ReferenceBlocks { + if referenceString, ok := hcl_helpers.ResourceNameFromTraversal(referenceBlockType, v); ok { var blockName string if len(block.Labels) > 0 { blockName = block.Labels[0] diff --git a/parse/schema.go b/parse/schema.go index e3f6e06f..14f8483b 100644 --- a/parse/schema.go +++ b/parse/schema.go @@ -2,28 +2,38 @@ package parse import ( "github.com/hashicorp/hcl/v2" - "github.com/turbot/pipe-fittings/schema" + "github.com/turbot/pipe-fittings/modconfig" ) // cache resource schemas var resourceSchemaCache = make(map[string]*hcl.BodySchema) -// TODO [node_reuse] Replace all block type with consts https://github.com/turbot/steampipe/issues/2922 - var ConfigBlockSchema = &hcl.BodySchema{ Attributes: []hcl.AttributeSchema{}, Blocks: []hcl.BlockHeaderSchema{ { - Type: "connection", + Type: modconfig.BlockTypeConnection, LabelNames: []string{"name"}, }, - { - Type: "options", + Type: modconfig.BlockTypePlugin, + LabelNames: []string{"name"}, + }, + { + Type: modconfig.BlockTypeOptions, LabelNames: []string{"type"}, }, { - Type: "workspace", + Type: modconfig.BlockTypeWorkspaceProfile, + LabelNames: []string{"name"}, + }, + }, +} +var PluginBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{}, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: modconfig.BlockTypeRateLimiter, LabelNames: []string{"name"}, }, }, @@ -68,90 +78,80 @@ var WorkspaceBlockSchema = &hcl.BodySchema{ Attributes: []hcl.AttributeSchema{}, Blocks: []hcl.BlockHeaderSchema{ { - Type: string(schema.BlockTypeMod), + Type: string(modconfig.BlockTypeMod), LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeVariable, + Type: modconfig.BlockTypeVariable, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeQuery, + Type: modconfig.BlockTypeQuery, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeControl, + Type: modconfig.BlockTypeControl, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeBenchmark, + Type: modconfig.BlockTypeBenchmark, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeDashboard, + Type: modconfig.BlockTypeDashboard, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeCard, + Type: modconfig.BlockTypeCard, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeChart, + Type: modconfig.BlockTypeChart, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeFlow, + Type: modconfig.BlockTypeFlow, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeGraph, + Type: modconfig.BlockTypeGraph, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeHierarchy, + Type: modconfig.BlockTypeHierarchy, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeImage, + Type: modconfig.BlockTypeImage, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeInput, + Type: modconfig.BlockTypeInput, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeTable, + Type: modconfig.BlockTypeTable, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeText, + Type: modconfig.BlockTypeText, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeNode, + Type: modconfig.BlockTypeNode, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeEdge, + Type: modconfig.BlockTypeEdge, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeLocals, + Type: modconfig.BlockTypeLocals, }, { - Type: schema.BlockTypeCategory, + Type: modconfig.BlockTypeCategory, LabelNames: []string{"name"}, }, - - // Flowpipe - { - Type: schema.BlockTypePipeline, - LabelNames: []string{schema.LabelName}, - }, - { - Type: schema.BlockTypeTrigger, - LabelNames: []string{schema.LabelType, schema.LabelName}, - }, }, } @@ -159,48 +159,48 @@ var WorkspaceBlockSchema = &hcl.BodySchema{ var DashboardBlockSchema = &hcl.BodySchema{ Blocks: []hcl.BlockHeaderSchema{ { - Type: schema.BlockTypeInput, + Type: modconfig.BlockTypeInput, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeParam, + Type: modconfig.BlockTypeParam, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeWith, + Type: modconfig.BlockTypeWith, }, { - Type: schema.BlockTypeContainer, + Type: modconfig.BlockTypeContainer, }, { - Type: schema.BlockTypeCard, + Type: modconfig.BlockTypeCard, }, { - Type: schema.BlockTypeChart, + Type: modconfig.BlockTypeChart, }, { - Type: schema.BlockTypeBenchmark, + Type: modconfig.BlockTypeBenchmark, }, { - Type: schema.BlockTypeControl, + Type: modconfig.BlockTypeControl, }, { - Type: schema.BlockTypeFlow, + Type: modconfig.BlockTypeFlow, }, { - Type: schema.BlockTypeGraph, + Type: modconfig.BlockTypeGraph, }, { - Type: schema.BlockTypeHierarchy, + Type: modconfig.BlockTypeHierarchy, }, { - Type: schema.BlockTypeImage, + Type: modconfig.BlockTypeImage, }, { - Type: schema.BlockTypeTable, + Type: modconfig.BlockTypeTable, }, { - Type: schema.BlockTypeText, + Type: modconfig.BlockTypeText, }, }, } @@ -209,45 +209,45 @@ var DashboardBlockSchema = &hcl.BodySchema{ var DashboardContainerBlockSchema = &hcl.BodySchema{ Blocks: []hcl.BlockHeaderSchema{ { - Type: schema.BlockTypeInput, + Type: modconfig.BlockTypeInput, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeParam, + Type: modconfig.BlockTypeParam, LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeContainer, + Type: modconfig.BlockTypeContainer, }, { - Type: schema.BlockTypeCard, + Type: modconfig.BlockTypeCard, }, { - Type: schema.BlockTypeChart, + Type: modconfig.BlockTypeChart, }, { - Type: schema.BlockTypeBenchmark, + Type: modconfig.BlockTypeBenchmark, }, { - Type: schema.BlockTypeControl, + Type: modconfig.BlockTypeControl, }, { - Type: schema.BlockTypeFlow, + Type: modconfig.BlockTypeFlow, }, { - Type: schema.BlockTypeGraph, + Type: modconfig.BlockTypeGraph, }, { - Type: schema.BlockTypeHierarchy, + Type: modconfig.BlockTypeHierarchy, }, { - Type: schema.BlockTypeImage, + Type: modconfig.BlockTypeImage, }, { - Type: schema.BlockTypeTable, + Type: modconfig.BlockTypeTable, }, { - Type: schema.BlockTypeText, + Type: modconfig.BlockTypeText, }, }, } @@ -306,10 +306,10 @@ var NodeAndEdgeProviderSchema = &hcl.BodySchema{ LabelNames: []string{"name"}, }, { - Type: schema.BlockTypeNode, + Type: modconfig.BlockTypeNode, }, { - Type: schema.BlockTypeEdge, + Type: modconfig.BlockTypeEdge, }, }, } diff --git a/parse/unresolved_block.go b/parse/unresolved_block.go index cfe1c2c2..e3584bcb 100644 --- a/parse/unresolved_block.go +++ b/parse/unresolved_block.go @@ -2,19 +2,29 @@ package parse import ( "fmt" + "github.com/turbot/go-kit/hcl_helpers" "strings" - "github.com/turbot/pipe-fittings/modconfig" - "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/modconfig" ) type unresolvedBlock struct { Name string Block *hcl.Block + DeclRange hcl.Range Dependencies map[string]*modconfig.ResourceDependency } +func newUnresolvedBlock(block *hcl.Block, name string, dependencies map[string]*modconfig.ResourceDependency) *unresolvedBlock { + return &unresolvedBlock{ + Name: name, + Block: block, + Dependencies: dependencies, + DeclRange: hcl_helpers.BlockRange(block), + } +} + func (b unresolvedBlock) String() string { depStrings := make([]string, len(b.Dependencies)) idx := 0 diff --git a/parse/validate.go b/parse/validate.go index 60071e0b..e0e28658 100644 --- a/parse/validate.go +++ b/parse/validate.go @@ -2,7 +2,6 @@ package parse import ( "fmt" - "github.com/hashicorp/hcl/v2" "github.com/turbot/pipe-fittings/modconfig" ) @@ -18,28 +17,26 @@ func validateResource(resource modconfig.HclResource) hcl.Diagnostics { diags = append(diags, moreDiags...) } - // TODO: commented out: dashboard - // if wp, ok := resource.(modconfig.WithProvider); ok { - // moreDiags := validateRuntimeDependencyProvider(wp) - // diags = append(diags, moreDiags...) - // } + if wp, ok := resource.(modconfig.WithProvider); ok { + moreDiags := validateRuntimeDependencyProvider(wp) + diags = append(diags, moreDiags...) + } return diags } -// TODO: commented out: dashboard -// func validateRuntimeDependencyProvider(wp modconfig.WithProvider) hcl.Diagnostics { -// resource := wp.(modconfig.HclResource) -// var diags hcl.Diagnostics -// if len(wp.GetWiths()) > 0 && !resource.IsTopLevel() { -// diags = append(diags, &hcl.Diagnostic{ -// Severity: hcl.DiagError, -// Summary: "Only top level resources can have `with` blocks", -// Detail: fmt.Sprintf("%s contains 'with' blocks but is not a top level resource.", resource.Name()), -// Subject: resource.GetDeclRange(), -// }) -// } -// return diags -// } +func validateRuntimeDependencyProvider(wp modconfig.WithProvider) hcl.Diagnostics { + resource := wp.(modconfig.HclResource) + var diags hcl.Diagnostics + if len(wp.GetWiths()) > 0 && !resource.IsTopLevel() { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Only top level resources can have `with` blocks", + Detail: fmt.Sprintf("%s contains 'with' blocks but is not a top level resource.", resource.Name()), + Subject: resource.GetDeclRange(), + }) + } + return diags +} // validate that the provider does not contains both edges/nodes and a query/sql // enrich the loaded nodes and edges with the fully parsed resources from the resourceMapProvider @@ -48,26 +45,20 @@ func validateNodeAndEdgeProvider(resource modconfig.NodeAndEdgeProvider) hcl.Dia // https://github.com/turbot/steampipe/issues/2918 var diags hcl.Diagnostics - - // TODO: commented out: dashboard - // containsEdgesOrNodes := len(resource.GetEdges())+len(resource.GetNodes()) > 0 + containsEdgesOrNodes := len(resource.GetEdges())+len(resource.GetNodes()) > 0 definesQuery := resource.GetSQL() != nil || resource.GetQuery() != nil - // TODO: commented out: dashboard // cannot declare both edges/nodes AND sql/query - // if definesQuery && containsEdgesOrNodes { - // diags = append(diags, &hcl.Diagnostic{ - // Severity: hcl.DiagError, - // Summary: fmt.Sprintf("%s contains edges/nodes AND has a query", resource.Name()), - // Subject: resource.GetDeclRange(), - // }) - // } + if definesQuery && containsEdgesOrNodes { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("%s contains edges/nodes AND has a query", resource.Name()), + Subject: resource.GetDeclRange(), + }) + } // if resource is NOT top level must have either edges/nodes OR sql/query - - // TODO: commented out: dashboard - // if !resource.IsTopLevel() && !definesQuery && !containsEdgesOrNodes { - if !resource.IsTopLevel() && !definesQuery { + if !resource.IsTopLevel() && !definesQuery && !containsEdgesOrNodes { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("%s does not define a query or SQL, and has no edges/nodes", resource.Name()), diff --git a/parse/workspace_profile.go b/parse/workspace_profile.go index cf769735..5a62d556 100644 --- a/parse/workspace_profile.go +++ b/parse/workspace_profile.go @@ -1,8 +1,8 @@ package parse import ( - "context" "fmt" + "github.com/turbot/go-kit/hcl_helpers" "log" "github.com/hashicorp/hcl/v2" @@ -12,11 +12,10 @@ import ( "github.com/turbot/pipe-fittings/constants" "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/options" - "github.com/turbot/pipe-fittings/schema" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" ) -func LoadWorkspaceProfiles(ctx context.Context, workspaceProfilePath string) (profileMap map[string]*modconfig.WorkspaceProfile, err error) { +func LoadWorkspaceProfiles(workspaceProfilePath string) (profileMap map[string]*modconfig.WorkspaceProfile, err error) { defer func() { if r := recover(); r != nil { @@ -58,7 +57,7 @@ func LoadWorkspaceProfiles(ctx context.Context, workspaceProfilePath string) (pr return nil, plugin.DiagsToError("Failed to load workspace profiles", diags) } - parseCtx := NewWorkspaceProfileParseContext(ctx, workspaceProfilePath) + parseCtx := NewWorkspaceProfileParseContext(workspaceProfilePath) parseCtx.SetDecodeContent(content, fileData) // build parse context @@ -112,7 +111,7 @@ func decodeWorkspaceProfiles(parseCtx *WorkspaceProfileParseContext) (map[string parseCtx.ClearDependencies() for _, block := range blocksToDecode { - if block.Type == schema.BlockTypeWorkspaceProfile { + if block.Type == modconfig.BlockTypeWorkspaceProfile { workspaceProfile, res := decodeWorkspaceProfile(block, parseCtx) if res.Success() { @@ -160,7 +159,7 @@ func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseCon // fail diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Subject: &block.DefRange, + Subject: hcl_helpers.BlockRangePointer(block), Summary: fmt.Sprintf("Duplicate options type '%s'", optionsBlockType), }) } @@ -179,7 +178,7 @@ func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseCon diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("invalid block type '%s' - only 'options' blocks are supported for workspace profiles", block.Type), - Subject: &block.DefRange, + Subject: hcl_helpers.BlockRangePointer(block), }) } } diff --git a/parse/workspace_profile_parse_context.go b/parse/workspace_profile_parse_context.go index e2ec5126..15f0d0a3 100644 --- a/parse/workspace_profile_parse_context.go +++ b/parse/workspace_profile_parse_context.go @@ -1,9 +1,7 @@ package parse import ( - "context" "fmt" - "github.com/hashicorp/hcl/v2" "github.com/turbot/pipe-fittings/modconfig" "github.com/zclconf/go-cty/cty" @@ -15,8 +13,8 @@ type WorkspaceProfileParseContext struct { valueMap map[string]cty.Value } -func NewWorkspaceProfileParseContext(ctx context.Context, rootEvalPath string) *WorkspaceProfileParseContext { - parseContext := NewParseContext(ctx, rootEvalPath) +func NewWorkspaceProfileParseContext(rootEvalPath string) *WorkspaceProfileParseContext { + parseContext := NewParseContext(rootEvalPath) // TODO uncomment once https://github.com/turbot/steampipe/issues/2640 is done //parseContext.BlockTypes = []string{modconfig.BlockTypeWorkspaceProfile} c := &WorkspaceProfileParseContext{ @@ -59,6 +57,6 @@ func (c *WorkspaceProfileParseContext) buildEvalContext() { vars := map[string]cty.Value{ "workspace": cty.ObjectVal(c.valueMap), } - c.ParseContext.BuildEvalContext(vars) + c.ParseContext.buildEvalContext(vars) } diff --git a/parse_v/decode.go b/parse_v/decode.go new file mode 100644 index 00000000..6f1545e6 --- /dev/null +++ b/parse_v/decode.go @@ -0,0 +1,801 @@ +package parse_v + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/modconfig/var_config" + "github.com/turbot/pipe-fittings/schema" +) + +// A consistent detail message for all "not a valid identifier" diagnostics. +const badIdentifierDetail = "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes." + +var missingVariableErrors = []string{ + // returned when the context variables does not have top level 'type' node (locals/control/etc) + "Unknown variable", + // returned when the variables have the type object but a field has not yet been populated + "Unsupported attribute", + "Missing map element", +} + +func decode(parseCtx *ModParseContext) hcl.Diagnostics { + var diags hcl.Diagnostics + + blocks, err := parseCtx.BlocksToDecode() + // build list of blocks to decode + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to determine required dependency order", + Detail: err.Error()}) + return diags + } + + // now clear dependencies from run context - they will be rebuilt + parseCtx.ClearDependencies() + + for _, block := range blocks { + if block.Type == schema.BlockTypeLocals { + resources, res := decodeLocalsBlock(block, parseCtx) + if !res.Success() { + diags = append(diags, res.Diags...) + continue + } + for _, resource := range resources { + resourceDiags := addResourceToMod(resource, block, parseCtx) + diags = append(diags, resourceDiags...) + } + } else { + resource, res := decodeBlock(block, parseCtx) + diags = append(diags, res.Diags...) + if !res.Success() || resource == nil { + continue + } + + resourceDiags := addResourceToMod(resource, block, parseCtx) + diags = append(diags, resourceDiags...) + } + } + + return diags +} + +func addResourceToMod(resource modconfig.HclResource, block *hcl.Block, parseCtx *ModParseContext) hcl.Diagnostics { + if !shouldAddToMod(resource, block, parseCtx) { + return nil + } + return parseCtx.CurrentMod.AddResource(resource) + +} + +func shouldAddToMod(resource modconfig.HclResource, block *hcl.Block, parseCtx *ModParseContext) bool { + return true + // TODO: commented out due to dashboard + // switch resource.(type) { + // // do not add mods, withs + // case *modconfig.Mod, *modconfig.DashboardWith: + // return false + + // case *modconfig.DashboardCategory, *modconfig.DashboardInput: + // // if this is a dashboard category or dashboard input, only add top level blocks + // // this is to allow nested categories/inputs to have the same name as top level categories + // // (nested inputs are added by Dashboard.InitInputs) + // return parseCtx.IsTopLevelBlock(block) + // default: + // return true + // } +} + +// special case decode logic for locals +func decodeLocalsBlock(block *hcl.Block, parseCtx *ModParseContext) ([]modconfig.HclResource, *DecodeResult) { + var resources []modconfig.HclResource + var res = newDecodeResult() + + // TODO remove and call ShouldIncludeBlock from BlocksToDecode + // https://github.com/turbot/steampipe/issues/2640 + // if opts specifies block types, then check whether this type is included + if !parseCtx.ShouldIncludeBlock(block) { + return nil, res + } + + // check name is valid + diags := validateName(block) + if diags.HasErrors() { + res.addDiags(diags) + return nil, res + } + + var locals []*modconfig.Local + locals, res = decodeLocals(block, parseCtx) + for _, local := range locals { + resources = append(resources, local) + handleModDecodeResult(local, res, block, parseCtx) + } + + return resources, res +} + +func decodeBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) { + var resource modconfig.HclResource + var res = newDecodeResult() + + // TODO remove and call ShouldIncludeBlock from BlocksToDecode + // https://github.com/turbot/steampipe/issues/2640 + // if opts specifies block types, then check whether this type is included + if !parseCtx.ShouldIncludeBlock(block) { + return nil, res + } + + // has this block already been decoded? + // (this could happen if it is a child block and has been decoded before its parent as part of second decode phase) + if resource, ok := parseCtx.GetDecodedResourceForBlock(block); ok { + return resource, res + } + + // check name is valid + diags := validateName(block) + if diags.HasErrors() { + res.addDiags(diags) + return nil, res + } + + // now do the actual decode + switch { + case helpers.StringSliceContains(schema.NodeAndEdgeProviderBlocks, block.Type): + resource, res = decodeNodeAndEdgeProvider(block, parseCtx) + case helpers.StringSliceContains(schema.QueryProviderBlocks, block.Type): + resource, res = decodeQueryProvider(block, parseCtx) + default: + switch block.Type { + case schema.BlockTypeMod: + // decodeMode has slightly different args as this code is shared with ParseModDefinition + resource, res = decodeMod(block, parseCtx.EvalCtx, parseCtx.CurrentMod) + // TODO: commented out: dashboard + // case schema.BlockTypeDashboard: + // resource, res = decodeDashboard(block, parseCtx) + // case schema.BlockTypeContainer: + // resource, res = decodeDashboardContainer(block, parseCtx) + case schema.BlockTypeVariable: + resource, res = decodeVariable(block, parseCtx) + case schema.BlockTypeBenchmark: + resource, res = decodeBenchmark(block, parseCtx) + case schema.BlockTypePipeline: + resource, res = decodePipeline(parseCtx.CurrentMod, block, parseCtx) + case schema.BlockTypeTrigger: + resource, res = decodeTrigger(parseCtx.CurrentMod, block, parseCtx) + default: + // all other blocks are treated the same: + resource, res = decodeResource(block, parseCtx) + } + } + + // Note that an interface value that holds a nil concrete value is itself non-nil. + if !helpers.IsNil(resource) { + // handle the result + // - if there are dependencies, add to run context + handleModDecodeResult(resource, res, block, parseCtx) + } + + return resource, res +} + +func decodeMod(block *hcl.Block, evalCtx *hcl.EvalContext, mod *modconfig.Mod) (*modconfig.Mod, *DecodeResult) { + res := newDecodeResult() + // decode the body + diags := decodeHclBody(block.Body, evalCtx, mod, mod) + res.handleDecodeDiags(diags) + return mod, res +} + +// generic decode function for any resource we do not have custom decode logic for +func decodeResource(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) { + res := newDecodeResult() + // get shell resource + resource, diags := resourceForBlock(block, parseCtx) + res.handleDecodeDiags(diags) + if diags.HasErrors() { + return nil, res + } + + diags = decodeHclBody(block.Body, parseCtx.EvalCtx, parseCtx, resource) + if len(diags) > 0 { + res.handleDecodeDiags(diags) + } + return resource, res +} + +// return a shell resource for the given block +func resourceForBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, hcl.Diagnostics) { + var resource modconfig.HclResource + // parseCtx already contains the current mod + mod := parseCtx.CurrentMod + blockName := parseCtx.DetermineBlockName(block) + + factoryFuncs := map[string]func(*hcl.Block, *modconfig.Mod, string) modconfig.HclResource{ + // for block type mod, just use the current mod + schema.BlockTypeMod: func(*hcl.Block, *modconfig.Mod, string) modconfig.HclResource { return mod }, + schema.BlockTypeQuery: modconfig.NewQuery, + // schema.BlockTypeControl: modconfig.NewControl, + // schema.BlockTypeBenchmark: modconfig.NewBenchmark, + // schema.BlockTypeDashboard: modconfig.NewDashboard, + // schema.BlockTypeContainer: modconfig.NewDashboardContainer, + // schema.BlockTypeChart: modconfig.NewDashboardChart, + // schema.BlockTypeCard: modconfig.NewDashboardCard, + // schema.BlockTypeFlow: modconfig.NewDashboardFlow, + // schema.BlockTypeGraph: modconfig.NewDashboardGraph, + // schema.BlockTypeHierarchy: modconfig.NewDashboardHierarchy, + // schema.BlockTypeImage: modconfig.NewDashboardImage, + // schema.BlockTypeInput: modconfig.NewDashboardInput, + // schema.BlockTypeTable: modconfig.NewDashboardTable, + // schema.BlockTypeText: modconfig.NewDashboardText, + // schema.BlockTypeNode: modconfig.NewDashboardNode, + // schema.BlockTypeEdge: modconfig.NewDashboardEdge, + // schema.BlockTypeCategory: modconfig.NewDashboardCategory, + // schema.BlockTypeWith: modconfig.NewDashboardWith, + } + + factoryFunc, ok := factoryFuncs[block.Type] + if !ok { + return nil, hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("resourceForBlock called for unsupported block type %s", block.Type), + Subject: &block.DefRange, + }, + } + } + resource = factoryFunc(block, mod, blockName) + return resource, nil +} + +func decodeLocals(block *hcl.Block, parseCtx *ModParseContext) ([]*modconfig.Local, *DecodeResult) { + res := newDecodeResult() + attrs, diags := block.Body.JustAttributes() + if len(attrs) == 0 { + res.Diags = diags + return nil, res + } + + // build list of locals + locals := make([]*modconfig.Local, 0, len(attrs)) + for name, attr := range attrs { + if !hclsyntax.ValidIdentifier(name) { + res.Diags = append(res.Diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid local value name", + Detail: badIdentifierDetail, + Subject: &attr.NameRange, + }) + continue + } + // try to evaluate expression + val, diags := attr.Expr.Value(parseCtx.EvalCtx) + // handle any resulting diags, which may specify dependencies + res.handleDecodeDiags(diags) + + // add to our list + locals = append(locals, modconfig.NewLocal(name, val, attr.Range, parseCtx.CurrentMod)) + } + return locals, res +} + +func decodeVariable(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Variable, *DecodeResult) { + res := newDecodeResult() + + var variable *modconfig.Variable + content, diags := block.Body.Content(VariableBlockSchema) + res.handleDecodeDiags(diags) + + v, diags := var_config.DecodeVariableBlock(block, content, false) + res.handleDecodeDiags(diags) + + if res.Success() { + variable = modconfig.NewVariable(v, parseCtx.CurrentMod) + } + + return variable, res + +} + +func decodeQueryProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.QueryProvider, *DecodeResult) { + res := newDecodeResult() + + // TODO [node_reuse] need raise errors for invalid properties https://github.com/turbot/steampipe/issues/2923 + + // get shell resource + resource, diags := resourceForBlock(block, parseCtx) + res.handleDecodeDiags(diags) + if diags.HasErrors() { + return nil, res + } + // do a partial decode using an empty schema - use to pull out all body content in the remain block + _, remain, diags := block.Body.PartialContent(&hcl.BodySchema{}) + res.handleDecodeDiags(diags) + if !res.Success() { + return nil, res + } + + // decode the body into 'resource' to populate all properties that can be automatically decoded + diags = decodeHclBody(remain, parseCtx.EvalCtx, parseCtx, resource) + res.handleDecodeDiags(diags) + + // decode 'with',args and params blocks + res.Merge(decodeQueryProviderBlocks(block, remain.(*hclsyntax.Body), resource, parseCtx)) + + return resource.(modconfig.QueryProvider), res +} + +func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resource modconfig.HclResource, parseCtx *ModParseContext) *DecodeResult { + var diags hcl.Diagnostics + res := newDecodeResult() + queryProvider, ok := resource.(modconfig.QueryProvider) + if !ok { + // coding error + panic(fmt.Sprintf("block type %s not convertible to a QueryProvider", block.Type)) + } + + if attr, exists := content.Attributes[schema.AttributeTypeArgs]; exists { + args, runtimeDependencies, diags := decodeArgs(attr.AsHCLAttribute(), parseCtx.EvalCtx, queryProvider) + if diags.HasErrors() { + // handle dependencies + res.handleDecodeDiags(diags) + } else { + queryProvider.SetArgs(args) + queryProvider.AddRuntimeDependencies(runtimeDependencies) + } + } + + var params []*modconfig.ParamDef + for _, b := range content.Blocks { + block = b.AsHCLBlock() + switch block.Type { + case schema.BlockTypeParam: + paramDef, runtimeDependencies, moreDiags := decodeParam(block, parseCtx) + if !moreDiags.HasErrors() { + params = append(params, paramDef) + queryProvider.AddRuntimeDependencies(runtimeDependencies) + // add and references contained in the param block to the control refs + moreDiags = AddReferences(resource, block, parseCtx) + } + diags = append(diags, moreDiags...) + } + } + + queryProvider.SetParams(params) + res.handleDecodeDiags(diags) + return res +} + +func decodeNodeAndEdgeProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) { + res := newDecodeResult() + + // TODO [node_reuse] need raise errors for invalid properties https://github.com/turbot/steampipe/issues/2923 + + // get shell resource + resource, diags := resourceForBlock(block, parseCtx) + res.handleDecodeDiags(diags) + if diags.HasErrors() { + return nil, res + } + + nodeAndEdgeProvider, ok := resource.(modconfig.NodeAndEdgeProvider) + if !ok { + // coding error + panic(fmt.Sprintf("block type %s not convertible to a NodeAndEdgeProvider", block.Type)) + } + + // do a partial decode using an empty schema - use to pull out all body content in the remain block + _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) + body := r.(*hclsyntax.Body) + res.handleDecodeDiags(diags) + if !res.Success() { + return nil, res + } + + // decode the body into 'resource' to populate all properties that can be automatically decoded + diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, resource) + // handle any resulting diags, which may specify dependencies + res.handleDecodeDiags(diags) + + // decode sql args and params + res.Merge(decodeQueryProviderBlocks(block, body, resource, parseCtx)) + + // now decode child blocks + if len(body.Blocks) > 0 { + blocksRes := decodeNodeAndEdgeProviderBlocks(body, nodeAndEdgeProvider, parseCtx) + res.Merge(blocksRes) + } + + return resource, res +} + +func decodeNodeAndEdgeProviderBlocks(content *hclsyntax.Body, nodeAndEdgeProvider modconfig.NodeAndEdgeProvider, parseCtx *ModParseContext) *DecodeResult { + var res = newDecodeResult() + + for _, b := range content.Blocks { + block := b.AsHCLBlock() + switch block.Type { + // TODO: commented out: dashboard + // case schema.BlockTypeCategory: + // // decode block + // category, blockRes := decodeBlock(block, parseCtx) + // res.Merge(blockRes) + // if !blockRes.Success() { + // continue + // } + + // // add the category to the nodeAndEdgeProvider + // res.addDiags(nodeAndEdgeProvider.AddCategory(category.(*modconfig.DashboardCategory))) + + // DO NOT add the category to the mod + + case schema.BlockTypeNode, schema.BlockTypeEdge: + child, childRes := decodeQueryProvider(block, parseCtx) + + // TACTICAL if child has any runtime dependencies, claim them + // this is to ensure if this resource is used as base, we can be correctly identified + // as the publisher of the runtime dependencies + for _, r := range child.GetRuntimeDependencies() { + r.Provider = nodeAndEdgeProvider + } + + // populate metadata, set references and call OnDecoded + handleModDecodeResult(child, childRes, block, parseCtx) + res.Merge(childRes) + if res.Success() { + moreDiags := nodeAndEdgeProvider.AddChild(child) + res.addDiags(moreDiags) + } + // TODO: commented out: dashboard + // case schema.BlockTypeWith: + // with, withRes := decodeBlock(block, parseCtx) + // res.Merge(withRes) + // if res.Success() { + // moreDiags := nodeAndEdgeProvider.AddWith(with.(*modconfig.DashboardWith)) + // res.addDiags(moreDiags) + // } + } + + } + + return res +} + +// TODO: commented out: dashboard +// func decodeDashboard(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Dashboard, *DecodeResult) { +// res := newDecodeResult() +// dashboard := modconfig.NewDashboard(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.Dashboard) + +// // do a partial decode using an empty schema - use to pull out all body content in the remain block +// _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) +// body := r.(*hclsyntax.Body) +// res.handleDecodeDiags(diags) + +// // decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded +// diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, dashboard) +// // handle any resulting diags, which may specify dependencies +// res.handleDecodeDiags(diags) + +// if dashboard.Base != nil && len(dashboard.Base.ChildNames) > 0 { +// supportedChildren := []string{schema.BlockTypeContainer, schema.BlockTypeChart, schema.BlockTypeControl, schema.BlockTypeCard, schema.BlockTypeFlow, schema.BlockTypeGraph, schema.BlockTypeHierarchy, schema.BlockTypeImage, schema.BlockTypeInput, schema.BlockTypeTable, schema.BlockTypeText} +// // TACTICAL: we should be passing in the block for the Base resource - but this is only used for diags +// // and we do not expect to get any (as this function has already succeeded when the base was originally parsed) +// children, _ := resolveChildrenFromNames(dashboard.Base.ChildNames, block, supportedChildren, parseCtx) +// dashboard.Base.SetChildren(children) +// } +// if !res.Success() { +// return dashboard, res +// } + +// // now decode child blocks +// if len(body.Blocks) > 0 { +// blocksRes := decodeDashboardBlocks(body, dashboard, parseCtx) +// res.Merge(blocksRes) +// } + +// return dashboard, res +// } + +// func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboard, parseCtx *ModParseContext) *DecodeResult { +// var res = newDecodeResult() +// // set dashboard as parent on the run context - this is used when generating names for anonymous blocks +// parseCtx.PushParent(dashboard) +// defer func() { +// parseCtx.PopParent() +// }() + +// for _, b := range content.Blocks { +// block := b.AsHCLBlock() + +// // decode block +// resource, blockRes := decodeBlock(block, parseCtx) +// res.Merge(blockRes) +// if !blockRes.Success() { +// continue +// } + +// // we expect either inputs or child report nodes +// // add the resource to the mod +// res.addDiags(addResourceToMod(resource, block, parseCtx)) +// // add to the dashboard children +// // (we expect this cast to always succeed) +// if child, ok := resource.(modconfig.ModTreeItem); ok { +// dashboard.AddChild(child) +// } + +// } + +// moreDiags := dashboard.InitInputs() +// res.addDiags(moreDiags) + +// return res +// } + +// func decodeDashboardContainer(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.DashboardContainer, *DecodeResult) { +// res := newDecodeResult() +// container := modconfig.NewDashboardContainer(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.DashboardContainer) + +// // do a partial decode using an empty schema - use to pull out all body content in the remain block +// _, r, diags := block.Body.PartialContent(&hcl.BodySchema{}) +// body := r.(*hclsyntax.Body) +// res.handleDecodeDiags(diags) +// if !res.Success() { +// return nil, res +// } + +// // decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded +// diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, container) +// // handle any resulting diags, which may specify dependencies +// res.handleDecodeDiags(diags) + +// // now decode child blocks +// if len(body.Blocks) > 0 { +// blocksRes := decodeDashboardContainerBlocks(body, container, parseCtx) +// res.Merge(blocksRes) +// } + +// return container, res +// } + +// func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer *modconfig.DashboardContainer, parseCtx *ModParseContext) *DecodeResult { +// var res = newDecodeResult() + +// // set container as parent on the run context - this is used when generating names for anonymous blocks +// parseCtx.PushParent(dashboardContainer) +// defer func() { +// parseCtx.PopParent() +// }() + +// for _, b := range content.Blocks { +// block := b.AsHCLBlock() +// resource, blockRes := decodeBlock(block, parseCtx) +// res.Merge(blockRes) +// if !blockRes.Success() { +// continue +// } + +// // special handling for inputs +// if b.Type == schema.BlockTypeInput { +// input := resource.(*modconfig.DashboardInput) +// dashboardContainer.Inputs = append(dashboardContainer.Inputs, input) +// dashboardContainer.AddChild(input) +// // the input will be added to the mod by the parent dashboard + +// } else { +// // for all other children, add to mod and children +// res.addDiags(addResourceToMod(resource, block, parseCtx)) +// if child, ok := resource.(modconfig.ModTreeItem); ok { +// dashboardContainer.AddChild(child) +// } +// } +// } + +// return res +// } + +func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Benchmark, *DecodeResult) { + res := newDecodeResult() + benchmark := modconfig.NewBenchmark(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.Benchmark) + content, diags := block.Body.Content(BenchmarkBlockSchema) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "children", &benchmark.ChildNames, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "description", &benchmark.Description, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "documentation", &benchmark.Documentation, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "tags", &benchmark.Tags, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "title", &benchmark.Title, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "type", &benchmark.Type, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + diags = decodeProperty(content, "display", &benchmark.Display, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + + // now add children + if res.Success() { + supportedChildren := []string{schema.BlockTypeBenchmark, schema.BlockTypeControl} + children, diags := resolveChildrenFromNames(benchmark.ChildNames.StringList(), block, supportedChildren, parseCtx) + res.handleDecodeDiags(diags) + + // now set children and child name strings + benchmark.SetChildren(children) + benchmark.ChildNameStrings = getChildNameStringsFromModTreeItem(children) + } + + diags = decodeProperty(content, "base", &benchmark.Base, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + if benchmark.Base != nil && len(benchmark.Base.ChildNames) > 0 { + supportedChildren := []string{schema.BlockTypeBenchmark, schema.BlockTypeControl} + // TACTICAL: we should be passing in the block for the Base resource - but this is only used for diags + // and we do not expect to get any (as this function has already succeeded when the base was originally parsed) + children, _ := resolveChildrenFromNames(benchmark.Base.ChildNameStrings, block, supportedChildren, parseCtx) + benchmark.Base.SetChildren(children) + } + diags = decodeProperty(content, "width", &benchmark.Width, parseCtx.EvalCtx) + res.handleDecodeDiags(diags) + return benchmark, res +} + +func decodeProperty(content *hcl.BodyContent, property string, dest interface{}, evalCtx *hcl.EvalContext) hcl.Diagnostics { + var diags hcl.Diagnostics + if attr, ok := content.Attributes[property]; ok { + diags = gohcl.DecodeExpression(attr.Expr, evalCtx, dest) + } + return diags +} + +// handleModDecodeResult +// if decode was successful: +// - generate and set resource metadata +// - add resource to ModParseContext (which adds it to the mod)handleModDecodeResult +func handleModDecodeResult(resource modconfig.HclResource, res *DecodeResult, block *hcl.Block, parseCtx *ModParseContext) { + if !res.Success() { + if len(res.Depends) > 0 { + moreDiags := parseCtx.AddDependencies(block, resource.GetUnqualifiedName(), res.Depends) + res.addDiags(moreDiags) + } + return + } + // set whether this is a top level resource + resource.SetTopLevel(parseCtx.IsTopLevelBlock(block)) + + // call post decode hook + // NOTE: must do this BEFORE adding resource to run context to ensure we respect the base property + moreDiags := resource.OnDecoded(block, parseCtx) + res.addDiags(moreDiags) + + // add references + moreDiags = AddReferences(resource, block, parseCtx) + res.addDiags(moreDiags) + + // validate the resource + moreDiags = validateResource(resource) + res.addDiags(moreDiags) + // if we failed validation, return + if !res.Success() { + return + } + + // if resource is NOT anonymous, and this is a TOP LEVEL BLOCK, add into the run context + // NOTE: we can only reference resources defined in a top level block + if !resourceIsAnonymous(resource) && resource.IsTopLevel() { + moreDiags = parseCtx.AddResource(resource) + res.addDiags(moreDiags) + } + + // if resource supports metadata, save it + if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok { + body := block.Body.(*hclsyntax.Body) + moreDiags = addResourceMetadata(resourceWithMetadata, body.SrcRange, parseCtx) + res.addDiags(moreDiags) + } +} + +func resourceIsAnonymous(resource modconfig.HclResource) bool { + // (if a resource anonymous it must support ResourceWithMetadata) + resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata) + anonymousResource := ok && resourceWithMetadata.IsAnonymous() + return anonymousResource +} + +func addResourceMetadata(resourceWithMetadata modconfig.ResourceWithMetadata, srcRange hcl.Range, parseCtx *ModParseContext) hcl.Diagnostics { + metadata, err := GetMetadataForParsedResource(resourceWithMetadata.Name(), srcRange, parseCtx.FileData, parseCtx.CurrentMod) + if err != nil { + return hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: err.Error(), + Subject: &srcRange, + }} + } + // set on resource + resourceWithMetadata.SetMetadata(metadata) + return nil +} + +func validateName(block *hcl.Block) hcl.Diagnostics { + if len(block.Labels) == 0 { + return nil + } + + if !hclsyntax.ValidIdentifier(block.Labels[0]) { + return hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }} + } + return nil +} + +// Validate all blocks and attributes are supported +// We use partial decoding so that we can automatically decode as many properties as possible +// and only manually decode properties requiring special logic. +// The problem is the partial decode does not return errors for invalid attributes/blocks, so we must implement our own +func validateHcl(blockType string, body *hclsyntax.Body, schema *hcl.BodySchema) hcl.Diagnostics { + var diags hcl.Diagnostics + + // identify any blocks specified by hcl tags + var supportedBlocks = make(map[string]struct{}) + var supportedAttributes = make(map[string]struct{}) + for _, b := range schema.Blocks { + supportedBlocks[b.Type] = struct{}{} + } + for _, b := range schema.Attributes { + supportedAttributes[b.Name] = struct{}{} + } + + // now check for invalid blocks + for _, block := range body.Blocks { + if _, ok := supportedBlocks[block.Type]; !ok { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf(`Unsupported block type: Blocks of type '%s' are not expected here.`, block.Type), + Subject: &block.TypeRange, + }) + } + } + for _, attribute := range body.Attributes { + if _, ok := supportedAttributes[attribute.Name]; !ok { + // special case code for deprecated properties + subject := attribute.Range() + if isDeprecated(attribute, blockType) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: fmt.Sprintf(`Deprecated attribute: '%s' is deprecated for '%s' blocks and will be ignored.`, attribute.Name, blockType), + Subject: &subject, + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf(`Unsupported attribute: '%s' not expected here.`, attribute.Name), + Subject: &subject, + }) + } + } + } + + return diags +} + +func isDeprecated(attribute *hclsyntax.Attribute, blockType string) bool { + switch attribute.Name { + case "search_path", "search_path_prefix": + return blockType == schema.BlockTypeQuery || blockType == schema.BlockTypeControl + default: + return false + } +} diff --git a/parse_v/decode_args.go b/parse_v/decode_args.go new file mode 100644 index 00000000..a315b473 --- /dev/null +++ b/parse_v/decode_args.go @@ -0,0 +1,356 @@ +package parse_v + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/schema" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" +) + +func decodeArgs(attr *hcl.Attribute, evalCtx *hcl.EvalContext, resource modconfig.QueryProvider) (*modconfig.QueryArgs, []*modconfig.RuntimeDependency, hcl.Diagnostics) { + var runtimeDependencies []*modconfig.RuntimeDependency + var args = modconfig.NewQueryArgs() + var diags hcl.Diagnostics + + v, valDiags := attr.Expr.Value(evalCtx) + ty := v.Type() + // determine which diags are runtime dependencies (which we allow) and which are not + if valDiags.HasErrors() { + for _, diag := range diags { + dependency := diagsToDependency(diag) + if dependency == nil || !dependency.IsRuntimeDependency() { + diags = append(diags, diag) + } + } + } + // now diags contains all diags which are NOT runtime dependencies + if diags.HasErrors() { + return nil, nil, diags + } + + var err error + + switch { + case ty.IsObjectType(): + var argMap map[string]any + argMap, runtimeDependencies, err = ctyObjectToArgMap(attr, v, evalCtx) + if err == nil { + err = args.SetArgMap(argMap) + } + case ty.IsTupleType(): + var argList []any + argList, runtimeDependencies, err = ctyTupleToArgArray(attr, v) + if err == nil { + err = args.SetArgList(argList) + } + default: + err = fmt.Errorf("'params' property must be either a map or an array") + } + + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("%s has invalid parameter config", resource.Name()), + Detail: err.Error(), + Subject: &attr.Range, + }) + } + return args, runtimeDependencies, diags +} + +func ctyTupleToArgArray(attr *hcl.Attribute, val cty.Value) ([]any, []*modconfig.RuntimeDependency, error) { + // convert the attribute to a slice + values := val.AsValueSlice() + + // build output array + res := make([]any, len(values)) + var runtimeDependencies []*modconfig.RuntimeDependency + + for idx, v := range values { + // if the value is unknown, this is a runtime dependency + if !v.IsKnown() { + runtimeDependency, err := identifyRuntimeDependenciesFromArray(attr, idx, schema.AttributeTypeArgs) + if err != nil { + return nil, nil, err + } + + runtimeDependencies = append(runtimeDependencies, runtimeDependency) + } else { + // decode the value into a go type + val, err := hclhelpers.CtyToGo(v) + if err != nil { + err := fmt.Errorf("invalid value provided for arg #%d: %v", idx, err) + return nil, nil, err + } + + res[idx] = val + } + } + return res, runtimeDependencies, nil +} + +func ctyObjectToArgMap(attr *hcl.Attribute, val cty.Value, evalCtx *hcl.EvalContext) (map[string]any, []*modconfig.RuntimeDependency, error) { + res := make(map[string]any) + var runtimeDependencies []*modconfig.RuntimeDependency + it := val.ElementIterator() + for it.Next() { + k, v := it.Element() + + // decode key + var key string + if err := gocty.FromCtyValue(k, &key); err != nil { + return nil, nil, err + } + + // if the value is unknown, this is a runtime dependency + if !v.IsKnown() { + runtimeDependency, err := identifyRuntimeDependenciesFromObject(attr, key, schema.AttributeTypeArgs, evalCtx) + if err != nil { + return nil, nil, err + } + runtimeDependencies = append(runtimeDependencies, runtimeDependency) + } else if getWrappedUnknownVal(v) { + runtimeDependency, err := identifyRuntimeDependenciesFromObject(attr, key, schema.AttributeTypeArgs, evalCtx) + if err != nil { + return nil, nil, err + } + runtimeDependencies = append(runtimeDependencies, runtimeDependency) + } else { + // decode the value into a go type + val, err := hclhelpers.CtyToGo(v) + if err != nil { + err := fmt.Errorf("invalid value provided for param '%s': %v", key, err) + return nil, nil, err + } + res[key] = val + } + } + + return res, runtimeDependencies, nil +} + +// TACTICAL - is the cty value an array with a single unknown value +func getWrappedUnknownVal(v cty.Value) bool { + ty := v.Type() + + switch { + + case ty.IsTupleType(): + values := v.AsValueSlice() + if len(values) == 1 && !values[0].IsKnown() { + return true + } + } + return false +} + +func identifyRuntimeDependenciesFromObject(attr *hcl.Attribute, targetProperty, parentProperty string, evalCtx *hcl.EvalContext) (*modconfig.RuntimeDependency, error) { + // find the expression for this key + argsExpr, ok := attr.Expr.(*hclsyntax.ObjectConsExpr) + if !ok { + return nil, fmt.Errorf("could not extract runtime dependency for arg %s", targetProperty) + } + for _, item := range argsExpr.Items { + nameCty, valDiags := item.KeyExpr.Value(evalCtx) + if valDiags.HasErrors() { + return nil, fmt.Errorf("could not extract runtime dependency for arg %s", targetProperty) + } + var name string + if err := gocty.FromCtyValue(nameCty, &name); err != nil { + return nil, err + } + if name == targetProperty { + dep, err := getRuntimeDepFromExpression(item.ValueExpr, targetProperty, parentProperty) + if err != nil { + return nil, err + } + + return dep, nil + } + } + return nil, fmt.Errorf("could not extract runtime dependency for arg %s - not found in attribute map", targetProperty) +} + +func getRuntimeDepFromExpression(expr hcl.Expression, targetProperty, parentProperty string) (*modconfig.RuntimeDependency, error) { + isArray, propertyPath, err := propertyPathFromExpression(expr) + if err != nil { + return nil, err + } + + if propertyPath.ItemType == schema.BlockTypeInput { + // tactical: validate input dependency + if err := validateInputRuntimeDependency(propertyPath); err != nil { + return nil, err + } + } + ret := &modconfig.RuntimeDependency{ + PropertyPath: propertyPath, + ParentPropertyName: parentProperty, + TargetPropertyName: &targetProperty, + IsArray: isArray, + } + return ret, nil +} + +func propertyPathFromExpression(expr hcl.Expression) (bool, *modconfig.ParsedPropertyPath, error) { + var propertyPathStr string + var isArray bool + +dep_loop: + for { + switch e := expr.(type) { + case *hclsyntax.ScopeTraversalExpr: + propertyPathStr = hclhelpers.TraversalAsString(e.Traversal) + break dep_loop + case *hclsyntax.SplatExpr: + root := hclhelpers.TraversalAsString(e.Source.(*hclsyntax.ScopeTraversalExpr).Traversal) + var suffix string + // if there is a property path, add it + if each, ok := e.Each.(*hclsyntax.RelativeTraversalExpr); ok { + suffix = fmt.Sprintf(".%s", hclhelpers.TraversalAsString(each.Traversal)) + } + propertyPathStr = fmt.Sprintf("%s.*%s", root, suffix) + break dep_loop + case *hclsyntax.TupleConsExpr: + // TACTICAL + // handle the case where an arg value is given as a runtime dependency inside an array, for example + // arns = [input.arn] + // this is a common pattern where a runtime depdency gives a scalar value, but an array is needed for the arg + // NOTE: this code only supports a SINGLE item in the array + if len(e.Exprs) != 1 { + return false, nil, fmt.Errorf("unsupported runtime dependency expression - only a single runtime depdency item may be wrapped in an array") + } + isArray = true + expr = e.Exprs[0] + // fall through to rerun loop with updated expr + default: + // unhandled expression type + return false, nil, fmt.Errorf("unexpected runtime dependency expression type") + } + } + + propertyPath, err := modconfig.ParseResourcePropertyPath(propertyPathStr) + if err != nil { + return false, nil, err + } + return isArray, propertyPath, nil +} + +func identifyRuntimeDependenciesFromArray(attr *hcl.Attribute, idx int, parentProperty string) (*modconfig.RuntimeDependency, error) { + // find the expression for this key + argsExpr, ok := attr.Expr.(*hclsyntax.TupleConsExpr) + if !ok { + return nil, fmt.Errorf("could not extract runtime dependency for arg #%d", idx) + } + for i, item := range argsExpr.Exprs { + if i == idx { + isArray, propertyPath, err := propertyPathFromExpression(item) + if err != nil { + return nil, err + } + // tactical: validate input dependency + if propertyPath.ItemType == schema.BlockTypeInput { + if err := validateInputRuntimeDependency(propertyPath); err != nil { + return nil, err + } + } + ret := &modconfig.RuntimeDependency{ + PropertyPath: propertyPath, + ParentPropertyName: parentProperty, + TargetPropertyIndex: &idx, + IsArray: isArray, + } + + return ret, nil + } + } + return nil, fmt.Errorf("could not extract runtime dependency for arg %d - not found in attribute list", idx) +} + +// tactical - if runtime dependency is an input, validate it is of correct format +// TODO - include this with the main runtime dependency validation, when it is rewritten https://github.com/turbot/steampipe/issues/2925 +func validateInputRuntimeDependency(propertyPath *modconfig.ParsedPropertyPath) error { + // input references must be of form self.input..value + + // TODO: commented out: dashboard + // if propertyPath.Scope != modconfig.RuntimeDependencyDashboardScope { + // return fmt.Errorf("could not resolve runtime dependency resource %s", propertyPath.Original) + // } + return nil +} + +func decodeParam(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.ParamDef, []*modconfig.RuntimeDependency, hcl.Diagnostics) { + def := modconfig.NewParamDef(block) + var runtimeDependencies []*modconfig.RuntimeDependency + content, diags := block.Body.Content(ParamDefBlockSchema) + + if attr, exists := content.Attributes["description"]; exists { + moreDiags := gohcl.DecodeExpression(attr.Expr, parseCtx.EvalCtx, &def.Description) + diags = append(diags, moreDiags...) + } + if attr, exists := content.Attributes["default"]; exists { + defaultValue, deps, moreDiags := decodeParamDefault(attr, parseCtx, def.UnqualifiedName) + diags = append(diags, moreDiags...) + if !helpers.IsNil(defaultValue) { + err := def.SetDefault(defaultValue) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "invalid default config for " + def.UnqualifiedName, + Detail: err.Error(), + Subject: &attr.Range, + }) + return nil, nil, diags + } + } + runtimeDependencies = deps + } + return def, runtimeDependencies, diags +} + +func decodeParamDefault(attr *hcl.Attribute, parseCtx *ModParseContext, paramName string) (any, []*modconfig.RuntimeDependency, hcl.Diagnostics) { + v, diags := attr.Expr.Value(parseCtx.EvalCtx) + + if v.IsKnown() { + // convert the raw default into a string representation + val, err := hclhelpers.CtyToGo(v) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("%s has invalid default config", paramName), + Detail: err.Error(), + Subject: &attr.Range, + }) + return nil, nil, diags + } + return val, nil, nil + } + + // so value not known - is there a runtime dependency? + + // check for a runtime dependency + runtimeDependency, err := getRuntimeDepFromExpression(attr.Expr, "default", paramName) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("%s has invalid parameter default config", paramName), + Detail: err.Error(), + Subject: &attr.Range, + }) + return nil, nil, diags + } + if runtimeDependency == nil { + // return the original diags + return nil, nil, diags + } + + // so we have a runtime dependency + return nil, []*modconfig.RuntimeDependency{runtimeDependency}, nil +} diff --git a/parse_v/decode_body.go b/parse_v/decode_body.go new file mode 100644 index 00000000..61b76d14 --- /dev/null +++ b/parse_v/decode_body.go @@ -0,0 +1,301 @@ +package parse_v + +import ( + "reflect" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/schema" +) + +func decodeHclBody(body hcl.Body, evalCtx *hcl.EvalContext, resourceProvider modconfig.ResourceMapsProvider, resource modconfig.HclResource) (diags hcl.Diagnostics) { + defer func() { + if r := recover(); r != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "unexpected error in decodeHclBody", + Detail: helpers.ToError(r).Error()}) + } + }() + + nestedStructs, moreDiags := getNestedStructValsRecursive(resource) + diags = append(diags, moreDiags...) + // get the schema for this resource + schema := getResourceSchema(resource, nestedStructs) + // handle invalid block types + moreDiags = validateHcl(resource.BlockType(), body.(*hclsyntax.Body), schema) + diags = append(diags, moreDiags...) + + moreDiags = decodeHclBodyIntoStruct(body, evalCtx, resourceProvider, resource) + diags = append(diags, moreDiags...) + + for _, nestedStruct := range nestedStructs { + moreDiags := decodeHclBodyIntoStruct(body, evalCtx, resourceProvider, nestedStruct) + diags = append(diags, moreDiags...) + } + + return diags +} + +func decodeHclBodyIntoStruct(body hcl.Body, evalCtx *hcl.EvalContext, resourceProvider modconfig.ResourceMapsProvider, resource any) hcl.Diagnostics { + var diags hcl.Diagnostics + // call decodeHclBodyIntoStruct to do actual decode + moreDiags := gohcl.DecodeBody(body, evalCtx, resource) + diags = append(diags, moreDiags...) + + // resolve any resource references using the resource map, rather than relying on the EvalCtx + // (which does not work with nested struct vals) + moreDiags = resolveReferences(body, resourceProvider, resource) + diags = append(diags, moreDiags...) + return diags +} + +// build the hcl schema for this resource +func getResourceSchema(resource modconfig.HclResource, nestedStructs []any) *hcl.BodySchema { + t := reflect.TypeOf(helpers.DereferencePointer(resource)) + typeName := t.Name() + + if cachedSchema, ok := resourceSchemaCache[typeName]; ok { + return cachedSchema + } + var res = &hcl.BodySchema{} + + // ensure we cache before returning + defer func() { + resourceSchemaCache[typeName] = res + }() + + var schemas []*hcl.BodySchema + + // build schema for top level object + schemas = append(schemas, getSchemaForStruct(t)) + + // now get schemas for any nested structs (using cache) + for _, nestedStruct := range nestedStructs { + t := reflect.TypeOf(helpers.DereferencePointer(nestedStruct)) + typeName := t.Name() + + // is this cached? + nestedStructSchema, schemaCached := resourceSchemaCache[typeName] + if !schemaCached { + nestedStructSchema = getSchemaForStruct(t) + resourceSchemaCache[typeName] = nestedStructSchema + } + + // add to our list of schemas + schemas = append(schemas, nestedStructSchema) + } + + // TODO handle duplicates and required/optional + // now merge the schemas + for _, s := range schemas { + res.Blocks = append(res.Blocks, s.Blocks...) + res.Attributes = append(res.Attributes, s.Attributes...) + } + + // special cases for manually parsed attributes and blocks + switch resource.BlockType() { + case schema.BlockTypeMod: + res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: schema.BlockTypeRequire}) + case schema.BlockTypeDashboard, schema.BlockTypeContainer: + res.Blocks = append(res.Blocks, + hcl.BlockHeaderSchema{Type: schema.BlockTypeControl}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeBenchmark}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeCard}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeChart}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeContainer}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeFlow}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeGraph}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeHierarchy}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeImage}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeInput}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeTable}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeText}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeWith}, + ) + case schema.BlockTypeQuery: + // remove `Query` from attributes + var querySchema = &hcl.BodySchema{} + for _, a := range res.Attributes { + if a.Name != schema.AttributeQuery { + querySchema.Attributes = append(querySchema.Attributes, a) + } + } + res = querySchema + } + + if _, ok := resource.(modconfig.QueryProvider); ok { + res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: schema.BlockTypeParam}) + // if this is NOT query, add args + if resource.BlockType() != schema.BlockTypeQuery { + res.Attributes = append(res.Attributes, hcl.AttributeSchema{Name: schema.AttributeTypeArgs}) + } + } + if _, ok := resource.(modconfig.NodeAndEdgeProvider); ok { + res.Blocks = append(res.Blocks, + hcl.BlockHeaderSchema{Type: schema.BlockTypeCategory}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeNode}, + hcl.BlockHeaderSchema{Type: schema.BlockTypeEdge}) + } + // TODO: dashbaord related (WithProvider) + // if _, ok := resource.(modconfig.WithProvider); ok { + // res.Blocks = append(res.Blocks, hcl.BlockHeaderSchema{Type: schema.BlockTypeWith}) + // } + return res +} + +func getSchemaForStruct(t reflect.Type) *hcl.BodySchema { + var schema = &hcl.BodySchema{} + // get all hcl tags + for i := 0; i < t.NumField(); i++ { + tag := t.FieldByIndex([]int{i}).Tag.Get("hcl") + if tag == "" { + continue + } + if idx := strings.LastIndex(tag, ",block"); idx != -1 { + blockName := tag[:idx] + schema.Blocks = append(schema.Blocks, hcl.BlockHeaderSchema{Type: blockName}) + } else { + attributeName := strings.Split(tag, ",")[0] + if attributeName != "" { + schema.Attributes = append(schema.Attributes, hcl.AttributeSchema{Name: attributeName}) + } + } + } + return schema +} + +// rather than relying on the evaluation context to resolve resource references +// (which has the issue that when deserializing from cty we do not receive all base struct values) +// instead resolve the reference by parsing the resource name and finding the resource in the ResourceMap +// and use this resource to set the target property +func resolveReferences(body hcl.Body, resourceMapsProvider modconfig.ResourceMapsProvider, val any) (diags hcl.Diagnostics) { + defer func() { + if r := recover(); r != nil { + if r := recover(); r != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "unexpected error in resolveReferences", + Detail: helpers.ToError(r).Error()}) + } + } + }() + attributes := body.(*hclsyntax.Body).Attributes + rv := reflect.ValueOf(val) + for rv.Type().Kind() == reflect.Pointer { + rv = rv.Elem() + } + ty := rv.Type() + if ty.Kind() != reflect.Struct { + return + } + + ct := ty.NumField() + for i := 0; i < ct; i++ { + field := ty.Field(i) + fieldVal := rv.Field(i) + // get hcl attribute tag (if any) tag + hclAttribute := getHclAttributeTag(field) + if hclAttribute == "" { + continue + } + if fieldVal.Type().Kind() == reflect.Pointer && !fieldVal.IsNil() { + fieldVal = fieldVal.Elem() + } + if fieldVal.Kind() == reflect.Struct { + v := fieldVal.Addr().Interface() + if _, ok := v.(modconfig.HclResource); ok { + if hclVal, ok := attributes[hclAttribute]; ok { + if scopeTraversal, ok := hclVal.Expr.(*hclsyntax.ScopeTraversalExpr); ok { + path := hclhelpers.TraversalAsString(scopeTraversal.Traversal) + if parsedName, err := modconfig.ParseResourceName(path); err == nil { + if r, ok := resourceMapsProvider.GetResource(parsedName); ok { + f := rv.FieldByName(field.Name) + if f.IsValid() && f.CanSet() { + targetVal := reflect.ValueOf(r) + f.Set(targetVal) + } + } + } + } + } + } + } + } + return nil +} + +func getHclAttributeTag(field reflect.StructField) string { + tag := field.Tag.Get("hcl") + if tag == "" { + return "" + } + + comma := strings.Index(tag, ",") + var name, kind string + if comma != -1 { + name = tag[:comma] + kind = tag[comma+1:] + } else { + name = tag + kind = "attr" + } + + switch kind { + case "attr": + return name + default: + return "" + } +} + +func getNestedStructValsRecursive(val any) ([]any, hcl.Diagnostics) { + nested, diags := getNestedStructVals(val) + res := nested + + for _, n := range nested { + nestedVals, moreDiags := getNestedStructValsRecursive(n) + diags = append(diags, moreDiags...) + res = append(res, nestedVals...) + } + return res, diags + +} + +// GetNestedStructVals return a slice of any nested structs within val +func getNestedStructVals(val any) (_ []any, diags hcl.Diagnostics) { + defer func() { + if r := recover(); r != nil { + if r := recover(); r != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "unexpected error in resolveReferences", + Detail: helpers.ToError(r).Error()}) + } + } + }() + + rv := reflect.ValueOf(val) + for rv.Type().Kind() == reflect.Pointer { + rv = rv.Elem() + } + ty := rv.Type() + if ty.Kind() != reflect.Struct { + return nil, nil + } + ct := ty.NumField() + var res []any + for i := 0; i < ct; i++ { + field := ty.Field(i) + fieldVal := rv.Field(i) + if field.Anonymous && fieldVal.Kind() == reflect.Struct { + res = append(res, fieldVal.Addr().Interface()) + } + } + return res, nil +} diff --git a/parse_v/decode_children.go b/parse_v/decode_children.go new file mode 100644 index 00000000..f8d8779b --- /dev/null +++ b/parse_v/decode_children.go @@ -0,0 +1,94 @@ +package parse_v + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/modconfig" +) + +func resolveChildrenFromNames(childNames []string, block *hcl.Block, supportedChildren []string, parseCtx *ModParseContext) ([]modconfig.ModTreeItem, hcl.Diagnostics) { + var diags hcl.Diagnostics + diags = checkForDuplicateChildren(childNames, block) + if diags.HasErrors() { + return nil, diags + } + + // find the children in the eval context and populate control children + children := make([]modconfig.ModTreeItem, len(childNames)) + + for i, childName := range childNames { + parsedName, err := modconfig.ParseResourceName(childName) + if err != nil || !helpers.StringSliceContains(supportedChildren, parsedName.ItemType) { + diags = append(diags, childErrorDiagnostic(childName, block)) + continue + } + + // now get the resource from the parent mod + // find the mod which owns this resource - it may be either the current mod, or one of it's direct dependencies + var mod = parseCtx.GetMod(parsedName.Mod) + if mod == nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Could not resolve mod for child %s", childName), + Subject: &block.TypeRange, + }) + break + } + + resource, found := mod.GetResource(parsedName) + // ensure this item is a mod tree item + child, ok := resource.(modconfig.ModTreeItem) + if !found || !ok { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Could not resolve child %s", childName), + Subject: &block.TypeRange, + }) + continue + } + + children[i] = child + } + if diags.HasErrors() { + return nil, diags + } + + return children, nil +} + +func checkForDuplicateChildren(names []string, block *hcl.Block) hcl.Diagnostics { + var diags hcl.Diagnostics + // validate each child name appears only once + nameMap := make(map[string]int) + for _, n := range names { + nameCount := nameMap[n] + // raise an error if this name appears more than once (but only raise 1 error per name) + if nameCount == 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("'%s.%s' has duplicate child name '%s'", block.Type, block.Labels[0], n), + Subject: &block.DefRange}) + } + nameMap[n] = nameCount + 1 + } + + return diags +} + +func childErrorDiagnostic(childName string, block *hcl.Block) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Invalid child %s", childName), + Subject: &block.TypeRange, + } +} + +func getChildNameStringsFromModTreeItem(children []modconfig.ModTreeItem) []string { + res := make([]string, len(children)) + for i, n := range children { + res[i] = n.Name() + } + return res +} diff --git a/parse_v/decode_options.go b/parse_v/decode_options.go new file mode 100644 index 00000000..8f536b66 --- /dev/null +++ b/parse_v/decode_options.go @@ -0,0 +1,59 @@ +package parse_v + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/turbot/pipe-fittings/options" +) + +// DecodeOptions decodes an options block +func DecodeOptions(block *hcl.Block, overrides ...BlockMappingOverride) (options.Options, hcl.Diagnostics) { + var diags hcl.Diagnostics + mapping := defaultOptionsBlockMapping() + for _, applyOverride := range overrides { + applyOverride(mapping) + } + + destination, ok := mapping[block.Labels[0]] + if !ok { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Unexpected options type '%s'", block.Labels[0]), + Subject: &block.DefRange, + }) + return nil, diags + } + + diags = gohcl.DecodeBody(block.Body, nil, destination) + if diags.HasErrors() { + return nil, diags + } + + return destination, nil +} + +type OptionsBlockMapping = map[string]options.Options + +func defaultOptionsBlockMapping() OptionsBlockMapping { + mapping := OptionsBlockMapping{ + options.ConnectionBlock: &options.Connection{}, + options.DatabaseBlock: &options.Database{}, + options.TerminalBlock: &options.Terminal{}, + options.GeneralBlock: &options.General{}, + options.QueryBlock: &options.Query{}, + options.CheckBlock: &options.Check{}, + options.DashboardBlock: &options.GlobalDashboard{}, + } + return mapping +} + +type BlockMappingOverride func(OptionsBlockMapping) + +// WithOverride overrides the default block mapping for a single block type +func WithOverride(blockName string, destination options.Options) BlockMappingOverride { + return func(mapping OptionsBlockMapping) { + mapping[blockName] = destination + } +} diff --git a/parse_v/decode_result.go b/parse_v/decode_result.go new file mode 100644 index 00000000..26bdd4a5 --- /dev/null +++ b/parse_v/decode_result.go @@ -0,0 +1,59 @@ +package parse_v + +import ( + "slices" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/modconfig" +) + +// struct to hold the result of a decoding operation +type DecodeResult struct { + Diags hcl.Diagnostics + Depends map[string]*modconfig.ResourceDependency +} + +func newDecodeResult() *DecodeResult { + return &DecodeResult{Depends: make(map[string]*modconfig.ResourceDependency)} +} + +// Merge merges this decode result with another +func (p *DecodeResult) Merge(other *DecodeResult) *DecodeResult { + p.Diags = append(p.Diags, other.Diags...) + for k, v := range other.Depends { + p.Depends[k] = v + } + + return p +} + +// Success returns if the was parsing successful - true if there are no errors and no dependencies +func (p *DecodeResult) Success() bool { + return !p.Diags.HasErrors() && len(p.Depends) == 0 +} + +// if the diags contains dependency errors, add dependencies to the result +// otherwise add diags to the result +func (p *DecodeResult) handleDecodeDiags(diags hcl.Diagnostics) { + for _, diag := range diags { + if dependency := diagsToDependency(diag); dependency != nil { + p.Depends[dependency.String()] = dependency + } + } + // only register errors if there are NOT any missing variables + if len(p.Depends) == 0 { + p.addDiags(diags) + } +} + +// determine whether the diag is a dependency error, and if so, return a dependency object +func diagsToDependency(diag *hcl.Diagnostic) *modconfig.ResourceDependency { + if slices.Contains[[]string, string](missingVariableErrors, diag.Summary) { + return &modconfig.ResourceDependency{Range: diag.Expression.Range(), Traversals: diag.Expression.Variables()} + } + return nil +} + +func (p *DecodeResult) addDiags(diags hcl.Diagnostics) { + p.Diags = append(p.Diags, diags...) +} diff --git a/parse_v/metadata.go b/parse_v/metadata.go new file mode 100644 index 00000000..97ead49c --- /dev/null +++ b/parse_v/metadata.go @@ -0,0 +1,39 @@ +package parse_v + +import ( + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/modconfig" +) + +func GetMetadataForParsedResource(resourceName string, srcRange hcl.Range, fileData map[string][]byte, mod *modconfig.Mod) (*modconfig.ResourceMetadata, error) { + // convert the name into a short name + parsedName, err := modconfig.ParseResourceName(resourceName) + if err != nil { + return nil, err + } + m := &modconfig.ResourceMetadata{ + ResourceName: parsedName.Name, + FileName: srcRange.Filename, + StartLineNumber: srcRange.Start.Line, + EndLineNumber: srcRange.End.Line, + IsAutoGenerated: false, + SourceDefinition: getSourceDefinition(srcRange, fileData), + } + // update the 'ModName' and 'ModShortName' fields + m.SetMod(mod) + return m, nil +} + +func getSourceDefinition(sourceRange hcl.Range, fileData map[string][]byte) string { + filename := sourceRange.Filename + fileBytes, ok := fileData[filename] + if !ok { + return "" + } + + source := strings.Join( + strings.Split(string(fileBytes), "\n")[sourceRange.Start.Line-1:sourceRange.End.Line], "\n") + return source +} diff --git a/parse_v/mod.go b/parse_v/mod.go new file mode 100644 index 00000000..76f6c829 --- /dev/null +++ b/parse_v/mod.go @@ -0,0 +1,219 @@ +package parse_v + +import ( + "fmt" + "log" + "os" + "path" + "path/filepath" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/pipe-fittings/error_helpers" + "github.com/turbot/pipe-fittings/filepaths" + "github.com/turbot/pipe-fittings/funcs" + "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/perr" + "github.com/turbot/pipe-fittings/schema" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/zclconf/go-cty/cty" +) + +func LoadModfile(modPath string) (*modconfig.Mod, error) { + if !ModfileExists(modPath) { + return nil, nil + } + + // build an eval context just containing functions + evalCtx := &hcl.EvalContext{ + Functions: funcs.ContextFunctions(modPath), + Variables: make(map[string]cty.Value), + } + + mod, res := ParseModDefinition(modPath, evalCtx) + if res.Diags.HasErrors() { + return nil, plugin.DiagsToError("Failed to load mod", res.Diags) + } + + return mod, nil +} + +func ParseModDefinitionWithFileName(modPath string, modFileName string, evalCtx *hcl.EvalContext) (*modconfig.Mod, *DecodeResult) { + res := newDecodeResult() + + // if there is no mod at this location, return error + modFilePath := filepath.Join(modPath, modFileName) + + modFileFound := true + if _, err := os.Stat(modFilePath); os.IsNotExist(err) { + modFileFound = false + for _, file := range filepaths.PipesComponentValidModFiles { + modFilePath = filepath.Join(modPath, file) + if _, err := os.Stat(modFilePath); os.IsNotExist(err) { + + continue + } else { + modFileFound = true + break + } + } + } + + if !modFileFound { + res.Diags = append(res.Diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "no mod file found in " + modPath, + }) + return nil, res + } + + fileData, diags := LoadFileData(modFilePath) + res.addDiags(diags) + if diags.HasErrors() { + return nil, res + } + + body, diags := ParseHclFiles(fileData) + res.addDiags(diags) + if diags.HasErrors() { + return nil, res + } + + workspaceContent, diags := body.Content(WorkspaceBlockSchema) + res.addDiags(diags) + if diags.HasErrors() { + return nil, res + } + + block := hclhelpers.GetFirstBlockOfType(workspaceContent.Blocks, schema.BlockTypeMod) + if block == nil { + res.Diags = append(res.Diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("no mod definition found in %s", modPath), + }) + return nil, res + } + var defRange = block.DefRange + if hclBody, ok := block.Body.(*hclsyntax.Body); ok { + defRange = hclBody.SrcRange + } + mod := modconfig.NewMod(block.Labels[0], path.Dir(modFilePath), defRange) + // set modFilePath + mod.SetFilePath(modFilePath) + + mod, res = decodeMod(block, evalCtx, mod) + if res.Diags.HasErrors() { + return nil, res + } + + // NOTE: IGNORE DEPENDENCY ERRORS + + // call decode callback + diags = mod.OnDecoded(block, nil) + res.addDiags(diags) + + return mod, res +} + +// ParseModDefinition parses the modfile only +// it is expected the calling code will have verified the existence of the modfile by calling ModfileExists +// this is called before parsing the workspace to, for example, identify dependency mods +// +// This function only parse the "mod" block, and does not parse any resources in the mod file +func ParseModDefinition(modPath string, evalCtx *hcl.EvalContext) (*modconfig.Mod, *DecodeResult) { + return ParseModDefinitionWithFileName(modPath, filepaths.PipesComponentModsFileName, evalCtx) +} + +// ParseMod parses all source hcl files for the mod path and associated resources, and returns the mod object +// NOTE: the mod definition has already been parsed (or a default created) and is in opts.RunCtx.RootMod +func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableResource, parseCtx *ModParseContext) (*modconfig.Mod, *error_helpers.ErrorAndWarnings) { + body, diags := ParseHclFiles(fileData) + if diags.HasErrors() { + return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod source files", diags)) + } + + content, moreDiags := body.Content(WorkspaceBlockSchema) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to load mod", diags)) + } + + mod := parseCtx.CurrentMod + if mod == nil { + return nil, error_helpers.NewErrorsAndWarning(fmt.Errorf("ParseMod called with no Current Mod set in ModParseContext")) + } + // get names of all resources defined in hcl which may also be created as pseudo resources + hclResources, err := loadMappableResourceNames(content) + if err != nil { + return nil, error_helpers.NewErrorsAndWarning(err) + } + + // if variables were passed in parsecontext, add to the mod + if parseCtx.Variables != nil { + for _, v := range parseCtx.Variables.RootVariables { + if diags = mod.AddResource(v); diags.HasErrors() { + return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to add resource to mod", diags)) + } + } + } + + // add pseudo resources to the mod + addPseudoResourcesToMod(pseudoResources, hclResources, mod) + + // add the parsed content to the run context + parseCtx.SetDecodeContent(content, fileData) + + // add the mod to the run context + // - this it to ensure all pseudo resources get added and build the eval context with the variables we just added + + // ! This is the place where the child mods (dependent mods) resources are "pulled up" into this current evaluation + // ! context. + // ! + // ! Step through the code to find the place where the child mod resources are added to the "referencesValue" + // ! + // ! Note that this resource MUST implement ModItem interface, otherwise it will look "flat", i.e. it will be added + // ! to the current mod + // ! + // ! There's also a bug where we test for ModTreeItem, we added a new interface ModItem for resources that are mod + // ! resources but not necessarily need to be in the mod tree + // ! + if diags = parseCtx.AddModResources(mod); diags.HasErrors() { + return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to add mod to run context", diags)) + } + + // collect warnings as we parse + var res = &error_helpers.ErrorAndWarnings{} + + // we may need to decode more than once as we gather dependencies as we go + // continue decoding as long as the number of unresolved blocks decreases + prevUnresolvedBlocks := 0 + for attempts := 0; ; attempts++ { + diags = decode(parseCtx) + if diags.HasErrors() { + return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to decode all mod hcl files", diags)) + } + // now retrieve the warning strings + res.AddWarning(plugin.DiagsToWarnings(diags)...) + + // if there are no unresolved blocks, we are done + unresolvedBlocks := len(parseCtx.UnresolvedBlocks) + if unresolvedBlocks == 0 { + log.Printf("[TRACE] parse complete after %d decode passes", attempts+1) + break + } + // if the number of unresolved blocks has NOT reduced, fail + if prevUnresolvedBlocks != 0 && unresolvedBlocks >= prevUnresolvedBlocks { + str := parseCtx.FormatDependencies() + msg := fmt.Sprintf("Failed to resolve dependencies after %d passes. Unresolved blocks:\n%s", attempts+1, str) + return nil, error_helpers.NewErrorsAndWarning(perr.BadRequestWithTypeAndMessage(perr.ErrorCodeDependencyFailure, msg)) + } + // update prevUnresolvedBlocks + prevUnresolvedBlocks = unresolvedBlocks + } + + // now tell mod to build tree of resources + res.Error = mod.BuildResourceTree(parseCtx.GetTopLevelDependencyMods()) + + return mod, res +} diff --git a/parse_v/mod_dependency_config.go b/parse_v/mod_dependency_config.go new file mode 100644 index 00000000..da66011f --- /dev/null +++ b/parse_v/mod_dependency_config.go @@ -0,0 +1,27 @@ +package parse_v + +import ( + "fmt" + + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/versionmap" +) + +type ModDependencyConfig struct { + ModDependency *versionmap.ResolvedVersionConstraint + DependencyPath *string +} + +func (c ModDependencyConfig) SetModProperties(mod *modconfig.Mod) { + mod.Version = c.ModDependency.Version + mod.DependencyPath = c.DependencyPath + mod.DependencyName = c.ModDependency.Name +} + +func NewDependencyConfig(modDependency *versionmap.ResolvedVersionConstraint) *ModDependencyConfig { + d := fmt.Sprintf("%s@v%s", modDependency.Name, modDependency.Version.String()) + return &ModDependencyConfig{ + DependencyPath: &d, + ModDependency: modDependency, + } +} diff --git a/parse_v/mod_parse_context.go b/parse_v/mod_parse_context.go new file mode 100644 index 00000000..2ab803f4 --- /dev/null +++ b/parse_v/mod_parse_context.go @@ -0,0 +1,744 @@ +package parse_v + +import ( + "context" + "fmt" + "github.com/turbot/terraform-components/terraform" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + filehelpers "github.com/turbot/go-kit/files" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/inputvars" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/perr" + "github.com/turbot/pipe-fittings/schema" + "github.com/turbot/pipe-fittings/utils" + "github.com/turbot/pipe-fittings/versionmap" + "github.com/zclconf/go-cty/cty" +) + +const rootDependencyNode = "rootDependencyNode" + +type ParseModFlag uint32 + +const ( + CreateDefaultMod ParseModFlag = 1 << iota + CreatePseudoResources + CreateTransientLocalMod +) + +/* + ReferenceTypeValueMap is the raw data used to build the evaluation context + +When resolving hcl references like : +- query.q1 +- var.v1 +- mod1.query.my_query.sql + +ReferenceTypeValueMap is keyed by resource type, then by resource name +*/ +type ReferenceTypeValueMap map[string]map[string]cty.Value + +type ModParseContext struct { + ParseContext + + // PipelineHcls map[string]*modconfig.Pipeline + TriggerHcls map[string]*modconfig.Trigger + + // the mod which is currently being parsed + CurrentMod *modconfig.Mod + // the workspace lock data + WorkspaceLock *versionmap.WorkspaceLock + + Flags ParseModFlag + ListOptions *filehelpers.ListOptions + + // Variables are populated in an initial parse pass top we store them on the run context + // so we can set them on the mod when we do the main parse + + // Variables is a tree of maps of the variables in the current mod and child dependency mods + Variables *modconfig.ModVariableMap + + ParentParseCtx *ModParseContext + + // stack of parent resources for the currently parsed block + // (unqualified name) + parents []string + + // map of resource children, keyed by parent unqualified name + blockChildMap map[string][]string + + // map of top level blocks, for easy checking + topLevelBlocks map[*hcl.Block]struct{} + // map of block names, keyed by a hash of the blopck + blockNameMap map[string]string + // map of ReferenceTypeValueMaps keyed by mod name + // NOTE: all values from root mod are keyed with "local" + referenceValues map[string]ReferenceTypeValueMap + + // a map of just the top level dependencies of the CurrentMod, keyed my full mod DependencyName (with no version) + topLevelDependencyMods modconfig.ModMap + // if we are loading dependency mod, this contains the details + DependencyConfig *ModDependencyConfig +} + +func NewModParseContext(runContext context.Context, workspaceLock *versionmap.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *ModParseContext { + + parseContext := NewParseContext(runContext, rootEvalPath) + c := &ModParseContext{ + ParseContext: parseContext, + + // TODO: fix this issue + // TODO: temporary mapping until we sort out merging Flowpipe and Steampipe + // PipelineHcls: make(map[string]*modconfig.Pipeline), + TriggerHcls: make(map[string]*modconfig.Trigger), + + Flags: flags, + WorkspaceLock: workspaceLock, + ListOptions: listOptions, + + topLevelDependencyMods: make(modconfig.ModMap), + blockChildMap: make(map[string][]string), + blockNameMap: make(map[string]string), + // initialise reference maps - even though we later overwrite them + referenceValues: map[string]ReferenceTypeValueMap{ + "local": make(ReferenceTypeValueMap), + }, + } + // add root node - this will depend on all other nodes + c.dependencyGraph = c.newDependencyGraph() + c.buildEvalContext() + + return c +} + +func NewChildModParseContext(parent *ModParseContext, modVersion *versionmap.ResolvedVersionConstraint, rootEvalPath string) *ModParseContext { + // create a child run context + child := NewModParseContext( + parent.RunCtx, + parent.WorkspaceLock, + rootEvalPath, + parent.Flags, + parent.ListOptions) + // copy our block tpyes + child.BlockTypes = parent.BlockTypes + // set the child's parent + child.ParentParseCtx = parent + // set the dependency config + child.DependencyConfig = NewDependencyConfig(modVersion) + // set variables if parent has any + if parent.Variables != nil { + childVars, ok := parent.Variables.DependencyVariables[modVersion.Name] + if ok { + child.Variables = childVars + child.Variables.PopulatePublicVariables() + child.AddVariablesToEvalContext() + } + } + + return child +} + +func (m *ModParseContext) EnsureWorkspaceLock(mod *modconfig.Mod) error { + // if the mod has dependencies, there must a workspace lock object in the run context + // (mod MUST be the workspace mod, not a dependency, as we would hit this error as soon as we parse it) + if mod.HasDependentMods() && (m.WorkspaceLock.Empty() || m.WorkspaceLock.Incomplete()) { + // logger := fplog.Logger(m.RunCtx) + // logger.Error("mod has dependencies but no workspace lock file found", "mod", mod.Name(), "m.HasDependentMods()", mod.HasDependentMods(), "m.WorkspaceLock.Empty()", m.WorkspaceLock.Empty(), "m.WorkspaceLock.Incomplete()", m.WorkspaceLock.Incomplete()) + return perr.BadRequestWithTypeAndMessage(perr.ErrorCodeDependencyFailure, "not all dependencies are installed - run 'steampipe mod install'") + } + + return nil +} + +func (m *ModParseContext) PushParent(parent modconfig.ModTreeItem) { + m.parents = append(m.parents, parent.GetUnqualifiedName()) +} + +func (m *ModParseContext) PopParent() string { + n := len(m.parents) - 1 + res := m.parents[n] + m.parents = m.parents[:n] + return res +} + +func (m *ModParseContext) PeekParent() string { + if len(m.parents) == 0 { + return m.CurrentMod.Name() + } + return m.parents[len(m.parents)-1] +} + +// VariableValueCtyMap converts a map of variables to a map of the underlying cty value +func VariableValueCtyMap(variables map[string]*modconfig.Variable) map[string]cty.Value { + ret := make(map[string]cty.Value, len(variables)) + for k, v := range variables { + ret[k] = v.Value + } + return ret +} + +// AddInputVariableValues adds evaluated variables to the run context. +// This function is called for the root run context after loading all input variables +func (m *ModParseContext) AddInputVariableValues(inputVariables *modconfig.ModVariableMap) { + // store the variables + m.Variables = inputVariables + + // now add variables into eval context + m.AddVariablesToEvalContext() +} + +func (m *ModParseContext) AddVariablesToEvalContext() { + m.addRootVariablesToReferenceMap() + m.addDependencyVariablesToReferenceMap() + m.buildEvalContext() +} + +// addRootVariablesToReferenceMap sets the Variables property +// and adds the variables to the referenceValues map (used to build the eval context) +func (m *ModParseContext) addRootVariablesToReferenceMap() { + + variables := m.Variables.RootVariables + // write local variables directly into referenceValues map + // NOTE: we add with the name "var" not "variable" as that is how variables are referenced + m.referenceValues["local"]["var"] = VariableValueCtyMap(variables) +} + +// addDependencyVariablesToReferenceMap adds the dependency variables to the referenceValues map +// (used to build the eval context) +func (m *ModParseContext) addDependencyVariablesToReferenceMap() { + // retrieve the resolved dependency versions for the parent mod + resolvedVersions := m.WorkspaceLock.InstallCache[m.Variables.Mod.GetInstallCacheKey()] + + for depModName, depVars := range m.Variables.DependencyVariables { + alias := resolvedVersions[depModName].Alias + if m.referenceValues[alias] == nil { + m.referenceValues[alias] = make(ReferenceTypeValueMap) + } + m.referenceValues[alias]["var"] = VariableValueCtyMap(depVars.RootVariables) + } +} + +// AddModResources is used to add mod resources to the eval context +func (m *ModParseContext) AddModResources(mod *modconfig.Mod) hcl.Diagnostics { + if len(m.UnresolvedBlocks) > 0 { + // should never happen + panic("calling AddModResources on ModParseContext but there are unresolved blocks from a previous parse") + } + + var diags hcl.Diagnostics + + moreDiags := m.storeResourceInReferenceValueMap(mod) + diags = append(diags, moreDiags...) + + // do not add variables (as they have already been added) + // if the resource is for a dependency mod, do not add locals + shouldAdd := func(item modconfig.HclResource) bool { + if item.BlockType() == schema.BlockTypeVariable || + item.BlockType() == schema.BlockTypeLocals && item.(modconfig.ModItem).GetMod().ShortName != m.CurrentMod.ShortName { + return false + } + return true + } + + resourceFunc := func(item modconfig.HclResource) (bool, error) { + // add all mod resources (except those excluded) into cty map + if shouldAdd(item) { + moreDiags := m.storeResourceInReferenceValueMap(item) + diags = append(diags, moreDiags...) + } + // continue walking + return true, nil + } + err := mod.WalkResources(resourceFunc) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "error walking mod resources", + Detail: err.Error(), + }) + return diags + } + + // rebuild the eval context + m.buildEvalContext() + return diags +} + +func (m *ModParseContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) { + // put blocks into map as well + m.topLevelBlocks = make(map[*hcl.Block]struct{}, len(m.blocks)) + for _, b := range content.Blocks { + m.topLevelBlocks[b] = struct{}{} + } + m.ParseContext.SetDecodeContent(content, fileData) +} + +// AddDependencies :: the block could not be resolved as it has dependencies +// 1) store block as unresolved +// 2) add dependencies to our tree of dependencies +func (m *ModParseContext) AddDependencies(block *hcl.Block, name string, dependencies map[string]*modconfig.ResourceDependency) hcl.Diagnostics { + // TACTICAL if this is NOT a top level block, add a suffix to the block name + // this is needed to avoid circular dependency errors if a nested block references + // a top level block with the same name + if !m.IsTopLevelBlock(block) { + name = "nested." + name + } + return m.ParseContext.AddDependencies(block, name, dependencies) +} + +// ShouldCreateDefaultMod returns whether the flag is set to create a default mod if no mod definition exists +func (m *ModParseContext) ShouldCreateDefaultMod() bool { + return m.Flags&CreateDefaultMod == CreateDefaultMod +} + +func (m *ModParseContext) ShouldCreateCreateTransientLocalMod() bool { + return m.Flags&CreateTransientLocalMod == CreateTransientLocalMod +} + +// CreatePseudoResources returns whether the flag is set to create pseudo resources +func (m *ModParseContext) CreatePseudoResources() bool { + return m.Flags&CreatePseudoResources == CreatePseudoResources +} + +// AddResource stores this resource as a variable to be added to the eval context. +func (m *ModParseContext) AddResource(resource modconfig.HclResource) hcl.Diagnostics { + diagnostics := m.storeResourceInReferenceValueMap(resource) + if diagnostics.HasErrors() { + return diagnostics + } + + // rebuild the eval context + m.buildEvalContext() + + return nil +} + +// GetMod finds the mod with given short name, looking only in first level dependencies +// this is used to resolve resource references +// specifically when the 'children' property of dashboards and benchmarks refers to resource in a dependency mod +func (m *ModParseContext) GetMod(modShortName string) *modconfig.Mod { + if modShortName == m.CurrentMod.ShortName { + return m.CurrentMod + } + // we need to iterate through dependency mods of the current mod + key := m.CurrentMod.GetInstallCacheKey() + deps := m.WorkspaceLock.InstallCache[key] + for _, dep := range deps { + depMod, ok := m.topLevelDependencyMods[dep.Name] + if ok && depMod.ShortName == modShortName { + return depMod + } + } + return nil +} + +func (m *ModParseContext) GetResourceMaps() *modconfig.ResourceMaps { + // use the current mod as the base resource map + resourceMap := m.CurrentMod.GetResourceMaps() + // get a map of top level loaded dep mods + deps := m.GetTopLevelDependencyMods() + + dependencyResourceMaps := make([]*modconfig.ResourceMaps, 0, len(deps)) + + // merge in the top level resources of the dependency mods + for _, dep := range deps { + dependencyResourceMaps = append(dependencyResourceMaps, dep.GetResourceMaps().TopLevelResources()) + } + + resourceMap = resourceMap.Merge(dependencyResourceMaps) + return resourceMap +} + +func (m *ModParseContext) GetResource(parsedName *modconfig.ParsedResourceName) (resource modconfig.HclResource, found bool) { + return m.GetResourceMaps().GetResource(parsedName) +} + +// build the eval context from the cached reference values +func (m *ModParseContext) buildEvalContext() { + // convert reference values to cty objects + referenceValues := make(map[string]cty.Value) + + // now for each mod add all the values + for mod, modMap := range m.referenceValues { + // TODO: this code is from steampipe, looks like there's a special treatment if the mod is named "local"? + if mod == "local" { + for k, v := range modMap { + referenceValues[k] = cty.ObjectVal(v) + } + continue + } + + // mod map is map[string]map[string]cty.Value + // for each element (i.e. map[string]cty.Value) convert to cty object + refTypeMap := make(map[string]cty.Value) + // TODO: this code is from steampipe, looks like there's a special treatment if the mod is named "local"? + if mod == "local" { + for k, v := range modMap { + referenceValues[k] = cty.ObjectVal(v) + } + } else { + for refType, typeValueMap := range modMap { + refTypeMap[refType] = cty.ObjectVal(typeValueMap) + } + } + // now convert the referenceValues itself to a cty object + referenceValues[mod] = cty.ObjectVal(refTypeMap) + } + + // rebuild the eval context + m.ParseContext.BuildEvalContext(referenceValues) +} + +// store the resource as a cty value in the reference valuemap +func (m *ModParseContext) storeResourceInReferenceValueMap(resource modconfig.HclResource) hcl.Diagnostics { + // add resource to variable map + ctyValue, diags := m.getResourceCtyValue(resource) + if diags.HasErrors() { + return diags + } + + // add into the reference value map + if diags := m.addReferenceValue(resource, ctyValue); diags.HasErrors() { + return diags + } + + // remove this resource from unparsed blocks + delete(m.UnresolvedBlocks, resource.Name()) + + return nil +} + +// convert a HclResource into a cty value, taking into account nested structs +func (m *ModParseContext) getResourceCtyValue(resource modconfig.HclResource) (cty.Value, hcl.Diagnostics) { + ctyValue, err := resource.(modconfig.CtyValueProvider).CtyValue() + if err != nil { + return cty.Zero, m.errToCtyValueDiags(resource, err) + } + // if this is a value map, merge in the values of base structs + // if it is NOT a value map, the resource must have overridden CtyValue so do not merge base structs + if ctyValue.Type().FriendlyName() != "object" { + return ctyValue, nil + } + // TODO [node_reuse] fetch nested structs and serialise automatically https://github.com/turbot/steampipe/issues/2924 + valueMap := ctyValue.AsValueMap() + if valueMap == nil { + valueMap = make(map[string]cty.Value) + } + base := resource.GetHclResourceImpl() + if err := m.mergeResourceCtyValue(base, valueMap); err != nil { + return cty.Zero, m.errToCtyValueDiags(resource, err) + } + + if qp, ok := resource.(modconfig.QueryProvider); ok { + base := qp.GetQueryProviderImpl() + if err := m.mergeResourceCtyValue(base, valueMap); err != nil { + return cty.Zero, m.errToCtyValueDiags(resource, err) + } + } + + if treeItem, ok := resource.(modconfig.ModTreeItem); ok { + base := treeItem.GetModTreeItemImpl() + if err := m.mergeResourceCtyValue(base, valueMap); err != nil { + return cty.Zero, m.errToCtyValueDiags(resource, err) + } + } + return cty.ObjectVal(valueMap), nil +} + +// merge the cty value of the given interface into valueMap +// (note: this mutates valueMap) +func (m *ModParseContext) mergeResourceCtyValue(resource modconfig.CtyValueProvider, valueMap map[string]cty.Value) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic in mergeResourceCtyValue: %s", helpers.ToError(r).Error()) + } + }() + ctyValue, err := resource.CtyValue() + if err != nil { + return err + } + if ctyValue == cty.Zero { + return nil + } + // merge results + for k, v := range ctyValue.AsValueMap() { + valueMap[k] = v + } + return nil +} + +func (m *ModParseContext) errToCtyValueDiags(resource modconfig.HclResource, err error) hcl.Diagnostics { + return hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("failed to convert resource '%s' to its cty value", resource.Name()), + Detail: err.Error(), + Subject: resource.GetDeclRange(), + }} +} + +func (m *ModParseContext) addReferenceValue(resource modconfig.HclResource, value cty.Value) hcl.Diagnostics { + parsedName, err := modconfig.ParseResourceName(resource.Name()) + if err != nil { + return hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("failed to parse resource name %s", resource.Name()), + Detail: err.Error(), + Subject: resource.GetDeclRange(), + }} + } + + // TODO validate mod name clashes + // TODO mod reserved names + // TODO handle aliases + + key := parsedName.Name + typeString := parsedName.ItemType + + // most resources will have a mod property - use this if available + var mod *modconfig.Mod + if modTreeItem, ok := resource.(modconfig.ModItem); ok { + mod = modTreeItem.GetMod() + } + // fall back to current mod + if mod == nil { + mod = m.CurrentMod + } + + modName := mod.ShortName + if mod.ModPath == m.RootEvalPath { + modName = "local" + } + variablesForMod, ok := m.referenceValues[modName] + // do we have a map of reference values for this dep mod? + if !ok { + // no - create one + variablesForMod = make(ReferenceTypeValueMap) + m.referenceValues[modName] = variablesForMod + } + // do we have a map of reference values for this type + variablesForType, ok := variablesForMod[typeString] + if !ok { + // no - create one + variablesForType = make(map[string]cty.Value) + } + + // DO NOT update the cached cty values if the value already exists + // this can happen in the case of variables where we initialise the context with values read from file + // or passed on the command line, // does this item exist in the map + if _, ok := variablesForType[key]; !ok { + variablesForType[key] = value + variablesForMod[typeString] = variablesForType + m.referenceValues[modName] = variablesForMod + } + + return nil +} + +func (m *ModParseContext) IsTopLevelBlock(block *hcl.Block) bool { + _, isTopLevel := m.topLevelBlocks[block] + return isTopLevel +} + +func (m *ModParseContext) AddLoadedDependencyMod(mod *modconfig.Mod) { + m.topLevelDependencyMods[mod.DependencyName] = mod +} + +// GetTopLevelDependencyMods build a mod map of top level loaded dependencies, keyed by mod name +func (m *ModParseContext) GetTopLevelDependencyMods() modconfig.ModMap { + return m.topLevelDependencyMods +} + +func (m *ModParseContext) SetCurrentMod(mod *modconfig.Mod) error { + m.CurrentMod = mod + // now we have the mod, load any arg values from the mod require - these will be passed to dependency mods + return m.loadModRequireArgs() +} + +// when reloading a mod dependency tree to resolve require args values, this function is called after each mod is loaded +// to load the require arg values and update the variable values +func (m *ModParseContext) loadModRequireArgs() error { + //if we have not loaded variable definitions yet, do not load require args + if m.Variables == nil { + return nil + } + + depModVarValues, err := inputvars.CollectVariableValuesFromModRequire(m.CurrentMod, m.WorkspaceLock) + if err != nil { + return err + } + if len(depModVarValues) == 0 { + return nil + } + // if any mod require args have an unknown value, we have failed to resolve them - raise an error + if err := m.validateModRequireValues(depModVarValues); err != nil { + return err + } + // now update the variables map with the input values + inputvars.SetVariableValues(depModVarValues, m.Variables) + + // now add overridden variables into eval context - in case the root mod references any dependency variable values + m.AddVariablesToEvalContext() + + return nil +} + +func (m *ModParseContext) validateModRequireValues(depModVarValues terraform.InputValues) error { + if len(depModVarValues) == 0 { + return nil + } + var missingVarExpressions []string + requireBlock := m.getModRequireBlock() + if requireBlock == nil { + return fmt.Errorf("require args extracted but no require block found for %s", m.CurrentMod.Name()) + } + + for k, v := range depModVarValues { + // if we successfully resolved this value, continue + if v.Value.IsKnown() { + continue + } + parsedVarName, err := modconfig.ParseResourceName(k) + if err != nil { + return err + } + + // re-parse the require block manually to extract the range and unresolved arg value expression + var errorString string + errorString, err = m.getErrorStringForUnresolvedArg(parsedVarName, requireBlock) + if err != nil { + // if there was an error retrieving details, return less specific error string + errorString = fmt.Sprintf("\"%s\" (%s %s)", k, m.CurrentMod.Name(), m.CurrentMod.GetDeclRange().Filename) + } + + missingVarExpressions = append(missingVarExpressions, errorString) + } + + if errorCount := len(missingVarExpressions); errorCount > 0 { + if errorCount == 1 { + return fmt.Errorf("failed to resolve dependency mod argument value: %s", missingVarExpressions[0]) + } + + return fmt.Errorf("failed to resolve %d dependency mod arguments %s:\n\t%s", errorCount, utils.Pluralize("value", errorCount), strings.Join(missingVarExpressions, "\n\t")) + } + return nil +} + +func (m *ModParseContext) getErrorStringForUnresolvedArg(parsedVarName *modconfig.ParsedResourceName, requireBlock *hclsyntax.Block) (_ string, err error) { + defer func() { + if r := recover(); r != nil { + err = helpers.ToError(r) + } + }() + // which mod and variable is this is this for + modShortName := parsedVarName.Mod + varName := parsedVarName.Name + var modDependencyName string + // determine the mod dependency name as that is how it will be keyed in the require map + for depName, modVersion := range m.WorkspaceLock.InstallCache[m.CurrentMod.GetInstallCacheKey()] { + if modVersion.Alias == modShortName { + modDependencyName = depName + break + } + } + + // iterate through require blocks looking for mod blocks + for _, b := range requireBlock.Body.Blocks { + // only interested in mod blocks + if b.Type != "mod" { + continue + } + // if this is not the mod we're looking for, continue + if b.Labels[0] != modDependencyName { + continue + } + // now find the failed arg + argsAttr, ok := b.Body.Attributes["args"] + if !ok { + return "", fmt.Errorf("no args block found for %s", modDependencyName) + } + // iterate over args looking for the correctly named item + for _, a := range argsAttr.Expr.(*hclsyntax.ObjectConsExpr).Items { + thisVarName, err := a.KeyExpr.Value(&hcl.EvalContext{}) + if err != nil { + return "", err + } + + // is this the var we are looking for? + if thisVarName.AsString() != varName { + continue + } + + // this is the var, get the value expression + expr, ok := a.ValueExpr.(*hclsyntax.ScopeTraversalExpr) + if !ok { + return "", fmt.Errorf("failed to get args details for %s", parsedVarName.ToResourceName()) + } + // ok we have the expression - build the error string + exprString := hclhelpers.TraversalAsString(expr.Traversal) + r := expr.Range() + sourceRange := fmt.Sprintf("%s:%d", r.Filename, r.Start.Line) + res := fmt.Sprintf("\"%s = %s\" (%s %s)", + parsedVarName.ToResourceName(), + exprString, + m.CurrentMod.Name(), + sourceRange) + return res, nil + + } + } + return "", fmt.Errorf("failed to get args details for %s", parsedVarName.ToResourceName()) +} + +func (m *ModParseContext) getModRequireBlock() *hclsyntax.Block { + for _, b := range m.CurrentMod.ResourceWithMetadataBaseRemain.(*hclsyntax.Body).Blocks { + if b.Type == schema.BlockTypeRequire { + return b + } + } + return nil + +} + +// TODO: transition period +// AddPipeline stores this resource as a variable to be added to the eval context. It alse +func (m *ModParseContext) AddPipeline(pipelineHcl *modconfig.Pipeline) hcl.Diagnostics { + + // Split and get the last part for pipeline name + // pipelineFullName := pipelineHcl.Name() + // parts := strings.Split(pipelineFullName, ".") + // pipelineNameOnly := parts[len(parts)-1] + + // m.PipelineHcls[pipelineNameOnly] = pipelineHcl + + diags := m.addReferenceValue(pipelineHcl, pipelineHcl.AsCtyValue()) + if diags.HasErrors() { + return diags + } + + // remove this resource from unparsed blocks + delete(m.UnresolvedBlocks, pipelineHcl.Name()) + + m.buildEvalContext() + return nil +} + +func (m *ModParseContext) AddTrigger(trigger *modconfig.Trigger) hcl.Diagnostics { + + // Split and get the last part for pipeline name + parts := strings.Split(trigger.Name(), ".") + triggerNameOnly := parts[len(parts)-1] + + m.TriggerHcls[triggerNameOnly] = trigger + + // remove this resource from unparsed blocks + delete(m.UnresolvedBlocks, trigger.Name()) + + m.buildEvalContext() + return nil +} + +// TODO: transition period diff --git a/parse_v/mod_parse_context_blocks.go b/parse_v/mod_parse_context_blocks.go new file mode 100644 index 00000000..df753fdf --- /dev/null +++ b/parse_v/mod_parse_context_blocks.go @@ -0,0 +1,94 @@ +package parse_v + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/modconfig" +) + +func (m *ModParseContext) DetermineBlockName(block *hcl.Block) string { + var shortName string + + // have we cached a name for this block (i.e. is this the second decode pass) + if name, ok := m.GetCachedBlockShortName(block); ok { + return name + } + + // if there is a parent set in the parent stack, this block is a child of that parent + parentName := m.PeekParent() + + anonymous := len(block.Labels) == 0 + if anonymous { + shortName = m.getUniqueName(block.Type, parentName) + } else { + shortName = block.Labels[0] + } + // build unqualified name + unqualifiedName := fmt.Sprintf("%s.%s", block.Type, shortName) + m.addChildBlockForParent(parentName, unqualifiedName) + // cache this name for the second decode pass + m.cacheBlockName(block, unqualifiedName) + return shortName +} + +func (m *ModParseContext) GetCachedBlockName(block *hcl.Block) (string, bool) { + name, ok := m.blockNameMap[m.blockHash(block)] + return name, ok +} + +func (m *ModParseContext) GetCachedBlockShortName(block *hcl.Block) (string, bool) { + unqualifiedName, ok := m.blockNameMap[m.blockHash(block)] + if ok { + parsedName, err := modconfig.ParseResourceName(unqualifiedName) + if err != nil { + return "", false + } + return parsedName.Name, true + } + return "", false +} + +func (m *ModParseContext) GetDecodedResourceForBlock(block *hcl.Block) (modconfig.HclResource, bool) { + if name, ok := m.GetCachedBlockName(block); ok { + // see whether the mod contains this resource already + parsedName, err := modconfig.ParseResourceName(name) + if err == nil { + return m.CurrentMod.GetResource(parsedName) + } + } + return nil, false +} + +func (m *ModParseContext) cacheBlockName(block *hcl.Block, shortName string) { + m.blockNameMap[m.blockHash(block)] = shortName +} + +func (m *ModParseContext) blockHash(block *hcl.Block) string { + return helpers.GetMD5Hash(block.DefRange.String()) +} + +// getUniqueName returns a name unique within the scope of this execution tree +func (m *ModParseContext) getUniqueName(blockType string, parent string) string { + // count how many children of this block type the parent has + childCount := 0 + + for _, childName := range m.blockChildMap[parent] { + parsedName, err := modconfig.ParseResourceName(childName) + if err != nil { + // we do not expect this + continue + } + if parsedName.ItemType == blockType { + childCount++ + } + } + sanitisedParentName := strings.ReplaceAll(parent, ".", "_") + return fmt.Sprintf("%s_anonymous_%s_%d", sanitisedParentName, blockType, childCount) +} + +func (m *ModParseContext) addChildBlockForParent(parent, child string) { + m.blockChildMap[parent] = append(m.blockChildMap[parent], child) +} diff --git a/parse_v/parse_context.go b/parse_v/parse_context.go new file mode 100644 index 00000000..bc20dc91 --- /dev/null +++ b/parse_v/parse_context.go @@ -0,0 +1,241 @@ +package parse_v + +import ( + "context" + "fmt" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/stevenle/topsort" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/funcs" + "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/zclconf/go-cty/cty" +) + +type ParseContext struct { + // This is the running application context + RunCtx context.Context + UnresolvedBlocks map[string]*unresolvedBlock + FileData map[string][]byte + + // the eval context used to decode references in HCL + EvalCtx *hcl.EvalContext + + Diags hcl.Diagnostics + + RootEvalPath string + + // if set, only decode these blocks + BlockTypes []string + // if set, exclude these block types + BlockTypeExclusions []string + + dependencyGraph *topsort.Graph + blocks hcl.Blocks +} + +func NewParseContext(runContext context.Context, rootEvalPath string) ParseContext { + c := ParseContext{ + UnresolvedBlocks: make(map[string]*unresolvedBlock), + RootEvalPath: rootEvalPath, + RunCtx: runContext, + } + // add root node - this will depend on all other nodes + c.dependencyGraph = c.newDependencyGraph() + + return c +} + +func (r *ParseContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) { + r.blocks = content.Blocks + r.FileData = fileData +} + +func (r *ParseContext) ClearDependencies() { + r.UnresolvedBlocks = make(map[string]*unresolvedBlock) + r.dependencyGraph = r.newDependencyGraph() +} + +// AddDependencies is called when a block could not be resolved as it has dependencies +// 1) store block as unresolved +// 2) add dependencies to our tree of dependencies +func (r *ParseContext) AddDependencies(block *hcl.Block, name string, dependencies map[string]*modconfig.ResourceDependency) hcl.Diagnostics { + var diags hcl.Diagnostics + + if r.UnresolvedBlocks[name] != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("duplicate unresolved block name '%s'", name), + Detail: fmt.Sprintf("block '%s' already exists. This could mean that there are unresolved duplicate resources,", name), + Subject: &block.DefRange, + }) + return diags + } + + // store unresolved block + r.UnresolvedBlocks[name] = &unresolvedBlock{Name: name, Block: block, Dependencies: dependencies} + + // store dependency in tree - d + if !r.dependencyGraph.ContainsNode(name) { + r.dependencyGraph.AddNode(name) + } + // add root dependency + if err := r.dependencyGraph.AddEdge(rootDependencyNode, name); err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to add root dependency to graph", + Detail: err.Error(), + Subject: &block.DefRange, + }) + } + + for _, dep := range dependencies { + // each dependency object may have multiple traversals + for _, t := range dep.Traversals { + parsedPropertyPath, err := modconfig.ParseResourcePropertyPath(hclhelpers.TraversalAsString(t)) + + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to parse dependency", + Detail: err.Error(), + Subject: &block.DefRange, + }) + continue + + } + + // 'd' may be a property path - when storing dependencies we only care about the resource names + dependencyResourceName := parsedPropertyPath.ToResourceName() + if !r.dependencyGraph.ContainsNode(dependencyResourceName) { + r.dependencyGraph.AddNode(dependencyResourceName) + } + if err := r.dependencyGraph.AddEdge(name, dependencyResourceName); err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to add dependency to graph", + Detail: err.Error(), + Subject: &block.DefRange, + }) + } + } + } + return diags +} + +// BlocksToDecode builds a list of blocks to decode, the order of which is determined by the dependency order +func (r *ParseContext) BlocksToDecode() (hcl.Blocks, error) { + depOrder, err := r.getDependencyOrder() + if err != nil { + return nil, err + } + if len(depOrder) == 0 { + return r.blocks, nil + } + + // NOTE: a block may appear more than once in unresolved blocks + // if it defines multiple unresolved resources, e.g a locals block + + // make a map of blocks we have already included, keyed by the block def range + blocksMap := make(map[string]bool) + var blocksToDecode hcl.Blocks + for _, name := range depOrder { + // depOrder is all the blocks required to resolve dependencies. + // if this one is unparsed, added to list + block, ok := r.UnresolvedBlocks[name] + if ok && !blocksMap[block.Block.DefRange.String()] { + blocksToDecode = append(blocksToDecode, block.Block) + // add to map + blocksMap[block.Block.DefRange.String()] = true + } + } + return blocksToDecode, nil +} + +// EvalComplete returns whether all elements in the dependency tree fully evaluated +func (r *ParseContext) EvalComplete() bool { + return len(r.UnresolvedBlocks) == 0 +} + +func (r *ParseContext) FormatDependencies() string { + // first get the dependency order + dependencyOrder, err := r.getDependencyOrder() + if err != nil { + return err.Error() + } + // build array of dependency strings - processes dependencies in reverse order for presentation reasons + numDeps := len(dependencyOrder) + depStrings := make([]string, numDeps) + for i := 0; i < len(dependencyOrder); i++ { + srcIdx := len(dependencyOrder) - i - 1 + resourceName := dependencyOrder[srcIdx] + // find dependency + dep, ok := r.UnresolvedBlocks[resourceName] + + if ok { + depStrings[i] = dep.String() + } else { + // this could happen if there is a dependency on a missing item + depStrings[i] = fmt.Sprintf(" MISSING: %s", resourceName) + } + } + + return helpers.Tabify(strings.Join(depStrings, "\n"), " ") +} + +func (r *ParseContext) ShouldIncludeBlock(block *hcl.Block) bool { + if len(r.BlockTypes) > 0 && !helpers.StringSliceContains(r.BlockTypes, block.Type) { + return false + } + if len(r.BlockTypeExclusions) > 0 && helpers.StringSliceContains(r.BlockTypeExclusions, block.Type) { + return false + } + return true +} + +func (r *ParseContext) newDependencyGraph() *topsort.Graph { + dependencyGraph := topsort.NewGraph() + // add root node - this will depend on all other nodes + dependencyGraph.AddNode(rootDependencyNode) + return dependencyGraph +} + +// return the optimal run order required to resolve dependencies + +func (r *ParseContext) getDependencyOrder() ([]string, error) { + rawDeps, err := r.dependencyGraph.TopSort(rootDependencyNode) + if err != nil { + return nil, err + } + + // now remove the variable names and dedupe + var deps []string + for _, d := range rawDeps { + if d == rootDependencyNode { + continue + } + + propertyPath, err := modconfig.ParseResourcePropertyPath(d) + if err != nil { + return nil, err + } + dep := modconfig.BuildModResourceName(propertyPath.ItemType, propertyPath.Name) + if !helpers.StringSliceContains(deps, dep) { + deps = append(deps, dep) + } + } + return deps, nil +} + +// eval functions +func (r *ParseContext) BuildEvalContext(variables map[string]cty.Value) { + + // create evaluation context + r.EvalCtx = &hcl.EvalContext{ + Variables: variables, + // use the mod path as the file root for functions + Functions: funcs.ContextFunctions(r.RootEvalPath), + } +} diff --git a/parse_v/parser.go b/parse_v/parser.go new file mode 100644 index 00000000..51ccb9b4 --- /dev/null +++ b/parse_v/parser.go @@ -0,0 +1,203 @@ +package parse_v + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/json" + "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/filepaths" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/schema" + "github.com/turbot/pipe-fittings/utils" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "sigs.k8s.io/yaml" +) + +// LoadFileData builds a map of filepath to file data +func LoadFileData(paths ...string) (map[string][]byte, hcl.Diagnostics) { + var diags hcl.Diagnostics + var fileData = map[string][]byte{} + + for _, configPath := range paths { + data, err := os.ReadFile(configPath) + + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("failed to read config file %s", configPath), + Detail: err.Error()}) + continue + } + fileData[configPath] = data + } + return fileData, diags +} + +// ParseHclFiles parses hcl file data and returns the hcl body object +func ParseHclFiles(fileData map[string][]byte) (hcl.Body, hcl.Diagnostics) { + var parsedConfigFiles []*hcl.File + var diags hcl.Diagnostics + parser := hclparse.NewParser() + + // build ordered list of files so that we parse in a repeatable order + filePaths := buildOrderedFileNameList(fileData) + + for _, filePath := range filePaths { + var file *hcl.File + var moreDiags hcl.Diagnostics + ext := filepath.Ext(filePath) + if ext == constants.JsonExtension { + file, moreDiags = json.ParseFile(filePath) + } else if constants.IsYamlExtension(ext) { + file, moreDiags = parseYamlFile(filePath) + } else { + data := fileData[filePath] + file, moreDiags = parser.ParseHCL(data, filePath) + } + + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + continue + } + parsedConfigFiles = append(parsedConfigFiles, file) + } + + return hcl.MergeFiles(parsedConfigFiles), diags +} + +func buildOrderedFileNameList(fileData map[string][]byte) []string { + filePaths := make([]string, len(fileData)) + idx := 0 + for filePath := range fileData { + filePaths[idx] = filePath + idx++ + } + sort.Strings(filePaths) + return filePaths +} + +// ModfileExists returns whether a mod file exists at the specified path +func ModfileExists(modPath string) bool { + modFilePath := filepath.Join(modPath, filepaths.PipesComponentModsFileName) + if _, err := os.Stat(modFilePath); os.IsNotExist(err) { + return false + } + return true +} + +// parse a yaml file into a hcl.File object +func parseYamlFile(filename string) (*hcl.File, hcl.Diagnostics) { + f, err := os.Open(filename) + if err != nil { + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Failed to open file", + Detail: fmt.Sprintf("The file %q could not be opened.", filename), + }, + } + } + defer f.Close() + + src, err := io.ReadAll(f) + if err != nil { + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Failed to read file", + Detail: fmt.Sprintf("The file %q was opened, but an error occured while reading it.", filename), + }, + } + } + jsonData, err := yaml.YAMLToJSON(src) + if err != nil { + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Failed to read convert YAML to JSON", + Detail: fmt.Sprintf("The file %q was opened, but an error occured while converting it to JSON.", filename), + }, + } + } + return json.Parse(jsonData, filename) +} + +func addPseudoResourcesToMod(pseudoResources []modconfig.MappableResource, hclResources map[string]bool, mod *modconfig.Mod) { + var duplicates []string + for _, r := range pseudoResources { + // is there a hcl resource with the same name as this pseudo resource - it takes precedence + name := r.GetUnqualifiedName() + if _, ok := hclResources[name]; ok { + duplicates = append(duplicates, r.GetDeclRange().Filename) + continue + } + // add pseudo resource to mod + mod.AddResource(r.(modconfig.HclResource)) //nolint:errcheck // TODO: handle error + // add to map of existing resources + hclResources[name] = true + } + numDupes := len(duplicates) + if numDupes > 0 { + log.Printf("[TRACE] %d %s not converted into resources as hcl resources of same name are defined: %v", numDupes, utils.Pluralize("file", numDupes), duplicates) + } +} + +// get names of all resources defined in hcl which may also be created as pseudo resources +// if we find a mod block, build a shell mod +func loadMappableResourceNames(content *hcl.BodyContent) (map[string]bool, error) { + hclResources := make(map[string]bool) + + for _, block := range content.Blocks { + // if this is a mod, build a shell mod struct (with just the name populated) + switch block.Type { + case schema.BlockTypeQuery: + // for any mappable resource, store the resource name + name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) + hclResources[name] = true + } + } + return hclResources, nil +} + +// ParseModResourceNames parses all source hcl files for the mod path and associated resources, +// and returns the resource names +func ParseModResourceNames(fileData map[string][]byte) (*modconfig.WorkspaceResources, error) { + var resources = modconfig.NewWorkspaceResources() + body, diags := ParseHclFiles(fileData) + if diags.HasErrors() { + return nil, plugin.DiagsToError("Failed to load all mod source files", diags) + } + + content, moreDiags := body.Content(WorkspaceBlockSchema) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + return nil, plugin.DiagsToError("Failed to load mod", diags) + } + + for _, block := range content.Blocks { + // if this is a mod, build a shell mod struct (with just the name populated) + switch block.Type { + + case schema.BlockTypeQuery: + // for any mappable resource, store the resource name + name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) + resources.Query[name] = true + case schema.BlockTypeControl: + // for any mappable resource, store the resource name + name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) + resources.Control[name] = true + case schema.BlockTypeBenchmark: + // for any mappable resource, store the resource name + name := modconfig.BuildModResourceName(block.Type, block.Labels[0]) + resources.Benchmark[name] = true + } + } + return resources, nil +} diff --git a/parse/pipeline_decode.go b/parse_v/pipeline_decode.go similarity index 99% rename from parse/pipeline_decode.go rename to parse_v/pipeline_decode.go index 62ab3d52..b33daa44 100644 --- a/parse/pipeline_decode.go +++ b/parse_v/pipeline_decode.go @@ -1,4 +1,4 @@ -package parse +package parse_v import ( "fmt" diff --git a/parse_v/query_invocation.go b/parse_v/query_invocation.go new file mode 100644 index 00000000..8a031111 --- /dev/null +++ b/parse_v/query_invocation.go @@ -0,0 +1,185 @@ +package parse_v + +import ( + "fmt" + "github.com/turbot/go-kit/type_conversion" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" +) + +// ParseQueryInvocation parses a query invocation and extracts the args (if any) +// supported formats are: +// +// 1) positional args +// query.my_prepared_statement('val1','val1') +// +// 2) named args +// query.my_prepared_statement(my_arg1 => 'test', my_arg2 => 'test2') +func ParseQueryInvocation(arg string) (string, *modconfig.QueryArgs, error) { + // TODO strip non printing chars + args := &modconfig.QueryArgs{} + + arg = strings.TrimSpace(arg) + query := arg + var err error + openBracketIdx := strings.Index(arg, "(") + closeBracketIdx := strings.LastIndex(arg, ")") + if openBracketIdx != -1 && closeBracketIdx == len(arg)-1 { + argsString := arg[openBracketIdx+1 : len(arg)-1] + args, err = parseArgs(argsString) + query = strings.TrimSpace(arg[:openBracketIdx]) + } + return query, args, err +} + +// parse the actual args string, i.e. the contents of the bracket +// supported formats are: +// +// 1) positional args +// 'val1','val1' +// +// 2) named args +// my_arg1 => 'val1', my_arg2 => 'val2' +func parseArgs(argsString string) (*modconfig.QueryArgs, error) { + res := modconfig.NewQueryArgs() + if len(argsString) == 0 { + return res, nil + } + + // split on comma to get each arg string (taking quotes and brackets into account) + splitArgs, err := splitArgString(argsString) + if err != nil { + // return empty result, even if we have an error + return res, err + } + + // first check for named args + argMap, err := parseNamedArgs(splitArgs) + if err != nil { + return res, err + } + if err := res.SetArgMap(argMap); err != nil { + return res, err + } + + if res.Empty() { + // no named args - fall back on positional + argList, err := parsePositionalArgs(splitArgs) + if err != nil { + return res, err + } + if err := res.SetArgList(argList); err != nil { + return res, err + } + } + // return empty result, even if we have an error + return res, err +} + +func splitArgString(argsString string) ([]string, error) { + var argsList []string + openElements := map[string]int{ + "quote": 0, + "curly": 0, + "square": 0, + } + var currentWord string + for _, c := range argsString { + // should we split - are we in a block + if c == ',' && + openElements["quote"] == 0 && openElements["curly"] == 0 && openElements["square"] == 0 { + if len(currentWord) > 0 { + argsList = append(argsList, currentWord) + currentWord = "" + } + } else { + currentWord = currentWord + string(c) + } + + // handle brackets and quotes + switch c { + case '{': + if openElements["quote"] == 0 { + openElements["curly"]++ + } + case '}': + if openElements["quote"] == 0 { + openElements["curly"]-- + if openElements["curly"] < 0 { + return nil, fmt.Errorf("bad arg syntax") + } + } + case '[': + if openElements["quote"] == 0 { + openElements["square"]++ + } + case ']': + if openElements["quote"] == 0 { + openElements["square"]-- + if openElements["square"] < 0 { + return nil, fmt.Errorf("bad arg syntax") + } + } + case '"': + if openElements["quote"] == 0 { + openElements["quote"] = 1 + } else { + openElements["quote"] = 0 + } + } + } + if len(currentWord) > 0 { + argsList = append(argsList, currentWord) + } + return argsList, nil +} + +func parseArg(v string) (any, error) { + b, diags := hclsyntax.ParseExpression([]byte(v), "", hcl.Pos{}) + if diags.HasErrors() { + return "", plugin.DiagsToError("bad arg syntax", diags) + } + val, diags := b.Value(nil) + if diags.HasErrors() { + return "", plugin.DiagsToError("bad arg syntax", diags) + } + return type_conversion.CtyToGo(val) +} + +func parseNamedArgs(argsList []string) (map[string]any, error) { + var res = make(map[string]any) + for _, p := range argsList { + argTuple := strings.Split(strings.TrimSpace(p), "=>") + if len(argTuple) != 2 { + // not all args have valid syntax - give up + return nil, nil + } + k := strings.TrimSpace(argTuple[0]) + val, err := parseArg(argTuple[1]) + if err != nil { + return nil, err + } + res[k] = val + } + return res, nil +} + +func parsePositionalArgs(argsList []string) ([]any, error) { + // convert to pointer array + res := make([]any, len(argsList)) + // just treat args as positional args + // strip spaces + for i, v := range argsList { + valStr, err := parseArg(v) + if err != nil { + return nil, err + } + res[i] = valStr + } + + return res, nil +} diff --git a/parse_v/references.go b/parse_v/references.go new file mode 100644 index 00000000..0a224804 --- /dev/null +++ b/parse_v/references.go @@ -0,0 +1,38 @@ +package parse_v + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/schema" +) + +// AddReferences populates the 'References' resource field, used for the introspection tables +func AddReferences(resource modconfig.HclResource, block *hcl.Block, parseCtx *ModParseContext) hcl.Diagnostics { + resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata) + if !ok { + return nil + } + + var diags hcl.Diagnostics + for _, attr := range block.Body.(*hclsyntax.Body).Attributes { + for _, v := range attr.Expr.Variables() { + for _, referenceBlockType := range schema.ReferenceBlocks { + if referenceString, ok := hclhelpers.ResourceNameFromTraversal(referenceBlockType, v); ok { + var blockName string + if len(block.Labels) > 0 { + blockName = block.Labels[0] + } + reference := modconfig.NewResourceReference(resource, block, referenceString, blockName, attr) + + moreDiags := addResourceMetadata(reference, attr.SrcRange, parseCtx) + diags = append(diags, moreDiags...) + resourceWithMetadata.AddReference(reference) + break + } + } + } + } + return diags +} diff --git a/parse_v/schema.go b/parse_v/schema.go new file mode 100644 index 00000000..03ff3504 --- /dev/null +++ b/parse_v/schema.go @@ -0,0 +1,344 @@ +package parse_v + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/schema" +) + +// cache resource schemas +var resourceSchemaCache = make(map[string]*hcl.BodySchema) + +// TODO [node_reuse] Replace all block type with consts https://github.com/turbot/steampipe/issues/2922 + +var ConfigBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{}, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "connection", + LabelNames: []string{"name"}, + }, + + { + Type: "options", + LabelNames: []string{"type"}, + }, + { + Type: "workspace", + LabelNames: []string{"name"}, + }, + }, +} + +var WorkspaceProfileBlockSchema = &hcl.BodySchema{ + + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "options", + LabelNames: []string{"type"}, + }, + }, +} + +var ConnectionBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "plugin", + Required: true, + }, + { + Name: "type", + }, + { + Name: "connections", + }, + { + Name: "import_schema", + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "options", + LabelNames: []string{"type"}, + }, + }, +} + +// WorkspaceBlockSchema is the top level schema for all workspace resources +var WorkspaceBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{}, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: string(schema.BlockTypeMod), + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeVariable, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeQuery, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeControl, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeBenchmark, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeDashboard, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeCard, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeChart, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeFlow, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeGraph, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeHierarchy, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeImage, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeInput, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeTable, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeText, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeNode, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeEdge, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeLocals, + }, + { + Type: schema.BlockTypeCategory, + LabelNames: []string{"name"}, + }, + + // Flowpipe + { + Type: schema.BlockTypePipeline, + LabelNames: []string{schema.LabelName}, + }, + { + Type: schema.BlockTypeTrigger, + LabelNames: []string{schema.LabelType, schema.LabelName}, + }, + }, +} + +// DashboardBlockSchema is only used to validate the blocks of a Dashboard +var DashboardBlockSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: schema.BlockTypeInput, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeParam, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeWith, + }, + { + Type: schema.BlockTypeContainer, + }, + { + Type: schema.BlockTypeCard, + }, + { + Type: schema.BlockTypeChart, + }, + { + Type: schema.BlockTypeBenchmark, + }, + { + Type: schema.BlockTypeControl, + }, + { + Type: schema.BlockTypeFlow, + }, + { + Type: schema.BlockTypeGraph, + }, + { + Type: schema.BlockTypeHierarchy, + }, + { + Type: schema.BlockTypeImage, + }, + { + Type: schema.BlockTypeTable, + }, + { + Type: schema.BlockTypeText, + }, + }, +} + +// DashboardContainerBlockSchema is only used to validate the blocks of a DashboardContainer +var DashboardContainerBlockSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: schema.BlockTypeInput, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeParam, + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeContainer, + }, + { + Type: schema.BlockTypeCard, + }, + { + Type: schema.BlockTypeChart, + }, + { + Type: schema.BlockTypeBenchmark, + }, + { + Type: schema.BlockTypeControl, + }, + { + Type: schema.BlockTypeFlow, + }, + { + Type: schema.BlockTypeGraph, + }, + { + Type: schema.BlockTypeHierarchy, + }, + { + Type: schema.BlockTypeImage, + }, + { + Type: schema.BlockTypeTable, + }, + { + Type: schema.BlockTypeText, + }, + }, +} + +var BenchmarkBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + {Name: "children"}, + {Name: "description"}, + {Name: "documentation"}, + {Name: "tags"}, + {Name: "title"}, + // for report benchmark blocks + {Name: "width"}, + {Name: "base"}, + {Name: "type"}, + {Name: "display"}, + }, +} + +// QueryProviderBlockSchema schema for all blocks satisfying QueryProvider interface +// NOTE: these are just the blocks/attributes that are explicitly decoded +// other query provider properties are implicitly decoded using tags +var QueryProviderBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + {Name: "args"}, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "param", + LabelNames: []string{"name"}, + }, + { + Type: "with", + LabelNames: []string{"name"}, + }, + }, +} + +// NodeAndEdgeProviderSchema is used to decode graph/hierarchy/flow +// (EXCEPT categories) +var NodeAndEdgeProviderSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + {Name: "args"}, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "param", + LabelNames: []string{"name"}, + }, + { + Type: "category", + LabelNames: []string{"name"}, + }, + { + Type: "with", + LabelNames: []string{"name"}, + }, + { + Type: schema.BlockTypeNode, + }, + { + Type: schema.BlockTypeEdge, + }, + }, +} + +var ParamDefBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + {Name: "description"}, + {Name: "default"}, + }, +} + +var VariableBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "description", + }, + { + Name: "default", + }, + { + Name: "type", + }, + { + Name: "sensitive", + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "validation", + }, + }, +} diff --git a/parse_v/unresolved_block.go b/parse_v/unresolved_block.go new file mode 100644 index 00000000..ff5dbd42 --- /dev/null +++ b/parse_v/unresolved_block.go @@ -0,0 +1,26 @@ +package parse_v + +import ( + "fmt" + "strings" + + "github.com/turbot/pipe-fittings/modconfig" + + "github.com/hashicorp/hcl/v2" +) + +type unresolvedBlock struct { + Name string + Block *hcl.Block + Dependencies map[string]*modconfig.ResourceDependency +} + +func (b unresolvedBlock) String() string { + depStrings := make([]string, len(b.Dependencies)) + idx := 0 + for _, dep := range b.Dependencies { + depStrings[idx] = fmt.Sprintf(`%s -> %s`, b.Name, dep.String()) + idx++ + } + return strings.Join(depStrings, "\n") +} diff --git a/parse_v/validate.go b/parse_v/validate.go new file mode 100644 index 00000000..643aa7a2 --- /dev/null +++ b/parse_v/validate.go @@ -0,0 +1,133 @@ +package parse_v + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/modconfig" +) + +// validate the resource +func validateResource(resource modconfig.HclResource) hcl.Diagnostics { + var diags hcl.Diagnostics + if qp, ok := resource.(modconfig.NodeAndEdgeProvider); ok { + moreDiags := validateNodeAndEdgeProvider(qp) + diags = append(diags, moreDiags...) + } else if qp, ok := resource.(modconfig.QueryProvider); ok { + moreDiags := validateQueryProvider(qp) + diags = append(diags, moreDiags...) + } + + // TODO: commented out: dashboard + // if wp, ok := resource.(modconfig.WithProvider); ok { + // moreDiags := validateRuntimeDependencyProvider(wp) + // diags = append(diags, moreDiags...) + // } + return diags +} + +// TODO: commented out: dashboard +// func validateRuntimeDependencyProvider(wp modconfig.WithProvider) hcl.Diagnostics { +// resource := wp.(modconfig.HclResource) +// var diags hcl.Diagnostics +// if len(wp.GetWiths()) > 0 && !resource.IsTopLevel() { +// diags = append(diags, &hcl.Diagnostic{ +// Severity: hcl.DiagError, +// Summary: "Only top level resources can have `with` blocks", +// Detail: fmt.Sprintf("%s contains 'with' blocks but is not a top level resource.", resource.Name()), +// Subject: resource.GetDeclRange(), +// }) +// } +// return diags +// } + +// validate that the provider does not contains both edges/nodes and a query/sql +// enrich the loaded nodes and edges with the fully parsed resources from the resourceMapProvider +func validateNodeAndEdgeProvider(resource modconfig.NodeAndEdgeProvider) hcl.Diagnostics { + // TODO [node_reuse] add NodeAndEdgeProviderImpl and move validate there + // https://github.com/turbot/steampipe/issues/2918 + + var diags hcl.Diagnostics + + // TODO: commented out: dashboard + // containsEdgesOrNodes := len(resource.GetEdges())+len(resource.GetNodes()) > 0 + definesQuery := resource.GetSQL() != nil || resource.GetQuery() != nil + + // TODO: commented out: dashboard + // cannot declare both edges/nodes AND sql/query + // if definesQuery && containsEdgesOrNodes { + // diags = append(diags, &hcl.Diagnostic{ + // Severity: hcl.DiagError, + // Summary: fmt.Sprintf("%s contains edges/nodes AND has a query", resource.Name()), + // Subject: resource.GetDeclRange(), + // }) + // } + + // if resource is NOT top level must have either edges/nodes OR sql/query + + // TODO: commented out: dashboard + // if !resource.IsTopLevel() && !definesQuery && !containsEdgesOrNodes { + if !resource.IsTopLevel() && !definesQuery { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("%s does not define a query or SQL, and has no edges/nodes", resource.Name()), + Subject: resource.GetDeclRange(), + }) + } + + diags = append(diags, validateSqlAndQueryNotBothSet(resource)...) + + diags = append(diags, validateParamAndQueryNotBothSet(resource)...) + + return diags +} + +func validateQueryProvider(resource modconfig.QueryProvider) hcl.Diagnostics { + var diags hcl.Diagnostics + + diags = append(diags, resource.ValidateQuery()...) + + diags = append(diags, validateSqlAndQueryNotBothSet(resource)...) + + diags = append(diags, validateParamAndQueryNotBothSet(resource)...) + + return diags +} + +func validateParamAndQueryNotBothSet(resource modconfig.QueryProvider) hcl.Diagnostics { + var diags hcl.Diagnostics + + // param block cannot be set if a query property is set - it is only valid if inline SQL ids defined + if len(resource.GetParams()) > 0 { + if resource.GetQuery() != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: fmt.Sprintf("Deprecated usage: %s has 'query' property set so should not define 'param' blocks", resource.Name()), + Subject: resource.GetDeclRange(), + }) + } + if !resource.IsTopLevel() && !resource.ParamsInheritedFromBase() { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Deprecated usage: Only top level resources can have 'param' blocks", + Detail: fmt.Sprintf("%s contains 'param' blocks but is not a top level resource.", resource.Name()), + Subject: resource.GetDeclRange(), + }) + } + } + return diags +} + +func validateSqlAndQueryNotBothSet(resource modconfig.QueryProvider) hcl.Diagnostics { + var diags hcl.Diagnostics + // are both sql and query set? + if resource.GetSQL() != nil && resource.GetQuery() != nil { + // either Query or SQL property may be set - if Query property already set, error + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("%s has both 'SQL' and 'query' property set - only 1 of these may be set", resource.Name()), + Subject: resource.GetDeclRange(), + }) + } + return diags +} diff --git a/parse_v/workspace_profile.go b/parse_v/workspace_profile.go new file mode 100644 index 00000000..9580b48b --- /dev/null +++ b/parse_v/workspace_profile.go @@ -0,0 +1,208 @@ +package parse_v + +import ( + "context" + "fmt" + "log" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + filehelpers "github.com/turbot/go-kit/files" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/options" + "github.com/turbot/pipe-fittings/schema" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" +) + +func LoadWorkspaceProfiles(ctx context.Context, workspaceProfilePath string) (profileMap map[string]*modconfig.WorkspaceProfile, err error) { + + defer func() { + if r := recover(); r != nil { + err = helpers.ToError(r) + } + // be sure to return the default + if profileMap != nil && profileMap["default"] == nil { + profileMap["default"] = &modconfig.WorkspaceProfile{ProfileName: "default"} + } + }() + + // create profile map to populate + profileMap = map[string]*modconfig.WorkspaceProfile{} + + configPaths, err := filehelpers.ListFiles(workspaceProfilePath, &filehelpers.ListOptions{ + Flags: filehelpers.FilesFlat, + Include: filehelpers.InclusionsFromExtensions([]string{constants.ConfigExtension}), + }) + if err != nil { + return nil, err + } + if len(configPaths) == 0 { + return profileMap, nil + } + + fileData, diags := LoadFileData(configPaths...) + if diags.HasErrors() { + return nil, plugin.DiagsToError("Failed to load workspace profiles", diags) + } + + body, diags := ParseHclFiles(fileData) + if diags.HasErrors() { + return nil, plugin.DiagsToError("Failed to load workspace profiles", diags) + } + + // do a partial decode + content, diags := body.Content(ConfigBlockSchema) + if diags.HasErrors() { + return nil, plugin.DiagsToError("Failed to load workspace profiles", diags) + } + + parseCtx := NewWorkspaceProfileParseContext(ctx, workspaceProfilePath) + parseCtx.SetDecodeContent(content, fileData) + + // build parse context + return parseWorkspaceProfiles(parseCtx) + +} +func parseWorkspaceProfiles(parseCtx *WorkspaceProfileParseContext) (map[string]*modconfig.WorkspaceProfile, error) { + // we may need to decode more than once as we gather dependencies as we go + // continue decoding as long as the number of unresolved blocks decreases + prevUnresolvedBlocks := 0 + for attempts := 0; ; attempts++ { + _, diags := decodeWorkspaceProfiles(parseCtx) + if diags.HasErrors() { + return nil, plugin.DiagsToError("Failed to decode all workspace profile files", diags) + } + + // if there are no unresolved blocks, we are done + unresolvedBlocks := len(parseCtx.UnresolvedBlocks) + if unresolvedBlocks == 0 { + log.Printf("[TRACE] parse complete after %d decode passes", attempts+1) + break + } + // if the number of unresolved blocks has NOT reduced, fail + if prevUnresolvedBlocks != 0 && unresolvedBlocks >= prevUnresolvedBlocks { + str := parseCtx.FormatDependencies() + return nil, fmt.Errorf("failed to resolve workspace profile dependencies after %d attempts\nDependencies:\n%s", attempts+1, str) + } + // update prevUnresolvedBlocks + prevUnresolvedBlocks = unresolvedBlocks + } + + return parseCtx.workspaceProfiles, nil + +} + +func decodeWorkspaceProfiles(parseCtx *WorkspaceProfileParseContext) (map[string]*modconfig.WorkspaceProfile, hcl.Diagnostics) { + profileMap := map[string]*modconfig.WorkspaceProfile{} + + var diags hcl.Diagnostics + blocksToDecode, err := parseCtx.BlocksToDecode() + // build list of blocks to decode + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to determine required dependency order", + Detail: err.Error()}) + return nil, diags + } + + // now clear dependencies from run context - they will be rebuilt + parseCtx.ClearDependencies() + + for _, block := range blocksToDecode { + if block.Type == schema.BlockTypeWorkspaceProfile { + workspaceProfile, res := decodeWorkspaceProfile(block, parseCtx) + + if res.Success() { + // success - add to map + profileMap[workspaceProfile.ProfileName] = workspaceProfile + } + diags = append(diags, res.Diags...) + } + } + return profileMap, diags +} + +// decodeWorkspaceProfileOption decodes an options block as a workspace profile property +// setting the necessary overrides for special handling of the "dashboard" option which is different +// from the global "dashboard" option +func decodeWorkspaceProfileOption(block *hcl.Block) (options.Options, hcl.Diagnostics) { + return DecodeOptions(block, WithOverride(constants.CmdNameDashboard, &options.WorkspaceProfileDashboard{})) +} + +func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseContext) (*modconfig.WorkspaceProfile, *DecodeResult) { + res := newDecodeResult() + // get shell resource + resource := modconfig.NewWorkspaceProfile(block) + + // do a partial decode to get options blocks into workspaceProfileOptions, with all other attributes in rest + workspaceProfileOptions, rest, diags := block.Body.PartialContent(WorkspaceProfileBlockSchema) + if diags.HasErrors() { + res.handleDecodeDiags(diags) + return nil, res + } + + diags = gohcl.DecodeBody(rest, parseCtx.EvalCtx, resource) + if len(diags) > 0 { + res.handleDecodeDiags(diags) + } + // use a map keyed by a string for fast lookup + // we use an empty struct as the value type, so that + // we don't use up unnecessary memory + foundOptions := map[string]struct{}{} + for _, block := range workspaceProfileOptions.Blocks { + switch block.Type { + case "options": + optionsBlockType := block.Labels[0] + if _, found := foundOptions[optionsBlockType]; found { + // fail + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Subject: &block.DefRange, + Summary: fmt.Sprintf("Duplicate options type '%s'", optionsBlockType), + }) + } + opts, moreDiags := decodeWorkspaceProfileOption(block) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + break + } + moreDiags = resource.SetOptions(opts, block) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + } + foundOptions[optionsBlockType] = struct{}{} + default: + // this should never happen + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("invalid block type '%s' - only 'options' blocks are supported for workspace profiles", block.Type), + Subject: &block.DefRange, + }) + } + } + + handleWorkspaceProfileDecodeResult(resource, res, block, parseCtx) + return resource, res +} + +func handleWorkspaceProfileDecodeResult(resource *modconfig.WorkspaceProfile, res *DecodeResult, block *hcl.Block, parseCtx *WorkspaceProfileParseContext) { + if res.Success() { + // call post decode hook + // NOTE: must do this BEFORE adding resource to run context to ensure we respect the base property + moreDiags := resource.OnDecoded() + res.addDiags(moreDiags) + + moreDiags = parseCtx.AddResource(resource) + res.addDiags(moreDiags) + return + } + + // failure :( + if len(res.Depends) > 0 { + moreDiags := parseCtx.AddDependencies(block, resource.Name(), res.Depends) + res.addDiags(moreDiags) + } +} diff --git a/parse_v/workspace_profile_parse_context.go b/parse_v/workspace_profile_parse_context.go new file mode 100644 index 00000000..5b4f6a48 --- /dev/null +++ b/parse_v/workspace_profile_parse_context.go @@ -0,0 +1,64 @@ +package parse_v + +import ( + "context" + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/zclconf/go-cty/cty" +) + +type WorkspaceProfileParseContext struct { + ParseContext + workspaceProfiles map[string]*modconfig.WorkspaceProfile + valueMap map[string]cty.Value +} + +func NewWorkspaceProfileParseContext(ctx context.Context, rootEvalPath string) *WorkspaceProfileParseContext { + parseContext := NewParseContext(ctx, rootEvalPath) + // TODO uncomment once https://github.com/turbot/steampipe/issues/2640 is done + //parseContext.BlockTypes = []string{modconfig.BlockTypeWorkspaceProfile} + c := &WorkspaceProfileParseContext{ + ParseContext: parseContext, + workspaceProfiles: make(map[string]*modconfig.WorkspaceProfile), + valueMap: make(map[string]cty.Value), + } + + c.buildEvalContext() + + return c +} + +// AddResource stores this resource as a variable to be added to the eval context. It alse +func (c *WorkspaceProfileParseContext) AddResource(workspaceProfile *modconfig.WorkspaceProfile) hcl.Diagnostics { + ctyVal, err := workspaceProfile.CtyValue() + if err != nil { + return hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("failed to convert workspaceProfile '%s' to its cty value", workspaceProfile.ProfileName), + Detail: err.Error(), + Subject: &workspaceProfile.DeclRange, + }} + } + + c.workspaceProfiles[workspaceProfile.ProfileName] = workspaceProfile + c.valueMap[workspaceProfile.ProfileName] = ctyVal + + // remove this resource from unparsed blocks + delete(c.UnresolvedBlocks, workspaceProfile.ProfileName) + + c.buildEvalContext() + + return nil +} + +func (c *WorkspaceProfileParseContext) buildEvalContext() { + // rebuild the eval context + // build a map with a single key - workspace + vars := map[string]cty.Value{ + "workspace": cty.ObjectVal(c.valueMap), + } + c.ParseContext.BuildEvalContext(vars) + +} diff --git a/workspace/workspace.go b/workspace/workspace.go index debd9fbf..c400374f 100644 --- a/workspace/workspace.go +++ b/workspace/workspace.go @@ -337,7 +337,6 @@ func (w *Workspace) getParseContext(ctx context.Context) (*parse.ModParseContext return nil, err } parseCtx := parse.NewModParseContext( - ctx, workspaceLock, w.Path, parseFlag,