patched parameter detection

cmd/ast/definition.go

@@ -17,15 +17,31 @@ type (
 		Required bool   `json:",omitempty" yaml:",omitempty"`
 		Type     string `json:",omitempty" yaml:",omitempty"`
 		fullName string
+		Assumed  bool
+		Typer    Typer `json:",omitempty" yaml:",omitempty"`
 	}
 )
 
 func (m *ViewMeta) addParameter(param *Parameter) {
 	if index, ok := m.index[param.Id]; ok {
-		m.Parameters[index].Required = m.Parameters[index].Required || param.Required
+		parameter := m.Parameters[index]
+		parameter.Required = parameter.Required || param.Required
+		if parameter.Assumed {
+			parameter.Type = param.Type
+		}
+
 		return
 	}
 
 	m.index[param.Id] = len(m.Parameters)
 	m.Parameters = append(m.Parameters, param)
 }
+
+func (m *ViewMeta) ParamByName(name string) (*Parameter, bool) {
+	index, ok := m.index[name]
+	if !ok {
+		return nil, false
+	}
+
+	return m.Parameters[index], true
+}

cmd/ast/lex.go

@@ -18,10 +18,23 @@ const (
 	paramToken
 	exprGroupToken
 	identityToken
+
+	anyToken
+	endToken
+	elseToken
+	assignToken
+	elseIfToken
+	forEachToken
+	ifToken
+
+	numberToken
+	boolToken
+	stringToken
 )
 
 var whitespaceMatcher = parsly.NewToken(whitespaceToken, "Whitespace", matcher.NewWhiteSpace())
-var wordMatcher = parsly.NewToken(wordToken, "Word", matchers.NewWordMatcher())
+var wordMatcher = parsly.NewToken(wordToken, "Word", matchers.NewWordMatcher(false))
+var fullWordMatcher = parsly.NewToken(wordToken, "Word", matchers.NewWordMatcher(true))
 var colonMatcher = parsly.NewToken(colonToken, "Colon", matcher.NewByte(':'))
 
 var squareBracketsMatcher = parsly.NewToken(squareBracketsToken, "Square brackets", matcher.NewBlock('[', ']', '\\'))
@@ -33,3 +46,20 @@ var paramMatcher = parsly.NewToken(paramToken, "Parameter", matcher.NewFragments
 var identityMatcher = parsly.NewToken(identityToken, "Identity", matchers.NewIdentity())
 var condBlockMatcher = parsly.NewToken(condBlockToken, "#if .... #end", matcher.NewSeqBlock("#if", "#end"))
 var exprGroupMatcher = parsly.NewToken(exprGroupToken, "( .... )", matcher.NewBlock('(', ')', '\\'))
+
+var anyMatcher = parsly.NewToken(anyToken, "Any", matchers.NewAny())
+var endMatcher = parsly.NewToken(endToken, "End", matcher.NewFragment("#end"))
+var elseMatcher = parsly.NewToken(elseToken, "Else", matcher.NewFragment("#else"))
+var elseIfMatcher = parsly.NewToken(elseToken, "ElseIf", matcher.NewFragment("#elseif"))
+var assignMatcher = parsly.NewToken(assignToken, "Set", matcher.NewFragment("#set"))
+var forEachMatcher = parsly.NewToken(forEachToken, "ForEach", matcher.NewFragment("#foreach"))
+var ifMatcher = parsly.NewToken(ifToken, "If", matcher.NewFragment("#if"))
+
+var numberMatcher = parsly.NewToken(numberToken, "Number", matcher.NewNumber())
+var boolMatcher = parsly.NewToken(boolToken, "Boolean", matcher.NewFragmentsFold([]byte("true"), []byte("false")))
+var boolTokenMatcher = parsly.NewToken(boolToken, "Boolean", matcher.NewFragments(
+	[]byte("&&"), []byte("||"),
+))
+
+var singleQuoteStringMatcher = parsly.NewToken(stringToken, "String", matcher.NewBlock('\'', '\'', '\\'))
+var doubleQuoteStringMatcher = parsly.NewToken(stringToken, "String", matcher.NewBlock('"', '"', '\\'))

cmd/ast/matchers/word.go

@@ -2,16 +2,18 @@ package matchers
 
 import (
 	"github.com/viant/parsly"
+	"github.com/viant/parsly/matcher"
 )
 
 type word struct {
+	matchAny bool
 }
 
 func (w *word) Match(cursor *parsly.Cursor) (matched int) {
 	var current byte
 	for i := cursor.Pos; i < cursor.InputSize; i++ {
 		current = cursor.Input[i]
-		if current >= 'a' && current <= 'z' || current >= 'A' && current <= 'Z' {
+		if (current >= 'a' && current <= 'z' || current >= 'A' && current <= 'Z') || (w.matchAny && !matcher.IsWhiteSpace(current)) {
 			matched++
 			continue
 		}
@@ -22,6 +24,6 @@ func (w *word) Match(cursor *parsly.Cursor) (matched int) {
 	return matched
 }
 
-func NewWordMatcher() parsly.Matcher {
-	return &word{}
+func NewWordMatcher(matchAny bool) parsly.Matcher {
+	return &word{matchAny: matchAny}
 }

cmd/ast/param.go

@@ -0,0 +1,78 @@
+package ast
+
+import (
+	"bytes"
+	"github.com/viant/parsly"
+	"github.com/viant/sqlx/io/read/cache/ast"
+	"reflect"
+	"strings"
+)
+
+var resetWords = []string{"AND", "OR", "WITH", "HAVING", "LIMIT", "OFFSET", "WHERE", "SELECT", "UNION", "ALL", "AS", "BETWEEN", "IN"}
+
+type (
+	Typer interface{}
+
+	LiteralType struct {
+		RType reflect.Type
+	}
+
+	ColumnType struct {
+		ColumnName string
+	}
+)
+
+func correctUntyped(SQL string, variables map[string]bool, meta *ViewMeta) {
+	var typer Typer
+	var untyped []string
+
+	cursor := parsly.NewCursor("", []byte(SQL), 0)
+outer:
+	for cursor.Pos < cursor.InputSize {
+		matched := cursor.MatchAfterOptional(whitespaceMatcher, doubleQuoteStringMatcher, singleQuoteStringMatcher, boolTokenMatcher, boolMatcher, numberMatcher, fullWordMatcher)
+		switch matched.Code {
+		case numberToken:
+			typer = &LiteralType{RType: ast.Float64Type}
+		case boolToken:
+			typer = &LiteralType{RType: ast.BoolType}
+		case stringToken:
+			typer = &LiteralType{RType: ast.StringType}
+		case parsly.EOF:
+			//Do nothing
+		default:
+			text := matched.Text(cursor)
+			for _, word := range resetWords {
+				if strings.EqualFold(text, word) {
+					typer = nil
+					untyped = nil
+					continue outer
+				}
+			}
+
+			firstLetter := bytes.ToUpper([]byte{text[0]})[0]
+			if (firstLetter < 'A' || firstLetter > 'Z') && firstLetter != '$' {
+				continue outer
+			}
+
+			if text[0] == '$' {
+				_, paramName := getParamName(text)
+				if isParameter(variables, paramName) {
+					untyped = append(untyped, paramName)
+				}
+			} else {
+				typer = &ColumnType{ColumnName: text}
+			}
+		}
+
+		if typer != nil {
+			for _, paramName := range untyped {
+				param, ok := meta.ParamByName(paramName)
+				if !ok {
+					continue
+				}
+
+				param.Typer = typer
+			}
+		}
+	}
+}

cmd/ast/parse.go

@@ -8,6 +8,7 @@ import (
 	"github.com/viant/velty/ast/expr"
 	"github.com/viant/velty/ast/stmt"
 	"github.com/viant/velty/parser"
+	"reflect"
 	"strconv"
 	"strings"
 )
@@ -36,13 +37,37 @@ func Parse(SQL string) (*ViewMeta, error) {
 	}
 
 	variables := map[string]bool{}
-	implyDefaultParams(variables, block.Statements(), viewMeta, true)
+	implyDefaultParams(variables, block.Statements(), viewMeta, true, nil)
+
+	SQL = removeVeltySyntax(string(from))
+	correctUntyped(SQL, variables, viewMeta)
 
 	viewMeta.Source = actualSource
 	return viewMeta, nil
 }
 
-func implyDefaultParams(variables map[string]bool, statements []ast.Statement, meta *ViewMeta, required bool) {
+func removeVeltySyntax(SQL string) string {
+	cursor := parsly.NewCursor("", []byte(SQL), 0)
+	sb := strings.Builder{}
+
+outer:
+	for cursor.Pos < cursor.InputSize {
+		matched := cursor.MatchAny(whitespaceMatcher, ifMatcher, assignMatcher, elseIfMatcher, elseMatcher, forEachMatcher, endMatcher, anyMatcher)
+		switch matched.Code {
+		case endToken, elseToken:
+			continue outer
+		case elseIfToken, assignToken, forEachToken, ifToken:
+			cursor.MatchOne(exprGroupMatcher)
+			continue outer
+		}
+
+		sb.WriteString(matched.Text(cursor))
+	}
+
+	return sb.String()
+}
+
+func implyDefaultParams(variables map[string]bool, statements []ast.Statement, meta *ViewMeta, required bool, rType reflect.Type) {
 	for _, statement := range statements {
 		switch actual := statement.(type) {
 		case stmt.ForEach:
@@ -55,11 +80,11 @@ func implyDefaultParams(variables map[string]bool, statements []ast.Statement, m
 
 			y, ok := actual.Y.(*expr.Select)
 			if ok && !variables[y.ID] {
-				indexParameter(variables, y, meta, required)
+				indexParameter(variables, y, meta, required, rType)
 			}
 
 		case *expr.Select:
-			indexParameter(variables, actual, meta, required)
+			indexParameter(variables, actual, meta, required, rType)
 
 		case *stmt.Statement:
 			x, ok := actual.X.(*expr.Select)
@@ -69,53 +94,75 @@ func implyDefaultParams(variables map[string]bool, statements []ast.Statement, m
 			variables[x.ID] = true
 		case *stmt.ForEach:
 			variables[actual.Item.ID] = true
+		case *expr.Unary:
+			implyDefaultParams(variables, []ast.Statement{actual}, meta, false, actual.Type())
+		case *expr.Binary:
+			xType := actual.X.Type()
+			if xType == nil {
+				xType = actual.Y.Type()
+			}
 
+			implyDefaultParams(variables, []ast.Statement{actual.X, actual.Y}, meta, false, xType)
+		case *expr.Parentheses:
+			implyDefaultParams(variables, []ast.Statement{actual.P}, meta, false, actual.Type())
 		case *stmt.If:
-			switch condition := actual.Condition.(type) {
-			case *expr.Unary:
-				implyDefaultParams(variables, []ast.Statement{condition.X}, meta, false)
-			case *expr.Binary:
-				implyDefaultParams(variables, []ast.Statement{condition.X, condition.Y}, meta, false)
-			case *expr.Parentheses:
-				implyDefaultParams(variables, []ast.Statement{condition.P}, meta, false)
-			default:
-				implyDefaultParams(variables, []ast.Statement{actual.Condition}, meta, false)
-			}
+			implyDefaultParams(variables, []ast.Statement{actual.Condition}, meta, false, actual.Type())
 		}
 
 		switch actual := statement.(type) {
 		case ast.StatementContainer:
-			implyDefaultParams(variables, actual.Statements(), meta, false)
+			implyDefaultParams(variables, actual.Statements(), meta, false, nil)
 		}
 	}
 }
 
-func indexParameter(variables map[string]bool, actual *expr.Select, meta *ViewMeta, required bool) {
-	paramName := paramId(actual)
-	prefix, paramName := removePrefixIfNeeded(paramName)
-	paramName = withoutPath(paramName)
+func indexParameter(variables map[string]bool, actual *expr.Select, meta *ViewMeta, required bool, rType reflect.Type) {
+	prefix, paramName := getParamName(actual.FullName)
 
-	if isVariable := variables[paramName]; isVariable {
+	if !isParameter(variables, paramName) {
 		return
 	}
 
-	switch paramName {
-	case keywords.Criteria[1:], keywords.SelectorCriteria[1:], keywords.Pagination[1:], keywords.ColumnsIn[1:]:
-		return
+	pType := "string"
+	assumed := true
+	if rType != nil && prefix != keywords.ParamsMetadataKey {
+		pType = rType.String()
+		assumed = false
 	}
 
 	meta.addParameter(&Parameter{
+		Assumed:  assumed,
 		Id:       paramName,
 		Name:     paramName,
 		Kind:     "query",
-		Type:     "string",
+		Type:     pType,
 		fullName: actual.FullName,
 		Required: required && prefix != keywords.ParamsMetadataKey,
 	})
 }
 
-func paramId(actual *expr.Select) string {
-	paramName := actual.FullName[1:]
+func getParamName(identifier string) (string, string) {
+	paramName := paramId(identifier)
+	prefix, paramName := removePrefixIfNeeded(paramName)
+	paramName = withoutPath(paramName)
+	return prefix, paramName
+}
+
+func isParameter(variables map[string]bool, paramName string) bool {
+	if isVariable := variables[paramName]; isVariable {
+		return false
+	}
+
+	switch paramName {
+	case keywords.Criteria[1:], keywords.SelectorCriteria[1:], keywords.Pagination[1:], keywords.ColumnsIn[1:]:
+		return false
+	}
+
+	return true
+}
+
+func paramId(identifier string) string {
+	paramName := identifier[1:]
 	if paramName[0] == '{' {
 		paramName = paramName[1 : len(paramName)-1]
 	}

cmd/ast/parse_test.go

@@ -71,16 +71,24 @@ func TestParse(t *testing.T) {
 			continue
 		}
 
-		expected := &ast.ViewMeta{}
-		_ = yaml.Unmarshal(outputData, expected)
-		if !assertly.AssertValues(t, expected, viewMeta, testcase.description) {
-			actualBytes, _ := yaml.Marshal(viewMeta)
-			fmt.Println(string(actualBytes))
-			fmt.Println(string(outputData))
+		actualMeta, _ := yaml.Marshal(viewMeta)
+		expected := normalize(outputData)
+		actual := normalize(actualMeta)
+		if !assertly.AssertValues(t, string(expected), string(actual), testcase.description) {
+			fmt.Println(string(expected))
+			fmt.Println(string(actual))
 		}
 	}
 }
 
+func normalize(b []byte) []byte {
+	aMap := map[string]interface{}{}
+
+	_ = yaml.Unmarshal(b, aMap)
+	result, _ := yaml.Marshal(aMap)
+	return result
+}
+
 func TestExtractCondBlock(t *testing.T) {
 	var testCases = []struct {
 		description string

cmd/ast/testdata/case003/output.yaml

@@ -4,6 +4,9 @@ parameters:
     kind: query
     required: true
     type: string
+    assumed: true
+    typer:
+      columnname: id
 source: |-
   SELECT *, (
       SELECT abc