Merge pull request #21 from viant/sqlparser

Sqlparser

cmd/builder.go

@@ -15,7 +15,7 @@ import (
 	"github.com/viant/datly/template/sanitize"
 	"github.com/viant/datly/view"
 	"github.com/viant/parsly"
-	"github.com/viant/sqlx/metadata/ast/query"
+	"github.com/viant/sqlparser/query"
 	"github.com/viant/toolbox"
 	"github.com/viant/toolbox/format"
 	"github.com/viant/xreflect"

cmd/configurer.go

@@ -10,10 +10,10 @@ import (
 	"github.com/viant/datly/template/sanitize"
 	"github.com/viant/datly/view"
 	"github.com/viant/parsly"
-	"github.com/viant/sqlx/metadata/ast/expr"
-	"github.com/viant/sqlx/metadata/ast/node"
-	"github.com/viant/sqlx/metadata/ast/parser"
-	"github.com/viant/sqlx/metadata/ast/query"
+	"github.com/viant/sqlparser"
+	"github.com/viant/sqlparser/expr"
+	"github.com/viant/sqlparser/node"
+	"github.com/viant/sqlparser/query"
 	rdata "github.com/viant/toolbox/data"
 	"net/http"
 	"os"
@@ -194,7 +194,7 @@ func buildExpandedTable(viewName string, table *Table, expandMap rdata.Map, opt
 		return table, nil
 	}
 
-	aQuery, err := parser.ParseQuery(expandMap.ExpandAsText(table.SQL))
+	aQuery, err := sqlparser.ParseQuery(expandMap.ExpandAsText(table.SQL))
 	if err != nil {
 		fmt.Printf("[WARN] couldn't parse epanded SQL for %v\n", viewName)
 	}
@@ -227,7 +227,7 @@ func buildExpandMap(paramsIndex *ParametersIndex) (rdata.Map, error) {
 }
 
 func (c *ViewConfigurer) prepareUnexpanded(viewName string, SQL string, opt *option.RouteConfig, parent *query.Join) (*viewConfig, []*viewParamConfig, error) {
-	aQuery, err := parser.ParseQuery(SQL)
+	aQuery, err := sqlparser.ParseQuery(SQL)
 	if err != nil {
 		fmt.Printf("[WARN] couldn't parse properly SQL for %v\n", viewName)
 	}
@@ -275,7 +275,7 @@ func (c *ViewConfigurer) prepareUnexpanded(viewName string, SQL string, opt *opt
 		if isMetaTable(relViewConfig.unexpandedTable.Name) {
 			holder := getMetaTemplateHolder(relViewConfig.unexpandedTable.Name)
 			result.AddMetaTemplate(join.Alias, holder, relViewConfig.unexpandedTable)
-		} else if isParamPredicate(parser.Stringify(join.On.X)) || relViewConfig.unexpandedTable.ViewConfig.DataViewParameter != nil {
+		} else if isParamPredicate(sqlparser.Stringify(join.On.X)) || relViewConfig.unexpandedTable.ViewConfig.DataViewParameter != nil {
 			relViewConfig.fileName = join.Alias
 			paramOption, err := c.paramIndex.ParamsMetaWithComment(relViewConfig.viewName, join.Comments)
 			if err != nil {
@@ -321,7 +321,7 @@ func (c *ViewConfigurer) prepareUnexpanded(viewName string, SQL string, opt *opt
 }
 
 func (c *ViewConfigurer) getAlias(asStarExpr *expr.Star) string {
-	stringify := parser.Stringify(asStarExpr.X)
+	stringify := sqlparser.Stringify(asStarExpr.X)
 	if index := strings.Index(stringify, "."); index != -1 {
 		return stringify[:index]
 	}
@@ -364,7 +364,7 @@ func isSQLXRelation(rel node.Node) bool {
 		return false
 	}
 
-	candidate := parser.Stringify(rel)
+	candidate := sqlparser.Stringify(rel)
 	return columns.ContainsSelect(candidate) || !columns.CanBeTableName(candidate)
 }
 
@@ -416,7 +416,7 @@ func updateExecViewConfig(stmtType byte, SQLStmt string, view *viewConfig) error
 
 	switch stmtType | ' ' {
 	case 'i':
-		stmt, err := parser.ParseInsert(rawSQL)
+		stmt, err := sqlparser.ParseInsert(rawSQL)
 		if stmt != nil {
 			inheritFromTarget(stmt.Target.X, view, stmt.Target.Comments)
 		}
@@ -425,7 +425,7 @@ func updateExecViewConfig(stmtType byte, SQLStmt string, view *viewConfig) error
 			return err
 		}
 	case 'u':
-		stmt, err := parser.ParseUpdate(rawSQL)
+		stmt, err := sqlparser.ParseUpdate(rawSQL)
 		if stmt != nil {
 			inheritFromTarget(stmt.Target.X, view, stmt.Target.Comments)
 
@@ -436,7 +436,7 @@ func updateExecViewConfig(stmtType byte, SQLStmt string, view *viewConfig) error
 		}
 
 	case 'd':
-		stmt, err := parser.ParseDelete(rawSQL)
+		stmt, err := sqlparser.ParseDelete(rawSQL)
 		if err != nil {
 			return err
 		}
@@ -448,7 +448,7 @@ func updateExecViewConfig(stmtType byte, SQLStmt string, view *viewConfig) error
 }
 
 func inheritFromTarget(target node.Node, view *viewConfig, tableNameComment string) {
-	tableName := parser.Stringify(target)
+	tableName := sqlparser.Stringify(target)
 	view.ensureTableName(tableName)
 	view.ensureOuterAlias(tableName)
 	view.ensureInnerAlias(tableName)

cmd/datly/datly.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/viant/afsc/s3"
 	_ "github.com/viant/bigquery"
 	"github.com/viant/datly/cmd"
+	_ "github.com/viant/dyndb"
 	_ "github.com/viant/scy/kms/blowfish"
 	_ "github.com/viant/sqlx/metadata/product/bigquery"
 	_ "github.com/viant/sqlx/metadata/product/mysql"

cmd/prepare.go

@@ -259,7 +259,6 @@ func (s *Builder) readSinkColumns(ctx context.Context, db *sql.DB, tableName str
 	if err != nil {
 		return nil, err
 	}
-
 	return sinkColumns, nil
 }
 

cmd/sqlx.go

@@ -7,10 +7,10 @@ import (
 	"github.com/viant/datly/cmd/option"
 	"github.com/viant/datly/view"
 	"github.com/viant/parsly"
-	"github.com/viant/sqlx/metadata/ast/expr"
-	"github.com/viant/sqlx/metadata/ast/node"
-	"github.com/viant/sqlx/metadata/ast/parser"
-	"github.com/viant/sqlx/metadata/ast/query"
+	"github.com/viant/sqlparser"
+	"github.com/viant/sqlparser/expr"
+	"github.com/viant/sqlparser/node"
+	"github.com/viant/sqlparser/query"
 	rdata "github.com/viant/toolbox/data"
 	"strings"
 )
@@ -52,7 +52,7 @@ func newViewConfig(viewName string, fileName string, parent *query.Join, aTable
 }
 
 func buildTableFromSQL(SQL string, routeOpt *option.RouteConfig) (*Table, error) {
-	aQuery, err := parser.ParseQuery(SQL)
+	aQuery, err := sqlparser.ParseQuery(SQL)
 	if err != nil {
 		return nil, err
 	}
@@ -115,7 +115,7 @@ func buildTable(x node.Node, routeOpt *option.RouteConfig) (*Table, error) {
 func extractTableName(node node.Node) (name string, SQL string) {
 	switch actual := node.(type) {
 	case *expr.Selector, *expr.Ident:
-		return parser.Stringify(actual), ""
+		return sqlparser.Stringify(actual), ""
 	}
 
 	return "", ""
@@ -140,15 +140,15 @@ func extractTableSQL(actual *expr.Raw) (name string, SQL string) {
 func UpdateTableSettings(table *Table, routeOpt *option.RouteConfig) error {
 	tableSQL := expandConsts(table.SQL, routeOpt)
 	innerSQL, _ := ExtractCondBlock(tableSQL)
-	innerQuery, err := parser.ParseQuery(innerSQL)
+	innerQuery, err := sqlparser.ParseQuery(innerSQL)
 	fmt.Printf("innerSQL %v %v\n", tableSQL, err)
 
 	if innerQuery != nil && innerQuery.From.X != nil {
-		table.Name = strings.Trim(parser.Stringify(innerQuery.From.X), "`")
+		table.Name = strings.Trim(sqlparser.Stringify(innerQuery.From.X), "`")
 		table.Inner = selectItemToColumn(innerQuery, routeOpt)
 		if len(innerQuery.Joins) > 0 {
 			for _, join := range innerQuery.Joins {
-				table.Deps[Alias(join.Alias)] = TableName(strings.Trim(parser.Stringify(join.With), "`"))
+				table.Deps[Alias(join.Alias)] = TableName(strings.Trim(sqlparser.Stringify(join.With), "`"))
 			}
 		}
 
@@ -220,7 +220,7 @@ func newKey(s *expr.Selector, table *Table) (*key, error) {
 	tableName := table.Name
 	byAlias := table.Inner.ByAlias()
 
-	colName := parser.Stringify(s.X)
+	colName := sqlparser.Stringify(s.X)
 	field := colName
 
 	for _, column := range table.Inner {
@@ -268,7 +268,7 @@ func fieldName(byAlias map[string]*Column, alias string) (string, bool) {
 }
 
 func hasOneCardinalityPredicate(n node.Node) bool {
-	predicate := parser.Stringify(n)
+	predicate := sqlparser.Stringify(n)
 	return strings.Contains(predicate, " 1 = 1")
 }
 
@@ -320,7 +320,7 @@ func appendItem(item *query.Item, result *[]*Column, route *option.RouteConfig)
 func getColumn(item *query.Item) (*Column, error) {
 	switch actual := item.Expr.(type) {
 	case *expr.Call:
-		call := parser.Stringify(actual)
+		call := sqlparser.Stringify(actual)
 		lcCall := strings.ToLower(call)
 		if item.DataType == "" {
 			item.DataType = "string"
@@ -336,24 +336,24 @@ func getColumn(item *query.Item) (*Column, error) {
 	case *expr.Ident:
 		return &Column{Name: actual.Name, Alias: item.Alias, DataType: item.DataType}, nil
 	case *expr.Selector:
-		return &Column{Name: parser.Stringify(actual.X), Ns: actual.Name, DataType: item.DataType, Alias: item.Alias}, nil
+		return &Column{Name: sqlparser.Stringify(actual.X), Ns: actual.Name, DataType: item.DataType, Alias: item.Alias}, nil
 	case *expr.Star:
 		switch star := actual.X.(type) {
 		case *expr.Ident:
 			return &Column{Name: star.Name, Except: actual.Except}, nil
 		case *expr.Selector:
-			return &Column{Name: parser.Stringify(star.X), Ns: star.Name, Except: actual.Except, Comments: actual.Comments}, nil
+			return &Column{Name: sqlparser.Stringify(star.X), Ns: star.Name, Except: actual.Except, Comments: actual.Comments}, nil
 		}
 	case *expr.Literal:
 		return &Column{Name: "", Alias: item.Alias, DataType: actual.Kind}, nil
 	case *expr.Binary:
-		enExpr := parser.Stringify(actual)
+		enExpr := sqlparser.Stringify(actual)
 		if item.DataType == "" || (strings.Contains(enExpr, "+") || strings.Contains(enExpr, "-") || strings.Contains(enExpr, "/") || strings.Contains(enExpr, "*")) {
 			item.DataType = "float64"
 		}
 		return &Column{Name: enExpr, Alias: item.Alias, DataType: item.DataType}, nil
 	case *expr.Parenthesis:
-		return &Column{Name: parser.Stringify(actual), Alias: item.Alias, DataType: item.DataType}, nil
+		return &Column{Name: sqlparser.Stringify(actual), Alias: item.Alias, DataType: item.DataType}, nil
 	}
 	return nil, fmt.Errorf("invalid type: %T", item.Expr)
 }

cmd/view.go

@@ -7,6 +7,8 @@ import (
 	"github.com/viant/datly/shared"
 	"github.com/viant/datly/template/columns"
 	"github.com/viant/datly/view"
+	"github.com/viant/sqlparser"
+	"github.com/viant/sqlparser/expr"
 	"github.com/viant/toolbox/format"
 )
 
@@ -38,7 +40,6 @@ func (s *Builder) buildAndAddView(ctx context.Context, viewConfig *viewConfig, s
 	if tableName, err := s.readColumnTypes(ctx, db, table); err != nil {
 		fmt.Printf("[WARN] %v", fmt.Errorf("couldn't read table %v column types %w ", tableName, err).Error())
 	}
-
 	template, err := s.buildTemplate(ctx, viewConfig, parameters)
 	if err != nil {
 		return nil, err
@@ -92,7 +93,7 @@ func (s *Builder) buildAndAddView(ctx context.Context, viewConfig *viewConfig, s
 }
 
 func (s *Builder) readColumnTypes(ctx context.Context, db *sql.DB, table *Table) (string, error) {
-	if err := s.indexColumnsWithLog(ctx, db, table.Name); err != nil {
+	if err := s.indexColumnsWithLog(ctx, db, table); err != nil {
 		return table.Name, err
 	}
 
@@ -101,7 +102,7 @@ func (s *Builder) readColumnTypes(ctx context.Context, db *sql.DB, table *Table)
 			continue
 		}
 
-		if err := s.indexColumnsWithLog(ctx, db, string(v)); err != nil {
+		if err := s.indexColumnsWithLog(ctx, db, &Table{Name: string(v)}); err != nil {
 			return string(v), err
 		}
 	}
@@ -130,7 +131,9 @@ func (s *Builder) ConnectorRef(name string) (*view.Connector, error) {
 	}, nil
 }
 
-func (s *Builder) indexColumnsWithLog(ctx context.Context, db *sql.DB, tableName string) error {
+func (s *Builder) indexColumnsWithLog(ctx context.Context, db *sql.DB, table *Table) error {
+	tableName := table.Name
+
 	if columns.ContainsSelect(tableName) || tableName == "" {
 		return nil
 	}
@@ -140,7 +143,7 @@ func (s *Builder) indexColumnsWithLog(ctx context.Context, db *sql.DB, tableName
 	}
 
 	fmt.Printf("[INFO] reading %v table column types\n", tableName)
-	err := s.indexColumns(ctx, db, tableName)
+	err := s.indexColumns(ctx, db, table)
 	if err != nil {
 		fmt.Printf("[WARN] couldn't read table %v column types\n", tableName)
 	} else {
@@ -150,22 +153,51 @@ func (s *Builder) indexColumnsWithLog(ctx context.Context, db *sql.DB, tableName
 	return err
 }
 
-func (s *Builder) indexColumns(ctx context.Context, db *sql.DB, tableName string) error {
+func (s *Builder) indexColumns(ctx context.Context, db *sql.DB, table *Table) error {
+	tableName := table.Name
 	tableMeta := s.tablesMeta.TableMeta(tableName)
+	if table.SQL != "" {
+		s.discoverySQLColumns(db, table, tableMeta)
+	}
 	ioColumns, err := columns.DetectColumns(context.Background(), db, tableName)
 	if err != nil {
 		return err
 	}
 	tableMeta.AddIoColumns(ioColumns)
-
 	sinkColumns, err := s.readSinkColumns(ctx, db, tableName)
 	if err != nil {
 		return err
 	}
-
 	return tableMeta.AddSinkColumns(sinkColumns)
 }
 
+func (s *Builder) discoverySQLColumns(db *sql.DB, table *Table, tableMeta *TableMeta) {
+	SQL, err := normalizeSQL(table)
+	if err != nil {
+		return
+	}
+	ioColumns, err := columns.DetectColumns(context.Background(), db, SQL)
+	if len(ioColumns) > 0 {
+		tableMeta.AddIoColumns(ioColumns)
+		if len(ioColumns) == len(table.Columns) {
+			for i, column := range ioColumns {
+				table.Columns[i].DataType = column.ScanType().String()
+			}
+		}
+	}
+}
+
+func normalizeSQL(table *Table) (string, error) {
+	aQuery, err := sqlparser.ParseQuery(table.SQL)
+	if err != nil {
+		return "", err
+	}
+	aQuery.Limit = &expr.Literal{Value: "0", Kind: "int"}
+	aQuery.Qualify = nil
+	SQL := sqlparser.Stringify(aQuery)
+	return SQL, nil
+}
+
 func stringsPtr(args ...string) *[]string {
 	return &args
 }

e2e/local/regression/cases/030_insert_acl/Events.go

@@ -0,0 +1,18 @@
+package tmp
+
+import (
+	"time"
+)
+
+type Events struct {
+	Id                int                  `sqlx:"name=ID"`
+	Quantity          int                  `sqlx:"name=QUANTITY"`
+	EventsPerformance []*EventsPerformance `typeName:"EventsPerformance" sqlx:"-"`
+}
+
+type EventsPerformance struct {
+	Id        int       `sqlx:"name=ID"`
+	Price     int       `sqlx:"name=PRICE"`
+	EventId   int       `sqlx:"-"`
+	Timestamp time.Time `sqlx:"name=TIMESTAMP"`
+}

e2e/local/regression/cases/030_insert_acl/expect_t0.json

@@ -0,0 +1,16 @@
+{
+  "id": "@exists@",
+  "quantity": 0,
+  "eventsPerformance": [
+    {
+      "id": "@exists@",
+      "price": 125,
+      "eventId": "@exists@"
+    },
+    {
+      "id": "@exists@",
+      "price": 250,
+      "eventId": "@exists@"
+    }
+  ]
+}

e2e/local/regression/cases/030_insert_acl/expect_t1.json

@@ -0,0 +1,3 @@
+{
+  "Message": "price can't be negative"
+}

e2e/local/regression/cases/030_insert_acl/gen.json

@@ -0,0 +1,5 @@
+{
+  "Name": "$tagId",
+  "URL": "$path/user_acl.sql",
+  "Args": ""
+}