feat: Migrate stage to the new sdk

pkg/sdk/client.go

@@ -50,6 +50,7 @@ type Client struct {
 	SessionPolicies  SessionPolicies
 	Sessions         Sessions
 	Shares           Shares
+	Stages           Stages
 	Streams          Streams
 	Tags             Tags
 	Tasks            Tasks
@@ -177,6 +178,7 @@ func (c *Client) initialize() {
 	c.SessionPolicies = &sessionPolicies{client: c}
 	c.Sessions = &sessions{client: c}
 	c.Shares = &shares{client: c}
+	c.Stages = &stages{client: c}
 	c.Streams = &streams{client: c}
 	c.SystemFunctions = &systemFunctions{client: c}
 	c.Tags = &tags{client: c}

pkg/sdk/file_format.go

@@ -201,7 +201,7 @@ func (row FileFormatRow) convert() *FileFormat {
 
 type FileFormatType string
 
-const (
+var (
 	FileFormatTypeCSV     FileFormatType = "CSV"
 	FileFormatTypeJSON    FileFormatType = "JSON"
 	FileFormatTypeAvro    FileFormatType = "AVRO"
@@ -329,6 +329,7 @@ type CreateFileFormatOptions struct {
 	IfNotExists *bool                  `ddl:"keyword" sql:"IF NOT EXISTS"`
 	name        SchemaObjectIdentifier `ddl:"identifier"`
 	Type        FileFormatType         `ddl:"parameter" sql:"TYPE"`
+	Comment     *string                `ddl:"parameter,single_quotes" sql:"SET COMMENT"`
 
 	FileFormatTypeOptions
 }
@@ -376,8 +377,9 @@ type AlterFileFormatOptions struct {
 	IfExists   *bool                  `ddl:"keyword" sql:"IF EXISTS"`
 	name       SchemaObjectIdentifier `ddl:"identifier"`
 
-	Rename *AlterFileFormatRenameOptions
-	Set    *FileFormatTypeOptions `ddl:"list,no_comma" sql:"SET"`
+	Rename     *AlterFileFormatRenameOptions
+	Set        *FileFormatTypeOptions `ddl:"list,no_comma" sql:"SET"`
+	SetComment *string                `ddl:"parameter,single_quotes" sql:"SET COMMENT"`
 }
 
 func (opts *AlterFileFormatOptions) validate() error {
@@ -466,8 +468,6 @@ type FileFormatTypeOptions struct {
 	XMLDisableAutoConvert       *bool           `ddl:"parameter" sql:"DISABLE_AUTO_CONVERT"`
 	XMLReplaceInvalidCharacters *bool           `ddl:"parameter" sql:"REPLACE_INVALID_CHARACTERS"`
 	XMLSkipByteOrderMark        *bool           `ddl:"parameter" sql:"SKIP_BYTE_ORDER_MARK"`
-
-	Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
 }
 
 func (opts *FileFormatTypeOptions) fieldsByType() map[FileFormatType][]any {

pkg/sdk/file_format_test.go

@@ -142,7 +142,6 @@ func TestFileFormatsCreate(t *testing.T) {
 			name:        NewSchemaObjectIdentifier("db4", "schema5", "format6"),
 			IfNotExists: Bool(true),
 			Type:        FileFormatTypeXML,
-
 			FileFormatTypeOptions: FileFormatTypeOptions{
 				XMLCompression:          &XMLCompressionZstd,
 				XMLIgnoreUTF8Errors:     Bool(true),
@@ -151,9 +150,8 @@ func TestFileFormatsCreate(t *testing.T) {
 				XMLDisableSnowflakeData: Bool(true),
 				XMLDisableAutoConvert:   Bool(true),
 				XMLSkipByteOrderMark:    Bool(true),
-
-				Comment: String("test file format"),
 			},
+			Comment: String("test file format"),
 		}
 		assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE TEMPORARY FILE FORMAT IF NOT EXISTS "db4"."schema5"."format6" TYPE = XML COMPRESSION = ZSTD IGNORE_UTF8_ERRORS = true PRESERVE_SPACE = true STRIP_OUTER_ELEMENT = true DISABLE_SNOWFLAKE_DATA = true DISABLE_AUTO_CONVERT = true SKIP_BYTE_ORDER_MARK = true COMMENT = 'test file format'`)
 	})
@@ -162,7 +160,6 @@ func TestFileFormatsCreate(t *testing.T) {
 		opts := &CreateFileFormatOptions{
 			name: NewSchemaObjectIdentifier("test_db", "test_schema", "test_file_format"),
 			Type: FileFormatTypeCSV,
-
 			FileFormatTypeOptions: FileFormatTypeOptions{
 				CSVNullIf:                     &[]NullString{{"NULL"}},
 				CSVSkipBlankLines:             Bool(false),
@@ -171,9 +168,8 @@ func TestFileFormatsCreate(t *testing.T) {
 				CSVReplaceInvalidCharacters:   Bool(false),
 				CSVEmptyFieldAsNull:           Bool(false),
 				CSVSkipByteOrderMark:          Bool(false),
-
-				Comment: String("great comment"),
 			},
+			Comment: String("great comment"),
 		}
 		assertOptsValidAndSQLEquals(t, opts, `CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = CSV SKIP_BLANK_LINES = false TRIM_SPACE = false NULL_IF = ('NULL') ERROR_ON_COLUMN_COUNT_MISMATCH = true REPLACE_INVALID_CHARACTERS = false EMPTY_FIELD_AS_NULL = false SKIP_BYTE_ORDER_MARK = false COMMENT = 'great comment'`)
 	})

pkg/sdk/poc/README.md

@@ -99,6 +99,13 @@ find a better solution to solve the issue (add more logic to the templates ?)
 - cleanup the design of builders in DSL (e.g. why transformer has to be always added?)
 - generate getters for requests, at least for identifier/name
 - generate integration tests in child package (because now we keep them in `testint` package)
+- struct_to_builder is not supporting templated-like values. See stages_def.go where in SQL there could be value, where 'n' can be replaced with any number
+  - `SKIP_FILE_n` - this looks more like keyword without a space between SQL prefix and int
+  - `SKIP_FILE_n%` (e.g. `SKIP_FILE_123%`) - this is more template-like behaviour, notice that 'n' is inside the value (we cannot reproduce that right now with struct_to_builder capabilities)
+- fix builder generation 
+  - we can add `flatten` option in cases where some sql structs had to be nested to create correct sql representation
+    - for example encryption options in `stages_def.go` (instead of calling `.WithEncryption(NewEncryptionRequest(encryption))` we could call `.WithEncryption(encryption)`)
+  - operation names (or their sql struct names) should dictate more how constructors are made
 
 ##### Known issues
 - generating two converts when Show and Desc use the same data structure

pkg/sdk/poc/generator/field.go

@@ -138,7 +138,7 @@ func (f *Field) PathWithRoot() string {
 	if f.IsRoot() {
 		return f.Name
 	} else {
-		return fmt.Sprintf("%s.%s", f.Parent.Path(), f.Name)
+		return fmt.Sprintf("%s.%s", f.Parent.PathWithRoot(), f.Name)
 	}
 }
 

pkg/sdk/poc/generator/field_transformers.go

@@ -132,6 +132,7 @@ type ListTransformer struct {
 	sqlPrefix   string
 	parentheses string
 	equals      string
+	comma       string
 }
 
 func ListOptions() *ListTransformer {
@@ -158,6 +159,11 @@ func (v *ListTransformer) NoEquals() *ListTransformer {
 	return v
 }
 
+func (v *ListTransformer) NoComma() *ListTransformer {
+	v.equals = "no_comma"
+	return v
+}
+
 func (v *ListTransformer) SQL(sqlPrefix string) *ListTransformer {
 	v.sqlPrefix = sqlPrefix
 	return v
@@ -171,6 +177,7 @@ func (v *ListTransformer) Transform(f *Field) *Field {
 	addTagIfMissing(f.Tags, "sql", v.sqlPrefix)
 	addTagIfMissing(f.Tags, "ddl", v.parentheses)
 	addTagIfMissing(f.Tags, "ddl", v.equals)
+	addTagIfMissing(f.Tags, "ddl", v.comma)
 	return f
 }
 

pkg/sdk/poc/generator/keyword_builders.go

@@ -5,6 +5,11 @@ func (v *QueryStruct) OptionalSQL(sql string) *QueryStruct {
 	return v
 }
 
+func (v *QueryStruct) NamedList(sql string, itemKind string) *QueryStruct {
+	v.fields = append(v.fields, NewField(sqlToFieldName(sql, true), KindOfSlice(itemKind), Tags().Keyword().SQL(sql), nil))
+	return v
+}
+
 func (v *QueryStruct) OrReplace() *QueryStruct {
 	return v.OptionalSQL("OR REPLACE")
 }
@@ -42,24 +47,17 @@ func (v *QueryStruct) OptionalNumber(name string, transformer *KeywordTransforme
 }
 
 func (v *QueryStruct) OptionalLimitFrom() *QueryStruct {
-	v.fields = append(v.fields, NewField("Limit", "*LimitFrom", Tags().Keyword().SQL("LIMIT"), nil))
-	return v
-}
-
-// SessionParameters *SessionParameters `ddl:"list,no_parentheses"`
-func (v *QueryStruct) SessionParameters() *QueryStruct {
-	v.fields = append(v.fields, NewField("SessionParameters", "*SessionParameters", Tags().List().NoParentheses(), nil).withValidations(NewValidation(ValidateValue, "SessionParameters")))
-	return v
+	return v.PredefinedQueryStructField("Limit", "*LimitFrom", KeywordOptions().SQL("LIMIT"))
 }
 
 func (v *QueryStruct) OptionalSessionParameters() *QueryStruct {
-	v.fields = append(v.fields, NewField("SessionParameters", "*SessionParameters", Tags().List().NoParentheses(), nil).withValidations(NewValidation(ValidateValue, "SessionParameters")))
-	return v
+	return v.PredefinedQueryStructField("SessionParameters", "*SessionParameters", ListOptions().NoParentheses()).
+		WithValidation(ValidateValue, "SessionParameters")
 }
 
 func (v *QueryStruct) OptionalSessionParametersUnset() *QueryStruct {
-	v.fields = append(v.fields, NewField("SessionParametersUnset", "*SessionParametersUnset", Tags().List().NoParentheses(), nil).withValidations(NewValidation(ValidateValue, "SessionParametersUnset")))
-	return v
+	return v.PredefinedQueryStructField("SessionParametersUnset", "*SessionParametersUnset", ListOptions().NoParentheses()).
+		WithValidation(ValidateValue, "SessionParametersUnset")
 }
 
 func (v *QueryStruct) NamedListWithParens(sqlPrefix string, listItemKind string, transformer *KeywordTransformer) *QueryStruct {
@@ -85,8 +83,7 @@ func (v *QueryStruct) OptionalSetTags() *QueryStruct {
 }
 
 func (v *QueryStruct) setTags(transformer *KeywordTransformer) *QueryStruct {
-	v.fields = append(v.fields, NewField("SetTags", "[]TagAssociation", Tags().Keyword().SQL("SET TAG"), transformer))
-	return v
+	return v.PredefinedQueryStructField("SetTags", "[]TagAssociation", transformer)
 }
 
 func (v *QueryStruct) UnsetTags() *QueryStruct {
@@ -98,28 +95,23 @@ func (v *QueryStruct) OptionalUnsetTags() *QueryStruct {
 }
 
 func (v *QueryStruct) unsetTags(transformer *KeywordTransformer) *QueryStruct {
-	v.fields = append(v.fields, NewField("UnsetTags", "[]ObjectIdentifier", Tags().Keyword().SQL("UNSET TAG"), transformer))
-	return v
+	return v.PredefinedQueryStructField("UnsetTags", "[]ObjectIdentifier", transformer)
 }
 
 func (v *QueryStruct) OptionalLike() *QueryStruct {
-	v.fields = append(v.fields, NewField("Like", "*Like", Tags().Keyword().SQL("LIKE"), nil))
-	return v
+	return v.PredefinedQueryStructField("Like", "*Like", KeywordOptions().SQL("LIKE"))
 }
 
 func (v *QueryStruct) OptionalIn() *QueryStruct {
-	v.fields = append(v.fields, NewField("In", "*In", Tags().Keyword().SQL("IN"), nil))
-	return v
+	return v.PredefinedQueryStructField("In", "*In", KeywordOptions().SQL("IN"))
 }
 
 func (v *QueryStruct) OptionalStartsWith() *QueryStruct {
-	v.fields = append(v.fields, NewField("StartsWith", "*string", Tags().Parameter().NoEquals().SingleQuotes().SQL("STARTS WITH"), nil))
-	return v
+	return v.PredefinedQueryStructField("StartsWith", "*string", ParameterOptions().NoEquals().SingleQuotes().SQL("STARTS WITH"))
 }
 
 func (v *QueryStruct) OptionalLimit() *QueryStruct {
-	v.fields = append(v.fields, NewField("Limit", "*LimitFrom", Tags().Keyword().SQL("LIMIT"), nil))
-	return v
+	return v.PredefinedQueryStructField("Limit", "*LimitFrom", KeywordOptions().SQL("LIMIT"))
 }
 
 func (v *QueryStruct) OptionalCopyGrants() *QueryStruct {

pkg/sdk/poc/generator/list_builders.go

@@ -0,0 +1,6 @@
+package generator
+
+func (v *QueryStruct) List(name string, itemKind string, transformer *ListTransformer) *QueryStruct {
+	v.fields = append(v.fields, NewField(name, KindOfSlice(itemKind), Tags().List(), transformer))
+	return v
+}

pkg/sdk/poc/generator/parameter_builders.go

@@ -34,7 +34,7 @@ func (v *QueryStruct) OptionalNumberAssignment(sqlPrefix string, transformer *Pa
 }
 
 func (v *QueryStruct) TextAssignment(sqlPrefix string, transformer *ParameterTransformer) *QueryStruct {
-	return v.Assignment(sqlPrefix, "string", transformer)
+	return v.Assignment(sqlPrefix, "string", transformer.Required())
 }
 
 func (v *QueryStruct) OptionalTextAssignment(sqlPrefix string, transformer *ParameterTransformer) *QueryStruct {

pkg/sdk/poc/generator/query_struct.go

@@ -32,6 +32,11 @@ func (v *QueryStruct) WithValidation(validationType ValidationType, fieldNames .
 	return v
 }
 
+func (v *QueryStruct) PredefinedQueryStructField(name string, kind string, transformer FieldTransformer) *QueryStruct {
+	v.fields = append(v.fields, NewField(name, kind, Tags(), transformer))
+	return v
+}
+
 func (v *QueryStruct) QueryStructField(name string, queryStruct *QueryStruct, transformer FieldTransformer) *QueryStruct {
 	return v.queryStructField(name, queryStruct, "", transformer)
 }

pkg/sdk/poc/main.go

@@ -23,6 +23,7 @@ var definitionMapping = map[string]*generator.Interface{
 	"streams_def.go":           sdk.StreamsDef,
 	"application_roles_def.go": sdk.ApplicationRolesDef,
 	"views_def.go":             sdk.ViewsDef,
+	"stages_def.go":            sdk.StagesDef,
 }
 
 func main() {