Skip to content

Commit

Permalink
Add support for building binary columns with records.Build[T]
Browse files Browse the repository at this point in the history
This commit adds `TYPE_BINARY` for schema `v1alpha1` that represents columns of
go type `[]byte`.

`[]byte` columns are mapped to `*array.Binary`. All other options i.e
`encoding`, `compression` etc works just like other supported column types.
  • Loading branch information
gernest committed Jan 25, 2024
1 parent 7dda70b commit b2ead40
Show file tree
Hide file tree
Showing 8 changed files with 265 additions and 124 deletions.
123 changes: 65 additions & 58 deletions gen/proto/go/frostdb/schema/v1alpha1/schema.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

61 changes: 51 additions & 10 deletions internal/records/record_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,11 @@ func NewBuild[T any](mem memory.Allocator) *Build[T] {
)
case reflect.Slice:
switch {
case isBinary(fty):
typ, styp = baseType(fty, dictionary)
fr.typ = styp
fr.nullable = true
fr.build = newFieldBuild(typ, mem, name, true)
case isUUIDSlice(fty):
fr.typ = schemapb.StorageLayout_TYPE_STRING
fr.build = newUUIDSliceField(mem, name)
Expand Down Expand Up @@ -511,7 +516,11 @@ func baseType(fty reflect.Type, dictionary bool) (typ arrow.DataType, sty schema
typ = arrow.BinaryTypes.String
sty = schemapb.StorageLayout_TYPE_STRING
default:
panic("frostdb/dynschema: " + fty.String() + " is npt supported")
if !isBinary(fty) {
panic("frostdb/dynschema: " + fty.String() + " is npt supported")
}
typ = arrow.BinaryTypes.Binary
sty = schemapb.StorageLayout_TYPE_BINARY
}
if dictionary {
typ = &arrow.DictionaryType{
Expand Down Expand Up @@ -616,16 +625,19 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
e.Append(v.Interface().(string))
return nil
}

case *array.BinaryBuilder:
f.buildFunc = func(v reflect.Value) error {
e.Append(v.Bytes())
return nil
}

case *array.BinaryDictionaryBuilder:
f.buildFunc = func(v reflect.Value) error {
if nullable {
if v.IsNil() {
e.AppendNull()
return nil
}
v = v.Elem()
if v.Kind() == reflect.String {
return e.AppendString(v.String())
}
return e.AppendString(v.Interface().(string))
return e.Append(v.Bytes())
}
case *array.ListBuilder:
switch build := e.ValueBuilder().(type) {
Expand Down Expand Up @@ -690,6 +702,19 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
return nil
})
}
case *array.BinaryBuilder:
f.buildFunc = func(v reflect.Value) error {
if v.IsNil() {
e.AppendNull()
return nil
}
e.Append(true)
build.Reserve(v.Len())
return applyBinary(v, func(i []byte) error {
build.Append(i)
return nil
})
}
case *array.BinaryDictionaryBuilder:
f.buildFunc = func(v reflect.Value) error {
if v.Len() == 0 {
Expand All @@ -698,7 +723,10 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
}
e.Append(true)
build.Reserve(v.Len())
return applyString(v, build.AppendString)
if v.Type().Elem().Kind() == reflect.String {
return applyString(v, build.AppendString)
}
return applyBinary(v, build.Append)
}
case *array.BooleanBuilder:
f.buildFunc = func(v reflect.Value) error {
Expand All @@ -720,6 +748,12 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
return
}

func applyBinary(v reflect.Value, apply func([]byte) error) error {
return listApply[[]byte](v, func(v reflect.Value) []byte {
return v.Bytes()
}, apply)
}

func applyString(v reflect.Value, apply func(string) error) error {
return listApply[string](v, func(v reflect.Value) string {
return v.Interface().(string)
Expand Down Expand Up @@ -836,12 +870,19 @@ func walkTag(tag string, f func(key, value string)) {
walkTag(tag, f)
}

var uuidSliceType = reflect.TypeOf([]uuid.UUID{})
var (
uuidSliceType = reflect.TypeOf([]uuid.UUID{})
binaryType = reflect.TypeOf([]byte{})
)

func isUUIDSlice(typ reflect.Type) bool {
return typ.AssignableTo(uuidSliceType)
}

func isBinary(typ reflect.Type) bool {
return typ == binaryType
}

var (
matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])")
Expand Down
Loading

0 comments on commit b2ead40

Please sign in to comment.