Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support nullable field #440

Merged
merged 1 commit into from
Oct 16, 2024
Merged

Support nullable field #440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions core/backup_impl_create_backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"strings"
"time"

"github.com/golang/protobuf/proto"
jsoniter "github.com/json-iterator/go"
"github.com/milvus-io/milvus-sdk-go/v2/entity"
"github.com/samber/lo"
Expand Down Expand Up @@ -236,7 +237,7 @@ func (b *BackupContext) backupCollectionPrepare(ctx context.Context, backupInfo
}
fields := make([]*backuppb.FieldSchema, 0)
for _, field := range completeCollection.Schema.Fields {
fields = append(fields, &backuppb.FieldSchema{
fieldBak := &backuppb.FieldSchema{
FieldID: field.ID,
Name: field.Name,
IsPrimaryKey: field.PrimaryKey,
Expand All @@ -247,8 +248,18 @@ func (b *BackupContext) backupCollectionPrepare(ctx context.Context, backupInfo
IndexParams: utils.MapToKVPair(field.IndexParams),
IsDynamic: field.IsDynamic,
IsPartitionKey: field.IsPartitionKey,
Nullable: field.Nullable,
ElementType: backuppb.DataType(field.ElementType),
})
}
defaultValue := field.DefaultValue
if defaultValue != nil {
bytes, err := proto.Marshal(field.DefaultValue)
if err != nil {
return err
}
fieldBak.DefaultValueProto = string(bytes)
}
fields = append(fields, fieldBak)
}
schema := &backuppb.CollectionSchema{
Name: completeCollection.Schema.CollectionName,
Expand Down
17 changes: 15 additions & 2 deletions core/backup_impl_restore_backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"time"

"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
jsoniter "github.com/json-iterator/go"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
gomilvus "github.com/milvus-io/milvus-sdk-go/v2/client"
"github.com/milvus-io/milvus-sdk-go/v2/entity"
"github.com/samber/lo"
Expand Down Expand Up @@ -420,7 +422,7 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
fields := make([]*entity.Field, 0)
hasPartitionKey := false
for _, field := range task.GetCollBackup().GetSchema().GetFields() {
fields = append(fields, &entity.Field{
fieldRestore := &entity.Field{
ID: field.GetFieldID(),
Name: field.GetName(),
PrimaryKey: field.GetIsPrimaryKey(),
Expand All @@ -431,8 +433,19 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
IndexParams: utils.KvPairsMap(field.GetIndexParams()),
IsDynamic: field.GetIsDynamic(),
IsPartitionKey: field.GetIsPartitionKey(),
Nullable: field.GetNullable(),
ElementType: entity.FieldType(field.GetElementType()),
})
}
if field.DefaultValueProto != "" {
defaultValue := &schemapb.ValueField{}
err := proto.Unmarshal([]byte(field.DefaultValueProto), defaultValue)
if err != nil {
return nil, err
}
fieldRestore.DefaultValue = defaultValue
}

fields = append(fields, fieldRestore)
if field.GetIsPartitionKey() {
hasPartitionKey = true
}
Expand Down
2 changes: 2 additions & 0 deletions core/proto/backup.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,8 @@ message FieldSchema {
ValueField default_value = 11; // default_value only support scalars except array and json for now
bool is_dynamic = 12; // mark whether this field is the dynamic field
bool is_partition_key = 13; // enable logic partitions
bool nullable = 14;
string default_value_proto = 15; // json not support marshall oneof, so use proto string
}

/**
Expand Down
444 changes: 231 additions & 213 deletions core/proto/backuppb/backup.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions docs/docs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,9 @@ const docTemplate = `{
"default_value": {
"$ref": "#/definitions/backuppb.ValueField"
},
"default_value_proto": {
"type": "string"
},
"description": {
"type": "string"
},
Expand Down Expand Up @@ -652,6 +655,9 @@ const docTemplate = `{
"name": {
"type": "string"
},
"nullable": {
"type": "boolean"
},
"state": {
"$ref": "#/definitions/backuppb.FieldState"
},
Expand Down
6 changes: 6 additions & 0 deletions docs/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,9 @@
"default_value": {
"$ref": "#/definitions/backuppb.ValueField"
},
"default_value_proto": {
"type": "string"
},
"description": {
"type": "string"
},
Expand Down Expand Up @@ -644,6 +647,9 @@
"name": {
"type": "string"
},
"nullable": {
"type": "boolean"
},
"state": {
"$ref": "#/definitions/backuppb.FieldState"
},
Expand Down
4 changes: 4 additions & 0 deletions docs/swagger.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,8 @@ definitions:
$ref: '#/definitions/backuppb.DataType'
default_value:
$ref: '#/definitions/backuppb.ValueField'
default_value_proto:
type: string
description:
type: string
element_type:
Expand All @@ -286,6 +288,8 @@ definitions:
type: boolean
name:
type: string
nullable:
type: boolean
state:
$ref: '#/definitions/backuppb.FieldState'
type_params:
Expand Down
31 changes: 31 additions & 0 deletions example/null_support/prepare_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
from pymilvus import CollectionSchema, FieldSchema, Collection, connections, DataType, Partition, utility
import numpy as np
import random
import time

connections.connect()
dim = 128
int64_field = FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True)
double_field = FieldSchema(name="nullableFid", dtype=DataType.DOUBLE, nullable=True, is_primary=False, is_clustering_key=True)
int32_field = FieldSchema(name="int32", dtype=DataType.INT64, default_value=10)
string_field = FieldSchema(name="string", dtype=DataType.VARCHAR, max_length=1000, default_value="10")
float_vector = FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim, mmap_enabled=True)
schema = CollectionSchema(fields=[int64_field, double_field, int32_field, string_field, float_vector])
utility.drop_collection("null_test")
collection = Collection("null_test", schema=schema)
res = collection.schema
print(res)


nb = 10000
slice = 100
for i in range(int(nb/slice)):
vectors = [[random.random() for _ in range(dim)] for _ in range(slice)]
data = [[j for j in range(i*slice,(i+1)*slice)], [None for _ in range(slice)],[], ["1" for _ in range(slice)], vectors]
collection.insert(data=data)
print("inserted %d %d" %(i, slice))


collection.flush()
res = collection.num_entities
print(res)
8 changes: 3 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
//github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3
github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265
github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16
github.com/minio/minio-go/v7 v7.0.61
github.com/pkg/errors v0.9.1
github.com/samber/lo v1.39.0
Expand All @@ -33,14 +33,14 @@ require (
go.uber.org/atomic v1.10.0
go.uber.org/zap v1.17.0
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
golang.org/x/sync v0.3.0
golang.org/x/sync v0.8.0
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
google.golang.org/grpc v1.48.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.1
)

require github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a
require github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd

require (
cloud.google.com/go v0.81.0 // indirect
Expand Down Expand Up @@ -112,5 +112,3 @@ require (
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

// replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20240814024750-1d41342d2c27
12 changes: 6 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,10 +378,10 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a h1:0B/8Fo66D8Aa23Il0yrQvg1KKz92tE/BJ5BvkUxxAAk=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265 h1:HxVtvxRflwbsKcGyIEr+2FCktUmm5Ga0oLSq7k0Rlg0=
github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265/go.mod h1:hCubTFcavPtRXyW7CpspzczPvIfq9k/2GB3/qrJkPUA=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd h1:x0b0+foTe23sKcVFseR1DE8+BB08EH6ViiRHaz8PEik=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16 h1:zsA/xt/HJ3HkWUgazbbkoIR8S5mcxAtypvoSKsemhMw=
github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16/go.mod h1:TdzShm5isV4F6kvrd+9V/CKowNIX+H+jvybF0WWts0Y=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI=
Expand Down Expand Up @@ -699,8 +699,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down
Loading