feat: support Milvus 2.5 function feature by bypassing SDK

Milvus 2.5 introduced the function feature, which is not well-supported by the older SDK versions. The new SDK hides many internal IDs used by the backup tool. To address this, the backup tool now interacts with Milvus directly via gRPC, bypassing the SDK.

Signed-off-by: huanghaoyuanhhy <haoyuan.huang@zilliz.com>

.github/workflows/main.yaml

@@ -41,7 +41,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Build
@@ -180,7 +180,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Build
@@ -270,7 +270,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Build
@@ -402,7 +402,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: "true"
 
       - name: Creating kind cluster
@@ -550,7 +550,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Build

.github/workflows/nightly.yaml

@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Creating kind cluster
@@ -147,7 +147,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Creating kind cluster

.github/workflows/perf.yaml

@@ -29,7 +29,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18.0'
+          go-version: '1.22'
           cache: true
 
       - name: Creating kind cluster

.github/workflows/release.yaml

@@ -18,7 +18,7 @@ jobs:
       - run: git fetch --force --tags
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.18
+          go-version: 1.22
           cache: true
       - uses: docker/login-action@v3
         with:
@@ -46,7 +46,7 @@ jobs:
       - run: git fetch --force --tags
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.22
           cache: true
 
       - name: Set up QEMU

Dockerfile

@@ -1,5 +1,4 @@
-
-FROM golang:1.18 AS builder
+FROM golang:1.22 AS builder
 
 ENV CGO_ENABLED=0
 

Makefile

@@ -1,5 +1,6 @@
 # Variables
 BINARY_NAME=milvus-backup
+PKG := github.com/zilliztech/milvus-backup
 VERSION=$(shell git describe --tags --always)
 COMMIT=$(shell git rev-parse --short HEAD)
 DATE=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
@@ -10,10 +11,15 @@ all: gen build
 # Build the binary
 build:
 	@echo "Building binary..."
-	GO111MODULE=on CGO_ENABLED=0 go build -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE)" -o $(BINARY_NAME)
+	GO111MODULE=on CGO_ENABLED=0 go build -ldflags "-X version.version=$(VERSION) -X version.commit=$(COMMIT) -X version.date=$(DATE)" -o $(BINARY_NAME)
 
 gen:
 	./scripts/gen_swag.sh
 	./scripts/gen_proto.sh
 
+fmt:
+	@echo Formatting code...
+	@goimports -w --local $(PKG) ./
+	@echo Format code done
+
 .PHONY: all build gen

cmd/backup_yaml.go

@@ -3,9 +3,11 @@ package cmd
 import (
 	"fmt"
 	"strings"
+
 	"github.com/spf13/cobra"
-	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"gopkg.in/yaml.v3"
+
+	"github.com/zilliztech/milvus-backup/core/paramtable"
 )
 
 var configCmd = &cobra.Command{

cmd/check.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 )
@@ -20,7 +21,7 @@ var checkCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		backupContext := core.CreateBackupContext(context, params)
+		backupContext := core.CreateBackupContext(context, &params)
 
 		resp := backupContext.Check(context)
 		fmt.Println(resp)

cmd/create.go

@@ -8,6 +8,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
@@ -35,7 +36,7 @@ var createBackupCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		backupContext := core.CreateBackupContext(context, params)
+		backupContext := core.CreateBackupContext(context, &params)
 
 		start := time.Now().Unix()
 		var collectionNameArr []string

cmd/delete.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
@@ -24,7 +25,7 @@ var deleteBackupCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		backupContext := core.CreateBackupContext(context, params)
+		backupContext := core.CreateBackupContext(context, &params)
 
 		resp := backupContext.DeleteBackup(context, &backuppb.DeleteBackupRequest{
 			BackupName: deleteBackName,

cmd/get.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
@@ -27,7 +28,7 @@ var getBackupCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		backupContext := core.CreateBackupContext(context, params)
+		backupContext := core.CreateBackupContext(context, &params)
 
 		resp := backupContext.GetBackup(context, &backuppb.GetBackupRequest{
 			BackupName:    getBackName,

cmd/list.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
@@ -25,7 +26,7 @@ var listBackupCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		backupContext := core.CreateBackupContext(context, params)
+		backupContext := core.CreateBackupContext(context, &params)
 
 		backups := backupContext.ListBackups(context, &backuppb.ListBackupsRequest{
 			CollectionName: collectionName,

cmd/restore.go

@@ -8,6 +8,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
@@ -43,7 +44,7 @@ var restoreBackupCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		backupContext := core.CreateBackupContext(context, params)
+		backupContext := core.CreateBackupContext(context, &params)
 		log.Info("restore cmd input args", zap.Strings("args", args))
 		start := time.Now().Unix()
 		var collectionNameArr []string

cmd/root.go

@@ -23,7 +23,7 @@ var rootCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		Error(cmd, args, errors.New("unrecognized command"))
 	},
-	PersistentPreRun: func(cmd *cobra.Command, args []string){
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		setEnvs(yamlOverrides)
 	},
 }

cmd/server.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 
 	"github.com/spf13/cobra"
+
 	"github.com/zilliztech/milvus-backup/core"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 )
@@ -31,7 +32,7 @@ var serverCmd = &cobra.Command{
 		params.Init()
 
 		context := context.Background()
-		server, err := core.NewServer(context, params, core.Port(port))
+		server, err := core.NewServer(context, &params, core.Port(port))
 		if err != nil {
 			fmt.Errorf("fail to create backup server, %s", err.Error())
 		}

core/backup_context.go

@@ -3,16 +3,15 @@ package core
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"path"
 	"sync"
 	"time"
 
-	gomilvus "github.com/milvus-io/milvus-sdk-go/v2/client"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
+	"github.com/zilliztech/milvus-backup/core/client"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
 	"github.com/zilliztech/milvus-backup/core/storage"
@@ -40,10 +39,10 @@ type BackupContext struct {
 	// lock to make sure only one backup is creating or restoring
 	mu      sync.Mutex
 	started bool
-	params  paramtable.BackupParams
+	params  *paramtable.BackupParams
 
 	// milvus client
-	milvusClient *MilvusClient
+	milvusClient client.Milvus
 
 	// data storage client
 	milvusStorageClient storage.ChunkManager
@@ -63,32 +62,36 @@ type BackupContext struct {
 	bulkinsertWorkerPools      map[string]*common.WorkerPool
 }
 
-func CreateMilvusClient(ctx context.Context, params paramtable.BackupParams) (gomilvus.Client, error) {
-	milvusEndpoint := params.MilvusCfg.Address + ":" + params.MilvusCfg.Port
-	log.Debug("Start Milvus client", zap.String("endpoint", milvusEndpoint))
-	var c gomilvus.Client
-	var err error
-	if params.MilvusCfg.AuthorizationEnabled && params.MilvusCfg.User != "" && params.MilvusCfg.Password != "" {
-		if params.MilvusCfg.TLSMode == 0 {
-			c, err = gomilvus.NewDefaultGrpcClientWithAuth(ctx, milvusEndpoint, params.MilvusCfg.User, params.MilvusCfg.Password)
-		} else if params.MilvusCfg.TLSMode == 1 || params.MilvusCfg.TLSMode == 2 {
-			c, err = gomilvus.NewDefaultGrpcClientWithTLSAuth(ctx, milvusEndpoint, params.MilvusCfg.User, params.MilvusCfg.Password)
-		} else {
-			log.Error("milvus.TLSMode is not illegal, support value 0, 1, 2")
-			return nil, errors.New("milvus.TLSMode is not illegal, support value 0, 1, 2")
-		}
-	} else {
-		c, err = gomilvus.NewGrpcClient(ctx, milvusEndpoint)
+func CreateMilvusClient(ctx context.Context, params *paramtable.BackupParams) (client.Milvus, error) {
+	ep := params.MilvusCfg.Address + ":" + params.MilvusCfg.Port
+	log.Debug("Start Milvus client", zap.String("endpoint", ep))
+
+	var tlsAuth bool
+	switch params.MilvusCfg.TLSMode {
+	case 0:
+		tlsAuth = false
+	case 1, 2:
+		tlsAuth = true
+	default:
+		log.Error("milvus.TLSMode is not illegal, support value 0, 1, 2")
 	}
+
+	cfg := &client.MilvusConfig{
+		Address:       ep,
+		EnableTLSAuth: tlsAuth,
+		Username:      params.MilvusCfg.User,
+		Password:      params.MilvusCfg.Password,
+	}
+	cli, err := client.NewMilvus(cfg)
 	if err != nil {
-		log.Error("failed to connect to milvus", zap.Error(err))
-		return nil, err
+		log.Error("failed to create milvus client", zap.Error(err))
+		return nil, fmt.Errorf("failed to create milvus client: %w", err)
 	}
-	return c, nil
+	return cli, nil
 }
 
 // Deprecated
-func createStorageClient(ctx context.Context, params paramtable.BackupParams) (storage.ChunkManager, error) {
+func createStorageClient(ctx context.Context, params *paramtable.BackupParams) (storage.ChunkManager, error) {
 	minioEndPoint := params.MinioCfg.Address + ":" + params.MinioCfg.Port
 	log.Debug("Start minio client",
 		zap.String("address", minioEndPoint),
@@ -128,7 +131,7 @@ func (b *BackupContext) Close() error {
 	return nil
 }
 
-func CreateBackupContext(ctx context.Context, params paramtable.BackupParams) *BackupContext {
+func CreateBackupContext(ctx context.Context, params *paramtable.BackupParams) *BackupContext {
 	return &BackupContext{
 		ctx:                   ctx,
 		params:                params,
@@ -141,16 +144,14 @@ func CreateBackupContext(ctx context.Context, params paramtable.BackupParams) *B
 	}
 }
 
-func (b *BackupContext) getMilvusClient() *MilvusClient {
+func (b *BackupContext) getMilvusClient() client.Milvus {
 	if b.milvusClient == nil {
 		milvusClient, err := CreateMilvusClient(b.ctx, b.params)
 		if err != nil {
 			log.Error("failed to initial milvus client", zap.Error(err))
 			panic(err)
 		}
-		b.milvusClient = &MilvusClient{
-			client: milvusClient,
-		}
+		b.milvusClient = milvusClient
 	}
 	return b.milvusClient
 }