rename minio.cloudProvider to minio.storageType

Signed-off-by: wayblink <anyang.wang@zilliz.com>
zilliztech · Dec 12, 2023 · 31632bc · 31632bc
1 parent 5214359
commit 31632bc
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 34 deletions.
diff --git a/configs/backup.yaml b/configs/backup.yaml
@@ -21,7 +21,8 @@ milvus:
 
 # Related configuration of minio, which is responsible for data persistence for Milvus.
 minio:
-  cloudProvider: "minio" # remote cloud storage provider: s3, gcp, aliyun, azure
+  # cloudProvider: "minio" # deprecated use storageType instead
+  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure
 
   address: localhost # Address of MinIO/S3
   port: 9000   # Port of MinIO/S3

diff --git a/core/paramtable/params.go b/core/paramtable/params.go
@@ -145,17 +145,21 @@ func (p *MilvusConfig) initTLSMode() {
 const (
 	Local               = "local"
 	Minio               = "minio"
+	S3                  = "s3"
 	CloudProviderAWS    = "aws"
 	CloudProviderGCP    = "gcp"
-	CloudProviderAliyun = "ali"
+	CloudProviderAli    = "ali"
+	CloudProviderAliyun = "aliyun"
 	CloudProviderAzure  = "azure"
 )
 
-var supportedCloudProvider = map[string]bool{
+var supportedStorageType = map[string]bool{
 	Local:               true,
 	Minio:               true,
+	S3:                  true,
 	CloudProviderAWS:    true,
 	CloudProviderGCP:    true,
+	CloudProviderAli:    true,
 	CloudProviderAliyun: true,
 	CloudProviderAzure:  true,
 }
@@ -249,7 +253,7 @@ func (p *MinioConfig) initUseIAM() {
 
 func (p *MinioConfig) initCloudProvider() {
 	p.CloudProvider = p.Base.LoadWithDefault("minio.cloudProvider", DefaultMinioCloudProvider)
-	if !supportedCloudProvider[p.CloudProvider] {
+	if !supportedStorageType[p.CloudProvider] {
 		panic("unsupported cloudProvider:" + p.CloudProvider)
 	}
 }
@@ -280,9 +284,10 @@ func (p *MinioConfig) initBackupRootPath() {
 }
 
 func (p *MinioConfig) initStorageType() {
-	engine := p.Base.LoadWithDefault("storage.type",
-		p.Base.LoadWithDefault("minio.type", DefaultStorageType))
-	if !supportedCloudProvider[engine] {
+	engine := p.Base.LoadWithDefault("storage.storageType",
+		p.Base.LoadWithDefault("minio.storageType",
+			p.Base.LoadWithDefault("minio.cloudProvider", DefaultStorageType)))
+	if !supportedStorageType[engine] {
 		panic("unsupported storage type:" + engine)
 	}
 	p.StorageType = engine

diff --git a/core/storage/chunk_manager.go b/core/storage/chunk_manager.go
@@ -7,28 +7,15 @@ import (
 )
 
 func NewChunkManager(ctx context.Context, params paramtable.BackupParams) (ChunkManager, error) {
-	engine := params.MinioCfg.CloudProvider
-	if engine == "azure" {
-		return newAzureChunkManagerWithParams(ctx, params)
-	} else if engine == "local" {
+	engine := params.MinioCfg.StorageType
+	switch engine {
+	case paramtable.Local:
 		return newLocalChunkManagerWithParams(ctx, params)
-	} else {
+	case paramtable.CloudProviderAzure:
+		return newAzureChunkManagerWithParams(ctx, params)
+	default:
 		return newMinioChunkManagerWithParams(ctx, params)
 	}
-	//switch engine {
-	//case "local":
-	//	return newMinioChunkManagerWithParams(ctx, params)
-	//	//return NewLocalChunkManager(RootPath(f.config.rootPath)), nil
-	//case "minio":
-	//case "s3":
-	//case "gcp":
-	//case "aliyun":
-	//	return newMinioChunkManagerWithParams(ctx, params)
-	//case "azure":
-	//	return newAzureChunkManagerWithParams(ctx, params)
-	//default:
-	//	return nil, errors.New("no chunk manager implemented with engine: " + engine)
-	//}
 }
 
 func newMinioChunkManagerWithParams(ctx context.Context, params paramtable.BackupParams) (*MinioChunkManager, error) {
@@ -39,8 +26,8 @@ func newMinioChunkManagerWithParams(ctx context.Context, params paramtable.Backu
 	c.useSSL = params.MinioCfg.UseSSL
 	c.bucketName = params.MinioCfg.BackupBucketName
 	c.rootPath = params.MinioCfg.RootPath
-	c.cloudProvider = params.MinioCfg.CloudProvider
-	c.storageEngine = params.MinioCfg.StorageType
+	//c.cloudProvider = params.MinioCfg.CloudProvider
+	c.storageType = params.MinioCfg.StorageType
 	c.useIAM = params.MinioCfg.UseIAM
 	c.iamEndpoint = params.MinioCfg.IAMEndpoint
 	c.createBucket = true
@@ -55,8 +42,8 @@ func newAzureChunkManagerWithParams(ctx context.Context, params paramtable.Backu
 	c.useSSL = params.MinioCfg.UseSSL
 	c.bucketName = params.MinioCfg.BucketName
 	c.rootPath = params.MinioCfg.RootPath
-	c.cloudProvider = params.MinioCfg.CloudProvider
-	c.storageEngine = params.MinioCfg.StorageType
+	//c.cloudProvider = params.MinioCfg.CloudProvider
+	c.storageType = params.MinioCfg.StorageType
 	c.useIAM = params.MinioCfg.UseIAM
 	c.iamEndpoint = params.MinioCfg.IAMEndpoint
 	c.createBucket = true
@@ -72,8 +59,8 @@ func newAzureChunkManagerWithParams(ctx context.Context, params paramtable.Backu
 func newLocalChunkManagerWithParams(ctx context.Context, params paramtable.BackupParams) (*LocalChunkManager, error) {
 	c := newDefaultConfig()
 	c.rootPath = params.MinioCfg.RootPath
-	c.cloudProvider = params.MinioCfg.CloudProvider
-	c.storageEngine = params.MinioCfg.StorageType
+	//c.cloudProvider = params.MinioCfg.CloudProvider
+	c.storageType = params.MinioCfg.StorageType
 	c.backupRootPath = params.MinioCfg.BackupRootPath
 
 	return NewLocalChunkManager(ctx, c)

diff --git a/core/storage/minio_chunk_manager.go b/core/storage/minio_chunk_manager.go
@@ -64,8 +64,9 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
 	var newMinioFn = minio.New
 	var bucketLookupType = minio.BucketLookupAuto
 
-	switch c.cloudProvider {
+	switch c.storageType {
 	case paramtable.CloudProviderAliyun:
+	case paramtable.CloudProviderAli:
 		// auto doesn't work for aliyun, so we set to dns deliberately
 		bucketLookupType = minio.BucketLookupDNS
 		if c.useIAM {

diff --git a/core/storage/options.go b/core/storage/options.go
@@ -12,8 +12,9 @@ type config struct {
 	useIAM            bool
 	iamEndpoint       string
 
+	// deprecated
 	cloudProvider string
-	storageEngine string
+	storageType   string
 
 	backupAccessKeyID       string
 	backupSecretAccessKeyID string