diff --git a/configs/backup.yaml b/configs/backup.yaml index d167f52b..c5de3ed0 100644 --- a/configs/backup.yaml +++ b/configs/backup.yaml @@ -49,6 +49,8 @@ backup: # which means if you set backup.parallelism = 2 backup.copydata.parallelism = 100, there will be 200 copy executing at the same time. # reduce it if blocks your storage's network bandwidth parallelism: 128 + listdata: + parallelism: 32 keepTempFiles: false restore: diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index e0375cda..827b6113 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -439,7 +439,7 @@ func (b *BackupContext) backupCollection(ctx context.Context, backupInfo *backup segmentBackupInfos := make([]*backuppb.SegmentBackupInfo, 0) partSegInfoMap := make(map[int64][]*backuppb.SegmentBackupInfo) mu := sync.Mutex{} - wp, err := common.NewWorkerPool(ctx, b.params.BackupCfg.BackupCopyDataParallelism, RPS) + wp, err := common.NewWorkerPool(ctx, b.params.BackupCfg.BackupListDataParallelism, RPS) if err != nil { return err } diff --git a/core/paramtable/params.go b/core/paramtable/params.go index f84f1867..05ef3e75 100644 --- a/core/paramtable/params.go +++ b/core/paramtable/params.go @@ -36,6 +36,7 @@ type BackupConfig struct { BackupParallelism int RestoreParallelism int + BackupListDataParallelism int BackupCopyDataParallelism int KeepTempFiles bool } @@ -46,6 +47,7 @@ func (p *BackupConfig) init(base *BaseTable) { p.initMaxSegmentGroupSize() p.initBackupParallelism() p.initRestoreParallelism() + p.initBackupListDataParallelism() p.initBackupCopyDataParallelism() p.initKeepTempFiles() } @@ -68,6 +70,11 @@ func (p *BackupConfig) initRestoreParallelism() { p.RestoreParallelism = size } +func (p *BackupConfig) initBackupListDataParallelism() { + size := p.Base.ParseIntWithDefault("backup.listdata.parallelism", 32) + p.BackupListDataParallelism = size +} + func (p *BackupConfig) initBackupCopyDataParallelism() { size := p.Base.ParseIntWithDefault("backup.copydata.parallelism", 128) p.BackupCopyDataParallelism = size