update redis lib

README.md

@@ -39,12 +39,11 @@ redis-dumper migrate <source> <destination> --pattern="prefix:*" --sourcePasswor
 
 #### Other flags:
 ```bash
-  --sourceUseTLS bool   "Enable TLS for source Redis" (default true)
-  --targetUseTLS bool   "Enable TLS for target Redis" (default true)
   --logInterval int     "Print current status every N seconds" (default 1)
   --scanCount int       "COUNT parameter for redis SCAN command" (default 1000)
   --parallelDumps int   "Number of parallel dump goroutines" (default 100)
   --pushRoutines int    "Number of parallel restore goroutines" (default 100)
+  --replaceExistingKeys bool    "Existing keys will be replaced" (default false)
 ```
 
 ### 2) Delete keys
@@ -54,7 +53,6 @@ redis-dumper delete <redis> --pattern="prefix:*" --password="Password"
 
 #### Other flags:
 ```bash
-  --useTLS bool           "Enable TLS" (default true)
   --logInterval int       "Print current status every N seconds" (default 1)
   --scanCount int         "COUNT parameter for redis SCAN command" (default 1000)
   --parallelDeletes int   "Number of parallel delete goroutines" (default 100)
@@ -67,7 +65,6 @@ redis-dumper generate <redis> --password="Password"
 
 #### Other flags:
 ```bash
-  --useTLS bool             "Enable TLS" (default true)
   --prefixes []string       "List of prefixes for generated keys" (default {"mykey:", "testkey:"})
   --prefixAmount []string   "Amount of keys to create for each prefix in one iteration" (default {"1", "2"})
   --entryCount int          "Iteration count to perform" (default 1)

go.mod

@@ -1,28 +1,33 @@
 module github.com/appit-online/redis-dumper
 
-go 1.17
+go 1.21
 
 require (
-	github.com/mediocregopher/radix/v4 v4.0.0
-	github.com/spf13/cobra v1.2.1
-	github.com/spf13/viper v1.9.0
+	github.com/redis/go-redis/v9 v9.3.0
+	github.com/spf13/cobra v1.8.0
+	github.com/spf13/viper v1.17.0
 )
 
 require (
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
-	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/tilinna/clock v1.0.2 // indirect
-	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	gopkg.in/ini.v1 v1.66.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

package.json

@@ -1,6 +1,6 @@
 {
   "name": "redis-dumper",
-  "version": "1.0.5",
+  "version": "2.0.0",
   "description": "CLI to migrate redis data from one instance to another, delete data by pattern and create new dummy data in seconds",
   "main": "index.js",
   "scripts": {

pkg/agent/delete.go

@@ -17,15 +17,17 @@ var deleteCmd = &cobra.Command{
 	Use:   "delete <redis>",
 	Short: "Delete keys from redis instance by given pattern",
 	Long: `Delete keys from redis instance by given pattern 
-Url can be provided as just "<host>:<port>" or in Redis URL format: "redis://[:<password>@]<host>:<port>[/<dbIndex>]"`,
+Url can be provided as just "<host>:<port>""`,
 	Args: cobra.MinimumNArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Start deleting keys")
 		ctx := context.Background()
 
 		// create redis clients
-		scannerClient := createClient(ctx, sourcePassword, args[0], sourceUseTLS)
-		deleterClient := createClient(ctx, sourcePassword, args[0], sourceUseTLS)
+		scannerClient := createClient(args[0], sourcePassword)
+		defer scannerClient.Close()
+		deleterClient := createClient(args[0], sourcePassword)
+		defer deleterClient.Close()
 
 		// init core services
 		logger := logger.CreateService()
@@ -34,12 +36,13 @@ Url can be provided as just "<host>:<port>" or in Redis URL format: "redis://[:<
 			scanner.Options{
 				SearchPattern:  pattern,
 				RedisScanCount: scanCount,
+				ParallelDumps:  parallelDeletes,
 			},
 			logger,
 		)
 		deleter := deleter.CreateService(
 			deleterClient,
-			scanner.GetScanChannel(),
+			scanner.GetDumperChannel(),
 			logger,
 		)
 
@@ -58,7 +61,6 @@ Url can be provided as just "<host>:<port>" or in Redis URL format: "redis://[:<
 func init() {
 	RootCmd.AddCommand(deleteCmd)
 
-	deleteCmd.Flags().BoolVar(&sourceUseTLS, "useTLS", true, "Enable TLS - default true")
 	deleteCmd.Flags().StringVar(&pattern, "pattern", "*", "Matching pattern for keys")
 	deleteCmd.Flags().StringVar(&sourcePassword, "password", "", "Password for redis")
 	deleteCmd.Flags().IntVar(&scanCount, "scanCount", 1000, "COUNT parameter for redis SCAN command")

pkg/agent/generator.go

@@ -9,7 +9,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/mediocregopher/radix/v4"
 	"github.com/spf13/cobra"
 )
 
@@ -20,7 +19,7 @@ var generatorCmd = &cobra.Command{
 	Use:   "generate <redis>",
 	Short: "Create random entries in redis instance",
 	Long: `Create random entries in redis instance
-Url can be provided as just "<host>:<port>" or in Redis URL format: "redis://[:<password>@]<host>:<port>[/<dbIndex>]"`,
+Url can be provided as just "<host>:<port>""`,
 	Args: cobra.MinimumNArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Start generating keys")
@@ -32,17 +31,16 @@ Url can be provided as just "<host>:<port>" or in Redis URL format: "redis://[:<
 		}
 
 		fmt.Println("Generated random values: ", args[0])
-		generatorClient := createClient(ctx, sourcePassword, args[0], sourceUseTLS)
+		generatorClient := createClient(args[0], sourcePassword)
 
 		rand.Seed(time.Now().UTC().UnixNano())
 		for j := 0; j < entryCount; j++ {
 			for prefix, number := range randomMap {
 				for i := 0; i < number; i++ {
 					randVal := strconv.Itoa(rand.Int())
-					action := radix.Cmd(nil, "SET", prefix+randVal, randVal)
-					err = generatorClient.Do(ctx, action)
+					err := generatorClient.Do(ctx, "SET", prefix+randVal, randVal).Err()
 					if err != nil {
-						fmt.Println(err)
+						fmt.Println(fmt.Errorf("could not create entry: %w", err))
 					}
 				}
 			}
@@ -78,7 +76,6 @@ func createRandomMap(prefix []string, prefixAmount []string) (map[string]int, er
 func init() {
 	RootCmd.AddCommand(generatorCmd)
 
-	generatorCmd.Flags().BoolVar(&sourceUseTLS, "useTLS", true, "Enable TLS - default true")
 	generatorCmd.Flags().StringVar(&sourcePassword, "password", "", "Password for redis")
 	generatorCmd.Flags().StringArrayVar(&keyPrefix, "prefixes", []string{"mykey:", "testkey:"}, "List of prefixes for generated keys")
 	generatorCmd.Flags().StringArrayVar(&prefixAmount, "prefixAmount", []string{"1", "2"}, "Amount of keys to create for each prefix in one iteration")

pkg/agent/migrate.go

@@ -14,22 +14,26 @@ import (
 )
 
 var parallelDumps, parallelRestores int
+var replaceExistingKeys bool
 
 var migrateCmd = &cobra.Command{
 	Use:   "migrate <source> <destination>",
 	Short: "Migrate keys from source redis instance to destination by given pattern",
 	Long: `Migrate keys from source redis instance to destination by given pattern <source> and <destination> 
 
-Can be provided as just ""<host>:<port>" or in Redis URL format: "redis://[:<password>@]<host>:<port>[/<dbIndex>]"`,
+Can be provided as just ""<host>:<port>""`,
 	Args: cobra.MinimumNArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Start migration")
 		ctx := context.Background()
 
 		// create redis clients
-		scannerClient := createClient(ctx, sourcePassword, args[0], sourceUseTLS)
-		dumperClient := createClient(ctx, sourcePassword, args[0], sourceUseTLS)
-		restorerClient := createClient(ctx, targetPassword, args[1], targetUseTLS)
+		scannerClient := createClient(args[0], sourcePassword)
+		defer scannerClient.Close()
+		dumperClient := createClient(args[0], sourcePassword)
+		defer dumperClient.Close()
+		restorerClient := createClient(args[1], targetPassword)
+		defer restorerClient.Close()
 
 		// init core services
 		logger := logger.CreateService()
@@ -38,15 +42,17 @@ Can be provided as just ""<host>:<port>" or in Redis URL format: "redis://[:<pas
 			scanner.Options{
 				SearchPattern:  pattern,
 				RedisScanCount: scanCount,
+				ParallelDumps:  parallelDumps,
 			},
 			logger,
 		)
 		dumper := dumper.CreateService(
 			dumperClient,
-			scanner.GetScanChannel(),
+			scanner.GetDumperChannel(),
 			logger,
+			parallelRestores,
 		)
-		restorer := restore.CreateService(restorerClient, dumper.GetDumpChannel(), logger)
+		restorer := restore.CreateService(restorerClient, dumper.GetRestorerChannel(), logger, replaceExistingKeys)
 
 		// start processing
 		wgRestore := new(sync.WaitGroup)
@@ -70,11 +76,10 @@ func init() {
 
 	migrateCmd.Flags().StringVar(&pattern, "pattern", "*", "Matching pattern for keys")
 	migrateCmd.Flags().StringVar(&sourcePassword, "sourcePassword", "", "Password of source redis")
-	migrateCmd.Flags().BoolVar(&sourceUseTLS, "sourceUseTLS", true, "Enable TLS for source redis- default true")
 	migrateCmd.Flags().StringVar(&targetPassword, "targetPassword", "", "Password of target redis")
-	migrateCmd.Flags().BoolVar(&targetUseTLS, "targetUseTLS", true, "Enable TLS for target redis- default true")
 	migrateCmd.Flags().IntVar(&scanCount, "scanCount", 1000, "COUNT parameter for redis SCAN command")
 	migrateCmd.Flags().IntVar(&logInterval, "logInterval", 1, "Print current status every N seconds")
 	migrateCmd.Flags().IntVar(&parallelDumps, "parallelDumps", 100, "Number of parallel dump goroutines")
 	migrateCmd.Flags().IntVar(&parallelRestores, "parallelRestores", 100, "Number of parallel restore goroutines")
+	migrateCmd.Flags().BoolVar(&replaceExistingKeys, "replaceExistingKeys", false, "Existing keys will be replaced")
 }

pkg/agent/service.go

@@ -1,25 +1,33 @@
 package agent
 
 import (
-	"context"
+	"crypto/tls"
 	"log"
+	"net"
 
-	"github.com/mediocregopher/radix/v4"
+	"github.com/redis/go-redis/v9"
 )
 
 var pattern, sourcePassword, targetPassword string
 var scanCount, logInterval int
-var sourceUseTLS, targetUseTLS bool
 
-func createClient(ctx context.Context, password string, redisUrl string, enableTLS bool) radix.Client {
-	scannerDialer := radix.Dialer{
-		AuthPass: password,
-	}
-	ctx = context.WithValue(ctx, "TLSEnabled", enableTLS)
-	client, err := scannerDialer.Dial(ctx, "tcp", redisUrl)
+func createClient(redisAddr string, password string) *redis.Client {
+	hostname, _, err := net.SplitHostPort(redisAddr)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	client := redis.NewClient(&redis.Options{
+		Addr:         redisAddr,
+		Password:     password,
+		ReadTimeout:  0,
+		WriteTimeout: 0,
+		PoolSize:     30,
+		MinIdleConns: 30,
+		TLSConfig: &tls.Config{
+			ServerName: hostname,
+		},
+	})
+
 	return client
 }

pkg/core/deleter/deleter.go

@@ -2,15 +2,15 @@ package deleter
 
 import (
 	"context"
-	"log"
+	"fmt"
 	"sync"
 
 	"github.com/appit-online/redis-dumper/pkg/core/logger"
-	"github.com/mediocregopher/radix/v4"
+	"github.com/redis/go-redis/v9"
 )
 
 type service struct {
-	client        radix.Client
+	client        *redis.Client
 	logger        logger.Service
 	deleteChannel <-chan string
 }
@@ -19,7 +19,7 @@ type Service interface {
 	Start(ctx context.Context, deleteRoutineCount int)
 }
 
-func CreateService(client radix.Client, deleteChannel <-chan string, logger logger.Service) Service {
+func CreateService(client *redis.Client, deleteChannel <-chan string, logger logger.Service) Service {
 	return &service{
 		client:        client,
 		logger:        logger,
@@ -41,8 +41,8 @@ func (s *service) Start(ctx context.Context, deleteRoutineCount int) {
 
 func (s *service) delete(ctx context.Context, wg *sync.WaitGroup) {
 	for key := range s.deleteChannel {
-		if err := s.client.Do(ctx, radix.Cmd(nil, "DEL", key)); err != nil {
-			log.Fatal(err)
+		if err := s.client.Del(ctx, key).Err(); err != nil {
+			fmt.Println(fmt.Errorf("could not delete entry: %w", err))
 		}
 
 		s.logger.IncDeletedCounter(1)