feat: add ftp destination

.github/workflows/check.yml

@@ -18,4 +18,5 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: '^1.21.1'
+      - run: pip install pyftpdlib
       - run: make SKIP_MARIADB_TESTS=1 test

destinations/ftp.go

@@ -0,0 +1,159 @@
+package destinations
+
+import (
+	"github.com/sloonz/uback/lib"
+
+	"fmt"
+	"io"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/secsy/goftp"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	ftpLog = logrus.WithFields(logrus.Fields{
+		"destination": "ftp",
+	})
+)
+
+type ftpDestination struct {
+	options *uback.Options
+	prefix  string
+	client  *goftp.Client
+}
+
+func isErrorCode(err error, code int) bool {
+	if _, ok := err.(goftp.Error); ok {
+		return err.(goftp.Error).Code() == code
+	}
+	return false
+}
+
+func newFTPDestination(options *uback.Options) (uback.Destination, error) {
+	u, err := url.Parse(options.String["URL"])
+	if err != nil {
+		ftpLog.Warnf("cannot parse URL: %v", err)
+		return nil, fmt.Errorf("invalid FTP URL: %v", err)
+	}
+
+	address := u.Host
+	username := u.User.Username()
+	password, _ := u.User.Password()
+	prefix := strings.Trim(options.String["Prefix"], "/") + "/"
+	if prefix == "/" {
+		prefix = ""
+	}
+
+	config := goftp.Config{
+		User:     username,
+		Password: password,
+	}
+
+	client, err := goftp.DialConfig(config, address)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to FTP server: %v", err)
+	}
+
+	return &ftpDestination{options: options, client: client, prefix: prefix}, nil
+}
+
+func (d *ftpDestination) makePrefix() error {
+	var err error
+
+	if d.prefix == "" {
+		return nil
+	}
+
+	dirs := strings.Split(strings.Trim(d.prefix, "/"), "/")
+	currentPath := ""
+
+	for _, dir := range dirs {
+		currentPath = path.Join(currentPath, dir)
+		_, err = d.client.Mkdir(currentPath)
+	}
+
+	return err
+}
+
+func (d *ftpDestination) ListBackups() ([]uback.Backup, error) {
+	var res []uback.Backup
+
+	_, err := d.client.Stat(d.prefix)
+	if err != nil && isErrorCode(err, 550) {
+		return nil, nil
+	}
+
+	files, err := d.client.ReadDir(d.prefix)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to list backups on FTP server: %v", err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() || strings.HasPrefix(file.Name(), ".") || strings.HasPrefix(file.Name(), "_") {
+			continue
+		}
+
+		backup, err := uback.ParseBackupFilename(file.Name(), true)
+		if err != nil {
+			ftpLog.WithFields(logrus.Fields{
+				"key": file.Name(),
+			}).Warnf("invalid backup file: %v", err)
+			continue
+		}
+
+		res = append(res, backup)
+	}
+
+	return res, nil
+}
+
+func (d *ftpDestination) RemoveBackup(backup uback.Backup) error {
+	filePath := path.Join(d.prefix, backup.Filename())
+	if err := d.client.Delete(filePath); err != nil {
+		return fmt.Errorf("failed to remove backup from FTP server: %v", err)
+	}
+	return nil
+}
+
+func (d *ftpDestination) SendBackup(backup uback.Backup, data io.Reader) error {
+	_, err := d.client.Stat(d.prefix)
+	if err != nil && isErrorCode(err, 550) {
+		if err = d.makePrefix(); err != nil {
+			return fmt.Errorf("failed to create directory: %v", err)
+		}
+	}
+
+	tmpFilePath := path.Join(d.prefix, "_tmp"+backup.Filename())
+	finalFilePath := path.Join(d.prefix, backup.Filename())
+	ftpLog.Printf("writing backup to temporary file %s", tmpFilePath)
+
+	if err = d.client.Store(tmpFilePath, data); err != nil {
+		return fmt.Errorf("failed to write temporary backup file to FTP server: %v", err)
+	}
+
+	ftpLog.Printf("renaming temporary file %s to %s", tmpFilePath, finalFilePath)
+	if err = d.client.Rename(tmpFilePath, finalFilePath); err != nil {
+		d.client.Delete(tmpFilePath) //nolint: errcheck
+		return fmt.Errorf("failed to rename temporary backup file on FTP server: %v", err)
+	}
+
+	return nil
+}
+
+func (d *ftpDestination) ReceiveBackup(backup uback.Backup) (io.ReadCloser, error) {
+	filePath := path.Join(d.prefix, backup.Filename())
+
+	reader, writer := io.Pipe()
+	go func() {
+		defer writer.Close()
+		if err := d.client.Retrieve(filePath, writer); err != nil {
+			writer.CloseWithError(fmt.Errorf("failed to read backup from FTP server: %v", err))
+		}
+	}()
+
+	return reader, nil
+}

destinations/new.go

@@ -12,6 +12,8 @@ func New(options *uback.Options) (uback.Destination, error) {
 		return newBtrfsDestination(options)
 	case "fs":
 		return newFSDestination(options)
+	case "ftp":
+		return newFTPDestination(options)
 	case "object-storage":
 		return newObjectStorageDestination(options)
 	case "command":

doc/dest-ftp.md

@@ -0,0 +1,6 @@
+# ftp Destination
+
+Store backups on a FTP server. It can be configured either by providing
+an URL, for example:
+
+`ftp://user:password@ftp.example.com:21/my-backups`

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/rs/xid v1.6.0 // indirect
+	github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.29.0 // indirect
 	golang.org/x/net v0.31.0 // indirect

go.sum

@@ -50,6 +50,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
+github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=

tests/dest_ftp_tests.py

@@ -0,0 +1,63 @@
+from .common import *
+from pyftpdlib.authorizers import DummyAuthorizer
+from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.servers import FTPServer
+import threading
+import unittest
+
+class DestFTPStorageTests(unittest.TestCase):
+    def setUp(self):
+        self.host = "127.0.0.1"
+        self.port = 2121
+        self.user = "testuser"
+        self.password = "testpass"
+        self.ftp_root = tempfile.mkdtemp()
+
+        authorizer = DummyAuthorizer()
+        authorizer.add_user(self.user, self.password, self.ftp_root, perm="elradfmw")
+        handler = FTPHandler
+        handler.authorizer = authorizer
+        self.ftp_server = FTPServer((self.host, self.port), handler)
+        threading.Thread(target=self.ftp_server.serve_forever, daemon=True).start()
+
+    def tearDown(self):
+        self.ftp_server.close_all()
+        shutil.rmtree(self.ftp_root)
+
+    def test_ftp_destination(self):
+        with tempfile.TemporaryDirectory() as d:
+            os.mkdir(f"{d}/restore")
+            os.mkdir(f"{d}/source")
+            subprocess.check_call([uback, "key", "gen", f"{d}/backup.key", f"{d}/backup.pub"])
+
+            source = f"type=tar,path={d}/source,key-file={d}/backup.pub,state-file={d}/state.json,snapshots-path={d}/snapshots,full-interval=weekly"
+            dest = f"id=test,type=ftp,@retention-policy=daily=3,key-file={d}/backup.key,url=ftp://{self.user}:{self.password}@{self.host}:{self.port},prefix=/test"
+
+            # Full 1
+            with open(f"{d}/source/a", "w+") as fd: fd.write("hello")
+            self.assertEqual(0, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines()))
+            subprocess.check_call([uback, "backup", "-n", "-f", source, dest])
+            self.assertEqual(1, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines()))
+            time.sleep(0.01)
+
+            # Full 2
+            subprocess.check_call([uback, "backup", "-n", "-f", source, dest])
+            self.assertEqual(2, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines()))
+
+            # Incremental
+            with open(f"{d}/source/b", "w+") as fd: fd.write("world")
+            subprocess.check_call([uback, "backup", "-n", source, dest])
+            self.assertEqual(3, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines()))
+
+            # Prune (remove full 1)
+            subprocess.check_call([uback, "prune", "backups", dest])
+            self.assertEqual(2, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines()))
+
+            # Restore full 2 + incremental
+            subprocess.check_call([uback, "restore", "-d", f"{d}/restore", dest])
+            self.assertEqual(b"hello", read_file(glob.glob(f"{d}/restore/*/a")[0]))
+            self.assertEqual(b"world", read_file(glob.glob(f"{d}/restore/*/b")[0]))
+
+            # Searching on "/" should not yield any result in the "/test/" prefix
+            parent_dest = f"id=test,type=ftp,@retention-policy=daily=3,key-file={d}/backup.key,url=ftp://{self.user}:{self.password}@{self.host}:{self.port}"
+            self.assertEqual(0, len(subprocess.check_output([uback, "list", "backups", parent_dest]).splitlines()))