diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 38c0ec6..ae736b2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -18,4 +18,5 @@ jobs: - uses: actions/setup-go@v4 with: go-version: '^1.21.1' + - run: pip install pyftpdlib - run: make SKIP_MARIADB_TESTS=1 test diff --git a/destinations/ftp.go b/destinations/ftp.go new file mode 100644 index 0000000..bfe14be --- /dev/null +++ b/destinations/ftp.go @@ -0,0 +1,159 @@ +package destinations + +import ( + "github.com/sloonz/uback/lib" + + "fmt" + "io" + "net/url" + "path" + "strings" + + "github.com/secsy/goftp" + "github.com/sirupsen/logrus" +) + +var ( + ftpLog = logrus.WithFields(logrus.Fields{ + "destination": "ftp", + }) +) + +type ftpDestination struct { + options *uback.Options + prefix string + client *goftp.Client +} + +func isErrorCode(err error, code int) bool { + if _, ok := err.(goftp.Error); ok { + return err.(goftp.Error).Code() == code + } + return false +} + +func newFTPDestination(options *uback.Options) (uback.Destination, error) { + u, err := url.Parse(options.String["URL"]) + if err != nil { + ftpLog.Warnf("cannot parse URL: %v", err) + return nil, fmt.Errorf("invalid FTP URL: %v", err) + } + + address := u.Host + username := u.User.Username() + password, _ := u.User.Password() + prefix := strings.Trim(options.String["Prefix"], "/") + "/" + if prefix == "/" { + prefix = "" + } + + config := goftp.Config{ + User: username, + Password: password, + } + + client, err := goftp.DialConfig(config, address) + if err != nil { + return nil, fmt.Errorf("failed to connect to FTP server: %v", err) + } + + return &ftpDestination{options: options, client: client, prefix: prefix}, nil +} + +func (d *ftpDestination) makePrefix() error { + var err error + + if d.prefix == "" { + return nil + } + + dirs := strings.Split(strings.Trim(d.prefix, "/"), "/") + currentPath := "" + + for _, dir := range dirs { + currentPath = path.Join(currentPath, dir) + _, err = d.client.Mkdir(currentPath) + } + + return err +} + +func (d *ftpDestination) ListBackups() ([]uback.Backup, error) { + var res []uback.Backup + + _, err := d.client.Stat(d.prefix) + if err != nil && isErrorCode(err, 550) { + return nil, nil + } + + files, err := d.client.ReadDir(d.prefix) + + if err != nil { + return nil, fmt.Errorf("failed to list backups on FTP server: %v", err) + } + + for _, file := range files { + if file.IsDir() || strings.HasPrefix(file.Name(), ".") || strings.HasPrefix(file.Name(), "_") { + continue + } + + backup, err := uback.ParseBackupFilename(file.Name(), true) + if err != nil { + ftpLog.WithFields(logrus.Fields{ + "key": file.Name(), + }).Warnf("invalid backup file: %v", err) + continue + } + + res = append(res, backup) + } + + return res, nil +} + +func (d *ftpDestination) RemoveBackup(backup uback.Backup) error { + filePath := path.Join(d.prefix, backup.Filename()) + if err := d.client.Delete(filePath); err != nil { + return fmt.Errorf("failed to remove backup from FTP server: %v", err) + } + return nil +} + +func (d *ftpDestination) SendBackup(backup uback.Backup, data io.Reader) error { + _, err := d.client.Stat(d.prefix) + if err != nil && isErrorCode(err, 550) { + if err = d.makePrefix(); err != nil { + return fmt.Errorf("failed to create directory: %v", err) + } + } + + tmpFilePath := path.Join(d.prefix, "_tmp"+backup.Filename()) + finalFilePath := path.Join(d.prefix, backup.Filename()) + ftpLog.Printf("writing backup to temporary file %s", tmpFilePath) + + if err = d.client.Store(tmpFilePath, data); err != nil { + return fmt.Errorf("failed to write temporary backup file to FTP server: %v", err) + } + + ftpLog.Printf("renaming temporary file %s to %s", tmpFilePath, finalFilePath) + if err = d.client.Rename(tmpFilePath, finalFilePath); err != nil { + d.client.Delete(tmpFilePath) //nolint: errcheck + return fmt.Errorf("failed to rename temporary backup file on FTP server: %v", err) + } + + return nil +} + +func (d *ftpDestination) ReceiveBackup(backup uback.Backup) (io.ReadCloser, error) { + filePath := path.Join(d.prefix, backup.Filename()) + + reader, writer := io.Pipe() + go func() { + defer writer.Close() + if err := d.client.Retrieve(filePath, writer); err != nil { + writer.CloseWithError(fmt.Errorf("failed to read backup from FTP server: %v", err)) + } + }() + + return reader, nil +} diff --git a/destinations/new.go b/destinations/new.go index b95860c..9c6376e 100644 --- a/destinations/new.go +++ b/destinations/new.go @@ -12,6 +12,8 @@ func New(options *uback.Options) (uback.Destination, error) { return newBtrfsDestination(options) case "fs": return newFSDestination(options) + case "ftp": + return newFTPDestination(options) case "object-storage": return newObjectStorageDestination(options) case "command": diff --git a/doc/dest-ftp.md b/doc/dest-ftp.md new file mode 100644 index 0000000..3b8f42c --- /dev/null +++ b/doc/dest-ftp.md @@ -0,0 +1,6 @@ +# ftp Destination + +Store backups on a FTP server. It can be configured either by providing +an URL, for example: + +`ftp://user:password@ftp.example.com:21/my-backups` diff --git a/go.mod b/go.mod index 5b5d3de..104e0c7 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/rs/xid v1.6.0 // indirect + github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.29.0 // indirect golang.org/x/net v0.31.0 // indirect diff --git a/go.sum b/go.sum index ff4a1a4..06293eb 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= diff --git a/tests/dest_ftp_tests.py b/tests/dest_ftp_tests.py new file mode 100644 index 0000000..176f4ca --- /dev/null +++ b/tests/dest_ftp_tests.py @@ -0,0 +1,63 @@ +from .common import * +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer +import threading +import unittest + +class DestFTPStorageTests(unittest.TestCase): + def setUp(self): + self.host = "127.0.0.1" + self.port = 2121 + self.user = "testuser" + self.password = "testpass" + self.ftp_root = tempfile.mkdtemp() + + authorizer = DummyAuthorizer() + authorizer.add_user(self.user, self.password, self.ftp_root, perm="elradfmw") + handler = FTPHandler + handler.authorizer = authorizer + self.ftp_server = FTPServer((self.host, self.port), handler) + threading.Thread(target=self.ftp_server.serve_forever, daemon=True).start() + + def tearDown(self): + self.ftp_server.close_all() + shutil.rmtree(self.ftp_root) + + def test_ftp_destination(self): + with tempfile.TemporaryDirectory() as d: + os.mkdir(f"{d}/restore") + os.mkdir(f"{d}/source") + subprocess.check_call([uback, "key", "gen", f"{d}/backup.key", f"{d}/backup.pub"]) + + source = f"type=tar,path={d}/source,key-file={d}/backup.pub,state-file={d}/state.json,snapshots-path={d}/snapshots,full-interval=weekly" + dest = f"id=test,type=ftp,@retention-policy=daily=3,key-file={d}/backup.key,url=ftp://{self.user}:{self.password}@{self.host}:{self.port},prefix=/test" + + # Full 1 + with open(f"{d}/source/a", "w+") as fd: fd.write("hello") + self.assertEqual(0, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines())) + subprocess.check_call([uback, "backup", "-n", "-f", source, dest]) + self.assertEqual(1, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines())) + time.sleep(0.01) + + # Full 2 + subprocess.check_call([uback, "backup", "-n", "-f", source, dest]) + self.assertEqual(2, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines())) + + # Incremental + with open(f"{d}/source/b", "w+") as fd: fd.write("world") + subprocess.check_call([uback, "backup", "-n", source, dest]) + self.assertEqual(3, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines())) + + # Prune (remove full 1) + subprocess.check_call([uback, "prune", "backups", dest]) + self.assertEqual(2, len(subprocess.check_output([uback, "list", "backups", dest]).splitlines())) + + # Restore full 2 + incremental + subprocess.check_call([uback, "restore", "-d", f"{d}/restore", dest]) + self.assertEqual(b"hello", read_file(glob.glob(f"{d}/restore/*/a")[0])) + self.assertEqual(b"world", read_file(glob.glob(f"{d}/restore/*/b")[0])) + + # Searching on "/" should not yield any result in the "/test/" prefix + parent_dest = f"id=test,type=ftp,@retention-policy=daily=3,key-file={d}/backup.key,url=ftp://{self.user}:{self.password}@{self.host}:{self.port}" + self.assertEqual(0, len(subprocess.check_output([uback, "list", "backups", parent_dest]).splitlines()))