From 73ac002dee50256ad0ddf7e1c843250d4bebebc4 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 9 Dec 2024 23:46:02 +0800 Subject: [PATCH 1/2] feat: ftp server support --- cmd/server.go | 29 +++ go.mod | 7 +- go.sum | 21 ++- internal/bootstrap/data/setting.go | 10 + internal/conf/config.go | 26 +++ internal/conf/const.go | 9 + internal/model/setting.go | 1 + internal/model/user.go | 8 + server/ftp.go | 285 +++++++++++++++++++++++++++++ server/ftp/afero.go | 91 +++++++++ server/ftp/fsmanage.go | 75 ++++++++ server/ftp/fsread.go | 184 +++++++++++++++++++ server/ftp/fsup.go | 91 +++++++++ 13 files changed, 833 insertions(+), 4 deletions(-) create mode 100644 server/ftp.go create mode 100644 server/ftp/afero.go create mode 100644 server/ftp/fsmanage.go create mode 100644 server/ftp/fsread.go create mode 100644 server/ftp/fsup.go diff --git a/cmd/server.go b/cmd/server.go index 8a7beafa7fd..66b57952b49 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" "net" "net/http" "os" @@ -112,6 +113,24 @@ the address is defined in config file`, } }() } + var ftpDriver *server.FtpMainDriver + var ftpServer *ftpserver.FtpServer + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { + var err error + ftpDriver, err = server.NewMainDriver() + if err != nil { + utils.Log.Fatalf("failed to start ftp driver: %s", err.Error()) + } else { + utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) + go func() { + ftpServer = ftpserver.NewFtpServer(ftpDriver) + err = ftpServer.ListenAndServe() + if err != nil { + utils.Log.Fatalf("problem ftp server listening: %s", err.Error()) + } + }() + } + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -152,6 +171,16 @@ the address is defined in config file`, } }() } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil { + wg.Add(1) + go func() { + defer wg.Done() + ftpDriver.Stop() + if err := ftpServer.Stop(); err != nil { + utils.Log.Fatal("FTP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/go.mod b/go.mod index 19bc7c2e627..2d55f7d33be 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,9 @@ require ( github.com/pquerna/otp v1.4.0 github.com/rclone/rclone v1.67.0 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 @@ -75,6 +76,7 @@ require ( require ( github.com/BurntSushi/toml v0.3.1 // indirect + github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2 // indirect github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -83,6 +85,7 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fclairamb/go-log v0.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect @@ -221,7 +224,7 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/tools v0.24.0 // indirect diff --git a/go.sum b/go.sum index 78ac273a5bf..769f392f178 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,11 @@ +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2 h1:P3MoQ1kDfbCjL6+MPd5K7wPdKB4nqMuLU6Mv0+tdWDA= +github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= @@ -144,6 +147,8 @@ github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJL github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= +github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= @@ -168,6 +173,10 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -393,6 +402,8 @@ github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= @@ -439,6 +450,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= @@ -457,6 +470,8 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:s github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -479,6 +494,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= @@ -512,8 +529,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= -github.com/xhofe/tache v0.1.2 h1:pHrXlrWcbTb4G7hVUDW7Rc+YTUnLJvnLBrdktVE1Fqg= -github.com/xhofe/tache v0.1.2/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/tache v0.1.3 h1:MipxzlljYX29E1YI/SLC7hVomVF+51iP1OUzlsuq1wE= github.com/xhofe/tache v0.1.3/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= @@ -634,6 +649,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 920a7a2d118..fbbb9900508 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -184,6 +184,16 @@ func InitialSettings() []model.SettingItem { {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + + //ftp settings + {Key: conf.FTPPublicHost, Value: "127.0.0.1", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPPasvPortMap, Value: "", Type: conf.TypeText, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPProxyUserAgent, Value: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/87.0.4280.88 Safari/537.36", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPMandatoryTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPImplicitTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, } initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { diff --git a/internal/conf/config.go b/internal/conf/config.go index aa29e1f506d..df6c0544e1e 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -71,6 +71,19 @@ type S3 struct { SSL bool `json:"ssl" env:"SSL"` } +type FTP struct { + Enable bool `json:"enable" env:"ENABLE"` + Listen string `json:"listen" env:"LISTEN"` + FindPasvPortAttempts int `json:"find_pasv_port_attempts" env:"FIND_PASV_PORT_ATTEMPTS"` + ActiveTransferPortNon20 bool `json:"active_transfer_port_non_20" env:"ACTIVE_TRANSFER_PORT_NON_20"` + IdleTimeout int `json:"idle_timeout" env:"IDLE_TIMEOUT"` + ConnectionTimeout int `json:"connection_timeout" env:"CONNECTION_TIMEOUT"` + DisableActiveMode bool `json:"disable_active_mode" env:"DISABLE_ACTIVE_MODE"` + DefaultTransferBinary bool `json:"default_transfer_binary" env:"DEFAULT_TRANSFER_BINARY"` + EnableActiveConnIPCheck bool `json:"enable_active_conn_ip_check" env:"ENABLE_ACTIVE_CONN_IP_CHECK"` + EnablePasvConnIPCheck bool `json:"enable_pasv_conn_ip_check" env:"ENABLE_PASV_CONN_IP_CHECK"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -90,6 +103,7 @@ type Config struct { Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` Cors Cors `json:"cors" envPrefix:"CORS_"` S3 S3 `json:"s3" envPrefix:"S3_"` + FTP FTP `json:"ftp" envPrefix:"FTP_"` } func DefaultConfig() *Config { @@ -159,5 +173,17 @@ func DefaultConfig() *Config { Port: 5246, SSL: false, }, + FTP: FTP{ + Enable: true, + Listen: ":5221", + FindPasvPortAttempts: 50, + ActiveTransferPortNon20: false, + IdleTimeout: 900, + ConnectionTimeout: 30, + DisableActiveMode: false, + DefaultTransferBinary: false, + EnableActiveConnIPCheck: true, + EnablePasvConnIPCheck: true, + }, } } diff --git a/internal/conf/const.go b/internal/conf/const.go index 13787b5e2ac..74ca79f945c 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -96,6 +96,15 @@ const ( // qbittorrent QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" + + // ftp + FTPPublicHost = "ftp_public_host" + FTPPasvPortMap = "ftp_pasv_port_map" + FTPProxyUserAgent = "ftp_proxy_user_agent" + FTPMandatoryTLS = "ftp_mandatory_tls" + FTPImplicitTLS = "ftp_implicit_tls" + FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" + FTPTLSPublicCertPath = "ftp_tls_public_cert_path" ) const ( diff --git a/internal/model/setting.go b/internal/model/setting.go index c474935ed49..9b60d98a76e 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -11,6 +11,7 @@ const ( SSO LDAP S3 + FTP ) const ( diff --git a/internal/model/user.go b/internal/model/user.go index 2d61a971c3d..b4e876a47ab 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -117,6 +117,14 @@ func (u *User) CanWebdavManage() bool { return u.IsAdmin() || (u.Permission>>9)&1 == 1 } +func (u *User) CanFTPAccess() bool { + return (u.Permission>>10)&1 == 1 +} + +func (u *User) CanFTPManage() bool { + return (u.Permission>>11)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/server/ftp.go b/server/ftp.go new file mode 100644 index 00000000000..161ea63c5b1 --- /dev/null +++ b/server/ftp.go @@ -0,0 +1,285 @@ +package server + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/ftp" + "math/rand" + "net" + "net/http" + "os" + "strconv" + "strings" + "sync" +) + +type FtpMainDriver struct { + settings *ftpserver.Settings + proxyHeader *http.Header + clients map[uint32]ftpserver.ClientContext + shutdownLock sync.RWMutex + isShutdown bool + tlsConfig *tls.Config +} + +func NewMainDriver() (*FtpMainDriver, error) { + header := &http.Header{} + header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent)) + transferType := ftpserver.TransferTypeASCII + if conf.Conf.FTP.DefaultTransferBinary { + transferType = ftpserver.TransferTypeBinary + } + activeConnCheck := ftpserver.IPMatchDisabled + if conf.Conf.FTP.EnableActiveConnIPCheck { + activeConnCheck = ftpserver.IPMatchRequired + } + pasvConnCheck := ftpserver.IPMatchDisabled + if conf.Conf.FTP.EnablePasvConnIPCheck { + pasvConnCheck = ftpserver.IPMatchRequired + } + tlsRequired := ftpserver.ClearOrEncrypted + if setting.GetBool(conf.FTPImplicitTLS) { + tlsRequired = ftpserver.ImplicitEncryption + } else if setting.GetBool(conf.FTPMandatoryTLS) { + tlsRequired = ftpserver.MandatoryEncryption + } + tlsConf, err := getTlsConf(setting.GetStr(conf.FTPTLSPrivateKeyPath), setting.GetStr(conf.FTPTLSPublicCertPath)) + if err != nil && tlsRequired != ftpserver.ClearOrEncrypted { + return nil, fmt.Errorf("FTP mandatory TLS has been enabled, but the certificate failed to load: %w", err) + } + return &FtpMainDriver{ + settings: &ftpserver.Settings{ + ListenAddr: conf.Conf.FTP.Listen, + PublicHost: lookupIP(setting.GetStr(conf.FTPPublicHost)), + PassiveTransferPortGetter: newPortMapper(setting.GetStr(conf.FTPPasvPortMap)), + FindPasvPortAttempts: conf.Conf.FTP.FindPasvPortAttempts, + ActiveTransferPortNon20: conf.Conf.FTP.ActiveTransferPortNon20, + IdleTimeout: conf.Conf.FTP.IdleTimeout, + ConnectionTimeout: conf.Conf.FTP.ConnectionTimeout, + DisableMLSD: false, + DisableMLST: false, + DisableMFMT: true, + Banner: setting.GetStr(conf.Announcement), + TLSRequired: tlsRequired, + DisableLISTArgs: false, + DisableSite: true, + DisableActiveMode: conf.Conf.FTP.DisableActiveMode, + EnableHASH: false, + DisableSTAT: false, + DisableSYST: false, + EnableCOMB: false, + DefaultTransferType: transferType, + ActiveConnectionsCheck: activeConnCheck, + PasvConnectionsCheck: pasvConnCheck, + }, + proxyHeader: header, + clients: make(map[uint32]ftpserver.ClientContext), + shutdownLock: sync.RWMutex{}, + isShutdown: false, + tlsConfig: tlsConf, + }, nil +} + +func (d *FtpMainDriver) GetSettings() (*ftpserver.Settings, error) { + return d.settings, nil +} + +func (d *FtpMainDriver) ClientConnected(cc ftpserver.ClientContext) (string, error) { + if d.isShutdown || !d.shutdownLock.TryRLock() { + return "", errors.New("server has shutdown") + } + defer d.shutdownLock.RUnlock() + d.clients[cc.ID()] = cc + return "AList FTP Endpoint", nil +} + +func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) { + err := cc.Close() + if err != nil { + utils.Log.Errorf("failed to close client: %v", err) + } + delete(d.clients, cc.ID()) +} + +func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + var userObj *model.User + var err error + if user == "anonymous" || user == "guest" { + userObj, err = op.GetGuest() + if err != nil { + return nil, err + } + } else { + userObj, err = op.GetUserByName(user) + if err != nil { + return nil, err + } + passHash := model.StaticHash(pass) + if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + return nil, err + } + } + if userObj.Disabled || !userObj.CanFTPAccess() { + return nil, errors.New("user not allowed to access FTP") + } + + ctx := context.Background() + ctx = context.WithValue(ctx, "user", userObj) + if user == "anonymous" || user == "guest" { + ctx = context.WithValue(ctx, "meta_pass", pass) + } else { + ctx = context.WithValue(ctx, "meta_pass", "") + } + ctx = context.WithValue(ctx, "client_ip", cc.RemoteAddr().String()) + ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader) + return ftp.NewAferoAdapter(ctx), nil +} + +func (d *FtpMainDriver) GetTLSConfig() (*tls.Config, error) { + if d.tlsConfig == nil { + return nil, errors.New("TLS config not provided") + } + return d.tlsConfig, nil +} + +func (d *FtpMainDriver) Stop() { + d.isShutdown = true + d.shutdownLock.Lock() + defer d.shutdownLock.Unlock() + for _, value := range d.clients { + _ = value.Close() + } +} + +func lookupIP(host string) string { + if host == "" || net.ParseIP(host) != nil { + return host + } + ips, err := net.LookupIP(host) + if err != nil || len(ips) == 0 { + utils.Log.Fatalf("given FTP public host is invalid, and the default value will be used: %v", err) + return "" + } + for _, ip := range ips { + if ip.To4() != nil { + return ip.String() + } + } + v6 := ips[0].String() + utils.Log.Warnf("no IPv4 record looked up, %s will be used as public host, and it might do not work.", v6) + return v6 +} + +func newPortMapper(str string) ftpserver.PasvPortGetter { + if str == "" { + return nil + } + pasvPortMappers := strings.Split(strings.Replace(str, "\n", ",", -1), ",") + type group struct { + ExposedStart int + ListenedStart int + Length int + } + groups := make([]group, len(pasvPortMappers)) + totalLength := 0 + convertToPorts := func(str string) (int, int, error) { + start, end, multi := strings.Cut(str, "-") + if multi { + si, err := strconv.Atoi(start) + if err != nil { + return 0, 0, err + } + ei, err := strconv.Atoi(end) + if err != nil { + return 0, 0, err + } + if ei < si || ei < 1024 || si < 1024 || ei > 65535 || si > 65535 { + return 0, 0, errors.New("invalid port") + } + return si, ei - si + 1, nil + } else { + ret, err := strconv.Atoi(str) + if err != nil { + return 0, 0, err + } else { + return ret, 1, nil + } + } + } + for i, mapper := range pasvPortMappers { + var err error + exposed, listened, mapped := strings.Cut(mapper, ":") + for { + if mapped { + var es, ls, el, ll int + es, el, err = convertToPorts(exposed) + if err != nil { + break + } + ls, ll, err = convertToPorts(listened) + if err != nil { + break + } + if el != ll { + err = errors.New("the number of exposed ports and listened ports does not match") + break + } + groups[i].ExposedStart = es + groups[i].ListenedStart = ls + groups[i].Length = el + totalLength += el + } else { + var start, length int + start, length, err = convertToPorts(mapper) + groups[i].ExposedStart = start + groups[i].ListenedStart = start + groups[i].Length = length + totalLength += length + } + break + } + if err != nil { + utils.Log.Fatalf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) + return nil + } + } + return func() (int, int, bool) { + idxPort := rand.Intn(totalLength) + for _, g := range groups { + if idxPort >= g.Length { + idxPort -= g.Length + } else { + return g.ExposedStart + idxPort, g.ListenedStart + idxPort, true + } + } + // unreachable + return 0, 0, false + } +} + +func getTlsConf(keyPath, certPath string) (*tls.Config, error) { + if keyPath == "" || certPath == "" { + return nil, errors.New("private key or certificate is not provided") + } + cert, err := os.ReadFile(certPath) + if err != nil { + return nil, err + } + key, err := os.ReadFile(keyPath) + if err != nil { + return nil, err + } + tlsCert, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, err + } + return &tls.Config{Certificates: []tls.Certificate{tlsCert}}, nil +} diff --git a/server/ftp/afero.go b/server/ftp/afero.go new file mode 100644 index 00000000000..6eb4bf8e4e4 --- /dev/null +++ b/server/ftp/afero.go @@ -0,0 +1,91 @@ +package ftp + +import ( + "context" + "errors" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/spf13/afero" + "os" + "time" +) + +type AferoAdapter struct { + ctx context.Context +} + +func NewAferoAdapter(ctx context.Context) *AferoAdapter { + return &AferoAdapter{ctx: ctx} +} + +func (a *AferoAdapter) Create(_ string) (afero.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (a *AferoAdapter) Mkdir(name string, _ os.FileMode) error { + return Mkdir(a.ctx, name) +} + +func (a *AferoAdapter) MkdirAll(path string, perm os.FileMode) error { + return a.Mkdir(path, perm) +} + +func (a *AferoAdapter) Open(_ string) (afero.File, error) { + // See also GetHandle and ReadDir + return nil, errs.NotImplement +} + +func (a *AferoAdapter) OpenFile(_ string, _ int, _ os.FileMode) (afero.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (a *AferoAdapter) Remove(name string) error { + return Remove(a.ctx, name) +} + +func (a *AferoAdapter) RemoveAll(path string) error { + return a.Remove(path) +} + +func (a *AferoAdapter) Rename(oldName, newName string) error { + return Rename(a.ctx, oldName, newName) +} + +func (a *AferoAdapter) Stat(name string) (os.FileInfo, error) { + return Stat(a.ctx, name) +} + +func (a *AferoAdapter) Name() string { + return "AList FTP Endpoint" +} + +func (a *AferoAdapter) Chmod(_ string, _ os.FileMode) error { + return errs.NotSupport +} + +func (a *AferoAdapter) Chown(_ string, _, _ int) error { + return errs.NotSupport +} + +func (a *AferoAdapter) Chtimes(_ string, _ time.Time, _ time.Time) error { + return errs.NotSupport +} + +func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) { + return List(a.ctx, name) +} + +func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { + if offset != 0 { + return nil, errors.New("offset") + } + if (flags & os.O_APPEND) > 0 { + return nil, errors.New("append") + } + if (flags & os.O_WRONLY) > 0 { + return OpenUpload(a.ctx, name) + } + return OpenDownload(a.ctx, name) +} diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go new file mode 100644 index 00000000000..5199a473b5b --- /dev/null +++ b/server/ftp/fsmanage.go @@ -0,0 +1,75 @@ +package ftp + +import ( + "context" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + stdpath "path" +) + +func Mkdir(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return err + } + if !user.CanWrite() || !user.CanFTPManage() { + meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + } + if !common.CanWrite(meta, reqPath) { + return errs.PermissionDenied + } + } + return fs.MakeDir(ctx, reqPath) +} + +func Remove(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + if !user.CanRemove() || !user.CanFTPManage() { + return errs.PermissionDenied + } + reqPath, err := user.JoinPath(path) + if err != nil { + return err + } + return fs.Remove(ctx, reqPath) +} + +func Rename(ctx context.Context, oldPath, newPath string) error { + user := ctx.Value("user").(*model.User) + srcPath, err := user.JoinPath(oldPath) + if err != nil { + return err + } + dstPath, err := user.JoinPath(newPath) + if err != nil { + return err + } + srcDir, srcBase := stdpath.Split(srcPath) + dstDir, dstBase := stdpath.Split(dstPath) + if srcDir == dstDir { + if !user.CanRename() || !user.CanFTPManage() { + return errs.PermissionDenied + } + return fs.Rename(ctx, srcPath, dstBase) + } else { + if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + return errs.PermissionDenied + } + if err := fs.Move(ctx, srcPath, dstDir); err != nil { + return err + } + if srcBase != dstBase { + return fs.Rename(ctx, stdpath.Join(dstDir, srcBase), dstBase) + } + return nil + } +} diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go new file mode 100644 index 00000000000..6c6510cba79 --- /dev/null +++ b/server/ftp/fsread.go @@ -0,0 +1,184 @@ +package ftp + +import ( + "context" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + "io" + fs2 "io/fs" + "net/http" + "os" + "time" +) + +type FileDownloadProxy struct { + ftpserver.FileTransfer + reader io.ReadCloser + closers *utils.Closers +} + +func OpenDownload(ctx context.Context, path string) (*FileDownloadProxy, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + + // directly use proxy + header := *(ctx.Value("proxy_header").(*http.Header)) + link, obj, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: ctx.Value("client_ip").(string), + Header: header, + }) + if err != nil { + return nil, err + } + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + return nil, err + } + if storage.GetStorage().ProxyRange { + common.ProxyRange(link, obj.GetSize()) + } + reader, closers, err := proxy(link) + if err != nil { + return nil, err + } + return &FileDownloadProxy{reader: reader, closers: closers}, nil +} + +func proxy(link *model.Link) (io.ReadCloser, *utils.Closers, error) { + if link.MFile != nil { + return link.MFile, nil, nil + } else if link.RangeReadCloser != nil { + rc, err := link.RangeReadCloser.RangeRead(context.Background(), http_range.Range{Length: -1}) + if err != nil { + return nil, nil, err + } + closers := link.RangeReadCloser.GetClosers() + return rc, &closers, nil + } else { + res, err := net.RequestHttp(context.Background(), http.MethodGet, link.Header, link.URL) + if err != nil { + return nil, nil, err + } + return res.Body, nil, nil + } +} + +func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { + return f.reader.Read(p) +} + +func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) { + return 0, errs.NotSupport +} + +func (f *FileDownloadProxy) Close() error { + defer func() { + if f.closers != nil { + _ = f.closers.Close() + } + }() + return f.reader.Close() +} + +type OsFileInfoAdapter struct { + obj model.Obj +} + +func (o *OsFileInfoAdapter) Name() string { + return o.obj.GetName() +} + +func (o *OsFileInfoAdapter) Size() int64 { + return o.obj.GetSize() +} + +func (o *OsFileInfoAdapter) Mode() fs2.FileMode { + return 0755 +} + +func (o *OsFileInfoAdapter) ModTime() time.Time { + return o.obj.ModTime() +} + +func (o *OsFileInfoAdapter) IsDir() bool { + return o.obj.IsDir() +} + +func (o *OsFileInfoAdapter) Sys() any { + return o.obj +} + +func Stat(ctx context.Context, path string) (os.FileInfo, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return nil, err + } + return &OsFileInfoAdapter{obj: obj}, nil +} + +func List(ctx context.Context, path string) ([]os.FileInfo, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) + if err != nil { + return nil, err + } + ret := make([]os.FileInfo, len(objs)) + for i, obj := range objs { + ret[i] = &OsFileInfoAdapter{obj: obj} + } + return ret, nil +} diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go new file mode 100644 index 00000000000..3042a3d2cea --- /dev/null +++ b/server/ftp/fsup.go @@ -0,0 +1,91 @@ +package ftp + +import ( + "context" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + "io" + "net/http" + "os" + stdpath "path" + "time" +) + +type FileUploadProxy struct { + ftpserver.FileTransfer + buffer *os.File + path string + ctx context.Context +} + +func OpenUpload(ctx context.Context, path string) (*FileUploadProxy, error) { + user := ctx.Value("user").(*model.User) + path, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(stdpath.Dir(path)) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) && + ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + return nil, errs.PermissionDenied + } + tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx}, nil +} + +func (f *FileUploadProxy) Read(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileUploadProxy) Write(p []byte) (n int, err error) { + return f.buffer.Write(p) +} + +func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) { + return 0, errs.NotSupport +} + +func (f *FileUploadProxy) Close() error { + dir, name := stdpath.Split(f.path) + size, err := f.buffer.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { + return err + } + arr := make([]byte, 512) + if _, err := f.buffer.Read(arr); err != nil { + return err + } + contentType := http.DetectContentType(arr) + if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { + return err + } + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: size, + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: false, + } + s.SetTmpFile(f.buffer) + return fs.PutDirectly(f.ctx, dir, s, true) +} From 56237d5e895e8b1defc1fc020934e82aeb5b8efc Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Tue, 10 Dec 2024 16:50:38 +0800 Subject: [PATCH 2/2] fix(ftp): incorrect mode for dirs in LIST returns --- server/ftp/fsread.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 6c6510cba79..6a9ba2ebb2a 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -118,7 +118,11 @@ func (o *OsFileInfoAdapter) Size() int64 { } func (o *OsFileInfoAdapter) Mode() fs2.FileMode { - return 0755 + var mode fs2.FileMode = 0755 + if o.IsDir() { + mode |= fs2.ModeDir + } + return mode } func (o *OsFileInfoAdapter) ModTime() time.Time {