From 39b8f28fc4986f3ebf47b998aac68878c978b8e8 Mon Sep 17 00:00:00 2001 From: Xhofe Date: Sun, 15 May 2022 16:17:52 +0800 Subject: [PATCH 1/9] fix: disable pprof while not debug --- server/static.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/static.go b/server/static.go index c60b98d47c5..8ef8fd07e97 100644 --- a/server/static.go +++ b/server/static.go @@ -50,7 +50,7 @@ func Static(r *gin.Engine) { c.Status(200) if strings.HasPrefix(c.Request.URL.Path, "/@manage") { _, _ = c.Writer.WriteString(conf.ManageHtml) - } else if strings.HasPrefix(c.Request.URL.Path, "/debug/pprof") { + } else if strings.HasPrefix(c.Request.URL.Path, "/debug/pprof") && conf.Debug { pprof.Index(c.Writer, c.Request) } else { _, _ = c.Writer.WriteString(conf.IndexHtml) From f2b2728be71ebb6658f5ed2a6493a37305929c7e Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 16 May 2022 09:48:33 +0800 Subject: [PATCH 2/9] fix(123,189pc,alidriver,xunlei):tempfile remove --- drivers/123/driver.go | 6 ++++-- drivers/189pc/driver.go | 19 ++++++++++++------- drivers/alidrive/driver.go | 6 ++++-- drivers/xunlei/driver.go | 6 ++++-- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 43cf7cf7efa..50fc39c356b 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -306,8 +306,10 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro if err != nil { return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() h := md5.New() if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { return err diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index e8ea2e1c9f2..29a1f5ab4c1 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -650,9 +650,10 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File if err != nil { return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) - + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() // 初始化上传 state := GetState(account) @@ -756,8 +757,10 @@ func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.Fi return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() fileMd5 := md5.New() if _, err = io.Copy(io.MultiWriter(fileMd5, tempFile), file); err != nil { @@ -805,8 +808,10 @@ func (driver Cloud189) uploadPerson(file *model.FileStream, parentFile *model.Fi return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() fileMd5 := md5.New() if _, err = io.Copy(io.MultiWriter(fileMd5, tempFile), file); err != nil { diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index 013d9a49f31..be964c906d3 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -467,8 +467,10 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() delete(reqBody, "pre_hash") h := sha1.New() diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index fa9db1cc36d..c2d461ed933 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -363,8 +363,10 @@ func (driver XunLeiCloud) Upload(file *model.FileStream, account *model.Account) return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() gcid, err := getGcid(io.TeeReader(file, tempFile), int64(file.Size)) if err != nil { From 6a66e39d5bd99f9d8cd90ba955e1de59f76167ed Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 16 May 2022 21:03:00 +0800 Subject: [PATCH 3/9] feat(123):add io stream upload --- drivers/123/driver.go | 61 +++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 50fc39c356b..47c8b21b200 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -1,7 +1,9 @@ package _23 import ( + "bytes" "crypto/md5" + "encoding/binary" "encoding/hex" "fmt" "io" @@ -68,6 +70,12 @@ func (driver Pan123) Items() []base.Item { Required: true, Default: "asc", }, + { + Name: "bool_1", + Label: "stream upload", + Type: base.TypeBool, + Description: "io stream upload (test)", + }, } } @@ -302,25 +310,44 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro return base.ErrNotFolder } - tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") - if err != nil { - return err - } - defer func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }() + const DEFAULT int64 = 10485760 + var uploadFile io.Reader h := md5.New() - if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { - return err - } - etag := hex.EncodeToString(h.Sum(nil)) + if account.Bool1 && file.GetSize() > uint64(DEFAULT) { + // 只计算前10MIB + buf := bytes.NewBuffer(make([]byte, 0, DEFAULT)) + if n, err := io.CopyN(io.MultiWriter(buf, h), file, DEFAULT); err != io.EOF && n == 0 { + return err + } + // 增加额外参数防止MD5碰撞 + h.Write([]byte(file.Name)) + num := make([]byte, 8) + binary.BigEndian.PutUint64(num, file.Size) + h.Write(num) + // 拼装 + uploadFile = io.MultiReader(buf, file) + } else { + // 计算完整文件MD5 + tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } + if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { + return err + } + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } + uploadFile = tempFile + } + etag := hex.EncodeToString(h.Sum(nil)) data := base.Json{ "driveId": 0, "duplicate": 2, // 2->覆盖 1->重命名 0->默认 @@ -354,7 +381,7 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, - Body: tempFile, + Body: uploadFile, } _, err = uploader.Upload(input) if err != nil { From 04e04a1aa61216f5aa7c55061906a3ec7b0dd24d Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 16 May 2022 23:33:12 +0800 Subject: [PATCH 4/9] fix(189pc): delete user-agent for upload --- drivers/189pc/driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 29a1f5ab4c1..5337f79d143 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -607,7 +607,7 @@ func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.Fi uploadData := uploadUrl.UploadUrls[fmt.Sprint("partNumber_", i)] req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, byteData) - req.Header.Del("User-Agent") + req.Header.Set("User-Agent", "") for k, v := range ParseHttpHeader(uploadData.RequestHeader) { req.Header.Set(k, v) } @@ -719,7 +719,7 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File for i := 1; i <= count; i++ { uploadData := uploadUrls.UploadUrls[fmt.Sprint("partNumber_", i)] req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, io.NewSectionReader(tempFile, int64(i-1)*DEFAULT, DEFAULT)) - req.Header.Del("User-Agent") + req.Header.Set("User-Agent", "") for k, v := range ParseHttpHeader(uploadData.RequestHeader) { req.Header.Set(k, v) } From 9207eb69ee47ead481785c2885596b3066181932 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Thu, 19 May 2022 15:31:40 +0800 Subject: [PATCH 5/9] feat: add `m4v` to default video types (close #1114) --- conf/var.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/var.go b/conf/var.go index cf1e9227439..4efc92eac1b 100644 --- a/conf/var.go +++ b/conf/var.go @@ -40,7 +40,7 @@ var ( "go", "sh", "c", "cpp", "h", "hpp", "tsx", "vtt", "srt", "ass"} DProxyTypes = []string{"m3u8"} OfficeTypes = []string{"doc", "docx", "xls", "xlsx", "ppt", "pptx", "pdf"} - VideoTypes = []string{"mp4", "mkv", "avi", "mov", "rmvb", "webm", "flv"} + VideoTypes = []string{"mp4", "mkv", "avi", "mov", "rmvb", "webm", "flv", "m4v"} AudioTypes = []string{"mp3", "flac", "ogg", "m4a", "wav", "opus"} ImageTypes = []string{"jpg", "tiff", "jpeg", "png", "gif", "bmp", "svg", "ico", "swf", "webp"} ) From eb549f2631fc03cdc25f483fe533aed6119e4d12 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Thu, 19 May 2022 15:31:47 +0800 Subject: [PATCH 6/9] feat: add `pdf viewer url` to settings (close #1109) --- bootstrap/setting.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bootstrap/setting.go b/bootstrap/setting.go index 1843325f0df..0927682f171 100644 --- a/bootstrap/setting.go +++ b/bootstrap/setting.go @@ -124,6 +124,13 @@ func InitSettings() { Access: model.PUBLIC, Group: model.FRONT, }, + { + Key: "pdf viewer url", + Type: "string", + Value: "https://alist-org.github.io/pdf.js/web/viewer.html?file=$url", + Access: model.PUBLIC, + Group: model.FRONT, + }, { Key: "autoplay video", Value: "false", From 6a7eb8b3ebca4ab334352ccbc1a096bbdd9f1f13 Mon Sep 17 00:00:00 2001 From: Xhofe Date: Sat, 21 May 2022 22:12:18 +0800 Subject: [PATCH 7/9] fix: don't save search files of balance account (close #1125) --- drivers/base/cache.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/base/cache.go b/drivers/base/cache.go index bde23a17217..b0d34189e6c 100644 --- a/drivers/base/cache.go +++ b/drivers/base/cache.go @@ -5,6 +5,7 @@ import ( "github.com/Xhofe/alist/model" "github.com/Xhofe/alist/utils" log "github.com/sirupsen/logrus" + "strings" ) func KeyCache(path string, account *model.Account) string { @@ -15,6 +16,9 @@ func KeyCache(path string, account *model.Account) string { } func SaveSearchFiles[T model.ISearchFile](key string, obj []T) { + if strings.Contains(key, ".balance") { + return + } err := model.DeleteSearchFilesByPath(key) if err != nil { log.Errorln("failed create search files", err) From 62aefc4f6826b88a4881627555c1ee93a14de188 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Sat, 28 May 2022 20:43:13 +0800 Subject: [PATCH 8/9] fix(189): new resty client --- drivers/189/189.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/drivers/189/189.go b/drivers/189/189.go index ac6ea189dd0..069c13cdc4c 100644 --- a/drivers/189/189.go +++ b/drivers/189/189.go @@ -17,7 +17,6 @@ import ( "io" "math" "net/http" - "net/http/cookiejar" "regexp" "strconv" "strings" @@ -94,18 +93,11 @@ type LoginResp struct { // Login refer to PanIndex func (driver Cloud189) Login(account *model.Account) error { - client, ok := client189Map[account.Name] - if !ok { - //cookieJar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) - client = resty.New() - //client.SetCookieJar(cookieJar) - client.SetTimeout(base.DefaultTimeout) - client.SetRetryCount(3) - client.SetHeader("Referer", "https://cloud.189.cn/") - } - // clear cookie - jar, _ := cookiejar.New(nil) - client.SetCookieJar(jar) + client := resty.New() + //client.SetCookieJar(cookieJar) + client.SetTimeout(base.DefaultTimeout) + client.SetRetryCount(3) + client.SetHeader("Referer", "https://cloud.189.cn/") url := "https://cloud.189.cn/api/portal/loginUrl.action?redirectURL=https%3A%2F%2Fcloud.189.cn%2Fmain.action" b := "" lt := "" @@ -131,7 +123,8 @@ func (driver Cloud189) Login(account *model.Account) error { } } if lt == "" { - return fmt.Errorf("get page: %s \nstatus: %d \nrequest url: %s", b, res.StatusCode(), res.RawResponse.Request.URL.String()) + return fmt.Errorf("get page: %s \nstatus: %d \nrequest url: %s\nredirect url: %s", + b, res.StatusCode(), res.RawResponse.Request.URL.String(), res.Header().Get("location")) } captchaToken := regexp.MustCompile(`captchaToken' value='(.+?)'`).FindStringSubmatch(b)[1] returnUrl := regexp.MustCompile(`returnUrl = '(.+?)'`).FindStringSubmatch(b)[1] From 323dad2a1c4fc9c90b052f5615d5120bf5801bd6 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Sat, 28 May 2022 21:01:04 +0800 Subject: [PATCH 9/9] fix(sftp): infinite loop while remove file (close #1094) --- drivers/sftp/driver.go | 2 +- drivers/sftp/sftp.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index a0b70b1b4fc..a531e826004 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -195,7 +195,7 @@ func (driver SFTP) Delete(path string, account *model.Account) error { if err != nil { return err } - return client.Remove(utils.Join(account.RootFolder, path)) + return client.remove(utils.Join(account.RootFolder, path)) } func (driver SFTP) Upload(file *model.FileStream, account *model.Account) error { diff --git a/drivers/sftp/sftp.go b/drivers/sftp/sftp.go index 8115dbd13a9..c314f55b41e 100644 --- a/drivers/sftp/sftp.go +++ b/drivers/sftp/sftp.go @@ -49,7 +49,7 @@ func (client *Client) Files(remotePath string) ([]os.FileInfo, error) { return client.ReadDir(remotePath) } -func (client *Client) Remove(remotePath string) error { +func (client *Client) remove(remotePath string) error { f, err := client.Stat(remotePath) if err != nil { return nil