Merge pull request #45 from AbdourahmaneGadio/gadio-patch-1

Add a file format formater

README.md

@@ -26,6 +26,7 @@ Usage:
 -U, --update                 Download missing tweet only
 -L, --login                  Log in to your account
 -o, --output   DIR           Output directory
+-f, --file-format FORMAT     Formatted name for the downloaded file
 -p, --proxy    PROXY         Use proxy (proto://ip:port)
 -V, --version                Print version and exit
 -B, --[no-]banner            Don't print banner

go.mod

@@ -4,7 +4,7 @@ go 1.22.0
 
 require (
 	github.com/andlabs/ui v0.0.0-20200610043537-70a69d6ae31e
-	github.com/imperatrona/twitter-scraper v0.0.0-20240309152646-e6d6dc71a98f
+	github.com/imperatrona/twitter-scraper v0.0.0-20240321062442-6a0f92e31498
 	github.com/mmpx12/optionparser v1.1.0
 	github.com/n0madic/twitter-scraper v0.0.0-20231104223941-296710769dd8
 	github.com/sqweek/dialog v0.0.0-20240226140203-065105509627
@@ -13,7 +13,7 @@ require (
 
 require (
 	github.com/AlexEidt/Vidio v1.5.1 // indirect
-	github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf // indirect
+	github.com/TheTitanrain/w32 v0.0.0-20200114052255-2654d97dbd3d // indirect
 	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect

go.sum

@@ -1,16 +1,17 @@
 github.com/AlexEidt/Vidio v1.5.1 h1:tovwvtgQagUz1vifiL9OeWkg1fP/XUzFazFKh7tFtaE=
 github.com/AlexEidt/Vidio v1.5.1/go.mod h1:djhIMnWMqPrC3X6nB6ymGX6uWWlgw+VayYGKE1bNwmI=
-github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf h1:FPsprx82rdrX2jiKyS17BH6IrTmUBYqZa/CXT4uvb+I=
 github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf/go.mod h1:peYoMncQljjNS6tZwI9WVyQB3qZS6u79/N3mBOcnd3I=
+github.com/TheTitanrain/w32 v0.0.0-20200114052255-2654d97dbd3d h1:2xp1BQbqcDDaikHnASWpVZRjibOxu7y9LhAv04whugI=
+github.com/TheTitanrain/w32 v0.0.0-20200114052255-2654d97dbd3d/go.mod h1:peYoMncQljjNS6tZwI9WVyQB3qZS6u79/N3mBOcnd3I=
 github.com/andlabs/ui v0.0.0-20200610043537-70a69d6ae31e h1:wSQCJiig/QkoUnpvelSPbLiZNWvh2yMqQTQvIQqSUkU=
 github.com/andlabs/ui v0.0.0-20200610043537-70a69d6ae31e/go.mod h1:5G2EjwzgZUPnnReoKvPWVneT8APYbyKkihDVAHUi0II=
 github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
 github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/imperatrona/twitter-scraper v0.0.0-20240309152646-e6d6dc71a98f h1:6YSLFbbkI4x5Vru+as7lrNJH7r6hnEQ+u1Dp1wiKvKk=
-github.com/imperatrona/twitter-scraper v0.0.0-20240309152646-e6d6dc71a98f/go.mod h1:+Z1pca7Faf2tzpHVRnRcratFxy/PuDKj2iaygdgZRLM=
+github.com/imperatrona/twitter-scraper v0.0.0-20240321062442-6a0f92e31498 h1:RXuNC5UJRKa+JZTocOtAKbDBW26yRi+mIcl1hod1nso=
+github.com/imperatrona/twitter-scraper v0.0.0-20240321062442-6a0f92e31498/go.mod h1:+Z1pca7Faf2tzpHVRnRcratFxy/PuDKj2iaygdgZRLM=
 github.com/mmpx12/optionparser v1.1.0 h1:CgfC8WBDxkHOlg9myndDMezNiyXeMzVRDLWDRIjdlf8=
 github.com/mmpx12/optionparser v1.1.0/go.mod h1:1Ub9+E2fDinPCmAU2lCuJcXE8x0HCmkurDv+lcXgRd8=
 github.com/n0madic/twitter-scraper v0.0.0-20231104223941-296710769dd8 h1:yToM7p7HL/WwEESkupFV3Nf7a8d80CHaKRoFNstGA2U=

twmd.go

@@ -20,20 +20,23 @@ import (
 	twitterscraper "github.com/imperatrona/twitter-scraper"
 	"github.com/mmpx12/optionparser"
 	"golang.org/x/term"
+	"unicode/utf8"
 )
 
 var (
-	usr     string
-	proxy   string
-	update  bool
-	onlyrtw bool
-	vidz    bool
-	imgs    bool
-	urlOnly bool
-	version = "1.12.0"
-	scraper *twitterscraper.Scraper
-	client  *http.Client
-	size    = "orig"
+	usr        string
+	format     string
+	formatName string
+	proxy      string
+	update     bool
+	onlyrtw    bool
+	vidz       bool
+	imgs       bool
+	urlOnly    bool
+	version    = "1.13.0"
+	scraper    *twitterscraper.Scraper
+	client     *http.Client
+	size       = "orig"
 )
 
 func download(wg *sync.WaitGroup, url string, filetype string, output string, dwn_type string) {
@@ -45,6 +48,9 @@ func download(wg *sync.WaitGroup, url string, filetype string, output string, dw
 		segments := strings.Split(name, "?")
 		name = segments[len(segments)-2]
 	}
+	if format != "" {
+		name = formatName + "_" + name
+	}
 	if urlOnly {
 		fmt.Println(url)
 		time.Sleep(2 * time.Millisecond)
@@ -260,6 +266,9 @@ func singleTweet(output string, id string) {
 		fmt.Println("Error retrieve tweet")
 		return
 	}
+	if format != "" {
+		getFormat(tweet)
+	}
 	if usr != "" {
 		if vidz {
 			videoSingle(tweet, output)
@@ -273,6 +282,129 @@ func singleTweet(output string, id string) {
 	}
 }
 
+func getFormat(tweet interface{}) {
+	var formatNew string
+	var tweetResult *twitterscraper.TweetResult
+	var tweetObj *twitterscraper.Tweet
+
+	switch t := tweet.(type) {
+	case *twitterscraper.TweetResult:
+		tweetResult = t
+	case *twitterscraper.Tweet:
+		tweetObj = t
+	default:
+		fmt.Println("Invalid tweet type")
+		return
+	}
+
+	formatParts := strings.Split(format, " ")
+
+	pattern := `[/\\:*?\"<>|]`
+
+	regex, err := regexp.Compile(pattern)
+	if err != nil {
+		fmt.Println("Error compiling regular expression:", err)
+		return
+	}
+
+	processText := func(text string, remainingChars int) string {
+		result := ""
+		for _, char := range text {
+			charStr := string(char)
+			if regex.MatchString(charStr) {
+				if utf8.RuneCountInString(result)+1 > remainingChars {
+					break
+				}
+				result += "_"
+				remainingChars--
+			} else if utf8.RuneCountInString(result)+1 <= remainingChars {
+				result += charStr
+				remainingChars--
+			} else {
+				break
+			}
+		}
+		return result
+	}
+
+	processPart := func(part string) {
+		switch part {
+		case "{DATE}":
+			var timestamp int64
+			if tweetResult != nil {
+				timestamp = tweetResult.Timestamp
+			} else if tweetObj != nil {
+				timestamp = tweetObj.Timestamp
+			} else {
+				fmt.Println("Error converting timestamp:", err)
+				return
+			}
+			t := time.Unix(timestamp, 0)
+			if err != nil {
+				fmt.Println("Error converting timestamp:", err)
+				return
+			}
+			date := t.Format("2006-01-02")
+			formatNew += date
+
+		case "{NAME}":
+			if tweetResult != nil {
+				formatNew += tweetResult.Name
+			} else if tweetObj != nil {
+				formatNew += tweetObj.Name
+			}
+
+		case "{USERNAME}":
+			if tweetResult != nil {
+				formatNew += tweetResult.Username
+			} else if tweetObj != nil {
+				formatNew += tweetObj.Username
+			}
+
+		case "{TITLE}":
+			var text string
+			var remainingChars int
+
+			if tweetResult != nil {
+				text = strings.ReplaceAll(tweetResult.Text, "/", "_")
+				remainingChars = 255 - len(formatNew) - len(tweetResult.Name) - len(tweetResult.Username) - len(tweetResult.ID)
+			} else if tweetObj != nil {
+				text = strings.ReplaceAll(tweetObj.Text, "/", "_")
+				remainingChars = 251 - len(formatNew) - len(tweetObj.ID) - 4
+			}
+
+			if text == "" {
+				formatNew += ""
+			} else if remainingChars > 0 && len(text) > remainingChars {
+				formatNew += processText(text, remainingChars)
+			} else {
+				formatNew += text
+			}
+
+		case "{ID}":
+			if tweetResult != nil {
+				formatNew += tweetResult.ID
+			} else if tweetObj != nil {
+				formatNew += tweetObj.ID
+			}
+		default:
+			fmt.Println("Invalid format part")
+			return
+		}
+	}
+
+	for i, part := range formatParts {
+		processPart(part)
+
+		if i != len(formatParts)-1 {
+			formatNew += "_"
+		}
+	}
+
+	formatName = formatNew
+
+}
+
 func main() {
 	var nbr, single, output string
 	var retweet, all, printversion, nologo, login, twofa bool
@@ -290,6 +422,7 @@ func main() {
 	op.On("-s", "--size SIZE", "Choose size between small|normal|large (default large)", &size)
 	op.On("-U", "--update", "Download missing tweet only", &update)
 	op.On("-o", "--output DIR", "Output directory", &output)
+	op.On("-f", "--file-format FORMAT", "Formatted name for the downloaded file, {DATE} {USERNAME} {NAME} {TITLE} {ID}", &format)
 	op.On("-L", "--login", "Login (needed for NSFW tweets)", &login)
 	op.On("-2", "--2fa", "Use 2fa", &twofa)
 	op.On("-p", "--proxy PROXY", "Use proxy (proto://ip:port)", &proxy)
@@ -299,6 +432,7 @@ func main() {
 	op.Exemple("twmd -u Spraytrains -o ~/Downlaods -R -U -n 300")
 	op.Exemple("twmd --proxy socks5://127.0.0.1:9050 -t 156170319961391104")
 	op.Exemple("twmd -t 156170319961391104")
+	op.Exemple("twmd -t 156170319961391104 -f \"{DATE} {ID}\"")
 	op.Parse()
 
 	if printversion {
@@ -321,8 +455,14 @@ func main() {
 		op.Help()
 		os.Exit(1)
 	}
+	var re = regexp.MustCompile(`{ID}|{DATE}|{NAME}|{USERNAME}|{TITLE}`)
+	if format != "" && !re.MatchString(format) {
+		fmt.Println("You must specify a format (-f --format)")
+		op.Help()
+		os.Exit(1)
+	}
 
-	re := regexp.MustCompile("small|normal|large")
+	re = regexp.MustCompile("small|normal|large")
 	if !re.MatchString(size) && size != "orig" {
 		print("Error in size, setting up to normal\n")
 		size = ""
@@ -387,6 +527,9 @@ func main() {
 			fmt.Println(tweet.Error)
 			os.Exit(1)
 		}
+		if format != "" {
+			getFormat(tweet)
+		}
 		if vidz {
 			wg.Add(1)
 			go videoUser(&wg, tweet, output, retweet)