Replace forbidden filename characters with full-width alternatives (#…

…1094) * Replace invalid path chars with full-width alternatives * More appropriate name * Replace timestamp separators with underscores * Use long instead of string for view template * Sanitize video ID just in case * Update tests * Sanitize output file name in CLI * Actually that may not be a great idea
lay295 · Jun 10, 2024 · 741fa72 · 741fa72
1 parent 88c3f46
commit 741fa72
Show file tree

Hide file tree

Showing 7 changed files with 85 additions and 46 deletions.
diff --git a/TwitchDownloaderCore.Tests/ToolTests/FilenameServiceTests.cs b/TwitchDownloaderCore.Tests/ToolTests/FilenameServiceTests.cs
@@ -5,8 +5,8 @@ namespace TwitchDownloaderCore.Tests.ToolTests
 {
     public class FilenameServiceTests
     {
-        private static (string title, string id, DateTime date, string channel, TimeSpan trimStart, TimeSpan trimEnd, string viewCount, string game) GetExampleInfo() =>
-            ("A Title", "abc123", new DateTime(1984, 11, 1, 9, 43, 21), "streamer8", new TimeSpan(0, 1, 2, 3, 4), new TimeSpan(0, 5, 6, 7, 8), "123456789", "A Game");
+        private static (string title, string id, DateTime date, string channel, TimeSpan trimStart, TimeSpan trimEnd, int viewCount, string game) GetExampleInfo() =>
+            ("A Title", "abc123", new DateTime(1984, 11, 1, 9, 43, 21), "streamer8", new TimeSpan(0, 1, 2, 3, 4), new TimeSpan(0, 5, 6, 7, 8), 123456789, "A Game");
 
         [Theory]
         [InlineData("{title}", "A Title")]
@@ -81,19 +81,17 @@ public void CorrectlyGeneratesSubFolders_WithBackSlash()
         }
 
         [Theory]
-        [InlineData("{title}")]
-        [InlineData("{id}")]
-        [InlineData("{channel}")]
-        [InlineData("{views}")]
-        [InlineData("{game}")]
-        public void CorrectlyReplacesInvalidCharactersForNonCustomTemplates(string template)
+        [InlineData("{title}", "＂＊：＜＞？｜／＼")]
+        [InlineData("{id}", "＂＊：＜＞？｜／＼")]
+        [InlineData("{channel}", "＂＊：＜＞？｜／＼")]
+        [InlineData("{game}", "＂＊：＜＞？｜／＼")]
+        public void CorrectlyReplacesInvalidCharactersForNonCustomTemplates(string template, string expected)
         {
-            const char EXPECTED = '_';
-            var invalidChars = new string(Path.GetInvalidFileNameChars());
+            const string INVALID_CHARS = "\"*:<>?|/\\";
 
-            var result = FilenameService.GetFilename(template, invalidChars, invalidChars, default, invalidChars, default, default, invalidChars, invalidChars);
+            var result = FilenameService.GetFilename(template, INVALID_CHARS, INVALID_CHARS, default, INVALID_CHARS, default, default, default, INVALID_CHARS);
 
-            Assert.All(result, c => Assert.Equal(EXPECTED, c));
+            Assert.Equal(expected, result);
         }
 
         [Theory]
@@ -103,27 +101,25 @@ public void CorrectlyReplacesInvalidCharactersForNonCustomTemplates(string templ
         [InlineData("{length_custom=\"'")]
         public void CorrectlyReplacesInvalidCharactersForCustomTemplates(string templateStart)
         {
-            const char EXPECTED = '_';
-            var invalidChars = new string(Path.GetInvalidFileNameChars());
-            var template = string.Concat(
-                templateStart,
-                invalidChars.ReplaceAny("\r\n", EXPECTED), // newline chars are not supported by the custom parameters. This will not change.
-                "'\"}");
+            const string EXPECTED = "＂＊：＜＞？｜／＼";
+            const string INVALID_CHARS = "\"*:<>?|/\\\\";
+            var template = templateStart + INVALID_CHARS + "'\"}";
 
-            var result = FilenameService.GetFilename(template, invalidChars, invalidChars, default, invalidChars, default, default, invalidChars, invalidChars);
+            var result = FilenameService.GetFilename(template, INVALID_CHARS, INVALID_CHARS, default, INVALID_CHARS, default, default, default, INVALID_CHARS);
 
-            Assert.All(result, c => Assert.Equal(EXPECTED, c));
+            Assert.Equal(EXPECTED, result);
         }
 
         [Fact]
         public void CorrectlyReplacesInvalidCharactersForSubFolders()
         {
-            var invalidChars = new string(Path.GetInvalidPathChars());
-            var template = invalidChars + "\\{title}";
-            var expected = Path.Combine(new string('_', invalidChars.Length), "A Title");
+            const string INVALID_CHARS = "\"*:<>?|";
+            const string FULL_WIDTH_CHARS = "＂＊：＜＞？｜";
+            const string TEMPLATE = INVALID_CHARS + "\\{title}";
+            var expected = Path.Combine(FULL_WIDTH_CHARS, "A Title");
             var (title, id, date, channel, trimStart, trimEnd, viewCount, game) = GetExampleInfo();
 
-            var result = FilenameService.GetFilename(template, title, id, date, channel, trimStart, trimEnd, viewCount, game);
+            var result = FilenameService.GetFilename(TEMPLATE, title, id, date, channel, trimStart, trimEnd, viewCount, game);
 
             Assert.Equal(expected, result);
         }
@@ -152,6 +148,24 @@ public void DoesNotInterpretBogusTemplateParameter()
             Assert.Equal(EXPECTED, result);
         }
 
+        [Theory]
+        [InlineData("\"", "＂")]
+        [InlineData("*", "＊")]
+        [InlineData(":", "：")]
+        [InlineData("<", "＜")]
+        [InlineData(">", "＞")]
+        [InlineData("?", "？")]
+        [InlineData("|", "｜")]
+        [InlineData("/", "／")]
+        [InlineData("\\", "＼")]
+        [InlineData("\0", "_")]
+        public void CorrectlyReplacesInvalidFilenameCharacters(string str, string expected)
+        {
+            var actual = FilenameService.ReplaceInvalidFilenameChars(str);
+
+            Assert.Equal(expected, actual);
+        }
+
         [Fact]
         public void GetNonCollidingNameWorks_WhenNoCollisionExists()
         {

diff --git a/TwitchDownloaderCore/Tools/FilenameService.cs b/TwitchDownloaderCore/Tools/FilenameService.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Globalization;
 using System.IO;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -8,21 +9,21 @@ namespace TwitchDownloaderCore.Tools
 {
     public static class FilenameService
     {
-        public static string GetFilename(string template, string title, string id, DateTime date, string channel, TimeSpan trimStart, TimeSpan trimEnd, string viewCount, string game)
+        public static string GetFilename(string template, string title, string id, DateTime date, string channel, TimeSpan trimStart, TimeSpan trimEnd, long viewCount, string game)
         {
             var videoLength = trimEnd - trimStart;
 
             var stringBuilder = new StringBuilder(template)
-                .Replace("{title}", RemoveInvalidFilenameChars(title))
-                .Replace("{id}", id)
-                .Replace("{channel}", RemoveInvalidFilenameChars(channel))
+                .Replace("{title}", ReplaceInvalidFilenameChars(title))
+                .Replace("{id}", ReplaceInvalidFilenameChars(id))
+                .Replace("{channel}", ReplaceInvalidFilenameChars(channel))
                 .Replace("{date}", date.ToString("M-d-yy"))
                 .Replace("{random_string}", Path.GetRandomFileName().Remove(8)) // Remove the period
                 .Replace("{trim_start}", TimeSpanHFormat.ReusableInstance.Format(@"HH\-mm\-ss", trimStart))
                 .Replace("{trim_end}", TimeSpanHFormat.ReusableInstance.Format(@"HH\-mm\-ss", trimEnd))
                 .Replace("{length}", TimeSpanHFormat.ReusableInstance.Format(@"HH\-mm\-ss", videoLength))
-                .Replace("{views}", viewCount)
-                .Replace("{game}", RemoveInvalidFilenameChars(game));
+                .Replace("{views}", viewCount.ToString(CultureInfo.CurrentCulture))
+                .Replace("{game}", ReplaceInvalidFilenameChars(game));
 
             if (template.Contains("{date_custom="))
             {
@@ -50,7 +51,7 @@ public static string GetFilename(string template, string title, string id, DateT
 
             var fileName = stringBuilder.ToString();
             var additionalSubfolders = GetTemplateSubfolders(ref fileName);
-            return Path.Combine(Path.Combine(additionalSubfolders), RemoveInvalidFilenameChars(fileName));
+            return Path.Combine(Path.Combine(additionalSubfolders), ReplaceInvalidFilenameChars(fileName));
         }
 
         private static void ReplaceCustomWithFormattable(StringBuilder sb, Regex regex, IFormattable formattable, IFormatProvider formatProvider = null)
@@ -65,7 +66,7 @@ private static void ReplaceCustomWithFormattable(StringBuilder sb, Regex regex,
 
                 var formatString = match.Groups[1].Value;
                 sb.Remove(match.Groups[0].Index, match.Groups[0].Length);
-                sb.Insert(match.Groups[0].Index, RemoveInvalidFilenameChars(formattable.ToString(formatString, formatProvider)));
+                sb.Insert(match.Groups[0].Index, ReplaceInvalidFilenameChars(formattable.ToString(formatString, formatProvider)));
             } while (true);
         }
 
@@ -77,15 +78,39 @@ private static string[] GetTemplateSubfolders(ref string fullPath)
 
             for (var i = 0; i < returnString.Length; i++)
             {
-                returnString[i] = RemoveInvalidFilenameChars(returnString[i]);
+                returnString[i] = ReplaceInvalidFilenameChars(returnString[i]);
             }
 
             return returnString;
         }
 
         private static readonly char[] FilenameInvalidChars = Path.GetInvalidFileNameChars();
 
-        private static string RemoveInvalidFilenameChars(string filename) => filename.ReplaceAny(FilenameInvalidChars, '_');
+        public static string ReplaceInvalidFilenameChars(string filename)
+        {
+            const string TIMESTAMP_PATTERN = /*lang=regex*/ @"(?<=\d):(?=\d\d)";
+            var newName = Regex.Replace(filename, TIMESTAMP_PATTERN, "_");
+
+            if (newName.AsSpan().IndexOfAny("\"*:<>?|/\\") != -1)
+            {
+                newName = string.Create(filename.Length, filename, (span, str) =>
+                {
+                    const int FULL_WIDTH_OFFSET = 0xFEE0; // https://en.wikipedia.org/wiki/Halfwidth_and_Fullwidth_Forms_(Unicode_block)
+                    for (var i = 0; i < str.Length; i++)
+                    {
+                        var ch = str[i];
+                        span[i] = ch switch
+                        {
+                            '\"' or '*' or ':' or '<' or '>' or '?' or '|' or '/' or '\\' => (char)(ch + FULL_WIDTH_OFFSET),
+                            _ => ch
+                        };
+                    }
+                });
+            }
+
+            // In case there are additional invalid chars such as control codes
+            return newName.ReplaceAny(FilenameInvalidChars, '_');
+        }
 
         public static FileInfo GetNonCollidingName(FileInfo fileInfo)
         {

diff --git a/TwitchDownloaderWPF/PageChatDownload.xaml.cs b/TwitchDownloaderWPF/PageChatDownload.xaml.cs
@@ -453,7 +453,7 @@ private async void SplitBtnDownload_Click(object sender, RoutedEventArgs e)
                 FileName = FilenameService.GetFilename(Settings.Default.TemplateChat, textTitle.Text, downloadId, currentVideoTime, textStreamer.Text,
                     CheckTrimStart.IsChecked == true ? new TimeSpan((int)numStartHour.Value, (int)numStartMinute.Value, (int)numStartSecond.Value) : TimeSpan.Zero,
                     CheckTrimEnd.IsChecked == true ? new TimeSpan((int)numEndHour.Value, (int)numEndMinute.Value, (int)numEndSecond.Value) : vodLength,
-                    viewCount.ToString(), game)
+                    viewCount, game)
             };
 
             if (radioJson.IsChecked == true)

diff --git a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs
@@ -487,7 +487,7 @@ private async void SplitBtnUpdate_Click(object sender, RoutedEventArgs e)
                     ChatJsonInfo.video.id ?? ChatJsonInfo.comments.FirstOrDefault()?.content_id ?? "-1", VideoCreatedAt, textStreamer.Text,
                     checkStart.IsChecked == true ? new TimeSpan((int)numStartHour.Value, (int)numStartMinute.Value, (int)numStartSecond.Value) : TimeSpan.FromSeconds(double.IsNegative(ChatJsonInfo.video.start) ? 0.0 : ChatJsonInfo.video.start),
                     checkEnd.IsChecked == true ? new TimeSpan((int)numEndHour.Value, (int)numEndMinute.Value, (int)numEndSecond.Value) : VideoLength,
-                    ViewCount.ToString(), Game)
+                    ViewCount, Game)
             };
 
             if (radioJson.IsChecked == true)

diff --git a/TwitchDownloaderWPF/PageClipDownload.xaml.cs b/TwitchDownloaderWPF/PageClipDownload.xaml.cs
@@ -202,7 +202,7 @@ private async void SplitBtnDownload_Click(object sender, RoutedEventArgs e)
             SaveFileDialog saveFileDialog = new SaveFileDialog
             {
                 Filter = "MP4 Files | *.mp4",
-                FileName = FilenameService.GetFilename(Settings.Default.TemplateClip, textTitle.Text, clipId, currentVideoTime, textStreamer.Text, TimeSpan.Zero, clipLength, viewCount.ToString(), game) + ".mp4"
+                FileName = FilenameService.GetFilename(Settings.Default.TemplateClip, textTitle.Text, clipId, currentVideoTime, textStreamer.Text, TimeSpan.Zero, clipLength, viewCount, game) + ".mp4"
             };
             if (saveFileDialog.ShowDialog() != true)
             {

diff --git a/TwitchDownloaderWPF/PageVodDownload.xaml.cs b/TwitchDownloaderWPF/PageVodDownload.xaml.cs
@@ -204,7 +204,7 @@ public VideoDownloadOptions GetOptions(string filename, string folder)
                 Filename = filename ?? Path.Combine(folder, FilenameService.GetFilename(Settings.Default.TemplateVod, textTitle.Text, currentVideoId.ToString(), currentVideoTime, textStreamer.Text,
                     checkStart.IsChecked == true ? new TimeSpan((int)numStartHour.Value, (int)numStartMinute.Value, (int)numStartSecond.Value) : TimeSpan.Zero,
                     checkEnd.IsChecked == true ? new TimeSpan((int)numEndHour.Value, (int)numEndMinute.Value, (int)numEndSecond.Value) : vodLength,
-                    viewCount.ToString(), game) + (comboQuality.Text.Contains("Audio", StringComparison.OrdinalIgnoreCase) ? ".m4a" : ".mp4")),
+                    viewCount, game) + (comboQuality.Text.Contains("Audio", StringComparison.OrdinalIgnoreCase) ? ".m4a" : ".mp4")),
                 Oauth = TextOauth.Text,
                 Quality = GetQualityWithoutSize(comboQuality.Text),
                 Id = currentVideoId,
@@ -412,7 +412,7 @@ private async void SplitBtnDownloader_Click(object sender, RoutedEventArgs e)
                 FileName = FilenameService.GetFilename(Settings.Default.TemplateVod, textTitle.Text, currentVideoId.ToString(), currentVideoTime, textStreamer.Text,
                     checkStart.IsChecked == true ? new TimeSpan((int)numStartHour.Value, (int)numStartMinute.Value, (int)numStartSecond.Value) : TimeSpan.Zero,
                     checkEnd.IsChecked == true ? new TimeSpan((int)numEndHour.Value, (int)numEndMinute.Value, (int)numEndSecond.Value) : vodLength,
-                    viewCount.ToString(), game) + ".mp4"
+                    viewCount, game) + ".mp4"
             };
             if (saveFileDialog.ShowDialog() == false)
             {

diff --git a/TwitchDownloaderWPF/WindowQueueOptions.xaml.cs b/TwitchDownloaderWPF/WindowQueueOptions.xaml.cs
@@ -225,7 +225,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e)
                     {
                         Filename = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateClip, clipDownloadPage.textTitle.Text, clipDownloadPage.clipId,
                             clipDownloadPage.currentVideoTime, clipDownloadPage.textStreamer.Text, TimeSpan.Zero, clipDownloadPage.clipLength,
-                            clipDownloadPage.viewCount.ToString(), clipDownloadPage.game) + ".mp4"),
+                            clipDownloadPage.viewCount, clipDownloadPage.game) + ".mp4"),
                         Id = clipDownloadPage.clipId,
                         Quality = clipDownloadPage.comboQuality.Text,
                         ThrottleKib = Settings.Default.DownloadThrottleEnabled
@@ -267,7 +267,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e)
                         chatOptions.EmbedData = checkEmbed.IsChecked.GetValueOrDefault();
                         chatOptions.Filename = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateChat, downloadTask.Info.Title, chatOptions.Id,
                             clipDownloadPage.currentVideoTime, clipDownloadPage.textStreamer.Text, TimeSpan.Zero, clipDownloadPage.clipLength,
-                            clipDownloadPage.viewCount.ToString(), clipDownloadPage.game) + "." + chatOptions.FileExtension);
+                            clipDownloadPage.viewCount, clipDownloadPage.game) + "." + chatOptions.FileExtension);
                         chatOptions.FileCollisionCallback = HandleFileCollisionCallback;
 
                         ChatDownloadTask chatTask = new ChatDownloadTask
@@ -333,7 +333,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e)
                     chatOptions.Filename = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateChat, chatDownloadPage.textTitle.Text, chatOptions.Id,chatDownloadPage.currentVideoTime, chatDownloadPage.textStreamer.Text,
                         chatOptions.TrimBeginning ? TimeSpan.FromSeconds(chatOptions.TrimBeginningTime) : TimeSpan.Zero,
                         chatOptions.TrimEnding ? TimeSpan.FromSeconds(chatOptions.TrimEndingTime) : chatDownloadPage.vodLength,
-                        chatDownloadPage.viewCount.ToString(), chatDownloadPage.game) + "." + chatOptions.FileExtension);
+                        chatDownloadPage.viewCount, chatDownloadPage.game) + "." + chatOptions.FileExtension);
                     chatOptions.FileCollisionCallback = HandleFileCollisionCallback;
 
                     ChatDownloadTask chatTask = new ChatDownloadTask
@@ -393,7 +393,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e)
                     chatOptions.OutputFile = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateChat, chatUpdatePage.textTitle.Text, chatUpdatePage.VideoId, chatUpdatePage.VideoCreatedAt, chatUpdatePage.textStreamer.Text,
                         chatOptions.TrimBeginning ? TimeSpan.FromSeconds(chatOptions.TrimBeginningTime) : TimeSpan.Zero,
                         chatOptions.TrimEnding ? TimeSpan.FromSeconds(chatOptions.TrimEndingTime) : chatUpdatePage.VideoLength,
-                        chatUpdatePage.ViewCount.ToString(), chatUpdatePage.Game) + "." + chatOptions.FileExtension);
+                        chatUpdatePage.ViewCount, chatUpdatePage.Game) + "." + chatOptions.FileExtension);
                     chatOptions.FileCollisionCallback = HandleFileCollisionCallback;
 
                     ChatUpdateTask chatTask = new ChatUpdateTask
@@ -495,7 +495,7 @@ private void EnqueueDataList()
                         downloadOptions.Filename = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateVod, taskData.Title, taskData.Id, taskData.Time, taskData.Streamer,
                             downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero,
                             downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : TimeSpan.FromSeconds(taskData.Length),
-                            taskData.Views.ToString(), taskData.Game) + ".mp4");
+                            taskData.Views, taskData.Game) + ".mp4");
 
                         VodDownloadTask downloadTask = new VodDownloadTask
                         {
@@ -520,7 +520,7 @@ private void EnqueueDataList()
                             Id = taskData.Id,
                             Quality = (ComboPreferredQuality.SelectedItem as ComboBoxItem)?.Content as string,
                             Filename = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateClip, taskData.Title, taskData.Id, taskData.Time, taskData.Streamer,
-                                TimeSpan.Zero, TimeSpan.FromSeconds(taskData.Length), taskData.Views.ToString(), taskData.Game) + ".mp4"),
+                                TimeSpan.Zero, TimeSpan.FromSeconds(taskData.Length), taskData.Views, taskData.Game) + ".mp4"),
                             ThrottleKib = Settings.Default.DownloadThrottleEnabled
                                 ? Settings.Default.MaximumBandwidthKib
                                 : -1,
@@ -569,7 +569,7 @@ private void EnqueueDataList()
                     downloadOptions.Filename = Path.Combine(folderPath, FilenameService.GetFilename(Settings.Default.TemplateChat, taskData.Title, taskData.Id, taskData.Time, taskData.Streamer,
                         downloadOptions.TrimBeginning ? TimeSpan.FromSeconds(downloadOptions.TrimBeginningTime) : TimeSpan.Zero,
                         downloadOptions.TrimEnding ? TimeSpan.FromSeconds(downloadOptions.TrimEndingTime) : TimeSpan.FromSeconds(taskData.Length),
-                        taskData.Views.ToString(), taskData.Game) + "." + downloadOptions.FileExtension);
+                        taskData.Views, taskData.Game) + "." + downloadOptions.FileExtension);
 
                     ChatDownloadTask downloadTask = new ChatDownloadTask
                     {