From 469336f938a4f6b941199dca6583615ca0ee161f Mon Sep 17 00:00:00 2001 From: Scrub <72096833+ScrubN@users.noreply.github.com> Date: Fri, 27 Oct 2023 23:33:55 -0400 Subject: [PATCH] Many chat updater fixes (#859) * Fix incorrect percentages in chat updater * Make crop lock object not static * Update video info if possible when updating chats * Compress chat crop updater temp files with gzip * Fix ArgumentOutOfRangeException when loading information from chat files with less than 2 comments * Add functionality to deserialize only the first and last comments * Fix chapter updating --- TwitchDownloaderCore/Chat/ChatJson.cs | 6 +- TwitchDownloaderCore/ChatRenderer.cs | 2 +- TwitchDownloaderCore/ChatUpdater.cs | 114 +++++++++++++++--- .../Tools/JsonElementExtensions.cs | 32 +++++ TwitchDownloaderWPF/PageChatUpdate.xaml.cs | 3 +- 5 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 TwitchDownloaderCore/Tools/JsonElementExtensions.cs diff --git a/TwitchDownloaderCore/Chat/ChatJson.cs b/TwitchDownloaderCore/Chat/ChatJson.cs index df0de55e..b156a184 100644 --- a/TwitchDownloaderCore/Chat/ChatJson.cs +++ b/TwitchDownloaderCore/Chat/ChatJson.cs @@ -29,7 +29,7 @@ public static class ChatJson /// A representation the deserialized chat json file. /// The file does not exist. /// The file is not a valid chat format. - public static async Task DeserializeAsync(string filePath, bool getComments = true, bool getEmbeds = true, CancellationToken cancellationToken = new()) + public static async Task DeserializeAsync(string filePath, bool getComments = true, bool onlyFirstAndLastComments = false, bool getEmbeds = true, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(filePath, nameof(filePath)); @@ -82,7 +82,9 @@ public static class ChatJson { if (jsonDocument.RootElement.TryGetProperty("comments", out JsonElement commentsElement)) { - returnChatRoot.comments = commentsElement.Deserialize>(options: _jsonSerializerOptions); + returnChatRoot.comments = onlyFirstAndLastComments + ? commentsElement.DeserializeFirstAndLastFromList(options: _jsonSerializerOptions) + : commentsElement.Deserialize>(options: _jsonSerializerOptions); } } diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index 1516082e..d9bb9276 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -1694,7 +1694,7 @@ private static bool IsRightToLeft(ReadOnlySpan message) public async Task ParseJsonAsync(CancellationToken cancellationToken = new()) { - chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, true, cancellationToken); + chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, false, true, cancellationToken); return chatRoot; } diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 2fb1b315..5e1dd01f 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -8,12 +8,14 @@ using TwitchDownloaderCore.Options; using TwitchDownloaderCore.Tools; using TwitchDownloaderCore.TwitchObjects; +using TwitchDownloaderCore.TwitchObjects.Gql; namespace TwitchDownloaderCore { public sealed class ChatUpdater { public ChatRoot chatRoot { get; internal set; } = new(); + private readonly object _cropChatRootLock = new(); private readonly ChatUpdateOptions _updateOptions; @@ -25,11 +27,6 @@ public ChatUpdater(ChatUpdateOptions updateOptions) "TwitchDownloader"); } - private static class SharedObjects - { - internal static object CropChatRootLock = new(); - } - public async Task UpdateAsync(IProgress progress, CancellationToken cancellationToken) { chatRoot.FileInfo = new() { Version = ChatRootVersion.CurrentVersion, CreatedAt = chatRoot.FileInfo.CreatedAt, UpdatedAt = DateTime.Now }; @@ -40,10 +37,13 @@ public async Task UpdateAsync(IProgress progress, CancellationTo // Dynamic step count setup int currentStep = 0; - int totalSteps = 1; + int totalSteps = 2; if (_updateOptions.CropBeginning || _updateOptions.CropEnding) totalSteps++; if (_updateOptions.EmbedMissing || _updateOptions.ReplaceEmbeds) totalSteps++; + currentStep++; + await UpdateVideoInfo(totalSteps, currentStep, progress, cancellationToken); + // If we are editing the chat crop if (_updateOptions.CropBeginning || _updateOptions.CropEnding) { @@ -60,7 +60,7 @@ public async Task UpdateAsync(IProgress progress, CancellationTo // Finally save the output to file! progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Writing Output File [{++currentStep}/{totalSteps}]")); - progress.Report(new ProgressReport(totalSteps / currentStep)); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); switch (_updateOptions.OutputFormat) { @@ -78,17 +78,100 @@ public async Task UpdateAsync(IProgress progress, CancellationTo } } + private async Task UpdateVideoInfo(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) + { + progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Video Info [{currentStep}/{totalSteps}]")); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); + + if (chatRoot.video.id.All(char.IsDigit)) + { + var videoId = int.Parse(chatRoot.video.id); + VideoInfo videoInfo = null; + try + { + videoInfo = (await TwitchHelper.GetVideoInfo(videoId)).data.video; + } + catch { /* Eat the exception */ } + + if (videoInfo is null) + { + progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch video info, deleted/expired VOD possibly?")); + return; + } + + chatRoot.video.title = videoInfo.title; + chatRoot.video.description = videoInfo.description; + chatRoot.video.created_at = videoInfo.createdAt; + chatRoot.video.length = videoInfo.lengthSeconds; + chatRoot.video.viewCount = videoInfo.viewCount; + chatRoot.video.game = videoInfo.game.displayName; + + var chaptersInfo = (await TwitchHelper.GetOrGenerateVideoChapters(videoId, videoInfo)).data.video.moments.edges; + foreach (var responseChapter in chaptersInfo) + { + chatRoot.video.chapters.Add(new VideoChapter + { + id = responseChapter.node.id, + startMilliseconds = responseChapter.node.positionMilliseconds, + lengthMilliseconds = responseChapter.node.durationMilliseconds, + _type = responseChapter.node._type, + description = responseChapter.node.description, + subDescription = responseChapter.node.subDescription, + thumbnailUrl = responseChapter.node.thumbnailURL, + gameId = responseChapter.node.details.game?.id, + gameDisplayName = responseChapter.node.details.game?.displayName, + gameBoxArtUrl = responseChapter.node.details.game?.boxArtURL + }); + } + } + else + { + var clipId = chatRoot.video.id; + Clip clipInfo = null; + try + { + clipInfo = (await TwitchHelper.GetClipInfo(clipId)).data.clip; + } + catch { /* Eat the exception */ } + + if (clipInfo is null) + { + progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch clip info, deleted possibly?")); + return; + } + + chatRoot.video.title = clipInfo.title; + chatRoot.video.created_at = clipInfo.createdAt; + chatRoot.video.length = clipInfo.durationSeconds; + chatRoot.video.viewCount = clipInfo.viewCount; + chatRoot.video.game = clipInfo.game.displayName; + + var clipChapter = TwitchHelper.GenerateClipChapter(clipInfo); + chatRoot.video.chapters.Add(new VideoChapter + { + id = clipChapter.node.id, + startMilliseconds = clipChapter.node.positionMilliseconds, + lengthMilliseconds = clipChapter.node.durationMilliseconds, + _type = clipChapter.node._type, + description = clipChapter.node.description, + subDescription = clipChapter.node.subDescription, + thumbnailUrl = clipChapter.node.thumbnailURL, + gameId = clipChapter.node.details.game?.id, + gameDisplayName = clipChapter.node.details.game?.displayName, + gameBoxArtUrl = clipChapter.node.details.game?.boxArtURL + }); + } + } + private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) { progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Chat Crop [{currentStep}/{totalSteps}]")); - progress.Report(new ProgressReport(totalSteps / currentStep)); - - chatRoot.video ??= new Video(); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); bool cropTaskVodExpired = false; var cropTaskProgress = new Progress(report => { - if (((string)report.Data).ToLower().Contains("vod is expired")) + if (((string)report.Data).Contains("vod is expired", StringComparison.OrdinalIgnoreCase)) { // If the user is moving both crops in one command, we only want to propagate a 'vod expired/id corrupt' report once if (cropTaskVodExpired) @@ -145,7 +228,7 @@ private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) { progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Updating Embeds [{currentStep}/{totalSteps}]")); - progress.Report(new ProgressReport(totalSteps / currentStep)); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); chatRoot.embeddedData ??= new EmbeddedData(); @@ -313,7 +396,7 @@ private async Task ChatEndingCropTask(IProgress progress, Cancel ChatDownloader chatDownloader = new ChatDownloader(downloadOptions); await chatDownloader.DownloadAsync(new Progress(), cancellationToken); - ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, getEmbeds: false, cancellationToken); + ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, onlyFirstAndLastComments: false, getEmbeds: false, cancellationToken); // Append the new comment section SortedSet commentsSet = new SortedSet(new SortedCommentComparer()); @@ -325,7 +408,7 @@ private async Task ChatEndingCropTask(IProgress progress, Cancel } } - lock (SharedObjects.CropChatRootLock) + lock (_cropChatRootLock) { foreach (var comment in chatRoot.comments) { @@ -345,6 +428,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi { Id = videoId, DownloadFormat = ChatFormat.Json, // json is required to parse as a new chatroot object + Compression = ChatCompression.Gzip, Filename = tempFile, CropBeginning = true, CropBeginningTime = sectionStart, @@ -361,7 +445,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi public async Task ParseJsonAsync(CancellationToken cancellationToken = new()) { - chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, true, cancellationToken); + chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, false, true, cancellationToken); return chatRoot; } } diff --git a/TwitchDownloaderCore/Tools/JsonElementExtensions.cs b/TwitchDownloaderCore/Tools/JsonElementExtensions.cs new file mode 100644 index 00000000..24f0e82e --- /dev/null +++ b/TwitchDownloaderCore/Tools/JsonElementExtensions.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Text.Json; + +namespace TwitchDownloaderCore.Tools +{ + public static class JsonElementExtensions + { + public static List DeserializeFirstAndLastFromList(this JsonElement arrayElement, JsonSerializerOptions options = null) + { + // It's not the prettiest, but for arrays with thousands of objects it can save whole seconds and prevent tons of fragmented memory + var list = new List(2); + JsonElement lastElement = default; + foreach (var element in arrayElement.EnumerateArray()) + { + if (list.Count == 0) + { + list.Add(element.Deserialize(options: options)); + continue; + } + + lastElement = element; + } + + if (lastElement.ValueKind != JsonValueKind.Undefined) + { + list.Add(lastElement.Deserialize(options: options)); + } + + return list; + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs index 0acbd49e..706ac3f3 100644 --- a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs +++ b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs @@ -66,8 +66,7 @@ private async void btnBrowse_Click(object sender, RoutedEventArgs e) try { - ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, false, CancellationToken.None); - ChatJsonInfo.comments.RemoveRange(1, ChatJsonInfo.comments.Count - 2); + ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, true, false, CancellationToken.None); GC.Collect(); } catch (Exception ex)