Skip to content

Commit

Permalink
Many chat updater fixes (#859)
Browse files Browse the repository at this point in the history 
* Fix incorrect percentages in chat updater

* Make crop lock object not static

* Update video info if possible when updating chats

* Compress chat crop updater temp files with gzip

* Fix ArgumentOutOfRangeException when loading information from chat files with less than 2 comments

* Add functionality to deserialize only the first and last comments

* Fix chapter updating
  • Loading branch information
@ScrubN
ScrubN authored Oct 28, 2023
1 parent c3db4a6 commit 469336f
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 20 deletions.
6 changes: 4 additions & 2 deletions TwitchDownloaderCore/Chat/ChatJson.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public static class ChatJson
/// <returns>A <see cref="ChatRoot"/> representation the deserialized chat json file.</returns>
/// <exception cref="IOException">The file does not exist.</exception>
/// <exception cref="NotSupportedException">The file is not a valid chat format.</exception>
public static async Task<ChatRoot> DeserializeAsync(string filePath, bool getComments = true, bool getEmbeds = true, CancellationToken cancellationToken = new())
public static async Task<ChatRoot> DeserializeAsync(string filePath, bool getComments = true, bool onlyFirstAndLastComments = false, bool getEmbeds = true, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(filePath, nameof(filePath));

Expand Down Expand Up @@ -82,7 +82,9 @@ public static class ChatJson
{
if (jsonDocument.RootElement.TryGetProperty("comments", out JsonElement commentsElement))
{
returnChatRoot.comments = commentsElement.Deserialize<List<Comment>>(options: _jsonSerializerOptions);
returnChatRoot.comments = onlyFirstAndLastComments
? commentsElement.DeserializeFirstAndLastFromList<Comment>(options: _jsonSerializerOptions)
: commentsElement.Deserialize<List<Comment>>(options: _jsonSerializerOptions);
}
}

Expand Down
2 changes: 1 addition & 1 deletion TwitchDownloaderCore/ChatRenderer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1694,7 +1694,7 @@ private static bool IsRightToLeft(ReadOnlySpan<char> message)

public async Task<ChatRoot> ParseJsonAsync(CancellationToken cancellationToken = new())
{
chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, true, cancellationToken);
chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, false, true, cancellationToken);
return chatRoot;
}

Expand Down
114 changes: 99 additions & 15 deletions TwitchDownloaderCore/ChatUpdater.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@
using TwitchDownloaderCore.Options;
using TwitchDownloaderCore.Tools;
using TwitchDownloaderCore.TwitchObjects;
using TwitchDownloaderCore.TwitchObjects.Gql;

namespace TwitchDownloaderCore
{
public sealed class ChatUpdater
{
public ChatRoot chatRoot { get; internal set; } = new();
private readonly object _cropChatRootLock = new();

private readonly ChatUpdateOptions _updateOptions;

Expand All @@ -25,11 +27,6 @@ public ChatUpdater(ChatUpdateOptions updateOptions)
"TwitchDownloader");
}

private static class SharedObjects
{
internal static object CropChatRootLock = new();
}

public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationToken cancellationToken)
{
chatRoot.FileInfo = new() { Version = ChatRootVersion.CurrentVersion, CreatedAt = chatRoot.FileInfo.CreatedAt, UpdatedAt = DateTime.Now };
Expand All @@ -40,10 +37,13 @@ public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationTo

// Dynamic step count setup
int currentStep = 0;
int totalSteps = 1;
int totalSteps = 2;
if (_updateOptions.CropBeginning || _updateOptions.CropEnding) totalSteps++;
if (_updateOptions.EmbedMissing || _updateOptions.ReplaceEmbeds) totalSteps++;

currentStep++;
await UpdateVideoInfo(totalSteps, currentStep, progress, cancellationToken);

// If we are editing the chat crop
if (_updateOptions.CropBeginning || _updateOptions.CropEnding)
{
Expand All @@ -60,7 +60,7 @@ public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationTo

// Finally save the output to file!
progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Writing Output File [{++currentStep}/{totalSteps}]"));
progress.Report(new ProgressReport(totalSteps / currentStep));
progress.Report(new ProgressReport(currentStep * 100 / totalSteps));

switch (_updateOptions.OutputFormat)
{
Expand All @@ -78,17 +78,100 @@ public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationTo
}
}

private async Task UpdateVideoInfo(int totalSteps, int currentStep, IProgress<ProgressReport> progress, CancellationToken cancellationToken)
{
progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Video Info [{currentStep}/{totalSteps}]"));
progress.Report(new ProgressReport(currentStep * 100 / totalSteps));

if (chatRoot.video.id.All(char.IsDigit))
{
var videoId = int.Parse(chatRoot.video.id);
VideoInfo videoInfo = null;
try
{
videoInfo = (await TwitchHelper.GetVideoInfo(videoId)).data.video;
}
catch { /* Eat the exception */ }

if (videoInfo is null)
{
progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch video info, deleted/expired VOD possibly?"));
return;
}

chatRoot.video.title = videoInfo.title;
chatRoot.video.description = videoInfo.description;
chatRoot.video.created_at = videoInfo.createdAt;
chatRoot.video.length = videoInfo.lengthSeconds;
chatRoot.video.viewCount = videoInfo.viewCount;
chatRoot.video.game = videoInfo.game.displayName;

var chaptersInfo = (await TwitchHelper.GetOrGenerateVideoChapters(videoId, videoInfo)).data.video.moments.edges;
foreach (var responseChapter in chaptersInfo)
{
chatRoot.video.chapters.Add(new VideoChapter
{
id = responseChapter.node.id,
startMilliseconds = responseChapter.node.positionMilliseconds,
lengthMilliseconds = responseChapter.node.durationMilliseconds,
_type = responseChapter.node._type,
description = responseChapter.node.description,
subDescription = responseChapter.node.subDescription,
thumbnailUrl = responseChapter.node.thumbnailURL,
gameId = responseChapter.node.details.game?.id,
gameDisplayName = responseChapter.node.details.game?.displayName,
gameBoxArtUrl = responseChapter.node.details.game?.boxArtURL
});
}
}
else
{
var clipId = chatRoot.video.id;
Clip clipInfo = null;
try
{
clipInfo = (await TwitchHelper.GetClipInfo(clipId)).data.clip;
}
catch { /* Eat the exception */ }

if (clipInfo is null)
{
progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch clip info, deleted possibly?"));
return;
}

chatRoot.video.title = clipInfo.title;
chatRoot.video.created_at = clipInfo.createdAt;
chatRoot.video.length = clipInfo.durationSeconds;
chatRoot.video.viewCount = clipInfo.viewCount;
chatRoot.video.game = clipInfo.game.displayName;

var clipChapter = TwitchHelper.GenerateClipChapter(clipInfo);
chatRoot.video.chapters.Add(new VideoChapter
{
id = clipChapter.node.id,
startMilliseconds = clipChapter.node.positionMilliseconds,
lengthMilliseconds = clipChapter.node.durationMilliseconds,
_type = clipChapter.node._type,
description = clipChapter.node.description,
subDescription = clipChapter.node.subDescription,
thumbnailUrl = clipChapter.node.thumbnailURL,
gameId = clipChapter.node.details.game?.id,
gameDisplayName = clipChapter.node.details.game?.displayName,
gameBoxArtUrl = clipChapter.node.details.game?.boxArtURL
});
}
}

private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress<ProgressReport> progress, CancellationToken cancellationToken)
{
progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Chat Crop [{currentStep}/{totalSteps}]"));
progress.Report(new ProgressReport(totalSteps / currentStep));

chatRoot.video ??= new Video();
progress.Report(new ProgressReport(currentStep * 100 / totalSteps));

bool cropTaskVodExpired = false;
var cropTaskProgress = new Progress<ProgressReport>(report =>
{
if (((string)report.Data).ToLower().Contains("vod is expired"))
if (((string)report.Data).Contains("vod is expired", StringComparison.OrdinalIgnoreCase))
{
// If the user is moving both crops in one command, we only want to propagate a 'vod expired/id corrupt' report once
if (cropTaskVodExpired)
Expand Down Expand Up @@ -145,7 +228,7 @@ private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress<Pro
private async Task UpdateEmbeds(int currentStep, int totalSteps, IProgress<ProgressReport> progress, CancellationToken cancellationToken)
{
progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Updating Embeds [{currentStep}/{totalSteps}]"));
progress.Report(new ProgressReport(totalSteps / currentStep));
progress.Report(new ProgressReport(currentStep * 100 / totalSteps));

chatRoot.embeddedData ??= new EmbeddedData();

Expand Down Expand Up @@ -313,7 +396,7 @@ private async Task ChatEndingCropTask(IProgress<ProgressReport> progress, Cancel
ChatDownloader chatDownloader = new ChatDownloader(downloadOptions);
await chatDownloader.DownloadAsync(new Progress<ProgressReport>(), cancellationToken);

ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, getEmbeds: false, cancellationToken);
ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, onlyFirstAndLastComments: false, getEmbeds: false, cancellationToken);

// Append the new comment section
SortedSet<Comment> commentsSet = new SortedSet<Comment>(new SortedCommentComparer());
Expand All @@ -325,7 +408,7 @@ private async Task ChatEndingCropTask(IProgress<ProgressReport> progress, Cancel
}
}

lock (SharedObjects.CropChatRootLock)
lock (_cropChatRootLock)
{
foreach (var comment in chatRoot.comments)
{
Expand All @@ -345,6 +428,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi
{
Id = videoId,
DownloadFormat = ChatFormat.Json, // json is required to parse as a new chatroot object
Compression = ChatCompression.Gzip,
Filename = tempFile,
CropBeginning = true,
CropBeginningTime = sectionStart,
Expand All @@ -361,7 +445,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi

public async Task<ChatRoot> ParseJsonAsync(CancellationToken cancellationToken = new())
{
chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, true, cancellationToken);
chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, false, true, cancellationToken);
return chatRoot;
}
}
Expand Down
32 changes: 32 additions & 0 deletions TwitchDownloaderCore/Tools/JsonElementExtensions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using System.Collections.Generic;
using System.Text.Json;

namespace TwitchDownloaderCore.Tools
{
public static class JsonElementExtensions
{
public static List<T> DeserializeFirstAndLastFromList<T>(this JsonElement arrayElement, JsonSerializerOptions options = null)
{
// It's not the prettiest, but for arrays with thousands of objects it can save whole seconds and prevent tons of fragmented memory
var list = new List<T>(2);
JsonElement lastElement = default;
foreach (var element in arrayElement.EnumerateArray())
{
if (list.Count == 0)
{
list.Add(element.Deserialize<T>(options: options));
continue;
}

lastElement = element;
}

if (lastElement.ValueKind != JsonValueKind.Undefined)
{
list.Add(lastElement.Deserialize<T>(options: options));
}

return list;
}
}
}
3 changes: 1 addition & 2 deletions TwitchDownloaderWPF/PageChatUpdate.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ private async void btnBrowse_Click(object sender, RoutedEventArgs e)

try
{
ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, false, CancellationToken.None);
ChatJsonInfo.comments.RemoveRange(1, ChatJsonInfo.comments.Count - 2);
ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, true, false, CancellationToken.None);
GC.Collect();
}
catch (Exception ex)
Expand Down

0 comments on commit 469336f

Please sign in to comment.