Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix being able to download chats before they start/after they end via the CLI #1272

Merged
merged 2 commits into from
Dec 26, 2024
Merged

Fix being able to download chats before they start/after they end via the CLI #1272

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5a38931
Fix being able to download chats before they start/after they end via…
ScrubN Dec 26, 2024
2c25e8c
Improve chatdownload error handling
ScrubN Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions TwitchDownloaderCore/ChatDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public ChatDownloader(ChatDownloadOptions chatDownloadOptions, ITaskProgress pro
"TwitchDownloader");
}

private static async Task<List<Comment>> DownloadSection(Range downloadRange, string videoId, IProgress<int> progress, ChatFormat format, CancellationToken cancellationToken)
private static async Task<List<Comment>> DownloadSection(Range downloadRange, string videoId, IProgress<int> progress, ITaskLogger logger, ChatFormat format, CancellationToken cancellationToken)
{
var comments = new List<Comment>();
int videoStart = downloadRange.Start.Value;
Expand All @@ -51,7 +51,8 @@ private static async Task<List<Comment>> DownloadSection(Range downloadRange, st
double latestMessage = videoStart - 1;
bool isFirst = true;
string cursor = "";
int errorCount = 0;
double errorCount = 0;
double nullCount = 0;

while (latestMessage < videoEnd)
{
Expand Down Expand Up @@ -83,26 +84,36 @@ private static async Task<List<Comment>> DownloadSection(Range downloadRange, st
httpResponse.EnsureSuccessStatusCode();
commentResponse = await httpResponse.Content.ReadFromJsonAsync<GqlCommentResponse>(options: null, cancellationToken);
}

errorCount = 0;
}
catch (HttpRequestException)
catch (HttpRequestException ex)
{
if (++errorCount > 10)
{
throw;
}

await Task.Delay(1_000 * errorCount, cancellationToken);
logger.LogVerbose($"Exception '{ex.Message}' thrown at {latestMessage}s ({cursor}) in range {downloadRange}. Current error factor: {errorCount}.");
await Task.Delay((int)(1_000 * errorCount), cancellationToken);
continue;
}

// video.comments can be null for some dumb reason
if (commentResponse.data.video.comments?.edges is null)
{
// video.comments can be null for some dumb reason, skip
if (++nullCount > 10)
{
throw new Exception("Received too many null comment lists. Try reducing your download threads.");
}

logger.LogVerbose($"Received null comment list at {latestMessage}s ({cursor}) in range {downloadRange}. Current null factor: {nullCount}.");
await Task.Delay((int)(100 * nullCount), cancellationToken);
continue;
}

const double BACK_OFF_FACTOR = 0.1;
nullCount = Math.Max(0, nullCount - BACK_OFF_FACTOR);
errorCount = Math.Max(0, errorCount - BACK_OFF_FACTOR);

var convertedComments = ConvertComments(commentResponse.data.video, format);
foreach (var comment in convertedComments)
{
Expand Down Expand Up @@ -336,8 +347,8 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF
chatRoot.video.description = videoInfoResponse.data.video.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd();
chatRoot.video.title = videoInfoResponse.data.video.title;
chatRoot.video.created_at = videoInfoResponse.data.video.createdAt;
chatRoot.video.start = downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : 0.0;
chatRoot.video.end = downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : videoInfoResponse.data.video.lengthSeconds;
chatRoot.video.start = downloadOptions.TrimBeginning ? Math.Max(0, downloadOptions.TrimBeginningTime) : 0.0;
chatRoot.video.end = downloadOptions.TrimEnding ? Math.Min(downloadOptions.TrimEndingTime, videoInfoResponse.data.video.lengthSeconds) : videoInfoResponse.data.video.lengthSeconds;
chatRoot.video.length = videoInfoResponse.data.video.lengthSeconds;
chatRoot.video.viewCount = videoInfoResponse.data.video.viewCount;
chatRoot.video.game = videoInfoResponse.data.video.game?.displayName ?? "Unknown";
Expand Down Expand Up @@ -437,7 +448,7 @@ private async Task<List<Comment>> DownloadComments(CancellationToken cancellatio
var start = videoStart + chunkSize * i;
var end = Math.Min(videoEnd, start + chunkSize);
var downloadRange = new Range(start, end);
downloadTasks.Add(DownloadSection(downloadRange, video.id, taskProgress, downloadOptions.DownloadFormat, cancellationToken));
downloadTasks.Add(DownloadSection(downloadRange, video.id, taskProgress, _progress, downloadOptions.DownloadFormat, cancellationToken));
}

await Task.WhenAll(downloadTasks);
Expand Down
Loading