From 9d7581366d9b173a90b6ccc916f6e265382aba18 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:20:28 -0500 Subject: [PATCH 1/9] Create ReadOnlySpanTryReplaceNonEscapedTests.DoesNotSupportReplacingEscapeChars --- .../ReadOnlySpanTryReplaceNonEscapedTests.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/TwitchDownloaderCore.Tests/ReadOnlySpanTryReplaceNonEscapedTests.cs b/TwitchDownloaderCore.Tests/ReadOnlySpanTryReplaceNonEscapedTests.cs index aa15d1a3..98977068 100644 --- a/TwitchDownloaderCore.Tests/ReadOnlySpanTryReplaceNonEscapedTests.cs +++ b/TwitchDownloaderCore.Tests/ReadOnlySpanTryReplaceNonEscapedTests.cs @@ -91,5 +91,16 @@ public void DoesNotEscapeDifferingQuotes() Assert.False(success); } + + [Fact] + public void DoesNotSupportReplacingEscapeChars() + { + ReadOnlySpan str = "\"SORRY FOR\" TRAFFIC NaM.\""; + var destination = new char[str.Length]; + + var success = str.TryReplaceNonEscaped(destination, '\"', 'W'); + + Assert.False(success); + } } } \ No newline at end of file From f2387edcddbc54efa55bb79848ce7050295099a2 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:22:21 -0500 Subject: [PATCH 2/9] Only target net6.0 for TwitchDownloaderCore.Tests --- TwitchDownloaderCore.Tests/TwitchDownloaderCore.Tests.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TwitchDownloaderCore.Tests/TwitchDownloaderCore.Tests.csproj b/TwitchDownloaderCore.Tests/TwitchDownloaderCore.Tests.csproj index c5e90bf8..7d386da3 100644 --- a/TwitchDownloaderCore.Tests/TwitchDownloaderCore.Tests.csproj +++ b/TwitchDownloaderCore.Tests/TwitchDownloaderCore.Tests.csproj @@ -1,13 +1,13 @@ - net6.0;net6.0-windows + net6.0 enable enable false true - latest + default From b537bf6ef7a73258c40fba1d4714c01f67fc2239 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:24:45 -0500 Subject: [PATCH 3/9] Improve performance of TimeSpanHFormat and return TimeSpan.ToString when format is null or empty --- .../TimeSpanHFormatTests.cs | 20 ++++++++--- TwitchDownloaderCore/Tools/TimeSpanHFormat.cs | 35 +++++++++---------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/TwitchDownloaderCore.Tests/TimeSpanHFormatTests.cs b/TwitchDownloaderCore.Tests/TimeSpanHFormatTests.cs index 8f78db92..8a9d4252 100644 --- a/TwitchDownloaderCore.Tests/TimeSpanHFormatTests.cs +++ b/TwitchDownloaderCore.Tests/TimeSpanHFormatTests.cs @@ -34,7 +34,7 @@ public void CustomFormatOverloadMatchesICustomFormatter() var timeSpan = new TimeSpan(17, 49, 12); const string FORMAT_STRING = @"HH\:mm\:ss"; - var resultICustomFormatter = ((ICustomFormatter)TimeSpanHFormat.ReusableInstance).Format(FORMAT_STRING, timeSpan,null); + var resultICustomFormatter = ((ICustomFormatter)TimeSpanHFormat.ReusableInstance).Format(FORMAT_STRING, timeSpan, null); var resultCustom = TimeSpanHFormat.ReusableInstance.Format(FORMAT_STRING, timeSpan); Assert.Equal(resultICustomFormatter, resultCustom); @@ -77,15 +77,27 @@ public void CorrectlyFormatsNull() } [Fact] - public void ReturnsEmptyString_WhenFormatIsEmpty() + public void ReturnsTimeSpanToString_WhenFormatIsEmpty() { var timeSpan = new TimeSpan(17, 49, 12); const string FORMAT_STRING = ""; - const string EXPECTED = ""; + var expected = timeSpan.ToString(); var resultCustom = TimeSpanHFormat.ReusableInstance.Format(FORMAT_STRING, timeSpan); - Assert.Equal(EXPECTED, resultCustom); + Assert.Equal(expected, resultCustom); + } + + [Fact] + public void ReturnsTimeSpanToString_WhenFormatIsNull() + { + var timeSpan = new TimeSpan(17, 49, 12); + const string FORMAT_STRING = null!; + var expected = timeSpan.ToString(); + + var resultCustom = TimeSpanHFormat.ReusableInstance.Format(FORMAT_STRING, timeSpan); + + Assert.Equal(expected, resultCustom); } [Fact] diff --git a/TwitchDownloaderCore/Tools/TimeSpanHFormat.cs b/TwitchDownloaderCore/Tools/TimeSpanHFormat.cs index de0f0cc5..6339d509 100644 --- a/TwitchDownloaderCore/Tools/TimeSpanHFormat.cs +++ b/TwitchDownloaderCore/Tools/TimeSpanHFormat.cs @@ -6,7 +6,7 @@ namespace TwitchDownloaderCore.Tools { /// Adds an 'H' parameter to string formatting. The 'H' parameter is equivalent to flooring .. /// - /// For optimal memory performance, resulting strings split about any 'H' parameters should be less than 256. + /// For optimal memory performance, resulting strings split about any 'H' parameters should be less than 256 chars in length. /// public class TimeSpanHFormat : IFormatProvider, ICustomFormatter { @@ -36,7 +36,7 @@ public string Format(string format, TimeSpan timeSpan, IFormatProvider formatPro { if (string.IsNullOrEmpty(format)) { - return ""; + return timeSpan.ToString(format, formatProvider); } if (!format.Contains('H')) @@ -100,14 +100,14 @@ private static string HandleBigHFormat(ReadOnlySpan format, TimeSpan timeS switch (readChar) { // If the current char is an escape we can skip the next char - case '\\' when i + 1 < formatLength: + case '\\': i++; continue; // If the current char is a quote we can skip the next quote, if it exists - case '\'' when i + 1 < formatLength: - case '\"' when i + 1 < formatLength: + case '\'': + case '\"': { - i = FindCloseQuoteMark(format, i, formatLength, readChar); + i = FindCloseQuoteChar(format, i, formatLength, readChar); if (i == -1) { @@ -134,30 +134,28 @@ private static string HandleBigHFormat(ReadOnlySpan format, TimeSpan timeS return sb.ToString(); } - private static int FindCloseQuoteMark(ReadOnlySpan format, int openQuoteIndex, int endIndex, char readChar) + private static int FindCloseQuoteChar(ReadOnlySpan destination, int openQuoteIndex, int endIndex, char openQuoteChar) { var i = openQuoteIndex + 1; - var quoteFound = false; while (i < endIndex) { - var readCharQuote = format[i]; + var readChar = destination[i]; i++; - if (readCharQuote == '\\') + if (readChar == '\\') { i++; continue; } - if (readCharQuote == readChar) + if (readChar == openQuoteChar) { i--; - quoteFound = true; - break; + return i; } } - return quoteFound ? i : -1; + return -1; } private static void AppendRegularFormat(StringBuilder sb, TimeSpan timeSpan, ReadOnlySpan format) @@ -178,16 +176,17 @@ private static void AppendBigHFormat(StringBuilder sb, TimeSpan timeSpan, int co { const int TIMESPAN_MAX_HOURS_LENGTH = 9; // The maximum integer hours a TimeSpan can hold is 256204778. Span destination = stackalloc char[TIMESPAN_MAX_HOURS_LENGTH]; - Span format = stackalloc char[count]; - format.Fill('0'); - if (((int)timeSpan.TotalHours).TryFormat(destination, out var charsWritten, format)) + if (((uint)timeSpan.TotalHours).TryFormat(destination, out var charsWritten)) { + sb.Append('0', Math.Max(0, count - charsWritten)); sb.Append(destination[..charsWritten]); } else { - sb.Append(((int)timeSpan.TotalHours).ToString(format.ToString())); + var foo = ((uint)timeSpan.TotalHours).ToString(); + sb.Append('0', Math.Max(0, count - foo.Length)); + sb.Append(foo); } } From d814a76a6b53111c1404e53320e5f18eec45647a Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:31:27 -0500 Subject: [PATCH 4/9] Update ReadOnlySpanExtensions --- .../Extensions/ReadOnlySpanExtensions.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs b/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs index 433553bd..22db9e17 100644 --- a/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs +++ b/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs @@ -9,6 +9,9 @@ public static bool TryReplaceNonEscaped(this ReadOnlySpan str, Span { const string ESCAPE_CHARS = @"\'"""; + if (oldChar is '\\' or '\'' or '\"') + return false; + if (destination.Length < str.Length) return false; @@ -40,7 +43,7 @@ public static bool TryReplaceNonEscaped(this ReadOnlySpan str, Span case '\'': case '\"': { - i = FindCloseQuoteMark(destination, i, lastIndex, readChar); + i = FindCloseQuoteChar(destination, i, lastIndex, readChar); if (i == -1) { @@ -65,30 +68,28 @@ public static bool TryReplaceNonEscaped(this ReadOnlySpan str, Span return true; } - private static int FindCloseQuoteMark(ReadOnlySpan destination, int openQuoteIndex, int endIndex, char readChar) + private static int FindCloseQuoteChar(ReadOnlySpan destination, int openQuoteIndex, int endIndex, char openQuoteChar) { var i = openQuoteIndex + 1; - var quoteFound = false; while (i < endIndex) { - var readCharQuote = destination[i]; + var readChar = destination[i]; i++; - if (readCharQuote == '\\') + if (readChar == '\\') { i++; continue; } - if (readCharQuote == readChar) + if (readChar == openQuoteChar) { i--; - quoteFound = true; - break; + return i; } } - return quoteFound ? i : -1; + return -1; } } } \ No newline at end of file From 4cf838a42cd2745bacd96639869ac3aa8f1223e6 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:33:46 -0500 Subject: [PATCH 5/9] Use non-memory overload to prevent internal reallocation --- TwitchDownloaderCore/Extensions/StreamExtensions.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/TwitchDownloaderCore/Extensions/StreamExtensions.cs b/TwitchDownloaderCore/Extensions/StreamExtensions.cs index 35bb2c7b..92f863f5 100644 --- a/TwitchDownloaderCore/Extensions/StreamExtensions.cs +++ b/TwitchDownloaderCore/Extensions/StreamExtensions.cs @@ -23,15 +23,14 @@ public static async Task ProgressCopyToAsync(this Stream source, Stream destinat } var rentedBuffer = ArrayPool.Shared.Rent(STREAM_DEFAULT_BUFFER_LENGTH); - var buffer = rentedBuffer.AsMemory(0, STREAM_DEFAULT_BUFFER_LENGTH); long totalBytesRead = 0; try { int bytesRead; - while ((bytesRead = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) != 0) + while ((bytesRead = await source.ReadAsync(rentedBuffer, 0, rentedBuffer.Length, cancellationToken).ConfigureAwait(false)) != 0) { - await destination.WriteAsync(buffer[..bytesRead], cancellationToken).ConfigureAwait(false); + await destination.WriteAsync(rentedBuffer, 0, bytesRead, cancellationToken).ConfigureAwait(false); totalBytesRead += bytesRead; progress.Report(new StreamCopyProgress(sourceLength.Value, totalBytesRead)); @@ -39,7 +38,7 @@ public static async Task ProgressCopyToAsync(this Stream source, Stream destinat } finally { - ArrayPool.Shared.Return(rentedBuffer); + ArrayPool.Shared.Return(rentedBuffer, true); } } } From e63469d2490d3f7533410c69587e52659c1c8ba6 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:38:24 -0500 Subject: [PATCH 6/9] Consolidate ChatCompression, ChatFormat, and TimestampFormat into Enums.cs --- .../Modes/Arguments/ChatDownloadArgs.cs | 2 +- .../Modes/Arguments/ChatUpdateArgs.cs | 2 +- TwitchDownloaderCLI/Modes/DownloadChat.cs | 1 - TwitchDownloaderCLI/Modes/UpdateChat.cs | 2 +- TwitchDownloaderCore/Chat/ChatCompression.cs | 9 ------- TwitchDownloaderCore/Chat/ChatFormat.cs | 9 ------- TwitchDownloaderCore/Chat/TimestampFormat.cs | 10 -------- .../Options/ChatDownloadOptions.cs | 2 +- .../Options/ChatUpdateOptions.cs | 2 +- TwitchDownloaderCore/Tools/Enums.cs | 24 +++++++++++++++++++ 10 files changed, 29 insertions(+), 34 deletions(-) delete mode 100644 TwitchDownloaderCore/Chat/ChatCompression.cs delete mode 100644 TwitchDownloaderCore/Chat/ChatFormat.cs delete mode 100644 TwitchDownloaderCore/Chat/TimestampFormat.cs create mode 100644 TwitchDownloaderCore/Tools/Enums.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs index 6e4d5a10..b83b7ae7 100644 --- a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs +++ b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs @@ -1,5 +1,5 @@ using CommandLine; -using TwitchDownloaderCore.Chat; +using TwitchDownloaderCore.Tools; namespace TwitchDownloaderCLI.Modes.Arguments { diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs index 0ebd86f8..507291cb 100644 --- a/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs +++ b/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs @@ -1,5 +1,5 @@ using CommandLine; -using TwitchDownloaderCore.Chat; +using TwitchDownloaderCore.Tools; namespace TwitchDownloaderCLI.Modes.Arguments { diff --git a/TwitchDownloaderCLI/Modes/DownloadChat.cs b/TwitchDownloaderCLI/Modes/DownloadChat.cs index cf6bb9a9..bd6a2159 100644 --- a/TwitchDownloaderCLI/Modes/DownloadChat.cs +++ b/TwitchDownloaderCLI/Modes/DownloadChat.cs @@ -4,7 +4,6 @@ using TwitchDownloaderCLI.Modes.Arguments; using TwitchDownloaderCLI.Tools; using TwitchDownloaderCore; -using TwitchDownloaderCore.Chat; using TwitchDownloaderCore.Options; using TwitchDownloaderCore.Tools; diff --git a/TwitchDownloaderCLI/Modes/UpdateChat.cs b/TwitchDownloaderCLI/Modes/UpdateChat.cs index bc214f9c..926e92e6 100644 --- a/TwitchDownloaderCLI/Modes/UpdateChat.cs +++ b/TwitchDownloaderCLI/Modes/UpdateChat.cs @@ -4,8 +4,8 @@ using TwitchDownloaderCLI.Modes.Arguments; using TwitchDownloaderCLI.Tools; using TwitchDownloaderCore; -using TwitchDownloaderCore.Chat; using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.Tools; namespace TwitchDownloaderCLI.Modes { diff --git a/TwitchDownloaderCore/Chat/ChatCompression.cs b/TwitchDownloaderCore/Chat/ChatCompression.cs deleted file mode 100644 index 1f6bf04c..00000000 --- a/TwitchDownloaderCore/Chat/ChatCompression.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace TwitchDownloaderCore.Chat -{ - // TODO: Add Bzip2 and possibly 7Zip support - public enum ChatCompression - { - None, - Gzip - } -} diff --git a/TwitchDownloaderCore/Chat/ChatFormat.cs b/TwitchDownloaderCore/Chat/ChatFormat.cs deleted file mode 100644 index 814e49e8..00000000 --- a/TwitchDownloaderCore/Chat/ChatFormat.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace TwitchDownloaderCore.Chat -{ - public enum ChatFormat - { - Json, - Text, - Html - } -} diff --git a/TwitchDownloaderCore/Chat/TimestampFormat.cs b/TwitchDownloaderCore/Chat/TimestampFormat.cs deleted file mode 100644 index 5386f812..00000000 --- a/TwitchDownloaderCore/Chat/TimestampFormat.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace TwitchDownloaderCore.Chat -{ - public enum TimestampFormat - { - Utc, - Relative, - None, - UtcFull - } -} diff --git a/TwitchDownloaderCore/Options/ChatDownloadOptions.cs b/TwitchDownloaderCore/Options/ChatDownloadOptions.cs index ee7c2c2b..7491a2bc 100644 --- a/TwitchDownloaderCore/Options/ChatDownloadOptions.cs +++ b/TwitchDownloaderCore/Options/ChatDownloadOptions.cs @@ -1,4 +1,4 @@ -using TwitchDownloaderCore.Chat; +using TwitchDownloaderCore.Tools; namespace TwitchDownloaderCore.Options { diff --git a/TwitchDownloaderCore/Options/ChatUpdateOptions.cs b/TwitchDownloaderCore/Options/ChatUpdateOptions.cs index 6b0eac01..fc4c1d51 100644 --- a/TwitchDownloaderCore/Options/ChatUpdateOptions.cs +++ b/TwitchDownloaderCore/Options/ChatUpdateOptions.cs @@ -1,4 +1,4 @@ -using TwitchDownloaderCore.Chat; +using TwitchDownloaderCore.Tools; namespace TwitchDownloaderCore.Options { diff --git a/TwitchDownloaderCore/Tools/Enums.cs b/TwitchDownloaderCore/Tools/Enums.cs new file mode 100644 index 00000000..1d2a4b94 --- /dev/null +++ b/TwitchDownloaderCore/Tools/Enums.cs @@ -0,0 +1,24 @@ +namespace TwitchDownloaderCore.Tools +{ + // TODO: Add Bzip2 and possibly 7Zip support + public enum ChatCompression + { + None, + Gzip + } + + public enum ChatFormat + { + Json, + Text, + Html + } + + public enum TimestampFormat + { + Utc, + Relative, + None, + UtcFull + } +} \ No newline at end of file From 3a947a9085ffd22e1993061e64c2c0e4b17b8be2 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:42:06 -0500 Subject: [PATCH 7/9] Relax IO access restrictions --- TwitchDownloaderCore/Tools/FfmpegMetadata.cs | 2 +- TwitchDownloaderCore/TwitchHelper.cs | 2 +- TwitchDownloaderCore/VideoDownloader.cs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index e3ce2107..a0a5ecca 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -16,7 +16,7 @@ public static class FfmpegMetadata public static async Task SerializeAsync(string filePath, string streamerName, string videoId, string videoTitle, DateTime videoCreation, int viewCount, string videoDescription = null, double startOffsetSeconds = 0, IEnumerable videoMomentEdges = null, CancellationToken cancellationToken = default) { - await using var fs = new FileStream(filePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); + await using var fs = new FileStream(filePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; await SerializeGlobalMetadata(sw, streamerName, videoId, videoTitle, videoCreation, viewCount, videoDescription); diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs index d7494e91..c5dc77fb 100644 --- a/TwitchDownloaderCore/TwitchHelper.cs +++ b/TwitchDownloaderCore/TwitchHelper.cs @@ -908,7 +908,7 @@ public static async Task GetUserInfo(List idList) //Let's save this image to the cache try { - await using var stream = File.Open(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + await using var stream = File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await stream.WriteAsync(imageBytes, cancellationToken); } catch { } diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 674f1441..dc58afd3 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -369,7 +369,7 @@ private static bool VerifyVideoPart(string downloadFolder, string part) return false; } - using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); + using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read); var fileLength = fs.Length; if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) { @@ -668,7 +668,7 @@ private async Task CombineVideoParts(string downloadFolder, List videoPa int partCount = videoParts.Count; int doneCount = 0; - await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); + await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); foreach (var part in videoParts) { await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); From cb011e63e6ab3ed770b30e4afb80bbef35ae84fa Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:15:33 -0500 Subject: [PATCH 8/9] Backport M3U8 parser and ReadOnlySpanExtensions: Count, UnEscapedIndexOf, and UnEscapedIndexOfAny & related tests --- TwitchDownloaderCore.Tests/M3U8Tests.cs | 423 ++++++++++++ .../ReadOnlySpanCountTests.cs | 44 ++ .../ReadOnlySpanUnEscapedIndexOfAnyTests.cs | 115 ++++ .../ReadOnlySpanUnEscapedIndexOfTests.cs | 115 ++++ .../Extensions/ReadOnlySpanExtensions.cs | 118 ++++ TwitchDownloaderCore/Tools/M3U8.cs | 637 ++++++++++++++++++ 6 files changed, 1452 insertions(+) create mode 100644 TwitchDownloaderCore.Tests/M3U8Tests.cs create mode 100644 TwitchDownloaderCore.Tests/ReadOnlySpanCountTests.cs create mode 100644 TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfAnyTests.cs create mode 100644 TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfTests.cs create mode 100644 TwitchDownloaderCore/Tools/M3U8.cs diff --git a/TwitchDownloaderCore.Tests/M3U8Tests.cs b/TwitchDownloaderCore.Tests/M3U8Tests.cs new file mode 100644 index 00000000..9251ba82 --- /dev/null +++ b/TwitchDownloaderCore.Tests/M3U8Tests.cs @@ -0,0 +1,423 @@ +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCore.Tests +{ + // ReSharper disable StringLiteralTypo + public class M3U8Tests + { + [Fact] + public void CorrectlyParsesTwitchM3U8OfTransportStreams() + { + const string ExampleM3U8Twitch = + "#EXTM3U" + + "\n#EXT-X-VERSION:3" + + "\n#EXT-X-TARGETDURATION:10" + + "\n#ID3-EQUIV-TDTG:2023-09-23T17:37:06" + + "\n#EXT-X-PLAYLIST-TYPE:EVENT" + + "\n#EXT-X-MEDIA-SEQUENCE:0" + + "\n#EXT-X-TWITCH-ELAPSED-SECS:0.000" + + "\n#EXT-X-TWITCH-TOTAL-SECS:500.000" + + "\n#EXTINF:10.000,\n0.ts\n#EXTINF:10.000,\n1.ts\n#EXTINF:10.000,\n2.ts\n#EXTINF:10.000,\n3.ts\n#EXTINF:10.000,\n4.ts\n#EXTINF:10.000,\n5.ts\n#EXTINF:10.000,\n6.ts\n#EXTINF:10.000,\n7.ts" + + "\n#EXTINF:10.000,\n8.ts\n#EXTINF:10.000,\n9.ts\n#EXTINF:10.000,\n10.ts\n#EXTINF:10.000,\n11.ts\n#EXTINF:10.000,\n12.ts\n#EXTINF:10.000,\n13.ts\n#EXTINF:10.000,\n14.ts\n#EXTINF:10.000,\n15.ts" + + "\n#EXTINF:10.000,\n16.ts\n#EXTINF:10.000,\n17.ts\n#EXTINF:10.000,\n18.ts\n#EXTINF:10.000,\n19.ts\n#EXTINF:10.000,\n20.ts\n#EXTINF:10.000,\n21.ts\n#EXTINF:10.000,\n22.ts\n#EXTINF:10.000,\n23.ts" + + "\n#EXTINF:10.000,\n24.ts\n#EXTINF:10.000,\n25.ts\n#EXTINF:10.000,\n26.ts\n#EXTINF:10.000,\n27.ts\n#EXTINF:10.000,\n28.ts\n#EXTINF:10.000,\n29.ts\n#EXTINF:10.000,\n30.ts\n#EXTINF:10.000,\n31.ts" + + "\n#EXTINF:10.000,\n32.ts\n#EXTINF:10.000,\n33.ts\n#EXTINF:10.000,\n34.ts\n#EXTINF:10.000,\n35.ts\n#EXTINF:10.000,\n36.ts\n#EXTINF:10.000,\n37.ts\n#EXTINF:10.000,\n38.ts\n#EXTINF:10.000,\n39.ts" + + "\n#EXTINF:10.000,\n40.ts\n#EXTINF:10.000,\n41.ts\n#EXTINF:10.000,\n42.ts\n#EXTINF:10.000,\n43.ts\n#EXTINF:10.000,\n44.ts\n#EXTINF:10.000,\n45.ts\n#EXTINF:10.000,\n46.ts\n#EXTINF:10.000,\n47.ts" + + "\n#EXTINF:10.000,\n48.ts\n#EXTINF:10.000,\n49.ts\n#EXT-X-ENDLIST"; + + var m3u8 = M3U8.Parse(ExampleM3U8Twitch); + + Assert.Equal(3u, m3u8.FileMetadata.Version); + Assert.Equal(10u, m3u8.FileMetadata.StreamTargetDuration); + Assert.Equal("2023-09-23T17:37:06", m3u8.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:").Value); + Assert.Equal(M3U8.Metadata.PlaylistType.Event, m3u8.FileMetadata.Type); + Assert.Equal(0u, m3u8.FileMetadata.MediaSequence); + Assert.Equal(0m, m3u8.FileMetadata.TwitchElapsedSeconds); + Assert.Equal(500m, m3u8.FileMetadata.TwitchTotalSeconds); + + Assert.Equal(50, m3u8.Streams.Length); + for (var i = 0; i < m3u8.Streams.Length; i++) + { + var stream = m3u8.Streams[i]; + Assert.Equal(10, stream.PartInfo.Duration); + Assert.False(stream.PartInfo.Live); + Assert.Equal($"{i}.ts", stream.Path); + } + } + + [Fact] + public void CorrectlyParsesTwitchM3U8OfLiveStreams() + { + const string ExampleM3U8Twitch = + "#EXTM3U" + + "\n#EXT-X-VERSION:3" + + "\n#EXT-X-TARGETDURATION:5" + + "\n#EXT-X-MEDIA-SEQUENCE:4815" + + "\n#EXT-X-TWITCH-LIVE-SEQUENCE:4997" + + "\n#EXT-X-TWITCH-ELAPSED-SECS:9994.338" + + "\n#EXT-X-TWITCH-TOTAL-SECS:10028.338" + + "\n#EXT-X-DATERANGE:ID=\"playlist-creation-1694908286\",CLASS=\"timestamp\",START-DATE=\"2023-09-16T23:51:26.423Z\",END-ON-NEXT=YES,X-SERVER-TIME=\"1694908286.42\"" + + "\n#EXT-X-DATERANGE:ID=\"playlist-session-1694908286\",CLASS=\"twitch-session\",START-DATE=\"2023-09-16T23:51:26.423Z\",END-ON-NEXT=YES,X-TV-TWITCH-SESSIONID=\"1234567890\"" + + "\n#EXT-X-DATERANGE:ID=\"1234567890\",CLASS=\"twitch-assignment\",START-DATE=\"2023-09-17T02:27:42.242Z\",END-ON-NEXT=YES,X-TV-TWITCH-SERVING-ID=\"1234567890\",X-TV-TWITCH-NODE=\"video-edge-foo.bar\",X-TV-TWITCH-CLUSTER=\"foo\"" + + "\n#EXT-X-DATERANGE:ID=\"source-1694916060\",CLASS=\"twitch-stream-source\",START-DATE=\"2023-09-17T02:01:00.242Z\",END-ON-NEXT=YES,X-TV-TWITCH-STREAM-SOURCE=\"live\"" + + "\n#EXT-X-DATERANGE:ID=\"trigger-1694908279\",CLASS=\"twitch-trigger\",START-DATE=\"2023-09-16T23:51:19.905Z\",END-ON-NEXT=YES,X-TV-TWITCH-TRIGGER-URL=\"https://video-weaver.bar.hls.ttvnw.net/trigger/abc-123DEF_456\"" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:31:48.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/abc-123DEF_456.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:31:50.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/ghi-789JKL_012.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:31:52.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/mno-345PQR_678.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:31:54.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/stu-901VWX_234.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:31:56.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/yza-567BCD_890.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:31:58.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/efg-123hij_456.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:00.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/klm-789nop_012.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:02.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/qrs-345TUV_678.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:04.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/wxy-901ZAB_234.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:06.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/cde-567FGH_890.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:08.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/ijk-123lmn_456.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:10.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/opq-789RST_012.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:12.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/uvx-345YZA_678.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:14.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/bcd-901EFG_234.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-09-17T02:32:16.242Z\n#EXTINF:2.000,live\nhttps://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/hij-567KLM_890.ts" + + "\n#EXT-X-TWITCH-PREFETCH:https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/nop-123QRS_456.ts" + + "\n#EXT-X-TWITCH-PREFETCH:https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/tuv-789WXY_012.ts"; + + var streamValues = new (DateTimeOffset programDateTime, decimal duration, bool isLive, string path)[] + { + (DateTimeOffset.Parse("2023-09-17T02:31:48.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/abc-123DEF_456.ts"), + (DateTimeOffset.Parse("2023-09-17T02:31:50.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/ghi-789JKL_012.ts"), + (DateTimeOffset.Parse("2023-09-17T02:31:52.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/mno-345PQR_678.ts"), + (DateTimeOffset.Parse("2023-09-17T02:31:54.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/stu-901VWX_234.ts"), + (DateTimeOffset.Parse("2023-09-17T02:31:56.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/yza-567BCD_890.ts"), + (DateTimeOffset.Parse("2023-09-17T02:31:58.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/efg-123hij_456.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:00.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/klm-789nop_012.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:02.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/qrs-345TUV_678.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:04.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/wxy-901ZAB_234.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:06.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/cde-567FGH_890.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:08.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/ijk-123lmn_456.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:10.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/opq-789RST_012.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:12.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/uvx-345YZA_678.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:14.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/bcd-901EFG_234.ts"), + (DateTimeOffset.Parse("2023-09-17T02:32:16.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/hij-567KLM_890.ts") + }; + + var m3u8 = M3U8.Parse(ExampleM3U8Twitch); + + Assert.Equal(3u, m3u8.FileMetadata.Version); + Assert.Equal(5u, m3u8.FileMetadata.StreamTargetDuration); + Assert.Equal(M3U8.Metadata.PlaylistType.Unknown, m3u8.FileMetadata.Type); + Assert.Equal(4815u, m3u8.FileMetadata.MediaSequence); + Assert.Equal(4997u, m3u8.FileMetadata.TwitchLiveSequence); + Assert.Equal(9994.338m, m3u8.FileMetadata.TwitchElapsedSeconds); + Assert.Equal(10028.338m, m3u8.FileMetadata.TwitchTotalSeconds); + + Assert.Equal(streamValues.Length, m3u8.Streams.Length); + for (var i = 0; i < m3u8.Streams.Length; i++) + { + var stream = m3u8.Streams[i]; + var expectedStream = streamValues[i]; + Assert.Equal(expectedStream.programDateTime, stream.ProgramDateTime); + Assert.Equal(expectedStream.duration, stream.PartInfo.Duration); + Assert.Equal(expectedStream.isLive, stream.PartInfo.Live); + Assert.Equal(expectedStream.path, stream.Path); + } + } + + [Fact] + public void CorrectlyParsesTwitchM3U8OfPlaylists() + { + const string ExampleM3U8Twitch = + "#EXTM3U" + + "\n#EXT-X-TWITCH-INFO:ORIGIN=\"s3\",B=\"false\",REGION=\"NA\",USER-IP=\"255.255.255.255\",SERVING-ID=\"123abc456def789ghi012jkl345mno67\",CLUSTER=\"cloudfront_vod\",USER-COUNTRY=\"US\",MANIFEST-CLUSTER=\"cloudfront_vod\"" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"chunked\",NAME=\"1080p60\",AUTOSELECT=NO,DEFAULT=NO" + + "\n#EXT-X-STREAM-INF:BANDWIDTH=5898203,CODECS=\"avc1.64002A,mp4a.40.2\",RESOLUTION=1920x1080,VIDEO=\"chunked\",FRAME-RATE=59.995" + + "\nhttps://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/chunked/index-dvr.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"720p60\",NAME=\"720p60\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:BANDWIDTH=3443956,CODECS=\"avc1.4D0020,mp4a.40.2\",RESOLUTION=1280x720,VIDEO=\"720p60\",FRAME-RATE=59.995" + + "\nhttps://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/720p60/index-dvr.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"480p30\",NAME=\"480p\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:BANDWIDTH=1454397,CODECS=\"avc1.4D001F,mp4a.40.2\",RESOLUTION=852x480,VIDEO=\"480p30\",FRAME-RATE=29.998" + + "\nhttps://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/480p30/index-dvr.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"audio_only\",NAME=\"Audio Only\",AUTOSELECT=NO,DEFAULT=NO" + + "\n#EXT-X-STREAM-INF:BANDWIDTH=220328,CODECS=\"mp4a.40.2\",VIDEO=\"audio_only\"" + + "\nhttps://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/audio_only/index-dvr.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"360p30\",NAME=\"360p\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:BANDWIDTH=708016,CODECS=\"avc1.4D001E,mp4a.40.2\",RESOLUTION=640x360,VIDEO=\"360p30\",FRAME-RATE=29.998" + + "\nhttps://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/360p30/index-dvr.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"160p30\",NAME=\"160p\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:BANDWIDTH=288409,CODECS=\"avc1.4D000C,mp4a.40.2\",RESOLUTION=284x160,VIDEO=\"160p30\",FRAME-RATE=29.998" + + "\nhttps://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/160p30/index-dvr.m3u8"; + + var streams = new M3U8.Stream[] + { + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "chunked", "1080p60", false, false), + new M3U8.Stream.ExtStreamInfo(0, 5898203, "avc1.64002A,mp4a.40.2", (1920, 1080), "chunked", 59.995m), + "https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/chunked/index-dvr.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "720p60", "720p60", true, true), + new M3U8.Stream.ExtStreamInfo(0, 3443956, "avc1.4D0020,mp4a.40.2", (1280, 720), "720p60", 59.995m), + "https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/720p60/index-dvr.m3u8"), + new(new M3U8.Stream.ExtMediaInfo (M3U8.Stream.ExtMediaInfo.MediaType.Video, "480p30", "480p", true, true), + new M3U8.Stream.ExtStreamInfo (0, 1454397, "avc1.4D001F,mp4a.40.2", (852, 480), "480p30", 29.998m), + "https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/480p30/index-dvr.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "audio_only", "Audio Only", false, false), + new M3U8.Stream.ExtStreamInfo(0, 220328, "mp4a.40.2", (0, 0), "audio_only", 0m), + "https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/audio_only/index-dvr.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "360p30", "360p", true, true), + new M3U8.Stream.ExtStreamInfo(0, 708016, "avc1.4D001E,mp4a.40.2", (640, 360), "360p30", 29.998m), + "https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/360p30/index-dvr.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "160p30", "160p", true, true), + new M3U8.Stream.ExtStreamInfo(0, 288409, "avc1.4D000C,mp4a.40.2", (284, 160), "160p30", 29.998m), + "https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/160p30/index-dvr.m3u8") + }; + + var m3u8 = M3U8.Parse(ExampleM3U8Twitch); + + Assert.Equal(streams.Length, m3u8.Streams.Length); + Assert.Equivalent(streams[0], m3u8.Streams[0], true); + Assert.Equivalent(streams[1], m3u8.Streams[1], true); + Assert.Equivalent(streams[2], m3u8.Streams[2], true); + Assert.Equivalent(streams[3], m3u8.Streams[3], true); + Assert.Equivalent(streams[4], m3u8.Streams[4], true); + Assert.Equivalent(streams[5], m3u8.Streams[5], true); + } + + [Theory] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"chunked\",NAME=\"1080p60\",AUTOSELECT=NO,DEFAULT=NO", M3U8.Stream.ExtMediaInfo.MediaType.Video, "chunked", "1080p60", false, false)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"720p60\",NAME=\"720p60\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "720p60", "720p60", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"480p30\",NAME=\"480p\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "480p30", "480p", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"audio_only\",NAME=\"Audio Only\",AUTOSELECT=NO,DEFAULT=NO", M3U8.Stream.ExtMediaInfo.MediaType.Video, "audio_only", "Audio Only", false, false)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"360p30\",NAME=\"360p\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "360p30", "360p", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"160p30\",NAME=\"160p\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "160p30", "160p", true, true)] + public void CorrectlyParsesTwitchM3U8MediaInfo(string mediaInfoString, M3U8.Stream.ExtMediaInfo.MediaType type, string groupId, string name, bool autoSelect, bool @default) + { + var mediaInfo = M3U8.Stream.ExtMediaInfo.Parse(mediaInfoString); + + Assert.Equal(type, mediaInfo.Type); + Assert.Equal(groupId, mediaInfo.GroupId); + Assert.Equal(name, mediaInfo.Name); + Assert.Equal(autoSelect, mediaInfo.AutoSelect); + Assert.Equal(@default, mediaInfo.Default); + } + + [Theory] + [InlineData("#EXT-X-STREAM-INF:BANDWIDTH=5898203,CODECS=\"avc1.64002A,mp4a.40.2\",RESOLUTION=1920x1080,VIDEO=\"chunked\",FRAME-RATE=59.995", 5898203, "avc1.64002A,mp4a.40.2", 1920, 1080, "chunked", 59.995)] + [InlineData("#EXT-X-STREAM-INF:BANDWIDTH=3443956,CODECS=\"avc1.4D0020,mp4a.40.2\",RESOLUTION=1280x720,VIDEO=\"720p60\",FRAME-RATE=59.995", 3443956, "avc1.4D0020,mp4a.40.2", 1280, 720, "720p60", 59.995)] + [InlineData("#EXT-X-STREAM-INF:BANDWIDTH=1454397,CODECS=\"avc1.4D001F,mp4a.40.2\",RESOLUTION=852x480,VIDEO=\"480p30\",FRAME-RATE=29.998", 1454397, "avc1.4D001F,mp4a.40.2", 852, 480, "480p30", 29.998)] + [InlineData("#EXT-X-STREAM-INF:BANDWIDTH=220328,CODECS=\"mp4a.40.2\",VIDEO=\"audio_only\"", 220328, "mp4a.40.2", 0, 0, "audio_only", 0)] + [InlineData("#EXT-X-STREAM-INF:BANDWIDTH=708016,CODECS=\"avc1.4D001E,mp4a.40.2\",RESOLUTION=640x360,VIDEO=\"360p30\",FRAME-RATE=29.998", 708016, "avc1.4D001E,mp4a.40.2", 640, 360, "360p30", 29.998)] + [InlineData("#EXT-X-STREAM-INF:BANDWIDTH=288409,CODECS=\"avc1.4D000C,mp4a.40.2\",RESOLUTION=284x160,VIDEO=\"160p30\",FRAME-RATE=29.998", 288409, "avc1.4D000C,mp4a.40.2", 284, 160, "160p30", 29.998)] + public void CorrectlyParsesTwitchM3U8StreamInfo(string streamInfoString, int bandwidth, string codecs, uint videoWidth, uint videoHeight, string video, decimal framerate) + { + var streamInfo = M3U8.Stream.ExtStreamInfo.Parse(streamInfoString); + + Assert.Equal(bandwidth, streamInfo.Bandwidth); + Assert.Equal(codecs, streamInfo.Codecs); + Assert.Equal((videoWidth, videoHeight), streamInfo.Resolution); + Assert.Equal(video, streamInfo.Video); + Assert.Equal(framerate, streamInfo.Framerate); + } + + [Fact] + public void CorrectlyParsesKickM3U8OfTransportStreams() + { + const string ExampleM3U8Kick = + "#EXTM3U" + + "\n#EXT-X-VERSION:4" + + "\n#EXT-X-MEDIA-SEQUENCE:0" + + "\n#EXT-X-TARGETDURATION:2" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:07.97Z\n#EXT-X-BYTERANGE:1601196@6470396\n#EXTINF:2.000,\n500.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:09.97Z\n#EXT-X-BYTERANGE:1588224@0\n#EXTINF:2.000,\n501.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:11.97Z\n#EXT-X-BYTERANGE:1579200@1588224\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:13.97Z\n#EXT-X-BYTERANGE:1646128@3167424\n#EXTINF:2.000,\n501.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:15.97Z\n#EXT-X-BYTERANGE:1587472@4813552\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:17.97Z\n#EXT-X-BYTERANGE:1594052@6401024\n#EXTINF:2.000,\n501.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:19.97Z\n#EXT-X-BYTERANGE:1851236@0\n#EXTINF:2.000,\n502.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:21.97Z\n#EXT-X-BYTERANGE:1437448@1851236\n#EXTINF:2.000,\n502.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:23.97Z\n#EXT-X-BYTERANGE:1535960@3288684\n#EXTINF:2.000,\n502.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:25.97Z\n#EXT-X-BYTERANGE:1568672@4824644\n#EXTINF:2.000,\n502.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:27.97Z\n#EXT-X-BYTERANGE:1625824@6393316\n#EXTINF:2.000,\n502.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:29.97Z\n#EXT-X-BYTERANGE:1583524@0\n#EXTINF:2.000,\n503.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:31.97Z\n#EXT-X-BYTERANGE:1597060@1583524\n#EXTINF:2.000,\n503.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:33.97Z\n#EXT-X-BYTERANGE:1642368@3180584\n#EXTINF:2.000,\n503.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:35.97Z\n#EXT-X-BYTERANGE:1556076@4822952\n#EXTINF:2.000,\n503.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:37.97Z\n#EXT-X-BYTERANGE:1669252@6379028\n#EXTINF:2.000,\n503.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:39.97Z\n#EXT-X-BYTERANGE:1544984@0\n#EXTINF:2.000,\n504.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:41.97Z\n#EXT-X-BYTERANGE:1601384@1544984\n#EXTINF:2.000,\n504.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:43.97Z\n#EXT-X-BYTERANGE:1672260@3146368\n#EXTINF:2.000,\n504.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:45.97Z\n#EXT-X-BYTERANGE:1623192@4818628\n#EXTINF:2.000,\n504.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:47.97Z\n#EXT-X-BYTERANGE:1526748@6441820\n#EXTINF:2.000,\n504.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:49.97Z\n#EXT-X-BYTERANGE:1731668@0\n#EXTINF:2.000,\n505.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:51.97Z\n#EXT-X-BYTERANGE:1454368@1731668\n#EXTINF:2.000,\n505.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:53.97Z\n#EXT-X-BYTERANGE:1572432@3186036\n#EXTINF:2.000,\n505.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:55.97Z\n#EXT-X-BYTERANGE:1625824@4758468\n#EXTINF:2.000,\n505.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:57.97Z\n#EXT-X-BYTERANGE:1616988@6384292\n#EXTINF:2.000,\n505.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:59.97Z\n#EXT-X-BYTERANGE:1632028@0\n#EXTINF:2.000,\n506.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:01.97Z\n#EXT-X-BYTERANGE:1543668@1632028\n#EXTINF:2.000,\n506.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:03.97Z\n#EXT-X-BYTERANGE:1768140@3175696\n#EXTINF:2.000,\n506.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:05.97Z\n#EXT-X-BYTERANGE:1519040@4943836\n#EXTINF:2.000,\n506.ts" + + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:07.97Z\n#EXT-X-BYTERANGE:1506068@6462876\n#EXTINF:2.000,\n506.ts\n#EXT-X-ENDLIST"; + + var streamValues = new (DateTimeOffset programDateTime, M3U8.Stream.ExtByteRange byteRange, string path)[] + { + (DateTimeOffset.Parse("2023-11-16T05:34:07.97Z"), (1601196, 6470396), "500.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:09.97Z"), (1588224, 0), "501.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:11.97Z"), (1579200, 1588224), "501.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:13.97Z"), (1646128, 3167424), "501.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:15.97Z"), (1587472, 4813552), "501.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:17.97Z"), (1594052, 6401024), "501.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:19.97Z"), (1851236, 0), "502.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:21.97Z"), (1437448, 1851236), "502.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:23.97Z"), (1535960, 3288684), "502.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:25.97Z"), (1568672, 4824644), "502.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:27.97Z"), (1625824, 6393316), "502.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:29.97Z"), (1583524, 0), "503.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:31.97Z"), (1597060, 1583524), "503.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:33.97Z"), (1642368, 3180584), "503.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:35.97Z"), (1556076, 4822952), "503.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:37.97Z"), (1669252, 6379028), "503.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:39.97Z"), (1544984, 0), "504.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:41.97Z"), (1601384, 1544984), "504.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:43.97Z"), (1672260, 3146368), "504.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:45.97Z"), (1623192, 4818628), "504.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:47.97Z"), (1526748, 6441820), "504.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:49.97Z"), (1731668, 0), "505.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:51.97Z"), (1454368, 1731668), "505.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:53.97Z"), (1572432, 3186036), "505.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:55.97Z"), (1625824, 4758468), "505.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:57.97Z"), (1616988, 6384292), "505.ts"), + (DateTimeOffset.Parse("2023-11-16T05:34:59.97Z"), (1632028, 0), "506.ts"), + (DateTimeOffset.Parse("2023-11-16T05:35:01.97Z"), (1543668, 1632028), "506.ts"), + (DateTimeOffset.Parse("2023-11-16T05:35:03.97Z"), (1768140, 3175696), "506.ts"), + (DateTimeOffset.Parse("2023-11-16T05:35:05.97Z"), (1519040, 4943836), "506.ts"), + (DateTimeOffset.Parse("2023-11-16T05:35:07.97Z"), (1506068, 6462876), "506.ts") + }; + + var m3u8 = M3U8.Parse(ExampleM3U8Kick); + + Assert.Equal(4u, m3u8.FileMetadata.Version); + Assert.Equal(2u, m3u8.FileMetadata.StreamTargetDuration); + Assert.Equal(M3U8.Metadata.PlaylistType.Unknown, m3u8.FileMetadata.Type); + Assert.Equal(0u, m3u8.FileMetadata.MediaSequence); + + Assert.Equal(streamValues.Length, m3u8.Streams.Length); + for (var i = 0; i < m3u8.Streams.Length; i++) + { + var stream = m3u8.Streams[i]; + Assert.Equal(2, stream.PartInfo.Duration); + Assert.False(stream.PartInfo.Live); + Assert.Equal(streamValues[i].programDateTime, stream.ProgramDateTime); + Assert.Equal(streamValues[i].byteRange, stream.ByteRange); + Assert.Equal(streamValues[i].path, stream.Path); + } + } + + [Fact] + public void CorrectlyParsesKickM3U8OfPlaylists() + { + const string ExampleM3U8Kick = + "#EXTM3U" + + "\n#EXT-X-SESSION-DATA:DATA-ID=\"net.live-video.content.id\",VALUE=\"AbC123dEf456\"" + + "\n#EXT-X-SESSION-DATA:DATA-ID=\"net.live-video.customer.id\",VALUE=\"123456789012\"" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"1080p60\",NAME=\"1080p60\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=9878400,CODECS=\"avc1.64002A,mp4a.40.2\",RESOLUTION=1920x1080,VIDEO=\"1080p60\"" + + "\n1080p60/playlist.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"720p60\",NAME=\"720p60\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=3330599,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=1280x720,VIDEO=\"720p60\"" + + "\n720p60/playlist.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"480p30\",NAME=\"480p\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1335600,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=852x480,VIDEO=\"480p30\"" + + "\n480p30/playlist.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"360p30\",NAME=\"360p\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=630000,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=640x360,VIDEO=\"360p30\"" + + "\n360p30/playlist.m3u8" + + "\n#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"160p30\",NAME=\"160p\",AUTOSELECT=YES,DEFAULT=YES" + + "\n#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=230000,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=284x160,VIDEO=\"160p30\"" + + "\n160p30/playlist.m3u8"; + + var streams = new M3U8.Stream[] + { + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "1080p60", "1080p60", true, true), + new M3U8.Stream.ExtStreamInfo(1, 9878400, "avc1.64002A,mp4a.40.2", (1920, 1080), "1080p60", 0m), + "1080p60/playlist.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "720p60", "720p60", true, true), + new M3U8.Stream.ExtStreamInfo(1, 3330599, "avc1.4D401F,mp4a.40.2", (1280, 720), "720p60", 0m), + "720p60/playlist.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "480p30", "480p", true, true), + new M3U8.Stream.ExtStreamInfo(1, 1335600, "avc1.4D401F,mp4a.40.2", (852, 480), "480p30", 0m), + "480p30/playlist.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "360p30", "360p", true, true), + new M3U8.Stream.ExtStreamInfo(1, 630000, "avc1.4D401F,mp4a.40.2", (640, 360), "360p30", 0m), + "360p30/playlist.m3u8"), + new(new M3U8.Stream.ExtMediaInfo(M3U8.Stream.ExtMediaInfo.MediaType.Video, "160p30", "160p", true, true), + new M3U8.Stream.ExtStreamInfo(1, 230000, "avc1.4D401F,mp4a.40.2", (284, 160), "160p30", 0m), + "160p30/playlist.m3u8") + }; + + var m3u8 = M3U8.Parse(ExampleM3U8Kick); + + Assert.Equal(streams.Length, m3u8.Streams.Length); + Assert.Equivalent(streams[0], m3u8.Streams[0], true); + Assert.Equivalent(streams[1], m3u8.Streams[1], true); + Assert.Equivalent(streams[2], m3u8.Streams[2], true); + Assert.Equivalent(streams[3], m3u8.Streams[3], true); + Assert.Equivalent(streams[4], m3u8.Streams[4], true); + } + + [Theory] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"1080p60\",NAME=\"1080p60\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "1080p60", "1080p60", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"720p60\",NAME=\"720p60\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "720p60", "720p60", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"480p30\",NAME=\"480p\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "480p30", "480p", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"360p30\",NAME=\"360p\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "360p30", "360p", true, true)] + [InlineData("#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID=\"160p30\",NAME=\"160p\",AUTOSELECT=YES,DEFAULT=YES", M3U8.Stream.ExtMediaInfo.MediaType.Video, "160p30", "160p", true, true)] + public void CorrectlyParsesKickM3U8MediaInfo(string mediaInfoString, M3U8.Stream.ExtMediaInfo.MediaType type, string groupId, string name, bool autoSelect, bool @default) + { + var mediaInfo = M3U8.Stream.ExtMediaInfo.Parse(mediaInfoString); + + Assert.Equal(type, mediaInfo.Type); + Assert.Equal(groupId, mediaInfo.GroupId); + Assert.Equal(name, mediaInfo.Name); + Assert.Equal(autoSelect, mediaInfo.AutoSelect); + Assert.Equal(@default, mediaInfo.Default); + } + + [Theory] + [InlineData("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=9878400,CODECS=\"avc1.64002A,mp4a.40.2\",RESOLUTION=1920x1080,VIDEO=\"1080p60\"", 9878400, "avc1.64002A,mp4a.40.2", 1920, 1080, "1080p60")] + [InlineData("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=3330599,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=1280x720,VIDEO=\"720p60\"", 3330599, "avc1.4D401F,mp4a.40.2", 1280, 720, "720p60")] + [InlineData("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1335600,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=852x480,VIDEO=\"480p30\"", 1335600, "avc1.4D401F,mp4a.40.2", 852, 480, "480p30")] + [InlineData("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=630000,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=640x360,VIDEO=\"360p30\"", 630000, "avc1.4D401F,mp4a.40.2", 640, 360, "360p30")] + [InlineData("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=230000,CODECS=\"avc1.4D401F,mp4a.40.2\",RESOLUTION=284x160,VIDEO=\"160p30\"", 230000, "avc1.4D401F,mp4a.40.2", 284, 160, "160p30")] + public void CorrectlyParsesKickM3U8StreamInfo(string streamInfoString, int bandwidth, string codecs, uint videoWidth, uint videoHeight, string video) + { + var streamInfo = M3U8.Stream.ExtStreamInfo.Parse(streamInfoString); + + Assert.Equal(bandwidth, streamInfo.Bandwidth); + Assert.Equal(codecs, streamInfo.Codecs); + Assert.Equal((videoWidth, videoHeight), streamInfo.Resolution); + Assert.Equal(video, streamInfo.Video); + } + + [Theory] + [InlineData(100, 200, "100@200")] + [InlineData(100, 200, "#EXT-X-BYTERANGE:100@200")] + public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeString) + { + var expected = new M3U8.Stream.ExtByteRange(start, length); + + var actual = M3U8.Stream.ExtByteRange.Parse(byteRangeString); + + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("429496729500@1")] + [InlineData("1@429496729500")] + [InlineData("42949672950000")] + public void CorrectlyThrowsFormatExceptionForBadByteRangeString(string byteRangeString) + { + Assert.Throws(() => M3U8.Stream.ExtByteRange.Parse(byteRangeString)); + } + + [Theory] + [InlineData(100, 200, "100x200")] + [InlineData(100, 200, "RESOLUTION=100x200")] + public void CorrectlyParsesResolution(uint start, uint length, string byteRangeString) + { + var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(start, length); + + var actual = M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString); + + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("429496729500x1")] + [InlineData("1x429496729500")] + [InlineData("42949672950000")] + public void CorrectlyThrowsFormatExceptionForBadResolutionString(string byteRangeString) + { + Assert.Throws(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString)); + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderCore.Tests/ReadOnlySpanCountTests.cs b/TwitchDownloaderCore.Tests/ReadOnlySpanCountTests.cs new file mode 100644 index 00000000..3145dcc4 --- /dev/null +++ b/TwitchDownloaderCore.Tests/ReadOnlySpanCountTests.cs @@ -0,0 +1,44 @@ +using TwitchDownloaderCore.Extensions; + +namespace TwitchDownloaderCore.Tests +{ + public class ReadOnlySpanCountTests + { + [Fact] + public void ReturnsNegativeOneWhenNotPresent() + { + ReadOnlySpan str = "SORRY FOR THE TRAFFIC NaM"; + const int EXPECTED = -1; + + var actual = str.Count('L'); + + Assert.Equal(EXPECTED, actual); + } + + [Fact] + public void ReturnsNegativeOneForEmptyString() + { + ReadOnlySpan str = ""; + const int EXPECTED = -1; + + var actual = str.Count('L'); + + Assert.Equal(EXPECTED, actual); + } + + [Theory] + [InlineData('S', 1)] + [InlineData('R', 4)] + [InlineData('a', 1)] + [InlineData('F', 3)] + [InlineData('M', 1)] + public void ReturnsCorrectCharacterCount(char character, int expectedCount) + { + ReadOnlySpan str = "SORRY FOR THE TRAFFIC NaM"; + + var actual = str.Count(character); + + Assert.Equal(expectedCount, actual); + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfAnyTests.cs b/TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfAnyTests.cs new file mode 100644 index 00000000..d5af065e --- /dev/null +++ b/TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfAnyTests.cs @@ -0,0 +1,115 @@ +using TwitchDownloaderCore.Extensions; + +namespace TwitchDownloaderCore.Tests +{ + public class ReadOnlySpanUnEscapedIndexOfAnyTests + { + [Fact] + public void CorrectlyFindsNextIndexWithoutEscapes() + { + ReadOnlySpan str = "SORRY FOR TRAFFIC NaM"; + const string CHARS_TO_FIND = "abc"; + const int CHAR_INDEX = 19; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void DoesNotFindAnIndexWhenNotPresent() + { + ReadOnlySpan str = "SORRY FOR TRAFFIC NaM"; + const string CHARS_TO_FIND = "LP"; + const int CHAR_INDEX = -1; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void CorrectlyFindsNextIndexWithBackslashEscapes() + { + ReadOnlySpan str = @"SORRY \FOR TRAFFIC NaM"; + const string CHARS_TO_FIND = "FT"; + const int CHAR_INDEX = 11; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void DoesNotFindIndexWithBackslashEscapes() + { + ReadOnlySpan str = @"SORRY \FOR TRA\F\F\IC NaM"; + const string CHARS_TO_FIND = "FI"; + const int CHAR_INDEX = -1; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void CorrectlyFindsNextIndexWithUnrelatedQuoteEscapes() + { + ReadOnlySpan str = "SORRY FOR \"TRAFFIC\" NaM"; + const string CHARS_TO_FIND = "abc"; + const int CHAR_INDEX = 21; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void CorrectlyFindsNextIndexWithQuoteEscapes() + { + ReadOnlySpan str = "SORRY \"FOR\" TRAFFIC NaM"; + const string CHARS_TO_FIND = "FM"; + const int CHAR_INDEX = 15; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void DoesNotFindAnIndexWithQuoteEscapes() + { + ReadOnlySpan str = "SORRY \"FOR\" \"TRAFFIC\" NaM"; + const string CHARS_TO_FIND = "FA"; + const int CHAR_INDEX = -1; + + var actual = str.UnEscapedIndexOfAny(CHARS_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Theory] + [InlineData("abc\\")] + [InlineData("abc\'")] + [InlineData("abc\"")] + public void Throws_WhenEscapeCharIsPassed(string charsToFind) + { + Assert.Throws(() => + { + ReadOnlySpan str = "SO\\R\\RY \'FOR\' \"TRAFFIC\" NaM"; + str.UnEscapedIndexOfAny(charsToFind); + }); + } + + [Fact] + public void Throws_WhenImbalancedQuoteChar() + { + Assert.Throws(() => + { + const string CHARS_TO_FIND = "FT"; + ReadOnlySpan str = "SORRY \"FOR TRAFFIC NaM"; + str.UnEscapedIndexOfAny(CHARS_TO_FIND); + }); + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfTests.cs b/TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfTests.cs new file mode 100644 index 00000000..d376991a --- /dev/null +++ b/TwitchDownloaderCore.Tests/ReadOnlySpanUnEscapedIndexOfTests.cs @@ -0,0 +1,115 @@ +using TwitchDownloaderCore.Extensions; + +namespace TwitchDownloaderCore.Tests +{ + public class ReadOnlySpanUnEscapedIndexOfTests + { + [Fact] + public void CorrectlyFindsNextIndexWithoutEscapes() + { + ReadOnlySpan str = "SORRY FOR TRAFFIC NaM"; + const char CHAR_TO_FIND = 'a'; + const int CHAR_INDEX = 19; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void DoesNotFindAnIndexWhenNotPresent() + { + ReadOnlySpan str = "SORRY FOR TRAFFIC NaM"; + const char CHAR_TO_FIND = 'L'; + const int CHAR_INDEX = -1; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void CorrectlyFindsNextIndexWithBackslashEscapes() + { + ReadOnlySpan str = @"SORRY \FOR TRAFFIC NaM"; + const char CHAR_TO_FIND = 'F'; + const int CHAR_INDEX = 14; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void DoesNotFindIndexWithBackslashEscapes() + { + ReadOnlySpan str = @"SORRY \FOR TRA\F\FIC NaM"; + const char CHAR_TO_FIND = 'F'; + const int CHAR_INDEX = -1; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void CorrectlyFindsNextIndexWithUnrelatedQuoteEscapes() + { + ReadOnlySpan str = "SORRY FOR \"TRAFFIC\" NaM"; + const char CHAR_TO_FIND = 'a'; + const int CHAR_INDEX = 21; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void CorrectlyFindsNextIndexWithQuoteEscapes() + { + ReadOnlySpan str = "SORRY \"FOR\" TRAFFIC NaM"; + const char CHAR_TO_FIND = 'F'; + const int CHAR_INDEX = 15; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Fact] + public void DoesNotFindAnIndexWithQuoteEscapes() + { + ReadOnlySpan str = "SORRY \"FOR\" \"TRAFFIC\" NaM"; + const char CHAR_TO_FIND = 'F'; + const int CHAR_INDEX = -1; + + var actual = str.UnEscapedIndexOf(CHAR_TO_FIND); + + Assert.Equal(CHAR_INDEX, actual); + } + + [Theory] + [InlineData('\\')] + [InlineData('\'')] + [InlineData('\"')] + public void Throws_WhenEscapeCharIsPassed(char charToFind) + { + Assert.Throws(() => + { + ReadOnlySpan str = "SO\\R\\RY \'FOR\' \"TRAFFIC\" NaM"; + str.UnEscapedIndexOf(charToFind); + }); + } + + [Fact] + public void Throws_WhenImbalancedQuoteChar() + { + Assert.Throws(() => + { + const char CHAR_TO_FIND = 'F'; + ReadOnlySpan str = "SORRY \"FOR TRAFFIC NaM"; + str.UnEscapedIndexOf(CHAR_TO_FIND); + }); + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs b/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs index 22db9e17..5b003864 100644 --- a/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs +++ b/TwitchDownloaderCore/Extensions/ReadOnlySpanExtensions.cs @@ -91,5 +91,123 @@ private static int FindCloseQuoteChar(ReadOnlySpan destination, int openQu return -1; } + + public static int UnEscapedIndexOf(this ReadOnlySpan str, char character) + { + if (character is '\\' or '\'' or '\"') + throw new ArgumentOutOfRangeException("Escape characters are not supported.", nameof(character)); + + var firstIndex = str.IndexOf(character); + if (firstIndex == -1) + return firstIndex; + + var firstEscapeIndex = str.IndexOfAny(@"\'"""); + if (firstEscapeIndex == -1 || firstEscapeIndex > firstIndex) + return firstIndex; + + var length = str.Length; + for (var i = firstEscapeIndex; i < length; i++) + { + var readChar = str[i]; + + switch (readChar) + { + case '\\': + i++; + break; + case '\'': + case '\"': + { + var closeQuoteMark = FindCloseQuoteChar(str, i, length, readChar); + if (closeQuoteMark == -1) + throw new FormatException($"Unbalanced quote mark at {i}."); + + i = closeQuoteMark; + + break; + } + default: + { + if (readChar == character) + { + return i; + } + + break; + } + } + } + + return -1; + } + + public static int UnEscapedIndexOfAny(this ReadOnlySpan str, ReadOnlySpan characters) + { + const string ESCAPE_CHARS = @"\'"""; + + if (characters.IndexOfAny(ESCAPE_CHARS) != -1) + throw new ArgumentOutOfRangeException("Escape characters are not supported.", nameof(characters)); + + var firstIndex = str.IndexOfAny(characters); + if (firstIndex == -1) + return firstIndex; + + var firstEscapeIndex = str.IndexOfAny(ESCAPE_CHARS); + if (firstEscapeIndex == -1 || firstEscapeIndex > firstIndex) + return firstIndex; + + var length = str.Length; + for (var i = firstEscapeIndex; i < length; i++) + { + var readChar = str[i]; + + switch (readChar) + { + case '\\': + i++; + break; + case '\'': + case '\"': + { + var closeQuoteMark = FindCloseQuoteChar(str, i, length, readChar); + if (closeQuoteMark == -1) + throw new FormatException($"Unbalanced quote mark at {i}."); + + i = closeQuoteMark; + + break; + } + default: + { + if (characters.Contains(readChar)) + { + return i; + } + + break; + } + } + } + + return -1; + } + + public static int Count(this ReadOnlySpan str, char character) + { + if (str.IsEmpty) + return -1; + + var count = 0; + var temp = str; + int index; + + while ((index = temp.IndexOf(character)) != -1) + { + count++; + temp = temp[(index + 1)..]; + } + + return count == 0 ? -1 : count; + } } } \ No newline at end of file diff --git a/TwitchDownloaderCore/Tools/M3U8.cs b/TwitchDownloaderCore/Tools/M3U8.cs new file mode 100644 index 00000000..648b6ffb --- /dev/null +++ b/TwitchDownloaderCore/Tools/M3U8.cs @@ -0,0 +1,637 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using TwitchDownloaderCore.Extensions; + +namespace TwitchDownloaderCore.Tools +{ + // https://en.wikipedia.org/wiki/M3U + // ReSharper disable StringLiteralTypo + public sealed record M3U8(M3U8.Metadata FileMetadata, M3U8.Stream[] Streams) + { + public static M3U8 Parse(ReadOnlySpan text, string basePath = "") + { + if (!ParsingHelpers.TryParseM3UHeader(ref text)) + { + throw new FormatException("Invalid playlist, M3U header is missing."); + } + + var streams = new List(); + + Stream.ExtMediaInfo currentExtMediaInfo = null; + Stream.ExtStreamInfo currentExtStreamInfo = null; + + Metadata.Builder metadataBuilder = new(); + DateTimeOffset currentExtProgramDateTime = default; + Stream.ExtByteRange currentByteRange = default; + Stream.ExtPartInfo currentExtPartInfo = null; + + var textStart = -1; + var textEnd = text.Length; + var lineEnd = -1; + var iterations = 0; + var maxIterations = text.Count('\n') + 1; + do + { + textStart++; + iterations++; + if (iterations > maxIterations) + throw new Exception("Infinite loop encountered while decoding M3U8 playlist."); + + if (textStart >= textEnd) + break; + + var workingSlice = text[textStart..]; + lineEnd = workingSlice.IndexOf('\n'); + if (lineEnd != -1) + workingSlice = workingSlice[..lineEnd]; + + if (workingSlice[0] != '#') + { + var path = Path.Combine(basePath, workingSlice.ToString()); + streams.Add(new Stream(currentExtMediaInfo, currentExtStreamInfo, currentExtPartInfo, currentExtProgramDateTime, currentByteRange, path)); + currentExtMediaInfo = null; + currentExtStreamInfo = null; + currentExtProgramDateTime = default; + currentByteRange = default; + currentExtPartInfo = null; + + if (lineEnd == -1) + break; + + continue; + } + + const string MEDIA_INFO_KEY = "#EXT-X-MEDIA:"; + const string STREAM_INFO_KEY = "#EXT-X-STREAM-INF:"; + const string PROGRAM_DATE_TIME_KEY = "#EXT-X-PROGRAM-DATE-TIME:"; + const string BYTE_RANGE_KEY = "#EXT-X-BYTERANGE:"; + const string PART_INFO_KEY = "#EXTINF:"; + const string END_LIST_KEY = "#EXT-X-ENDLIST"; + if (workingSlice.StartsWith(MEDIA_INFO_KEY)) + { + currentExtMediaInfo = Stream.ExtMediaInfo.Parse(workingSlice); + } + else if (workingSlice.StartsWith(STREAM_INFO_KEY)) + { + currentExtStreamInfo = Stream.ExtStreamInfo.Parse(workingSlice); + } + else if (workingSlice.StartsWith(PROGRAM_DATE_TIME_KEY)) + { + currentExtProgramDateTime = ParsingHelpers.ParseDateTimeOffset(workingSlice, PROGRAM_DATE_TIME_KEY); + } + else if (workingSlice.StartsWith(BYTE_RANGE_KEY)) + { + currentByteRange = Stream.ExtByteRange.Parse(workingSlice); + } + else if (workingSlice.StartsWith(PART_INFO_KEY)) + { + currentExtPartInfo = Stream.ExtPartInfo.Parse(workingSlice); + } + else if (workingSlice.StartsWith(END_LIST_KEY)) + { + break; + } + else + { + metadataBuilder.ParseAndAppend(workingSlice); + } + + if (lineEnd == -1) + break; + + } while ((textStart += lineEnd) < textEnd); + + return new M3U8(metadataBuilder.ToMetadata(), streams.ToArray()); + } + + public sealed record Metadata + { + public enum PlaylistType + { + Unknown, + Vod, + Event + } + + // Generic M3U headers + public uint Version { get; private set; } + public uint StreamTargetDuration { get; private set; } + public PlaylistType Type { get; private set; } = PlaylistType.Unknown; + public uint MediaSequence { get; private set; } + + // Twitch specific + public uint TwitchLiveSequence { get; private set; } + public decimal TwitchElapsedSeconds { get; private set; } + public decimal TwitchTotalSeconds { get; private set; } + + // Other headers that we don't have dedicated properties for. Useful for debugging. + private readonly List> _unparsedValues = new(); + public IReadOnlyList> UnparsedValues => _unparsedValues; + + public sealed class Builder + { + private Metadata _metadata; + + public Builder ParseAndAppend(ReadOnlySpan text) + { + text = text.Trim(); + + if (!text.IsEmpty) + { + ParseAndAppendCore(text); + } + + return this; + } + + private void ParseAndAppendCore(ReadOnlySpan text) + { + _metadata ??= new Metadata(); + + const string TARGET_VERSION_KEY = "#EXT-X-VERSION:"; + const string TARGET_DURATION_KEY = "#EXT-X-TARGETDURATION:"; + const string PLAYLIST_TYPE_KEY = "#EXT-X-PLAYLIST-TYPE:"; + const string MEDIA_SEQUENCE_KEY = "#EXT-X-MEDIA-SEQUENCE:"; + const string TWITCH_LIVE_SEQUENCE_KEY = "#EXT-X-TWITCH-LIVE-SEQUENCE:"; + const string TWITCH_ELAPSED_SECS_KEY = "#EXT-X-TWITCH-ELAPSED-SECS:"; + const string TWITCH_TOTAL_SECS_KEY = "#EXT-X-TWITCH-TOTAL-SECS:"; + const string TWITCH_INFO_KEY = "#EXT-X-TWITCH-INFO:"; + if (text.StartsWith(TARGET_VERSION_KEY)) + { + _metadata.Version = ParsingHelpers.ParseUIntValue(text, TARGET_VERSION_KEY); + } + else if (text.StartsWith(TARGET_DURATION_KEY)) + { + _metadata.StreamTargetDuration = ParsingHelpers.ParseUIntValue(text, TARGET_DURATION_KEY); + } + else if (text.StartsWith(PLAYLIST_TYPE_KEY)) + { + var temp = text[PLAYLIST_TYPE_KEY.Length..]; + if (temp.StartsWith("VOD")) + _metadata.Type = PlaylistType.Vod; + else if (temp.StartsWith("EVENT")) + _metadata.Type = PlaylistType.Event; + else + throw new FormatException($"Unable to parse PlaylistType from: {text}"); + } + else if (text.StartsWith(MEDIA_SEQUENCE_KEY)) + { + _metadata.MediaSequence = ParsingHelpers.ParseUIntValue(text, MEDIA_SEQUENCE_KEY); + } + else if (text.StartsWith(TWITCH_LIVE_SEQUENCE_KEY)) + { + _metadata.TwitchLiveSequence = ParsingHelpers.ParseUIntValue(text, TWITCH_LIVE_SEQUENCE_KEY); + } + else if (text.StartsWith(TWITCH_ELAPSED_SECS_KEY)) + { + _metadata.TwitchElapsedSeconds = ParsingHelpers.ParseDecimalValue(text, TWITCH_ELAPSED_SECS_KEY); + } + else if (text.StartsWith(TWITCH_TOTAL_SECS_KEY)) + { + _metadata.TwitchTotalSeconds = ParsingHelpers.ParseDecimalValue(text, TWITCH_TOTAL_SECS_KEY); + } + else if (text.StartsWith(TWITCH_INFO_KEY)) + { + // Do nothing. This header includes response related info that we don't need. + } + else if (text[0] == '#') + { + var colonIndex = text.IndexOf(':'); + if (colonIndex != -1) + { + var kvp = new KeyValuePair(text[..(colonIndex + 1)].ToString(), text[(colonIndex + 1)..].ToString()); + _metadata._unparsedValues.Add(kvp); + } + else + { + var kvp = new KeyValuePair("", text.ToString()); + _metadata._unparsedValues.Add(kvp); + } + } + } + + public Metadata ToMetadata() + { + return _metadata; + } + } + } + + public sealed record Stream(Stream.ExtMediaInfo MediaInfo, Stream.ExtStreamInfo StreamInfo, Stream.ExtPartInfo PartInfo, DateTimeOffset ProgramDateTime, Stream.ExtByteRange ByteRange, string Path) + { + public Stream(ExtMediaInfo mediaInfo, ExtStreamInfo streamInfo, string path) : this(mediaInfo, streamInfo, null, default, default, path) { } + + public Stream(ExtPartInfo partInfo, DateTimeOffset programDateTime, ExtByteRange byteRange, string path) : this(null, null, partInfo, programDateTime, byteRange, path) { } + + public bool IsPlaylist { get; } = Path.AsSpan().EndsWith(".m3u8") || Path.AsSpan().EndsWith(".m3u"); + + public override string ToString() + { + var sb = new StringBuilder(); + + if (MediaInfo != null) + sb.AppendLine(MediaInfo.ToString()); + + if (StreamInfo != null) + sb.AppendLine(StreamInfo.ToString()); + + if (PartInfo != null) + sb.AppendLine(PartInfo.ToString()); + + if (ProgramDateTime != default) + { + sb.Append("#EXT-X-PROGRAM-DATE-TIME:"); + sb.AppendLine(ProgramDateTime.ToString("O")); + } + + if (ByteRange != default) + sb.AppendLine(ByteRange.ToString()); + + if (!string.IsNullOrEmpty(Path)) + sb.AppendLine(Path); + + if (sb.Length == 0) + return ""; + + sb.Append("#EXT-X-ENDLIST"); + return sb.ToString(); + } + + public readonly record struct ExtByteRange(uint Start, uint Length) + { + public static implicit operator ExtByteRange((uint start, uint length) tuple) => new(tuple.start, tuple.length); + public override string ToString() => $"#EXT-X-BYTERANGE:{Start}@{Length}"; + + public static ExtByteRange Parse(ReadOnlySpan text) + { + if (text.StartsWith("#EXT-X-BYTERANGE:")) + text = text[17..]; + + var separatorIndex = text.IndexOf('@'); + if (separatorIndex == -1) + throw new FormatException($"Unable to parse ByteRange from {text}."); + + if (!uint.TryParse(text[..separatorIndex], out var start)) + throw new FormatException($"Unable to parse ByteRange from {text}."); + + if (!uint.TryParse(text[(separatorIndex + 1)..], out var end)) + throw new FormatException($"Unable to parse ByteRange from {text}."); + + return new ExtByteRange(start, end); + } + } + + public sealed class ExtMediaInfo + { + public enum MediaType + { + Unknown, + Video, + Audio + } + + private ExtMediaInfo() { } + + public ExtMediaInfo(MediaType type, string groupId, string name, bool autoSelect, bool @default) + { + Type = type; + GroupId = groupId; + Name = name; + AutoSelect = autoSelect; + Default = @default; + } + + public MediaType Type { get; private set; } = MediaType.Unknown; + public string GroupId { get; private set; } + public string Name { get; private set; } + public bool AutoSelect { get; private set; } + public bool Default { get; private set; } + + public override string ToString() + { + static string BooleanToWord(bool b) + { + return b ? "YES" : "NO"; + } + + return $"#EXT-X-MEDIA:TYPE={Type.ToString().ToUpper()},GROUP-ID=\"{GroupId}\",NAME=\"{Name}\",AUTOSELECT={BooleanToWord(AutoSelect)},DEFAULT={BooleanToWord(Default)}"; + } + + public static ExtMediaInfo Parse(ReadOnlySpan text) + { + var mediaInfo = new ExtMediaInfo(); + + if (text.StartsWith("#EXT-X-MEDIA:")) + text = text[13..]; + + const string KEY_TYPE = "TYPE="; + const string KEY_GROUP_ID = "GROUP-ID=\""; + const string KEY_NAME = "NAME=\""; + const string KEY_AUTOSELECT = "AUTOSELECT="; + const string KEY_DEFAULT = "DEFAULT="; + do + { + text = text.TrimStart(); + + if (text.StartsWith(KEY_TYPE)) + { + var temp = text[KEY_TYPE.Length..]; + if (temp.StartsWith("VIDEO")) + mediaInfo.Type = MediaType.Video; + else if (temp.StartsWith("AUDIO")) + mediaInfo.Type = MediaType.Audio; + else + throw new FormatException($"Unable to parse MediaType from: {text}"); + } + else if (text.StartsWith(KEY_GROUP_ID)) + { + mediaInfo.GroupId = ParsingHelpers.ParseStringValue(text, KEY_GROUP_ID); + } + else if (text.StartsWith(KEY_NAME)) + { + mediaInfo.Name = ParsingHelpers.ParseStringValue(text, KEY_NAME); + } + else if (text.StartsWith(KEY_AUTOSELECT)) + { + mediaInfo.AutoSelect = ParsingHelpers.ParseBooleanValue(text, KEY_AUTOSELECT); + } + else if (text.StartsWith(KEY_DEFAULT)) + { + mediaInfo.Default = ParsingHelpers.ParseBooleanValue(text, KEY_DEFAULT); + } + + var nextIndex = text.UnEscapedIndexOf(','); + if (nextIndex == -1) + break; + + text = text[(nextIndex + 1)..]; + } while (true); + + return mediaInfo; + } + } + + public sealed record ExtStreamInfo + { + public readonly record struct StreamResolution(uint Width, uint Height) + { + public static implicit operator StreamResolution((uint width, uint height) tuple) => new(tuple.width, tuple.height); + + public override string ToString() => $"{Width}x{Height}"; + + public static StreamResolution Parse(ReadOnlySpan text) + { + if (text.StartsWith("RESOLUTION=")) + text = text[11..]; + + var separatorIndex = text.IndexOfAny("x"); + if (separatorIndex == -1 || separatorIndex == text.Length) + throw new FormatException($"Unable to parse Resolution from {text}."); + + if (!uint.TryParse(text[..separatorIndex], out var width)) + throw new FormatException($"Unable to parse Resolution from {text}."); + + if (!uint.TryParse(text[(separatorIndex + 1)..], out var height)) + throw new FormatException($"Unable to parse Resolution from {text}."); + + return new StreamResolution(width, height); + } + } + + private ExtStreamInfo() { } + + public ExtStreamInfo(int programId, int bandwidth, string codecs, StreamResolution resolution, string video, decimal framerate) + { + ProgramId = programId; + Bandwidth = bandwidth; + Codecs = codecs; + Resolution = resolution; + Video = video; + Framerate = framerate; + } + + public int ProgramId { get; private set; } + public int Bandwidth { get; private set; } + public string Codecs { get; private set; } + public StreamResolution Resolution { get; private set; } + public string Video { get; private set; } + public decimal Framerate { get; private set; } + + public override string ToString() => $"#EXT-X-STREAM-INF:PROGRAM-ID={ProgramId},BANDWIDTH={Bandwidth},CODECS=\"{Codecs}\",RESOLUTION={Resolution},VIDEO=\"{Video}\",FRAME-RATE={Framerate}"; + + public static ExtStreamInfo Parse(ReadOnlySpan text) + { + var streamInfo = new ExtStreamInfo(); + + if (text.StartsWith("#EXT-X-STREAM-INF:")) + text = text[18..]; + + const string KEY_PROGRAM_ID = "PROGRAM-ID="; + const string KEY_BANDWIDTH = "BANDWIDTH="; + const string KEY_CODECS = "CODECS=\""; + const string KEY_RESOLUTION = "RESOLUTION="; + const string KEY_VIDEO = "VIDEO=\""; + const string KEY_FRAMERATE = "FRAME-RATE="; + do + { + text = text.TrimStart(); + + if (text.StartsWith(KEY_PROGRAM_ID)) + { + streamInfo.ProgramId = ParsingHelpers.ParseIntValue(text, KEY_PROGRAM_ID); + } + else if (text.StartsWith(KEY_BANDWIDTH)) + { + streamInfo.Bandwidth = ParsingHelpers.ParseIntValue(text, KEY_BANDWIDTH); + } + else if (text.StartsWith(KEY_CODECS)) + { + streamInfo.Codecs = ParsingHelpers.ParseStringValue(text, KEY_CODECS); + } + else if (text.StartsWith(KEY_RESOLUTION)) + { + streamInfo.Resolution = ParsingHelpers.ParseResolution(text, KEY_RESOLUTION); + } + else if (text.StartsWith(KEY_VIDEO)) + { + streamInfo.Video = ParsingHelpers.ParseStringValue(text, KEY_VIDEO); + } + else if (text.StartsWith(KEY_FRAMERATE)) + { + streamInfo.Framerate = ParsingHelpers.ParseDecimalValue(text, KEY_FRAMERATE); + } + + var nextIndex = text.UnEscapedIndexOf(','); + if (nextIndex == -1) + break; + + text = text[(nextIndex + 1)..]; + } while (true); + + return streamInfo; + } + } + + public sealed record ExtPartInfo + { + private ExtPartInfo() { } + + public ExtPartInfo(decimal duration, bool live) + { + Duration = duration; + Live = live; + } + + public decimal Duration { get; private set; } + public bool Live { get; private set; } + + public override string ToString() => $"#EXTINF:{Duration},{(Live ? "live" : "")}"; + + public static ExtPartInfo Parse(ReadOnlySpan text) + { + var partInfo = new ExtPartInfo(); + + if (text.StartsWith("#EXTINF:")) + text = text[8..]; + + do + { + text = text.TrimStart(); + + if (!text.IsEmpty && char.IsDigit(text[0])) + { + partInfo.Duration = ParsingHelpers.ParseDecimalValue(text, ""); + } + else if (text.StartsWith("live")) + { + partInfo.Live = true; + } + + var nextIndex = text.UnEscapedIndexOf(','); + if (nextIndex == -1) + break; + + text = text[(nextIndex + 1)..]; + } while (true); + + return partInfo; + } + } + } + + private static class ParsingHelpers + { + public static bool TryParseM3UHeader(ref ReadOnlySpan text) + { + const string M3U_HEADER = "#EXTM3U"; + if (!text.StartsWith(M3U_HEADER)) + { + return false; + } + + text = text[7..].TrimStart(" \r\n"); + return true; + } + + public static string ParseStringValue(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + + if (temp.Contains("\\\"", StringComparison.Ordinal)) + { + throw new NotSupportedException("Escaped quotes are not supported. Please report this as a bug: https://github.com/lay295/TwitchDownloader/issues/new/choose"); + } + + var closeQuote = temp.IndexOf('"'); + if (closeQuote == -1) + { + throw new FormatException("Expected close quote was not found."); + } + + return temp[..closeQuote].ToString(); + } + + public static int ParseIntValue(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + temp = temp[..NextKeyStart(temp)]; + + if (int.TryParse(temp, out var intValue)) + return intValue; + + throw new FormatException($"Unable to parse integer from: {text}"); + } + + public static uint ParseUIntValue(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + temp = temp[..NextKeyStart(temp)]; + + if (uint.TryParse(temp, out var uIntValue)) + return uIntValue; + + throw new FormatException($"Unable to parse integer from: {text}"); + } + + public static decimal ParseDecimalValue(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + temp = temp[..NextKeyStart(temp)]; + + if (decimal.TryParse(temp, out var decimalValue)) + return decimalValue; + + throw new FormatException($"Unable to parse decimal from: {text}"); + } + + public static bool ParseBooleanValue(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + + if (temp.StartsWith("NO")) + return false; + + if (temp.StartsWith("YES")) + return true; + + temp = temp[..NextKeyStart(temp)]; + + if (bool.TryParse(temp, out var booleanValue)) + return booleanValue; + + throw new FormatException($"Unable to parse boolean from: {text}"); + } + + public static Stream.ExtStreamInfo.StreamResolution ParseResolution(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + temp = temp[..NextKeyStart(temp)]; + + return Stream.ExtStreamInfo.StreamResolution.Parse(temp); + } + + public static DateTimeOffset ParseDateTimeOffset(ReadOnlySpan text, ReadOnlySpan keyName) + { + var temp = text[keyName.Length..]; + temp = temp[..NextKeyStart(temp)]; + + if (DateTimeOffset.TryParse(temp, out var dateTimeOffset)) + return dateTimeOffset; + + throw new FormatException($"Unable to parse DateTimeOffset from: {text}"); + } + + private static Index NextKeyStart(ReadOnlySpan text) + { + var nextKey = text.UnEscapedIndexOfAny(",\r\n"); + return nextKey switch + { + -1 => text.Length, // This is probably the last value + _ => nextKey + }; + } + } + } +} \ No newline at end of file From ea37a4275c53d3510a7706e279be413c41b6123a Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:44:40 -0500 Subject: [PATCH 9/9] Implement M3U8 parser --- TwitchDownloaderCore/TwitchHelper.cs | 8 +- TwitchDownloaderCore/VideoDownloader.cs | 227 ++++++++------------ TwitchDownloaderWPF/PageVodDownload.xaml.cs | 26 +-- 3 files changed, 101 insertions(+), 160 deletions(-) diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs index c5dc77fb..1b2f9b40 100644 --- a/TwitchDownloaderCore/TwitchHelper.cs +++ b/TwitchDownloaderCore/TwitchHelper.cs @@ -55,7 +55,7 @@ public static async Task GetVideoToken(int videoId, strin return await response.Content.ReadFromJsonAsync(); } - public static async Task GetVideoPlaylist(int videoId, string token, string sig) + public static async Task GetVideoPlaylist(int videoId, string token, string sig) { var request = new HttpRequestMessage() { @@ -63,8 +63,10 @@ public static async Task GetVideoPlaylist(int videoId, string token, s Method = HttpMethod.Get }; request.Headers.Add("Client-ID", "kimne78kx3ncx6brgo4mv6wki5h1ko"); - string playlist = await (await httpClient.SendAsync(request)).Content.ReadAsStringAsync(); - return playlist.Split('\n', StringSplitOptions.RemoveEmptyEntries); + var response = await httpClient.SendAsync(request); + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsStringAsync(); } public static async Task GetClipInfo(object clipId) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index dc58afd3..d24df926 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -55,14 +55,15 @@ public async Task DownloadAsync(CancellationToken cancellationToken) GqlVideoChapterResponse videoChapterResponse = await TwitchHelper.GetOrGenerateVideoChapters(downloadOptions.Id, videoInfoResponse.data.video); - var (playlistUrl, bandwidth) = await GetPlaylistUrl(); + var qualityPlaylist = await GetQualityPlaylist(videoInfoResponse); + + var playlistUrl = qualityPlaylist.Path; var baseUrl = new Uri(playlistUrl[..(playlistUrl.LastIndexOf('/') + 1)], UriKind.Absolute); var videoLength = TimeSpan.FromSeconds(videoInfoResponse.data.video.lengthSeconds); - CheckAvailableStorageSpace(bandwidth, videoLength); + CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); - List> videoList = new List>(); - (List videoPartsList, double vodAge) = await GetVideoPartsList(playlistUrl, videoList, cancellationToken); + var (playlist, videoListCrop, vodAge) = await GetVideoPlaylist(playlistUrl, cancellationToken); if (Directory.Exists(downloadFolder)) Directory.Delete(downloadFolder, true); @@ -70,35 +71,29 @@ public async Task DownloadAsync(CancellationToken cancellationToken) _progress.Report(new ProgressReport(ReportType.NewLineStatus, "Downloading 0% [2/5]")); - await DownloadVideoPartsAsync(videoPartsList, baseUrl, downloadFolder, vodAge, cancellationToken); + await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, vodAge, cancellationToken); _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Verifying Parts 0% [3/5]" }); - await VerifyDownloadedParts(videoPartsList, baseUrl, downloadFolder, vodAge, cancellationToken); + await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, vodAge, cancellationToken); _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [4/5]" }); - await CombineVideoParts(downloadFolder, videoPartsList, cancellationToken); + await CombineVideoParts(downloadFolder, playlist.Streams, videoListCrop, cancellationToken); _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Finalizing Video 0% [5/5]" }); - double startOffset = 0.0; + var startOffsetSeconds = (double)playlist.Streams + .Take(videoListCrop.Start.Value) + .Sum(x => x.PartInfo.Duration); - for (int i = 0; i < videoList.Count; i++) - { - if (videoList[i].Key == videoPartsList[0]) - break; - - startOffset += videoList[i].Value; - } - - double seekTime = downloadOptions.CropBeginningTime; - double seekDuration = Math.Round(downloadOptions.CropEndingTime - seekTime); + startOffsetSeconds -= downloadOptions.CropBeginningTime; + double seekDuration = Math.Round(downloadOptions.CropEndingTime - downloadOptions.CropBeginningTime); string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); VideoInfo videoInfo = videoInfoResponse.data.video; await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, downloadOptions.Id.ToString(), videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, - videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), startOffset, videoChapterResponse.data.video.moments.edges, cancellationToken); + videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), startOffsetSeconds, videoChapterResponse.data.video.moments.edges, cancellationToken); var finalizedFileDirectory = Directory.GetParent(Path.GetFullPath(downloadOptions.Filename))!; if (!finalizedFileDirectory.Exists) @@ -110,7 +105,7 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d var ffmpegRetries = 0; do { - ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, metadataPath, seekTime, startOffset, seekDuration), cancellationToken); + ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, metadataPath, startOffsetSeconds, seekDuration), cancellationToken); if (ffmpegExitCode != 0) { _progress.Report(new ProgressReport(ReportType.Log, $"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds...")); @@ -166,10 +161,10 @@ private void CheckAvailableStorageSpace(int bandwidth, TimeSpan videoLength) } } - private async Task DownloadVideoPartsAsync(List videoPartsList, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken) + private async Task DownloadVideoPartsAsync(IEnumerable playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken) { - var partCount = videoPartsList.Count; - var videoPartsQueue = new ConcurrentQueue(videoPartsList); + var partCount = videoListCrop.End.Value - videoListCrop.Start.Value; + var videoPartsQueue = new ConcurrentQueue(playlist.Take(videoListCrop).Select(x => x.Path)); var downloadTasks = new Task[downloadOptions.DownloadThreads]; for (var i = 0; i < downloadOptions.DownloadThreads; i++) @@ -325,15 +320,16 @@ private void LogDownloadThreadExceptions(IReadOnlyCollection download _progress.Report(new ProgressReport(ReportType.Log, sb.ToString())); } - private async Task VerifyDownloadedParts(List videoParts, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken) + private async Task VerifyDownloadedParts(IEnumerable playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken) { - var failedParts = new List(); - var partCount = videoParts.Count; + var failedParts = new List(); + var partCount = videoListCrop.End.Value - videoListCrop.Start.Value; var doneCount = 0; - foreach (var part in videoParts) + foreach (var part in playlist.Take(videoListCrop)) { - if (!VerifyVideoPart(downloadFolder, part)) + var filePath = Path.Combine(downloadFolder, RemoveQueryString(part.Path)); + if (!VerifyVideoPart(filePath)) { failedParts.Add(part); } @@ -348,28 +344,28 @@ private async Task VerifyDownloadedParts(List videoParts, Uri baseUrl, s if (failedParts.Count != 0) { - if (failedParts.Count == videoParts.Count) + if (failedParts.Count == partCount) { // Every video part returned corrupted, probably a false positive. return; } _progress.Report(new ProgressReport(ReportType.Log, $"The following parts will be redownloaded: {string.Join(", ", failedParts)}")); - await DownloadVideoPartsAsync(failedParts, baseUrl, downloadFolder, vodAge, cancellationToken); + await DownloadVideoPartsAsync(failedParts, videoListCrop, baseUrl, downloadFolder, vodAge, cancellationToken); } } - private static bool VerifyVideoPart(string downloadFolder, string part) + private static bool VerifyVideoPart(string filePath) { const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf - var partFile = Path.Combine(downloadFolder, RemoveQueryString(part)); - if (!File.Exists(partFile)) + + if (!File.Exists(filePath)) { return false; } - using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); var fileLength = fs.Length; if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) { @@ -379,7 +375,7 @@ private static bool VerifyVideoPart(string downloadFolder, string part) return true; } - private int RunFfmpegVideoCopy(string downloadFolder, string metadataPath, double seekTime, double startOffset, double seekDuration) + public int RunFfmpegVideoCopy(string downloadFolder, string metadataPath, double startOffset, double seekDuration) { var process = new Process { @@ -388,7 +384,7 @@ private int RunFfmpegVideoCopy(string downloadFolder, string metadataPath, doubl FileName = downloadOptions.FfmpegPath, Arguments = string.Format( "-hide_banner -stats -y -avoid_negative_ts make_zero " + (downloadOptions.CropBeginning ? "-ss {2} " : "") + "-i \"{0}\" -i \"{1}\" -map_metadata 1 -analyzeduration {3} -probesize {3} " + (downloadOptions.CropEnding ? "-t {4} " : "") + "-c:v copy \"{5}\"", - Path.Combine(downloadFolder, "output.ts"), metadataPath, (seekTime - startOffset).ToString(CultureInfo.InvariantCulture), int.MaxValue, seekDuration.ToString(CultureInfo.InvariantCulture), Path.GetFullPath(downloadOptions.Filename)), + Path.Combine(downloadFolder, "output.ts"), metadataPath, startOffset.ToString(CultureInfo.InvariantCulture), int.MaxValue, seekDuration.ToString(CultureInfo.InvariantCulture), Path.GetFullPath(downloadOptions.Filename)), UseShellExecute = false, CreateNoWindow = true, RedirectStandardInput = false, @@ -504,53 +500,62 @@ private static async Task DownloadVideoPartAsync(HttpClient httpClient, Uri base } } - private async Task<(List videoParts, double vodAge)> GetVideoPartsList(string playlistUrl, List> videoList, CancellationToken cancellationToken) + private async Task<(M3U8 playlist, Range cropRange, double vodAge)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken) { - string[] videoChunks = (await _httpClient.GetStringAsync(playlistUrl, cancellationToken)).Split('\n'); + var playlistString = await _httpClient.GetStringAsync(playlistUrl, cancellationToken); + var playlist = M3U8.Parse(playlistString); double vodAge = 25; - - try + var airDateKvp = playlist.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:"); + if (DateTimeOffset.TryParse(airDateKvp.Value, out var airDate)) { - vodAge = (DateTimeOffset.UtcNow - DateTimeOffset.Parse(videoChunks.First(x => x.StartsWith("#ID3-EQUIV-TDTG:")).Replace("#ID3-EQUIV-TDTG:", ""))).TotalHours; + vodAge = (DateTimeOffset.UtcNow - airDate).TotalHours; } - catch { } - for (int i = 0; i < videoChunks.Length; i++) + var videoListCrop = GetStreamListCrop(playlist.Streams, downloadOptions); + + return (playlist, videoListCrop, vodAge); + } + + private static Range GetStreamListCrop(IList streamList, VideoDownloadOptions downloadOptions) + { + var startCrop = TimeSpan.FromSeconds(downloadOptions.CropBeginningTime); + var endCrop = TimeSpan.FromSeconds(downloadOptions.CropEndingTime); + + var startIndex = 0; + if (downloadOptions.CropBeginning) { - if (videoChunks[i].StartsWith("#EXTINF")) + var startTime = 0m; + var cropTotalSeconds = (decimal)startCrop.TotalSeconds; + foreach (var videoPart in streamList) { - if (videoChunks[i + 1].StartsWith("#EXT-X-BYTERANGE")) - { - if (videoList.Any(x => x.Key == videoChunks[i + 2])) - { - KeyValuePair pair = videoList.Where(x => x.Key == videoChunks[i + 2]).First(); - pair = new KeyValuePair(pair.Key, pair.Value + Double.Parse(videoChunks[i].Remove(0, 8).TrimEnd(','), CultureInfo.InvariantCulture)); - } - else - { - videoList.Add(new KeyValuePair(videoChunks[i + 2], Double.Parse(videoChunks[i].Remove(0, 8).TrimEnd(','), CultureInfo.InvariantCulture))); - } - } - else - { - videoList.Add(new KeyValuePair(videoChunks[i + 1], Double.Parse(videoChunks[i].Remove(0, 8).TrimEnd(','), CultureInfo.InvariantCulture))); - } + if (startTime + videoPart.PartInfo.Duration > cropTotalSeconds) + break; + + startIndex++; + startTime += videoPart.PartInfo.Duration; } } - List> videoListCropped = GenerateCroppedVideoList(videoList, downloadOptions); - - List videoParts = new List(videoListCropped.Count); - foreach (var part in videoListCropped) + var endIndex = streamList.Count; + if (downloadOptions.CropEnding) { - videoParts.Add(part.Key); + var endTime = streamList.Sum(x => x.PartInfo.Duration); + var cropTotalSeconds = (decimal)endCrop.TotalSeconds; + for (var i = streamList.Count - 1; i >= 0; i--) + { + if (endTime - streamList[i].PartInfo.Duration < cropTotalSeconds) + break; + + endIndex--; + endTime -= streamList[i].PartInfo.Duration; + } } - return (videoParts, vodAge); + return new Range(startIndex, endIndex); } - private async Task<(string url, int bandwidth)> GetPlaylistUrl() + private async Task GetQualityPlaylist(GqlVideoResponse videoInfo) { GqlVideoTokenResponse accessToken = await TwitchHelper.GetVideoToken(downloadOptions.Id, downloadOptions.Oauth); @@ -559,39 +564,25 @@ private static async Task DownloadVideoPartAsync(HttpClient httpClient, Uri base throw new NullReferenceException("Invalid VOD, deleted/expired VOD possibly?"); } - string[] videoPlaylist = await TwitchHelper.GetVideoPlaylist(downloadOptions.Id, accessToken.data.videoPlaybackAccessToken.value, accessToken.data.videoPlaybackAccessToken.signature); - if (videoPlaylist[0].Contains("vod_manifest_restricted") || videoPlaylist[0].Contains("unauthorized_entitlements")) + var playlistString = await TwitchHelper.GetVideoPlaylist(downloadOptions.Id, accessToken.data.videoPlaybackAccessToken.value, accessToken.data.videoPlaybackAccessToken.signature); + if (playlistString.Contains("vod_manifest_restricted") || playlistString.Contains("unauthorized_entitlements")) { throw new NullReferenceException("Insufficient access to VOD, OAuth may be required."); } - var videoQualities = new List>(); + var m3u8 = M3U8.Parse(playlistString); - for (int i = 0; i < videoPlaylist.Length; i++) + for (var i = m3u8.Streams.Length - 1; i >= 0; i--) { - if (videoPlaylist[i].Contains("#EXT-X-MEDIA")) + var m3u8Stream = m3u8.Streams[i]; + if (m3u8Stream.MediaInfo.Name.StartsWith(downloadOptions.Quality, StringComparison.OrdinalIgnoreCase)) { - string lastPart = videoPlaylist[i].Substring(videoPlaylist[i].IndexOf("NAME=\"") + 6); - string stringQuality = lastPart.Substring(0, lastPart.IndexOf('"')); - - var bandwidthStartIndex = videoPlaylist[i + 1].IndexOf("BANDWIDTH=") + 10; - var bandwidthEndIndex = videoPlaylist[i + 1].IndexOf(',') - bandwidthStartIndex; - int.TryParse(videoPlaylist[i + 1].Substring(bandwidthStartIndex, bandwidthEndIndex), out var bandwidth); - - if (!videoQualities.Any(x => x.Key.Equals(stringQuality))) - { - videoQualities.Add(new KeyValuePair(stringQuality, (videoPlaylist[i + 2], bandwidth))); - } + return m3u8.Streams[i]; } } - if (downloadOptions.Quality != null && videoQualities.Any(x => x.Key.StartsWith(downloadOptions.Quality, StringComparison.OrdinalIgnoreCase))) - { - return videoQualities.Last(x => x.Key.StartsWith(downloadOptions.Quality, StringComparison.OrdinalIgnoreCase)).Value; - } - - // Unable to find specified quality, defaulting to highest quality - return videoQualities.First().Value; + // Unable to find specified quality, default to highest quality + return m3u8.Streams[0]; } /// @@ -660,23 +651,23 @@ private static async Task DownloadFileAsync(HttpClient httpClient, Uri url, stri cancellationTokenSource?.CancelAfter(TimeSpan.FromMilliseconds(uint.MaxValue - 1)); } - private async Task CombineVideoParts(string downloadFolder, List videoParts, CancellationToken cancellationToken) + private async Task CombineVideoParts(string downloadFolder, IEnumerable playlist, Range videoListCrop, CancellationToken cancellationToken) { DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); string outputFile = Path.Combine(downloadFolder, "output.ts"); - int partCount = videoParts.Count; + int partCount = videoListCrop.End.Value - videoListCrop.Start.Value; int doneCount = 0; await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); - foreach (var part in videoParts) + foreach (var part in playlist.Take(videoListCrop)) { await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); - string partFile = Path.Combine(downloadFolder, RemoveQueryString(part)); + string partFile = Path.Combine(downloadFolder, RemoveQueryString(part.Path)); if (File.Exists(partFile)) { - await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None)) + await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); } @@ -720,51 +711,5 @@ private static void Cleanup(string downloadFolder) } catch (IOException) { } // Directory is probably being used by another process } - - private static List> GenerateCroppedVideoList(List> videoList, VideoDownloadOptions downloadOptions) - { - List> returnList = new List>(videoList); - TimeSpan startCrop = TimeSpan.FromSeconds(downloadOptions.CropBeginningTime); - TimeSpan endCrop = TimeSpan.FromSeconds(downloadOptions.CropEndingTime); - - if (downloadOptions.CropBeginning) - { - double startTime = 0; - for (int i = 0; i < returnList.Count; i++) - { - if (startTime + returnList[i].Value < startCrop.TotalSeconds) - { - startTime += returnList[i].Value; - returnList.RemoveAt(i); - i--; - } - else - { - break; - } - } - } - - if (downloadOptions.CropEnding) - { - double endTime = 0.0; - videoList.ForEach(x => endTime += x.Value); - - for (int i = returnList.Count - 1; i >= 0; i--) - { - if (endTime - returnList[i].Value > endCrop.TotalSeconds) - { - endTime -= returnList[i].Value; - returnList.RemoveAt(i); - } - else - { - break; - } - } - } - - return returnList; - } } } \ No newline at end of file diff --git a/TwitchDownloaderWPF/PageVodDownload.xaml.cs b/TwitchDownloaderWPF/PageVodDownload.xaml.cs index 6250a4c3..841de879 100644 --- a/TwitchDownloaderWPF/PageVodDownload.xaml.cs +++ b/TwitchDownloaderWPF/PageVodDownload.xaml.cs @@ -109,30 +109,24 @@ private async Task GetVideoInfo() comboQuality.Items.Clear(); videoQualities.Clear(); - var playlist = await TwitchHelper.GetVideoPlaylist(videoId, taskAccessToken.Result.data.videoPlaybackAccessToken.value, taskAccessToken.Result.data.videoPlaybackAccessToken.signature); - if (playlist[0].Contains("vod_manifest_restricted") || playlist[0].Contains("unauthorized_entitlements")) + var playlistString = await TwitchHelper.GetVideoPlaylist(videoId, taskAccessToken.Result.data.videoPlaybackAccessToken.value, taskAccessToken.Result.data.videoPlaybackAccessToken.signature); + if (playlistString.Contains("vod_manifest_restricted") || playlistString.Contains("unauthorized_entitlements")) { throw new NullReferenceException(Translations.Strings.InsufficientAccessMayNeedOauth); } - for (int i = 0; i < playlist.Length; i++) + var videoPlaylist = M3U8.Parse(playlistString); + + //Add video qualities to combo quality + foreach (var stream in videoPlaylist.Streams) { - if (playlist[i].Contains("#EXT-X-MEDIA")) + if (!videoQualities.ContainsKey(stream.MediaInfo.Name)) { - string lastPart = playlist[i].Substring(playlist[i].IndexOf("NAME=\"", StringComparison.Ordinal) + 6); - string stringQuality = lastPart.Substring(0, lastPart.IndexOf('"')); - - var bandwidthStartIndex = playlist[i + 1].IndexOf("BANDWIDTH=", StringComparison.Ordinal) + 10; - var bandwidthEndIndex = playlist[i + 1].IndexOf(',') - bandwidthStartIndex; - int.TryParse(playlist[i + 1].Substring(bandwidthStartIndex, bandwidthEndIndex), out var bandwidth); - - if (!videoQualities.ContainsKey(stringQuality)) - { - videoQualities.Add(stringQuality, (playlist[i + 2], bandwidth)); - comboQuality.Items.Add(stringQuality); - } + videoQualities.Add(stream.MediaInfo.Name, (stream.Path, stream.StreamInfo.Bandwidth)); + comboQuality.Items.Add(stream.MediaInfo.Name); } } + comboQuality.SelectedIndex = 0; vodLength = TimeSpan.FromSeconds(taskVideoInfo.Result.data.video.lengthSeconds);