diff --git a/TwitchDownloaderCore.Tests/ToolTests/M3U8Tests.cs b/TwitchDownloaderCore.Tests/ToolTests/M3U8Tests.cs index b8b302c4..afeb3af0 100644 --- a/TwitchDownloaderCore.Tests/ToolTests/M3U8Tests.cs +++ b/TwitchDownloaderCore.Tests/ToolTests/M3U8Tests.cs @@ -66,6 +66,81 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream, string c } } + [Theory] + [InlineData(false, "en-US")] + [InlineData(true, "en-US")] + [InlineData(false, "ru-RU")] + [InlineData(true, "ru-RU")] + public void CorrectlyParsesTwitchM3U8OfMp4s(bool useStream, string culture) + { + const string EXAMPLE_M3U8_TWITCH = + "#EXTM3U" + + "\n#EXT-X-VERSION:6" + + "\n#EXT-X-TARGETDURATION:10" + + "\n#ID3-EQUIV-TDTG:2024-12-08T00:12:24" + + "\n#EXT-X-PLAYLIST-TYPE:EVENT" + + "\n#EXT-X-MEDIA-SEQUENCE:0" + + "\n#EXT-X-TWITCH-ELAPSED-SECS:0.000" + + "\n#EXT-X-TWITCH-TOTAL-SECS:1137.134" + + "\n#EXT-X-MAP:URI=\"init-0.mp4\"" + + "\n#EXTINF:10.000,\n0.mp4\n#EXTINF:10.000,\n1.mp4\n#EXTINF:10.000,\n2.mp4\n#EXTINF:10.000,\n3.mp4\n#EXTINF:10.000,\n4.mp4\n#EXTINF:10.000,\n5.mp4\n#EXTINF:10.000,\n6.mp4\n#EXTINF:10.000,\n7.mp4" + + "\n#EXTINF:10.000,\n8.mp4\n#EXTINF:10.000,\n9.mp4\n#EXTINF:10.000,\n10.mp4\n#EXTINF:10.000,\n11.mp4\n#EXTINF:10.000,\n12.mp4\n#EXTINF:10.000,\n13.mp4\n#EXTINF:10.000,\n14.mp4\n#EXTINF:10.000," + + "\n15.mp4\n#EXTINF:10.000,\n16.mp4\n#EXTINF:10.000,\n17.mp4\n#EXTINF:10.000,\n18.mp4\n#EXTINF:10.000,\n19.mp4\n#EXTINF:10.000,\n20.mp4\n#EXTINF:10.000,\n21.mp4\n#EXTINF:10.000,\n22.mp4" + + "\n#EXTINF:10.000,\n23.mp4\n#EXTINF:10.000,\n24.mp4\n#EXTINF:10.000,\n25.mp4\n#EXTINF:10.000,\n26.mp4\n#EXTINF:10.000,\n27.mp4\n#EXTINF:10.000,\n28.mp4\n#EXTINF:10.000,\n29.mp4\n#EXTINF:10.000," + + "\n30.mp4\n#EXTINF:10.000,\n31.mp4\n#EXTINF:10.000,\n32.mp4\n#EXTINF:10.000,\n33.mp4\n#EXTINF:10.000,\n34.mp4\n#EXTINF:10.000,\n35.mp4\n#EXTINF:10.000,\n36.mp4\n#EXTINF:10.000,\n37.mp4" + + "\n#EXTINF:10.000,\n38.mp4\n#EXTINF:10.000,\n39.mp4\n#EXTINF:10.000,\n40.mp4\n#EXTINF:10.000,\n41.mp4\n#EXTINF:10.000,\n42.mp4\n#EXTINF:10.000,\n43.mp4\n#EXTINF:10.000,\n44.mp4\n#EXTINF:10.000," + + "\n45.mp4\n#EXTINF:10.000,\n46.mp4\n#EXTINF:10.000,\n47.mp4\n#EXTINF:10.000,\n48.mp4\n#EXTINF:10.000,\n49.mp4\n#EXTINF:10.000,\n50.mp4\n#EXTINF:10.000,\n51.mp4\n#EXTINF:10.000,\n52.mp4" + + "\n#EXTINF:10.000,\n53.mp4\n#EXTINF:10.000,\n54.mp4\n#EXTINF:10.000,\n55.mp4\n#EXTINF:10.000,\n56.mp4\n#EXTINF:10.000,\n57.mp4\n#EXTINF:10.000,\n58.mp4\n#EXTINF:10.000,\n59.mp4\n#EXTINF:10.000," + + "\n60.mp4\n#EXTINF:10.000,\n61.mp4\n#EXTINF:10.000,\n62.mp4\n#EXTINF:10.000,\n63.mp4\n#EXTINF:10.000,\n64.mp4\n#EXTINF:10.000,\n65.mp4\n#EXTINF:10.000,\n66.mp4\n#EXTINF:10.000,\n67.mp4" + + "\n#EXTINF:10.000,\n68.mp4\n#EXTINF:10.000,\n69.mp4\n#EXTINF:10.000,\n70.mp4\n#EXTINF:10.000,\n71.mp4\n#EXTINF:10.000,\n72.mp4\n#EXTINF:10.000,\n73.mp4\n#EXTINF:10.000,\n74.mp4\n#EXTINF:10.000," + + "\n75.mp4\n#EXTINF:10.000,\n76.mp4\n#EXTINF:10.000,\n77.mp4\n#EXTINF:10.000,\n78.mp4\n#EXTINF:10.000,\n79.mp4\n#EXTINF:10.000,\n80.mp4\n#EXTINF:10.000,\n81.mp4\n#EXTINF:10.000,\n82.mp4" + + "\n#EXTINF:10.000,\n83.mp4\n#EXTINF:10.000,\n84.mp4\n#EXTINF:10.000,\n85.mp4\n#EXTINF:10.000,\n86.mp4\n#EXTINF:10.000,\n87.mp4\n#EXTINF:10.000,\n88.mp4\n#EXTINF:10.000,\n89.mp4\n#EXTINF:10.000," + + "\n90.mp4\n#EXTINF:10.000,\n91.mp4\n#EXTINF:10.000,\n92.mp4\n#EXTINF:10.000,\n93.mp4\n#EXTINF:10.000,\n94.mp4\n#EXTINF:10.000,\n95.mp4\n#EXTINF:10.000,\n96.mp4\n#EXTINF:10.000,\n97.mp4" + + "\n#EXTINF:10.000,\n98.mp4\n#EXTINF:10.000,\n99.mp4\n#EXTINF:10.000,\n100.mp4\n#EXTINF:10.000,\n101.mp4\n#EXTINF:10.000,\n102.mp4\n#EXTINF:10.000,\n103.mp4\n#EXTINF:10.000,\n104.mp4" + + "\n#EXTINF:10.000,\n105.mp4\n#EXTINF:10.000,\n106.mp4\n#EXTINF:10.000,\n107.mp4\n#EXTINF:10.000,\n108.mp4\n#EXTINF:10.000,\n109.mp4\n#EXTINF:10.000,\n110.mp4\n#EXTINF:10.000,\n111.mp4" + + "\n#EXTINF:10.000,\n112.mp4\n#EXTINF:7.134,\n113.mp4\n#EXT-X-ENDLIST"; + + var oldCulture = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = new CultureInfo(culture); + + M3U8 m3u8; + if (useStream) + { + var bytes = Encoding.Unicode.GetBytes(EXAMPLE_M3U8_TWITCH); + using var ms = new MemoryStream(bytes); + m3u8 = M3U8.Parse(ms, Encoding.Unicode); + } + else + { + m3u8 = M3U8.Parse(EXAMPLE_M3U8_TWITCH); + } + + CultureInfo.CurrentCulture = oldCulture; + + Assert.Equal(6u, m3u8.FileMetadata.Version); + Assert.Equal(10u, m3u8.FileMetadata.StreamTargetDuration); + Assert.Equal("2024-12-08T00:12:24", m3u8.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:").Value); + Assert.Equal(M3U8.Metadata.PlaylistType.Event, m3u8.FileMetadata.Type); + Assert.Equal(0u, m3u8.FileMetadata.MediaSequence); + Assert.Equal("init-0.mp4", m3u8.FileMetadata.Map.Uri); + Assert.Equal(default, m3u8.FileMetadata.Map.ByteRange); + Assert.Equal(0m, m3u8.FileMetadata.TwitchElapsedSeconds); + Assert.Equal(1137.134m, m3u8.FileMetadata.TwitchTotalSeconds); + + Assert.Equal(114, m3u8.Streams.Length); + + var duration = 1137.134m; + for (var i = 0; i < m3u8.Streams.Length; i++) + { + var stream = m3u8.Streams[i]; + Assert.Equal(duration > 10 ? 10 : duration, stream.PartInfo.Duration); + Assert.False(stream.PartInfo.Live); + Assert.Equal($"{i}.mp4", stream.Path); + + duration -= 10; + } + } + [Theory] [InlineData(false, "en-US")] [InlineData(true, "en-US")] @@ -301,7 +376,7 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream, string cul "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:03.97Z\n#EXT-X-BYTERANGE:1768140@3175696\n#EXTINF:2.000,\n506.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:05.97Z\n#EXT-X-BYTERANGE:1519040@4943836\n#EXTINF:2.000,\n506.ts" + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:07.97Z\n#EXT-X-BYTERANGE:1506068@6462876\n#EXTINF:2.000,\n506.ts\n#EXT-X-ENDLIST"; - var streamValues = new (DateTimeOffset programDateTime, M3U8.Stream.ExtByteRange byteRange, string path)[] + var streamValues = new (DateTimeOffset programDateTime, M3U8.ByteRange byteRange, string path)[] { (DateTimeOffset.Parse("2023-11-16T05:34:07.97Z"), (1601196, 6470396), "500.ts"), (DateTimeOffset.Parse("2023-11-16T05:34:09.97Z"), (1588224, 0), "501.ts"), @@ -474,13 +549,14 @@ public void CorrectlyParsesKickM3U8StreamInfo(string streamInfoString, int bandw } [Theory] - [InlineData(100, 200, "100@200")] - [InlineData(100, 200, "#EXT-X-BYTERANGE:100@200")] - public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeString) + [InlineData(100, 200, "100@200", "")] + [InlineData(100, 200, "#EXT-X-BYTERANGE:100@200", "#EXT-X-BYTERANGE:")] + [InlineData(100, 200, "BYTERANGE=100@200", "BYTERANGE=")] + public void CorrectlyParsesByteRange(uint length, uint start, string byteRangeString, string key) { - var expected = new M3U8.Stream.ExtByteRange(start, length); + var expected = new M3U8.ByteRange(length, start); - var actual = M3U8.Stream.ExtByteRange.Parse(byteRangeString); + var actual = M3U8.ByteRange.Parse(byteRangeString, key); Assert.Equal(expected, actual); } @@ -491,15 +567,15 @@ public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeSt [InlineData("42949672950000")] public void ThrowsFormatExceptionForBadByteRangeString(string byteRangeString) { - Assert.Throws(() => M3U8.Stream.ExtByteRange.Parse(byteRangeString)); + Assert.Throws(() => M3U8.ByteRange.Parse(byteRangeString, default)); } [Theory] [InlineData(100, 200, "100x200")] [InlineData(100, 200, "RESOLUTION=100x200")] - public void CorrectlyParsesResolution(uint start, uint length, string byteRangeString) + public void CorrectlyParsesResolution(uint width, uint height, string byteRangeString) { - var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(start, length); + var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(width, height); var actual = M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString); @@ -510,9 +586,9 @@ public void CorrectlyParsesResolution(uint start, uint length, string byteRangeS [InlineData("429496729500x1")] [InlineData("1x429496729500")] [InlineData("42949672950000")] - public void ThrowsFormatExceptionForBadResolutionString(string byteRangeString) + public void ThrowsFormatExceptionForBadResolutionString(string resolutionString) { - Assert.Throws(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString)); + Assert.Throws(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(resolutionString)); } [Theory] @@ -545,6 +621,7 @@ public void CorrectlyStringifiesInvariantOfCulture(string culture) "\n#EXT-X-VERSION:4" + "\n#EXT-X-MEDIA-SEQUENCE:0" + "\n#EXT-X-TARGETDURATION:2" + + "\n#EXT-X-MAP:URI=\"init-0.mp4\"" + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:07.97Z\n#EXT-X-BYTERANGE:1601196@6470396\n#EXTINF:2.000,\n500.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:09.97Z\n#EXT-X-BYTERANGE:1588224@0\n#EXTINF:2.000,\n501.ts" + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:11.97Z\n#EXT-X-BYTERANGE:1579200@1588224\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:13.97Z\n#EXT-X-BYTERANGE:1646128@3167424\n#EXTINF:2.000,\n501.ts" + "\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:15.97Z\n#EXT-X-BYTERANGE:1587472@4813552\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:17.97Z\n#EXT-X-BYTERANGE:1594052@6401024\n#EXTINF:2.000,\n501.ts" + diff --git a/TwitchDownloaderCore/Tools/DownloadTools.cs b/TwitchDownloaderCore/Tools/DownloadTools.cs index 227ff76a..a4a1f0e0 100644 --- a/TwitchDownloaderCore/Tools/DownloadTools.cs +++ b/TwitchDownloaderCore/Tools/DownloadTools.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Http; using System.Threading; @@ -15,20 +16,30 @@ public static class DownloadTools /// The to perform the download operation. /// The url of the file to download. /// The path to the file where download will be saved. + /// Path to a file whose contents will be written to the start of the destination file. /// The maximum download speed in kibibytes per second, or -1 for no maximum. /// Logger. /// A containing a to cancel the operation. /// The expected length of the downloaded file, or -1 if the content length header is not present. /// The may be canceled by this method. - public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null) + public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, [AllowNull] string headerFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null) { - var request = new HttpRequestMessage(HttpMethod.Get, url); + using var request = new HttpRequestMessage(HttpMethod.Get, url); var cancellationToken = cancellationTokenSource?.Token ?? CancellationToken.None; using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); + var fileMode = FileMode.Create; + if (!string.IsNullOrWhiteSpace(headerFile)) + { + await using var headerFs = new FileStream(headerFile, FileMode.Open, FileAccess.Read, FileShare.Read); + await using var destinationFs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read); + await headerFs.CopyToAsync(destinationFs, cancellationToken); + fileMode = FileMode.Append; + } + // Why are we setting a CTS CancelAfter timer? See lay295#265 const int SIXTY_SECONDS = 60; if (throttleKib == -1 || !response.Content.Headers.ContentLength.HasValue) @@ -48,7 +59,7 @@ public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, { case -1: { - await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read); + await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read); await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); break; } @@ -58,7 +69,7 @@ public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, { await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken); await using var throttledStream = new ThrottledStream(contentStream, throttleKib); - await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read); + await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read); await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); } catch (IOException ex) when (ex.Message.Contains("EOF")) diff --git a/TwitchDownloaderCore/Tools/FfmpegConcatList.cs b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs index e8c3b37d..f9e4785b 100644 --- a/TwitchDownloaderCore/Tools/FfmpegConcatList.cs +++ b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -12,7 +13,7 @@ public static class FfmpegConcatList { private const string LINE_FEED = "\u000A"; - public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, CancellationToken cancellationToken = default) + public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, StreamIds streamIds, CancellationToken cancellationToken = default) { await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; @@ -27,16 +28,29 @@ public static async Task SerializeAsync(string filePath, M3U8 playlist, Range vi await sw.WriteAsync(DownloadTools.RemoveQueryString(stream.Path)); await sw.WriteLineAsync('\''); - await sw.WriteLineAsync("stream"); - await sw.WriteLineAsync("exact_stream_id 0x100"); // Audio - await sw.WriteLineAsync("stream"); - await sw.WriteLineAsync("exact_stream_id 0x101"); // Video - await sw.WriteLineAsync("stream"); - await sw.WriteLineAsync("exact_stream_id 0x102"); // Subtitle + foreach (var id in streamIds.Ids) + { + await sw.WriteLineAsync("stream"); + await sw.WriteLineAsync($"exact_stream_id {id}"); + } await sw.WriteAsync("duration "); await sw.WriteLineAsync(stream.PartInfo.Duration.ToString(CultureInfo.InvariantCulture)); } } + + public record StreamIds + { + public static readonly StreamIds TransportStream = new("0x100", "0x101", "0x102"); + public static readonly StreamIds Mp4 = new("0x1", "0x2"); + public static readonly StreamIds None = new(); + + private StreamIds(params string[] ids) + { + Ids = ids; + } + + public IEnumerable Ids { get; } + } } } \ No newline at end of file diff --git a/TwitchDownloaderCore/Tools/M3U8.cs b/TwitchDownloaderCore/Tools/M3U8.cs index 064fbe17..acc7812b 100644 --- a/TwitchDownloaderCore/Tools/M3U8.cs +++ b/TwitchDownloaderCore/Tools/M3U8.cs @@ -48,6 +48,7 @@ public enum PlaylistType private const string TARGET_DURATION_KEY = "#EXT-X-TARGETDURATION:"; private const string PLAYLIST_TYPE_KEY = "#EXT-X-PLAYLIST-TYPE:"; private const string MEDIA_SEQUENCE_KEY = "#EXT-X-MEDIA-SEQUENCE:"; + private const string MAP_KEY = "#EXT-X-MAP:"; private const string TWITCH_LIVE_SEQUENCE_KEY = "#EXT-X-TWITCH-LIVE-SEQUENCE:"; private const string TWITCH_ELAPSED_SECS_KEY = "#EXT-X-TWITCH-ELAPSED-SECS:"; private const string TWITCH_TOTAL_SECS_KEY = "#EXT-X-TWITCH-TOTAL-SECS:"; @@ -58,6 +59,7 @@ public enum PlaylistType public uint? StreamTargetDuration { get; init; } public PlaylistType? Type { get; init; } public uint? MediaSequence { get; init; } + public ExtMap Map { get; init; } // Twitch specific public uint? TwitchLiveSequence { get; init; } @@ -94,6 +96,9 @@ public override string ToString() if (TwitchTotalSeconds.HasValue) sb.AppendKeyValue(TWITCH_TOTAL_SECS_KEY, TwitchTotalSeconds.Value, itemSeparator); + if (Map is not null) + sb.AppendKeyValue(MAP_KEY, Map.ToString(), itemSeparator); + foreach (var (key, value) in _unparsedValues) { sb.AppendKeyValue(key, value, itemSeparator); @@ -106,16 +111,42 @@ public override string ToString() return sb.TrimEnd(itemSeparator).ToString(); } + + public partial record ExtMap(string Uri, ByteRange ByteRange) + { + private const string URI_KEY = "URI=\""; + private const string BYTE_RANGE_KEY = "BYTERANGE="; + + public override string ToString() + { + var sb = new StringBuilder(); + ReadOnlySpan itemSeparator = stackalloc char[] { ',' }; + + if (!string.IsNullOrWhiteSpace(Uri)) + sb.AppendKeyQuoteValue(URI_KEY, Uri, itemSeparator); + + if (ByteRange != default) + sb.AppendKeyQuoteValue(BYTE_RANGE_KEY, ByteRange.ToString(), itemSeparator); + + if (sb.Length == 0) + return ""; + + return sb.TrimEnd(itemSeparator).ToString(); + } + } } - public partial record Stream(Stream.ExtMediaInfo MediaInfo, Stream.ExtStreamInfo StreamInfo, Stream.ExtPartInfo PartInfo, DateTimeOffset ProgramDateTime, Stream.ExtByteRange ByteRange, string Path) + public partial record Stream(Stream.ExtMediaInfo MediaInfo, Stream.ExtStreamInfo StreamInfo, Stream.ExtPartInfo PartInfo, DateTimeOffset ProgramDateTime, ByteRange ByteRange, string Path) { public Stream(ExtMediaInfo mediaInfo, ExtStreamInfo streamInfo, string path) : this(mediaInfo, streamInfo, null, default, default, path) { } - public Stream(ExtPartInfo partInfo, DateTimeOffset programDateTime, ExtByteRange byteRange, string path) : this(null, null, partInfo, programDateTime, byteRange, path) { } + public Stream(ExtPartInfo partInfo, DateTimeOffset programDateTime, ByteRange byteRange, string path) : this(null, null, partInfo, programDateTime, byteRange, path) { } public bool IsPlaylist { get; } = Path.AsSpan().EndsWith(".m3u8") || Path.AsSpan().EndsWith(".m3u"); + internal const string PROGRAM_DATE_TIME_KEY = "#EXT-X-PROGRAM-DATE-TIME:"; + internal const string BYTE_RANGE_KEY = "#EXT-X-BYTERANGE:"; + public override string ToString() { var sb = new StringBuilder(); @@ -130,10 +161,10 @@ public override string ToString() sb.AppendLine(PartInfo.ToString()); if (ProgramDateTime != default) - sb.AppendKeyValue("#EXT-X-PROGRAM-DATE-TIME:", ProgramDateTime.ToString("O"), default); + sb.AppendKeyValue(PROGRAM_DATE_TIME_KEY, ProgramDateTime.ToString("O"), Environment.NewLine); if (ByteRange != default) - sb.AppendLine(ByteRange.ToString()); + sb.AppendKeyValue(BYTE_RANGE_KEY, ByteRange.ToString(), Environment.NewLine); if (!string.IsNullOrEmpty(Path)) sb.Append(Path); @@ -144,15 +175,6 @@ public override string ToString() return sb.ToString(); } - public readonly partial record struct ExtByteRange(uint Start, uint Length) - { - internal const string BYTE_RANGE_KEY = "#EXT-X-BYTERANGE:"; - - public override string ToString() => $"{BYTE_RANGE_KEY}{Start}@{Length}"; - - public static implicit operator ExtByteRange((uint start, uint length) tuple) => new(tuple.start, tuple.length); - } - public partial record ExtMediaInfo { public enum MediaType @@ -264,7 +286,7 @@ public override string ToString() if (Framerate != default) sb.AppendKeyValue("FRAME-RATE=", Framerate, default); - return sb.ToString(); + return sb.TrimEnd(keyValueSeparator).ToString(); } } @@ -299,6 +321,19 @@ public override string ToString() } } } + + public readonly partial record struct ByteRange(uint Length, uint Start) + { + public override string ToString() + { + if (this == default) + return ""; + + return $"{Length}@{Start}"; + } + + public static implicit operator ByteRange((uint length, uint start) tuple) => new(tuple.length, tuple.start); + } } internal static class StringBuilderExtensions diff --git a/TwitchDownloaderCore/Tools/M3U8Parse.cs b/TwitchDownloaderCore/Tools/M3U8Parse.cs index ee1f2379..e1b4fc63 100644 --- a/TwitchDownloaderCore/Tools/M3U8Parse.cs +++ b/TwitchDownloaderCore/Tools/M3U8Parse.cs @@ -25,7 +25,7 @@ public static M3U8 Parse(System.IO.Stream stream, Encoding streamEncoding, strin Metadata.Builder metadataBuilder = new(); DateTimeOffset currentExtProgramDateTime = default; - Stream.ExtByteRange currentByteRange = default; + ByteRange currentByteRange = default; Stream.ExtPartInfo currentExtPartInfo = null; while (sr.ReadLine() is { } line) @@ -68,7 +68,7 @@ public static M3U8 Parse(ReadOnlySpan text, string basePath = "") Metadata.Builder metadataBuilder = new(); DateTimeOffset currentExtProgramDateTime = default; - Stream.ExtByteRange currentByteRange = default; + ByteRange currentByteRange = default; Stream.ExtPartInfo currentExtPartInfo = null; var textStart = -1; @@ -124,7 +124,7 @@ public static M3U8 Parse(ReadOnlySpan text, string basePath = "") } private static void ClearStreamMetadata(out Stream.ExtMediaInfo currentExtMediaInfo, out Stream.ExtStreamInfo currentExtStreamInfo, out DateTimeOffset currentExtProgramDateTime, - out Stream.ExtByteRange currentByteRange, out Stream.ExtPartInfo currentExtPartInfo) + out ByteRange currentByteRange, out Stream.ExtPartInfo currentExtPartInfo) { currentExtMediaInfo = null; currentExtStreamInfo = null; @@ -134,9 +134,8 @@ private static void ClearStreamMetadata(out Stream.ExtMediaInfo currentExtMediaI } private static bool ParseM3U8Key(ReadOnlySpan text, Metadata.Builder metadataBuilder, ref Stream.ExtMediaInfo extMediaInfo, ref Stream.ExtStreamInfo extStreamInfo, - ref DateTimeOffset extProgramDateTime, ref Stream.ExtByteRange byteRange, ref Stream.ExtPartInfo extPartInfo) + ref DateTimeOffset extProgramDateTime, ref ByteRange byteRange, ref Stream.ExtPartInfo extPartInfo) { - const string PROGRAM_DATE_TIME_KEY = "#EXT-X-PROGRAM-DATE-TIME:"; const string END_LIST_KEY = "#EXT-X-ENDLIST"; if (text.StartsWith(Stream.ExtMediaInfo.MEDIA_INFO_KEY)) { @@ -146,13 +145,13 @@ private static bool ParseM3U8Key(ReadOnlySpan text, Metadata.Builder metad { extStreamInfo = Stream.ExtStreamInfo.Parse(text); } - else if (text.StartsWith(PROGRAM_DATE_TIME_KEY)) + else if (text.StartsWith(Stream.PROGRAM_DATE_TIME_KEY)) { - extProgramDateTime = ParsingHelpers.ParseDateTimeOffset(text, PROGRAM_DATE_TIME_KEY, false); + extProgramDateTime = ParsingHelpers.ParseDateTimeOffset(text, Stream.PROGRAM_DATE_TIME_KEY, false); } - else if (text.StartsWith(Stream.ExtByteRange.BYTE_RANGE_KEY)) + else if (text.StartsWith(Stream.BYTE_RANGE_KEY)) { - byteRange = Stream.ExtByteRange.Parse(text); + byteRange = ByteRange.Parse(text, Stream.BYTE_RANGE_KEY); } else if (text.StartsWith(Stream.ExtPartInfo.PART_INFO_KEY)) { @@ -179,6 +178,7 @@ public sealed class Builder private uint? _streamTargetDuration; private PlaylistType? _type; private uint? _mediaSequence; + private ExtMap _map; // Twitch specific private uint? _twitchLiveSequence; @@ -224,6 +224,10 @@ private void ParseAndAppendCore(ReadOnlySpan text) { _mediaSequence = ParsingHelpers.ParseUIntValue(text, MEDIA_SEQUENCE_KEY); } + else if (text.StartsWith(MAP_KEY)) + { + _map = ExtMap.Parse(text); + } else if (text.StartsWith(TWITCH_LIVE_SEQUENCE_KEY)) { _twitchLiveSequence = ParsingHelpers.ParseUIntValue(text, TWITCH_LIVE_SEQUENCE_KEY); @@ -264,6 +268,7 @@ public Metadata ToMetadata() StreamTargetDuration = _streamTargetDuration, Type = _type, MediaSequence = _mediaSequence, + Map = _map, TwitchLiveSequence = _twitchLiveSequence, TwitchElapsedSeconds = _twitchElapsedSeconds, TwitchTotalSeconds = _twitchTotalSeconds, @@ -271,30 +276,44 @@ public Metadata ToMetadata() }; } } - } - public partial record Stream - { - public partial record struct ExtByteRange + public partial record ExtMap { - public static ExtByteRange Parse(ReadOnlySpan text) + public static ExtMap Parse(ReadOnlySpan text) { - if (text.StartsWith(BYTE_RANGE_KEY)) - text = text[17..]; - - var separatorIndex = text.IndexOf('@'); - if (separatorIndex != -1 - && separatorIndex != text.Length - && uint.TryParse(text[..separatorIndex], NumberStyles.Integer, CultureInfo.InvariantCulture, out var start) - && uint.TryParse(text[(separatorIndex + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var end)) + if (text.StartsWith(MAP_KEY)) + text = text[MAP_KEY.Length..]; + + ByteRange byteRange = default; + var uri = ""; + + do { - return new ExtByteRange(start, end); - } + text = text.TrimStart(); + + if (text.StartsWith(BYTE_RANGE_KEY)) + { + byteRange = ByteRange.Parse(text, BYTE_RANGE_KEY); + } + else if (text.StartsWith(URI_KEY)) + { + uri = ParsingHelpers.ParseStringValue(text, URI_KEY); + } - throw new FormatException($"Unable to parse ByteRange from {text}."); + var nextIndex = text.UnEscapedIndexOf(','); + if (nextIndex == -1) + break; + + text = text[(nextIndex + 1)..]; + } while (true); + + return new ExtMap(uri, byteRange); } } + } + public partial record Stream + { public partial record ExtMediaInfo { public static ExtMediaInfo Parse(ReadOnlySpan text) @@ -468,6 +487,26 @@ public static ExtPartInfo Parse(ReadOnlySpan text) } } + public partial record struct ByteRange + { + public static ByteRange Parse(ReadOnlySpan text, ReadOnlySpan keyName) + { + if (text.StartsWith(keyName)) + text = text[keyName.Length..]; + + var separatorIndex = text.IndexOf('@'); + if (separatorIndex != -1 + && separatorIndex != text.Length + && uint.TryParse(text[..separatorIndex], NumberStyles.Integer, CultureInfo.InvariantCulture, out var start) + && uint.TryParse(text[(separatorIndex + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var end)) + { + return new ByteRange(start, end); + } + + throw new FormatException($"Unable to parse ByteRange from {text}."); + } + } + private static class ParsingHelpers { public static bool TryParseM3UHeader(ReadOnlySpan text, out ReadOnlySpan textWithoutHeader) diff --git a/TwitchDownloaderCore/Tools/VideoDownloadThread.cs b/TwitchDownloaderCore/Tools/VideoDownloadThread.cs index 0b672cea..4b87f471 100644 --- a/TwitchDownloaderCore/Tools/VideoDownloadThread.cs +++ b/TwitchDownloaderCore/Tools/VideoDownloadThread.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net; using System.Net.Http; @@ -15,6 +16,7 @@ internal sealed record VideoDownloadThread private readonly HttpClient _client; private readonly Uri _baseUrl; private readonly string _cacheFolder; + private readonly string _headerFile; private readonly DateTimeOffset _vodAirDate; private TimeSpan VodAge => DateTimeOffset.UtcNow - _vodAirDate; private readonly int _throttleKib; @@ -22,8 +24,10 @@ internal sealed record VideoDownloadThread private readonly CancellationToken _cancellationToken; public Task ThreadTask { get; private set; } - public VideoDownloadThread(ConcurrentQueue videoPartsQueue, HttpClient httpClient, Uri baseUrl, string cacheFolder, DateTimeOffset vodAirDate, int throttleKib, ITaskLogger logger, CancellationToken cancellationToken) + public VideoDownloadThread(ConcurrentQueue videoPartsQueue, HttpClient httpClient, Uri baseUrl, string cacheFolder, [AllowNull] string headerFile, DateTimeOffset vodAirDate, int throttleKib, ITaskLogger logger, + CancellationToken cancellationToken) { + _headerFile = headerFile; _videoPartsQueue = videoPartsQueue; _client = httpClient; _baseUrl = baseUrl; @@ -98,43 +102,32 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke if (tryUnmute && videoPartName.Contains("-muted")) { var unmutedPartName = videoPartName.Replace("-muted", ""); - expectedLength = await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, unmutedPartName), partFile, _throttleKib, _logger, cancellationTokenSource); + expectedLength = await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, unmutedPartName), partFile, _headerFile, _throttleKib, _logger, cancellationTokenSource); } else { - expectedLength = await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, videoPartName), partFile, _throttleKib, _logger, cancellationTokenSource); + expectedLength = await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, videoPartName), partFile, _headerFile, _throttleKib, _logger, cancellationTokenSource); } - if (expectedLength is not -1) + // TODO: Support checking file length with header file + if (string.IsNullOrWhiteSpace(_headerFile) && expectedLength > 0) { // I would love to compare hashes here but unfortunately Twitch doesn't give us a ContentMD5 header var actualLength = new FileInfo(partFile).Length; - if (actualLength != expectedLength) + if (!VerifyFileLength(expectedLength, actualLength, partFile, ref lengthFailureCount)) { - const int MAX_RETRIES = 1; - - _logger.LogVerbose($"{partFile} failed to verify: expected {expectedLength:N0}B, got {actualLength:N0}B."); - if (++lengthFailureCount > MAX_RETRIES) - { - throw new Exception($"Failed to download {partFile}: expected {expectedLength:N0}B, got {actualLength:N0}B."); - } - await Delay(1_000, cancellationTokenSource.Token); continue; } - const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf - if (expectedLength % TS_PACKET_LENGTH != 0) - { - _logger.LogVerbose($"{partFile} contains malformed packets and may cause encoding issues."); - } + CheckTsLength(partFile, actualLength); } return; } catch (HttpRequestException ex) when (tryUnmute && ex.StatusCode is HttpStatusCode.Forbidden) { - _logger.LogVerbose($"Received {ex.StatusCode}: {ex.StatusCode} when trying to unmute {videoPartName}. Disabling {nameof(tryUnmute)}."); + _logger.LogVerbose($"Received HTTP {ex.StatusCode} when trying to unmute {videoPartName}. Disabling {nameof(tryUnmute)}."); tryUnmute = false; await Delay(100, cancellationTokenSource.Token); @@ -164,6 +157,36 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke await Delay(5_000 * timeoutCount, cancellationTokenSource.Token); } } + + bool VerifyFileLength(long expectedLength, long actualLength, string partFile, ref int failureCount) + { + if (actualLength != expectedLength) + { + const int MAX_RETRIES = 1; + + _logger.LogVerbose($"{partFile} failed to verify: expected {expectedLength:N0}B, got {actualLength:N0}B."); + if (++failureCount > MAX_RETRIES) + { + throw new Exception($"Failed to download {partFile}: expected {expectedLength:N0}B, got {actualLength:N0}B"); + } + + return false; + } + + return true; + } + } + + private void CheckTsLength(string partFile, long length) + { + if (partFile.EndsWith(".ts")) + { + const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf + if (length % TS_PACKET_LENGTH != 0) + { + _logger.LogVerbose($"{partFile} contains malformed packets and may cause encoding issues."); + } + } } private static Task Delay(int millis, CancellationToken cancellationToken) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index b2f74b92..282096ab 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Linq; @@ -94,13 +95,21 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF Directory.Delete(downloadFolder, true); TwitchHelper.CreateDirectory(downloadFolder); + if (qualityPlaylist.StreamInfo.Codecs.Any(x => x.StartsWith("av01"))) + { + _progress.LogWarning("AV1 VOD support is still experimental. " + + "If you encounter playback issues, try using an FFmpeg-based application like MPV, Kdenlive, or Blender, or re-encode the video file as H.264/AVC or H.265/HEVC with FFmpeg or Handbrake."); + } + + var headerFile = await GetHeaderFile(playlist, baseUrl, downloadFolder, cancellationToken); + _progress.SetTemplateStatus("Downloading {0}% [2/4]", 0); - await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); + await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, headerFile, airDate, cancellationToken); _progress.SetTemplateStatus("Verifying Parts {0}% [3/4]", 0); - await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); + await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, headerFile, airDate, cancellationToken); _progress.SetTemplateStatus("Finalizing Video {0}% [4/4]", 0); @@ -109,7 +118,8 @@ await FfmpegMetadata.SerializeAsync(metadataPath, downloadOptions.Id.ToString(), videoChapterResponse.data.video.moments.edges); var concatListPath = Path.Combine(downloadFolder, "concat.txt"); - await FfmpegConcatList.SerializeAsync(concatListPath, playlist, videoListCrop, cancellationToken); + var streamIds = GetStreamIds(playlist); + await FfmpegConcatList.SerializeAsync(concatListPath, playlist, videoListCrop, streamIds, cancellationToken); outputFs.Close(); @@ -145,6 +155,34 @@ await FfmpegMetadata.SerializeAsync(metadataPath, downloadOptions.Id.ToString(), } } + private async Task GetHeaderFile(M3U8 playlist, Uri baseUrl, string downloadFolder, CancellationToken cancellationToken) + { + var map = playlist.FileMetadata.Map; + if (string.IsNullOrWhiteSpace(map?.Uri)) + { + return null; + } + + if (map.ByteRange != default) + { + _progress.LogWarning($"Byte range was {map.ByteRange}, but is not yet implemented!"); + } + + var destinationFile = Path.Combine(downloadFolder, map.Uri); + + var uri = new Uri(baseUrl, map.Uri); + _progress.LogVerbose($"Downloading header file from '{uri}' to '{destinationFile}'"); + + using var request = new HttpRequestMessage(HttpMethod.Get, uri); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + response.EnsureSuccessStatusCode(); + + await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read); + await response.Content.CopyToAsync(fs, cancellationToken); + + return destinationFile; + } + private void CheckAvailableStorageSpace(int bandwidth, TimeSpan videoLength) { var videoSizeInBytes = VideoSizeEstimator.EstimateVideoSize(bandwidth, @@ -175,15 +213,15 @@ private void CheckAvailableStorageSpace(int bandwidth, TimeSpan videoLength) } } - private async Task DownloadVideoPartsAsync(IEnumerable playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, DateTimeOffset vodAirDate, CancellationToken cancellationToken) + private async Task DownloadVideoPartsAsync(IReadOnlyCollection playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, [AllowNull] string headerFile, DateTimeOffset vodAirDate, CancellationToken cancellationToken) { - var partCount = videoListCrop.End.Value - videoListCrop.Start.Value; + var partCount = videoListCrop.GetOffsetAndLength(playlist.Count).Length; var videoPartsQueue = new ConcurrentQueue(playlist.Take(videoListCrop).Select(x => x.Path)); var downloadThreads = new VideoDownloadThread[downloadOptions.DownloadThreads]; for (var i = 0; i < downloadOptions.DownloadThreads; i++) { - downloadThreads[i] = new VideoDownloadThread(videoPartsQueue, _httpClient, baseUrl, downloadFolder, vodAirDate, downloadOptions.ThrottleKib, _progress, cancellationToken); + downloadThreads[i] = new VideoDownloadThread(videoPartsQueue, _httpClient, baseUrl, downloadFolder, headerFile, vodAirDate, downloadOptions.ThrottleKib, _progress, cancellationToken); } var downloadExceptions = await WaitForDownloadThreads(downloadThreads, videoPartsQueue, partCount, cancellationToken); @@ -282,10 +320,10 @@ private void LogDownloadThreadExceptions(IReadOnlyCollection download _progress.LogInfo(sb.ToString()); } - private async Task VerifyDownloadedParts(ICollection playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, DateTimeOffset vodAirDate, CancellationToken cancellationToken) + private async Task VerifyDownloadedParts(IReadOnlyCollection playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, [AllowNull] string headerFile, DateTimeOffset vodAirDate, CancellationToken cancellationToken) { var failedParts = new List(); - var partCount = videoListCrop.End.Value - videoListCrop.Start.Value; + var partCount = videoListCrop.GetOffsetAndLength(playlist.Count).Length; var doneCount = 0; foreach (var part in playlist.Take(videoListCrop)) @@ -314,7 +352,23 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang } _progress.LogInfo($"The following parts will be redownloaded: {string.Join(", ", failedParts)}"); - await DownloadVideoPartsAsync(failedParts, Range.All, baseUrl, downloadFolder, vodAirDate, cancellationToken); + await DownloadVideoPartsAsync(failedParts, Range.All, baseUrl, downloadFolder, headerFile, vodAirDate, cancellationToken); + } + } + + private FfmpegConcatList.StreamIds GetStreamIds(M3U8 playlist) + { + var path = DownloadTools.RemoveQueryString(playlist.Streams.FirstOrDefault()?.Path ?? ""); + var extension = Path.GetExtension(path); + switch (extension) + { + case ".mp4": + return FfmpegConcatList.StreamIds.Mp4; + case ".ts": + return FfmpegConcatList.StreamIds.TransportStream; + default: + _progress.LogWarning("No file extension was found! Assuming TS."); + return FfmpegConcatList.StreamIds.TransportStream; } }