Skip to content

Commit

Permalink
Initial support for AV1/H.265 VODs (#1251)
Browse files Browse the repository at this point in the history 
* Support changing stream IDs depending on container type

* Fix ByteRange

* Support parsing EXT-X-MAP keys

* Update M3U8 tests

* Support adding header content to downloaded files

* Fix potential part count issues

* Add verbose log when downloading header file

* Rename VerifyTsLength to CheckTsLength

* Fix test variable names

* Add warning when downloading AV1 VODs

* Merge ByteRange structs

* Update tests

* Fix some M3U8 stringification issues

* Fix log message

* Use IReadOnlyCollection

* Use switch statement

* Do not read header file contents into memory
  • Loading branch information
@ScrubN
ScrubN authored Dec 26, 2024
1 parent 8bb9fc1 commit 8c4200c
Show file tree
Hide file tree
Showing 7 changed files with 342 additions and 89 deletions.
99 changes: 88 additions & 11 deletions TwitchDownloaderCore.Tests/ToolTests/M3U8Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,81 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream, string c
}
}

[Theory]
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfMp4s(bool useStream, string culture)
{
const string EXAMPLE_M3U8_TWITCH =
"#EXTM3U" +
"\n#EXT-X-VERSION:6" +
"\n#EXT-X-TARGETDURATION:10" +
"\n#ID3-EQUIV-TDTG:2024-12-08T00:12:24" +
"\n#EXT-X-PLAYLIST-TYPE:EVENT" +
"\n#EXT-X-MEDIA-SEQUENCE:0" +
"\n#EXT-X-TWITCH-ELAPSED-SECS:0.000" +
"\n#EXT-X-TWITCH-TOTAL-SECS:1137.134" +
"\n#EXT-X-MAP:URI=\"init-0.mp4\"" +
"\n#EXTINF:10.000,\n0.mp4\n#EXTINF:10.000,\n1.mp4\n#EXTINF:10.000,\n2.mp4\n#EXTINF:10.000,\n3.mp4\n#EXTINF:10.000,\n4.mp4\n#EXTINF:10.000,\n5.mp4\n#EXTINF:10.000,\n6.mp4\n#EXTINF:10.000,\n7.mp4" +
"\n#EXTINF:10.000,\n8.mp4\n#EXTINF:10.000,\n9.mp4\n#EXTINF:10.000,\n10.mp4\n#EXTINF:10.000,\n11.mp4\n#EXTINF:10.000,\n12.mp4\n#EXTINF:10.000,\n13.mp4\n#EXTINF:10.000,\n14.mp4\n#EXTINF:10.000," +
"\n15.mp4\n#EXTINF:10.000,\n16.mp4\n#EXTINF:10.000,\n17.mp4\n#EXTINF:10.000,\n18.mp4\n#EXTINF:10.000,\n19.mp4\n#EXTINF:10.000,\n20.mp4\n#EXTINF:10.000,\n21.mp4\n#EXTINF:10.000,\n22.mp4" +
"\n#EXTINF:10.000,\n23.mp4\n#EXTINF:10.000,\n24.mp4\n#EXTINF:10.000,\n25.mp4\n#EXTINF:10.000,\n26.mp4\n#EXTINF:10.000,\n27.mp4\n#EXTINF:10.000,\n28.mp4\n#EXTINF:10.000,\n29.mp4\n#EXTINF:10.000," +
"\n30.mp4\n#EXTINF:10.000,\n31.mp4\n#EXTINF:10.000,\n32.mp4\n#EXTINF:10.000,\n33.mp4\n#EXTINF:10.000,\n34.mp4\n#EXTINF:10.000,\n35.mp4\n#EXTINF:10.000,\n36.mp4\n#EXTINF:10.000,\n37.mp4" +
"\n#EXTINF:10.000,\n38.mp4\n#EXTINF:10.000,\n39.mp4\n#EXTINF:10.000,\n40.mp4\n#EXTINF:10.000,\n41.mp4\n#EXTINF:10.000,\n42.mp4\n#EXTINF:10.000,\n43.mp4\n#EXTINF:10.000,\n44.mp4\n#EXTINF:10.000," +
"\n45.mp4\n#EXTINF:10.000,\n46.mp4\n#EXTINF:10.000,\n47.mp4\n#EXTINF:10.000,\n48.mp4\n#EXTINF:10.000,\n49.mp4\n#EXTINF:10.000,\n50.mp4\n#EXTINF:10.000,\n51.mp4\n#EXTINF:10.000,\n52.mp4" +
"\n#EXTINF:10.000,\n53.mp4\n#EXTINF:10.000,\n54.mp4\n#EXTINF:10.000,\n55.mp4\n#EXTINF:10.000,\n56.mp4\n#EXTINF:10.000,\n57.mp4\n#EXTINF:10.000,\n58.mp4\n#EXTINF:10.000,\n59.mp4\n#EXTINF:10.000," +
"\n60.mp4\n#EXTINF:10.000,\n61.mp4\n#EXTINF:10.000,\n62.mp4\n#EXTINF:10.000,\n63.mp4\n#EXTINF:10.000,\n64.mp4\n#EXTINF:10.000,\n65.mp4\n#EXTINF:10.000,\n66.mp4\n#EXTINF:10.000,\n67.mp4" +
"\n#EXTINF:10.000,\n68.mp4\n#EXTINF:10.000,\n69.mp4\n#EXTINF:10.000,\n70.mp4\n#EXTINF:10.000,\n71.mp4\n#EXTINF:10.000,\n72.mp4\n#EXTINF:10.000,\n73.mp4\n#EXTINF:10.000,\n74.mp4\n#EXTINF:10.000," +
"\n75.mp4\n#EXTINF:10.000,\n76.mp4\n#EXTINF:10.000,\n77.mp4\n#EXTINF:10.000,\n78.mp4\n#EXTINF:10.000,\n79.mp4\n#EXTINF:10.000,\n80.mp4\n#EXTINF:10.000,\n81.mp4\n#EXTINF:10.000,\n82.mp4" +
"\n#EXTINF:10.000,\n83.mp4\n#EXTINF:10.000,\n84.mp4\n#EXTINF:10.000,\n85.mp4\n#EXTINF:10.000,\n86.mp4\n#EXTINF:10.000,\n87.mp4\n#EXTINF:10.000,\n88.mp4\n#EXTINF:10.000,\n89.mp4\n#EXTINF:10.000," +
"\n90.mp4\n#EXTINF:10.000,\n91.mp4\n#EXTINF:10.000,\n92.mp4\n#EXTINF:10.000,\n93.mp4\n#EXTINF:10.000,\n94.mp4\n#EXTINF:10.000,\n95.mp4\n#EXTINF:10.000,\n96.mp4\n#EXTINF:10.000,\n97.mp4" +
"\n#EXTINF:10.000,\n98.mp4\n#EXTINF:10.000,\n99.mp4\n#EXTINF:10.000,\n100.mp4\n#EXTINF:10.000,\n101.mp4\n#EXTINF:10.000,\n102.mp4\n#EXTINF:10.000,\n103.mp4\n#EXTINF:10.000,\n104.mp4" +
"\n#EXTINF:10.000,\n105.mp4\n#EXTINF:10.000,\n106.mp4\n#EXTINF:10.000,\n107.mp4\n#EXTINF:10.000,\n108.mp4\n#EXTINF:10.000,\n109.mp4\n#EXTINF:10.000,\n110.mp4\n#EXTINF:10.000,\n111.mp4" +
"\n#EXTINF:10.000,\n112.mp4\n#EXTINF:7.134,\n113.mp4\n#EXT-X-ENDLIST";

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
var bytes = Encoding.Unicode.GetBytes(EXAMPLE_M3U8_TWITCH);
using var ms = new MemoryStream(bytes);
m3u8 = M3U8.Parse(ms, Encoding.Unicode);
}
else
{
m3u8 = M3U8.Parse(EXAMPLE_M3U8_TWITCH);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(6u, m3u8.FileMetadata.Version);
Assert.Equal(10u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal("2024-12-08T00:12:24", m3u8.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:").Value);
Assert.Equal(M3U8.Metadata.PlaylistType.Event, m3u8.FileMetadata.Type);
Assert.Equal(0u, m3u8.FileMetadata.MediaSequence);
Assert.Equal("init-0.mp4", m3u8.FileMetadata.Map.Uri);
Assert.Equal(default, m3u8.FileMetadata.Map.ByteRange);
Assert.Equal(0m, m3u8.FileMetadata.TwitchElapsedSeconds);
Assert.Equal(1137.134m, m3u8.FileMetadata.TwitchTotalSeconds);

Assert.Equal(114, m3u8.Streams.Length);

var duration = 1137.134m;
for (var i = 0; i < m3u8.Streams.Length; i++)
{
var stream = m3u8.Streams[i];
Assert.Equal(duration > 10 ? 10 : duration, stream.PartInfo.Duration);
Assert.False(stream.PartInfo.Live);
Assert.Equal($"{i}.mp4", stream.Path);

duration -= 10;
}
}

[Theory]
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
Expand Down Expand Up @@ -301,7 +376,7 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream, string cul
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:03.97Z\n#EXT-X-BYTERANGE:1768140@3175696\n#EXTINF:2.000,\n506.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:05.97Z\n#EXT-X-BYTERANGE:1519040@4943836\n#EXTINF:2.000,\n506.ts" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:07.97Z\n#EXT-X-BYTERANGE:1506068@6462876\n#EXTINF:2.000,\n506.ts\n#EXT-X-ENDLIST";

var streamValues = new (DateTimeOffset programDateTime, M3U8.Stream.ExtByteRange byteRange, string path)[]
var streamValues = new (DateTimeOffset programDateTime, M3U8.ByteRange byteRange, string path)[]
{
(DateTimeOffset.Parse("2023-11-16T05:34:07.97Z"), (1601196, 6470396), "500.ts"),
(DateTimeOffset.Parse("2023-11-16T05:34:09.97Z"), (1588224, 0), "501.ts"),
Expand Down Expand Up @@ -474,13 +549,14 @@ public void CorrectlyParsesKickM3U8StreamInfo(string streamInfoString, int bandw
}

[Theory]
[InlineData(100, 200, "100@200")]
[InlineData(100, 200, "#EXT-X-BYTERANGE:100@200")]
public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeString)
[InlineData(100, 200, "100@200", "")]
[InlineData(100, 200, "#EXT-X-BYTERANGE:100@200", "#EXT-X-BYTERANGE:")]
[InlineData(100, 200, "BYTERANGE=100@200", "BYTERANGE=")]
public void CorrectlyParsesByteRange(uint length, uint start, string byteRangeString, string key)
{
var expected = new M3U8.Stream.ExtByteRange(start, length);
var expected = new M3U8.ByteRange(length, start);

var actual = M3U8.Stream.ExtByteRange.Parse(byteRangeString);
var actual = M3U8.ByteRange.Parse(byteRangeString, key);

Assert.Equal(expected, actual);
}
Expand All @@ -491,15 +567,15 @@ public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeSt
[InlineData("42949672950000")]
public void ThrowsFormatExceptionForBadByteRangeString(string byteRangeString)
{
Assert.Throws<FormatException>(() => M3U8.Stream.ExtByteRange.Parse(byteRangeString));
Assert.Throws<FormatException>(() => M3U8.ByteRange.Parse(byteRangeString, default));
}

[Theory]
[InlineData(100, 200, "100x200")]
[InlineData(100, 200, "RESOLUTION=100x200")]
public void CorrectlyParsesResolution(uint start, uint length, string byteRangeString)
public void CorrectlyParsesResolution(uint width, uint height, string byteRangeString)
{
var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(start, length);
var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(width, height);

var actual = M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString);

Expand All @@ -510,9 +586,9 @@ public void CorrectlyParsesResolution(uint start, uint length, string byteRangeS
[InlineData("429496729500x1")]
[InlineData("1x429496729500")]
[InlineData("42949672950000")]
public void ThrowsFormatExceptionForBadResolutionString(string byteRangeString)
public void ThrowsFormatExceptionForBadResolutionString(string resolutionString)
{
Assert.Throws<FormatException>(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString));
Assert.Throws<FormatException>(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(resolutionString));
}

[Theory]
Expand Down Expand Up @@ -545,6 +621,7 @@ public void CorrectlyStringifiesInvariantOfCulture(string culture)
"\n#EXT-X-VERSION:4" +
"\n#EXT-X-MEDIA-SEQUENCE:0" +
"\n#EXT-X-TARGETDURATION:2" +
"\n#EXT-X-MAP:URI=\"init-0.mp4\"" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:07.97Z\n#EXT-X-BYTERANGE:1601196@6470396\n#EXTINF:2.000,\n500.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:09.97Z\n#EXT-X-BYTERANGE:1588224@0\n#EXTINF:2.000,\n501.ts" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:11.97Z\n#EXT-X-BYTERANGE:1579200@1588224\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:13.97Z\n#EXT-X-BYTERANGE:1646128@3167424\n#EXTINF:2.000,\n501.ts" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:15.97Z\n#EXT-X-BYTERANGE:1587472@4813552\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:17.97Z\n#EXT-X-BYTERANGE:1594052@6401024\n#EXTINF:2.000,\n501.ts" +
Expand Down
19 changes: 15 additions & 4 deletions TwitchDownloaderCore/Tools/DownloadTools.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Net.Http;
using System.Threading;
Expand All @@ -15,20 +16,30 @@ public static class DownloadTools
/// <param name="httpClient">The <see cref="HttpClient"/> to perform the download operation.</param>
/// <param name="url">The url of the file to download.</param>
/// <param name="destinationFile">The path to the file where download will be saved.</param>
/// <param name="headerFile">Path to a file whose contents will be written to the start of the destination file.</param>
/// <param name="throttleKib">The maximum download speed in kibibytes per second, or -1 for no maximum.</param>
/// <param name="logger">Logger.</param>
/// <param name="cancellationTokenSource">A <see cref="CancellationTokenSource"/> containing a <see cref="CancellationToken"/> to cancel the operation.</param>
/// <returns>The expected length of the downloaded file, or -1 if the content length header is not present.</returns>
/// <remarks>The <paramref name="cancellationTokenSource"/> may be canceled by this method.</remarks>
public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, [AllowNull] string headerFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
using var request = new HttpRequestMessage(HttpMethod.Get, url);

var cancellationToken = cancellationTokenSource?.Token ?? CancellationToken.None;

using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();

var fileMode = FileMode.Create;
if (!string.IsNullOrWhiteSpace(headerFile))
{
await using var headerFs = new FileStream(headerFile, FileMode.Open, FileAccess.Read, FileShare.Read);
await using var destinationFs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await headerFs.CopyToAsync(destinationFs, cancellationToken);
fileMode = FileMode.Append;
}

// Why are we setting a CTS CancelAfter timer? See lay295#265
const int SIXTY_SECONDS = 60;
if (throttleKib == -1 || !response.Content.Headers.ContentLength.HasValue)
Expand All @@ -48,7 +59,7 @@ public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url,
{
case -1:
{
await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read);
await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
break;
}
Expand All @@ -58,7 +69,7 @@ public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url,
{
await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
await using var throttledStream = new ThrottledStream(contentStream, throttleKib);
await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read);
await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
}
catch (IOException ex) when (ex.Message.Contains("EOF"))
Expand Down
28 changes: 21 additions & 7 deletions TwitchDownloaderCore/Tools/FfmpegConcatList.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
Expand All @@ -12,7 +13,7 @@ public static class FfmpegConcatList
{
private const string LINE_FEED = "\u000A";

public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, CancellationToken cancellationToken = default)
public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, StreamIds streamIds, CancellationToken cancellationToken = default)
{
await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED };
Expand All @@ -27,16 +28,29 @@ public static async Task SerializeAsync(string filePath, M3U8 playlist, Range vi
await sw.WriteAsync(DownloadTools.RemoveQueryString(stream.Path));
await sw.WriteLineAsync('\'');

await sw.WriteLineAsync("stream");
await sw.WriteLineAsync("exact_stream_id 0x100"); // Audio
await sw.WriteLineAsync("stream");
await sw.WriteLineAsync("exact_stream_id 0x101"); // Video
await sw.WriteLineAsync("stream");
await sw.WriteLineAsync("exact_stream_id 0x102"); // Subtitle
foreach (var id in streamIds.Ids)
{
await sw.WriteLineAsync("stream");
await sw.WriteLineAsync($"exact_stream_id {id}");
}

await sw.WriteAsync("duration ");
await sw.WriteLineAsync(stream.PartInfo.Duration.ToString(CultureInfo.InvariantCulture));
}
}

public record StreamIds
{
public static readonly StreamIds TransportStream = new("0x100", "0x101", "0x102");
public static readonly StreamIds Mp4 = new("0x1", "0x2");
public static readonly StreamIds None = new();

private StreamIds(params string[] ids)
{
Ids = ids;
}

public IEnumerable<string> Ids { get; }
}
}
}
Loading

0 comments on commit 8c4200c

Please sign in to comment.