Ranged requests

ReplayBrowser/Helpers/HttpExtensions.cs

@@ -1,7 +1,30 @@
-namespace ReplayBrowser.Helpers;
+using System.IO.Compression;
+
+namespace ReplayBrowser.Helpers;
 
 public static class HttpExtensions
 {
+    /// <summary>
+    /// Checks if the server supports
+    /// </summary>
+    public static async Task<bool> SupportsRangeRequests(this HttpClient client, string requestUri)
+    {
+        var request = new HttpRequestMessage(HttpMethod.Head, requestUri);
+        var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+        return response.Headers.AcceptRanges.Contains("bytes");
+    }
+
+    /// <summary>
+    /// Returns the size of the file in bytes
+    /// </summary>
+    public static async Task<long> GetFileSizeAsync(this HttpClient client, string requestUri)
+    {
+        var request = new HttpRequestMessage(HttpMethod.Head, requestUri);
+        var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+        var contentLength = response.Content.Headers.ContentLength;
+        return contentLength ?? -1;
+    }
+
     public static async Task<Stream> GetStreamAsync(this HttpClient client, string requestUri, IProgress<double> progress, CancellationToken token)
     {
         var response = await client.GetAsync(requestUri, HttpCompletionOption.ResponseHeadersRead, token);

ReplayBrowser/Helpers/ZipDownloader.cs

@@ -0,0 +1,43 @@
+
+using System.Diagnostics;
+using System.Text;
+
+namespace ReplayBrowser.Helpers;
+
+public static class ZipDownloader
+{
+    public static async Task<Dictionary<string, Stream>> ExtractFilesFromZipAsync(string zipUrl, string[] filesToExtract)
+    {
+        // ok so i first tried doing this in c#, but then saw a python script "unzip_http" that does this, so now im just gonna call that
+
+        var files = new Dictionary<string, Stream>();
+
+        foreach (var file in filesToExtract)
+        {
+            var process = new Process();
+            process.StartInfo.FileName = "python";
+            process.StartInfo.UseShellExecute = false;
+            process.StartInfo.RedirectStandardOutput = true;
+            process.StartInfo.RedirectStandardError = true;
+            var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory + "Tools/unzip_http.py");
+            process.StartInfo.Arguments = $"{path} -o {zipUrl} {file}";
+
+            process.Start();
+
+            var output = await process.StandardOutput.ReadToEndAsync();
+            var error = await process.StandardError.ReadToEndAsync();
+
+            await process.WaitForExitAsync();
+
+            if (process.ExitCode != 0)
+            {
+                throw new Exception($"Failed to extract files from zip: {error}");
+            }
+
+            var stream = new MemoryStream(Encoding.UTF8.GetBytes(output));
+            files.Add(file, stream);
+        }
+
+        return files;
+    }
+}

ReplayBrowser/ReplayBrowser.csproj

@@ -6,6 +6,8 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Configurations>Debug;Release;Testing</Configurations>
         <Platforms>AnyCPU</Platforms>
+
+        <SolutionDir Condition=" '$(SolutionDir)' == '' ">$(MSBuildThisFileDirectory)..\</SolutionDir>
     </PropertyGroup>
 
     <ItemGroup>
@@ -33,8 +35,14 @@
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0"/>
         <PackageReference Include="YamlDotNet" Version="15.1.1" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.4" />
+
+        <ToolsFolder Include="$(SolutionDir)Tools\**\*.*" />
     </ItemGroup>
 
+    <Target Name="CopyToolsFolder" AfterTargets="Build">
+        <Copy SourceFiles="@(ToolsFolder)" DestinationFolder="$(OutputPath)Tools\%(RecursiveDir)" />
+    </Target>
+
     <ItemGroup>
       <_ContentIncludedByDefault Remove="wwwroot\lib\bootstrap\dist\css\bootstrap-grid.css" />
       <_ContentIncludedByDefault Remove="wwwroot\lib\bootstrap\dist\css\bootstrap-grid.css.map" />

ReplayBrowser/Services/ReplayParser/ReplayParserService.cs

@@ -3,6 +3,7 @@
 using System.IO.Compression;
 using System.Text;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Net.Http.Headers;
 using ReplayBrowser.Data;
 using ReplayBrowser.Data.Models;
 using ReplayBrowser.Helpers;
@@ -34,6 +35,11 @@
     private readonly IServiceScopeFactory _factory;
 
 
+    /// <summary>
+    /// In this case we wont just add it to the parsed replays, so it redownloads it every time.
+    /// </summary>
+    private const string YamlSerializerError = "Exception during deserialization";
+
     public ReplayParserService(IConfiguration configuration, IServiceScopeFactory factory)
     {
         _configuration = configuration;
@@ -163,18 +169,46 @@
                         });
                         client.DefaultRequestHeaders.Add("User-Agent", "ReplayBrowser");
                         Log.Information("Downloading " + replay);
-                        var stream = await client.GetStreamAsync(replay, progress, token);
-                        completed++;
-                        Details = $"{completed}/{total}";
+                        var fileSize = await client.GetFileSizeAsync(replay);
+                        // Check if the server supports range requests.
+                        var supportsRange = (await client.SupportsRangeRequests(replay) && fileSize != -1);
+
                         Replay? parsedReplay = null;
                         try
                         {
-                            parsedReplay = ParseReplay(stream, replay);
+                            if (supportsRange)
+                            {
+                                try
+                                {
+                                    // The server supports ranged processing!
+                                    string[] files = ["_replay/replay_final.yml"];
+                                    var extractedFiles = await ZipDownloader.ExtractFilesFromZipAsync(replay, files);
+                                    completed++;
+                                    Details = $"{completed}/{total}";
+                                    parsedReplay = FinalizeReplayParse(new StreamReader(extractedFiles["_replay/replay_final.yml"]), replay);
+                                }
+                                catch (Exception e)
+                                {
+                                    Log.Error(e, "Error while downloading " + replay);
+                                    // fuck it, we ball and try the normal method
+                                    supportsRange = false;
+                                }
+                            }
+
+                            if (!supportsRange)
+                            {
+                                var stream = await client.GetStreamAsync(replay, progress, token);
+                                completed++;
+                                Details = $"{completed}/{total}";
+                                parsedReplay = ParseReplay(stream, replay);
+                            }
                         }
                         catch (Exception e)
                         {
                             Log.Error(e, "Error while parsing " + replay);
-                            await AddParsedReplayToDb(replay); // Prevent circular download eating up all resources.
+                            if (e.Message.Contains(YamlSerializerError)) return;
+
+                            await AddParsedReplayToDb(replay);
                             return;
                         }
                         // See if the link matches the date regex, if it does set the date
@@ -187,12 +221,12 @@
                             {
                                 var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture);
                                 // Need to mark it as UTC, since the server is in UTC.
                                 parsedReplay.Date = date.ToUniversalTime();
                             }
                             catch (FormatException)
                             {
                                 var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy-MM-dd", CultureInfo.InvariantCulture);
                                 parsedReplay.Date = date.ToUniversalTime();
                             }
                         }
 
@@ -202,7 +236,7 @@
                             return;
                         }
 
                         await AddReplayToDb(parsedReplay);
                         await AddParsedReplayToDb(replay);
                         parsedReplays.Add(parsedReplay);
                         Log.Information("Parsed " + replay);
@@ -259,12 +293,17 @@
         var replayStream = replayFile.Open();
         var reader = new StreamReader(replayStream);
 
+        return FinalizeReplayParse(reader, replayLink);
+    }
+
+    private Replay FinalizeReplayParse(StreamReader stream, string replayLink)
+    {
         var deserializer = new DeserializerBuilder()
             .IgnoreUnmatchedProperties()
             .WithNamingConvention(CamelCaseNamingConvention.Instance)
             .Build();
 
-        var yamlReplay = deserializer.Deserialize<YamlReplay>(reader);
+        var yamlReplay = deserializer.Deserialize<YamlReplay>(stream);
         if (yamlReplay.Map == null && yamlReplay.Maps == null)
         {
             throw new Exception("Replay is not valid.");