fix performance issues and add polling to download

GithubBackup/GithubBackup.Cli.Tests/Commands/Github/Backup/BackupRunnerTests.cs

@@ -70,7 +70,7 @@ private BackupRunner CreateBackupRunner(bool quiet)
     {
         var globalArgs = new GlobalArgs(LogLevel.Debug, quiet, new FileInfo("test"));
         var migrateArgs = new MigrateArgs(new []{"test"}, false, false, false, false, false, false, false, new IntervalArgs(null), new LoginArgs(null, false));
-        var downloadArgs = new DownloadArgs(Array.Empty<long>(), false, new DirectoryInfo("test"), null, true, new IntervalArgs(null), new LoginArgs(null, false));
+        var downloadArgs = new DownloadArgs(Array.Empty<long>(), false, false, new DirectoryInfo("test"), null, true, new IntervalArgs(null), new LoginArgs(null, false));
         var backupArgs = new BackupArgs(migrateArgs, downloadArgs, new IntervalArgs(null), new LoginArgs(null, false));
 
         return new BackupRunner(

GithubBackup/GithubBackup.Cli.Tests/Commands/Github/Download/DownloadRunnerTests.RunAsync_NotQuietAndExportMigrationsAndUsePolling_DoWriteToConsoleAndDoDownload.verified.txt

@@ -0,0 +1,5 @@
+Downloading migrations...
+Downloading migration 2 to test...
+Downloaded migration 2 to 
+Downloading migration 1 to test...
+Downloaded migration 1 to 

GithubBackup/GithubBackup.Cli.Tests/Commands/Github/Download/DownloadRunnerTests.RunAsync_NotQuietAndExportMigrations_DoWriteToConsoleAndDoDownload.verified.txt

@@ -1,5 +1,5 @@
 Downloading migrations...
-Downloading migration 1 to test...
-Downloaded migration 1 to test1
 Downloading migration 2 to test...
 Downloaded migration 2 to test2
+Downloading migration 1 to test...
+Downloaded migration 1 to test1

GithubBackup/GithubBackup.Cli.Tests/Commands/Github/Download/DownloadRunnerTests.RunAsync_QuietAndExportMigrations_DoNotWriteToConsoleAndDoDownload.verified.txt

@@ -1,2 +1,2 @@
-test1
-test2
+test2
+test1

GithubBackup/GithubBackup.Cli.Tests/Commands/Github/Download/DownloadRunnerTests.cs

@@ -25,7 +25,7 @@ public class DownloadRunnerTests
     [Fact]
     public async Task RunAsync_QuietAndLatest_DoNotWriteToConsoleAndDownloadLatest()
     {
-        var runner = CreateRunner(true, true);
+        var runner = CreateRunner(true, true, false);
 
         const int id = 1;
 
@@ -47,6 +47,7 @@ public async Task RunAsync_QuietAndLatest_DoNotWriteToConsoleAndDownloadLatest()
         _logger.VerifyLogs(
             new LogEntry(LogLevel.Information, "Downloading latest migration"),
             new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1"),
             new LogEntry(LogLevel.Information, "Downloaded migration 1 to test")
         );
 
@@ -56,7 +57,7 @@ public async Task RunAsync_QuietAndLatest_DoNotWriteToConsoleAndDownloadLatest()
     [Fact]
     public async Task RunAsync_NotQuietAndLatest_DoWriteToConsoleAndDownloadLatest()
     {
-        var runner = CreateRunner(false, true);
+        var runner = CreateRunner(false, true, false);
 
         const int id = 1;
 
@@ -78,6 +79,7 @@ public async Task RunAsync_NotQuietAndLatest_DoWriteToConsoleAndDownloadLatest()
         _logger.VerifyLogs(
             new LogEntry(LogLevel.Information, "Downloading latest migration"),
             new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1"),
             new LogEntry(LogLevel.Information, "Downloaded migration 1 to test")
         );
 
@@ -87,7 +89,7 @@ public async Task RunAsync_NotQuietAndLatest_DoWriteToConsoleAndDownloadLatest()
     [Fact]
     public async Task RunAsync_QuietAndNoMigrations_DoNotWriteToConsoleAndDownloadLatest()
     {
-        var runner = CreateRunner(true, false);
+        var runner = CreateRunner(true, false, false);
 
         const int id = 1;
 
@@ -109,6 +111,7 @@ public async Task RunAsync_QuietAndNoMigrations_DoNotWriteToConsoleAndDownloadLa
         _logger.VerifyLogs(
             new LogEntry(LogLevel.Information, "No migration ids specified, downloading latest migration"),
             new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1"),
             new LogEntry(LogLevel.Information, "Downloaded migration 1 to test")
         );
 
@@ -118,7 +121,7 @@ public async Task RunAsync_QuietAndNoMigrations_DoNotWriteToConsoleAndDownloadLa
     [Fact]
     public async Task RunAsync_NotQuietAndLatestAndNoMigrations_DoWriteToConsoleAndDownloadLatest()
     {
-        var runner = CreateRunner(false, false);
+        var runner = CreateRunner(false, false, false);
 
         const int id = 1;
 
@@ -140,6 +143,7 @@ public async Task RunAsync_NotQuietAndLatestAndNoMigrations_DoWriteToConsoleAndD
         _logger.VerifyLogs(
             new LogEntry(LogLevel.Information, "No migration ids specified, downloading latest migration"),
             new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1"),
             new LogEntry(LogLevel.Information, "Downloaded migration 1 to test")
         );
 
@@ -149,7 +153,7 @@ public async Task RunAsync_NotQuietAndLatestAndNoMigrations_DoWriteToConsoleAndD
     [Fact]
     public async Task RunAsync_QuietAndLatestAndNoExportedMigrations_DoNotWriteToConsoleAndDoNotDownload()
     {
-        var runner = CreateRunner(true, true);
+        var runner = CreateRunner(true, true, false);
 
         _migrationService
             .GetMigrationsAsync(CancellationToken.None)
@@ -177,7 +181,7 @@ await _migrationService
     [Fact]
     public async Task RunAsync_NotQuietAndLatestAndNoExportedMigrations_DoWriteToConsoleAndDoNotDownload()
     {
-        var runner = CreateRunner(false, true);
+        var runner = CreateRunner(false, true, false);
 
         _migrationService
             .GetMigrationsAsync(CancellationToken.None)
@@ -205,7 +209,7 @@ await _migrationService
     [Fact]
     public async Task RunAsync_QuietAndExportMigrations_DoNotWriteToConsoleAndDoDownload()
     {
-        var runner = CreateRunner(true, true, new[] { 1L, 2L });
+        var runner = CreateRunner(true, true, false, new[] { 1L, 2L });
 
         _migrationService
             .GetMigrationsAsync(CancellationToken.None)
@@ -228,10 +232,12 @@ public async Task RunAsync_QuietAndExportMigrations_DoNotWriteToConsoleAndDoDown
 
         _logger.VerifyLogs(
             new LogEntry(LogLevel.Information, "Downloading migrations using ids"),
-            new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
-            new LogEntry(LogLevel.Information, "Downloaded migration 1 to test1"),
             new LogEntry(LogLevel.Information, "Downloading migration 2 to test"),
-            new LogEntry(LogLevel.Information, "Downloaded migration 2 to test2")
+            new LogEntry(LogLevel.Information, "Downloading migration 2"),
+            new LogEntry(LogLevel.Information, "Downloaded migration 2 to test2"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1"),
+            new LogEntry(LogLevel.Information, "Downloaded migration 1 to test1")
         );
 
         await Verify(_ansiConsole.Output);
@@ -240,7 +246,7 @@ public async Task RunAsync_QuietAndExportMigrations_DoNotWriteToConsoleAndDoDown
     [Fact]
     public async Task RunAsync_NotQuietAndExportMigrations_DoWriteToConsoleAndDoDownload()
     {
-        var runner = CreateRunner(false, true, new[] { 1L, 2L });
+        var runner = CreateRunner(false, true, false, new[] { 1L, 2L });
 
         _migrationService
             .GetMigrationsAsync(CancellationToken.None)
@@ -263,21 +269,61 @@ public async Task RunAsync_NotQuietAndExportMigrations_DoWriteToConsoleAndDoDown
 
         _logger.VerifyLogs(
             new LogEntry(LogLevel.Information, "Downloading migrations using ids"),
+            new LogEntry(LogLevel.Information, "Downloading migration 2 to test"),
+            new LogEntry(LogLevel.Information, "Downloading migration 2"),
+            new LogEntry(LogLevel.Information, "Downloaded migration 2 to test2"),
             new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
-            new LogEntry(LogLevel.Information, "Downloaded migration 1 to test1"),
+            new LogEntry(LogLevel.Information, "Downloading migration 1"),
+            new LogEntry(LogLevel.Information, "Downloaded migration 1 to test1")
+        );
+
+        await Verify(_ansiConsole.Output);
+    }
+
+    [Fact]
+    public async Task RunAsync_NotQuietAndExportMigrationsAndUsePolling_DoWriteToConsoleAndDoDownload()
+    {
+        var runner = CreateRunner(false, true, true, new[] { 1L, 2L });
+
+        _migrationService
+            .GetMigrationsAsync(CancellationToken.None)
+            .Returns(new List<Migration>
+            {
+                new(0, MigrationState.Failed, new DateTime(2022, 1, 1)),
+                new(1, MigrationState.Exported, new DateTime(2021, 1, 1)),
+                new(2, MigrationState.Exported, new DateTime(2020, 1, 1))
+            });
+
+        _migrationService
+            .PollAndDownloadMigrationAsync(Arg.Is<DownloadMigrationOptions>(o => o.Id == 1), _ => Task.CompletedTask, CancellationToken.None)
+            .Returns("test1");
+
+        _migrationService
+            .PollAndDownloadMigrationAsync(Arg.Is<DownloadMigrationOptions>(o => o.Id == 2), _ => Task.CompletedTask, CancellationToken.None)
+            .Returns("test2");
+
+        await runner.RunAsync(CancellationToken.None);
+
+        _logger.VerifyLogs(
+            new LogEntry(LogLevel.Information, "Downloading migrations using ids"),
             new LogEntry(LogLevel.Information, "Downloading migration 2 to test"),
-            new LogEntry(LogLevel.Information, "Downloaded migration 2 to test2")
+            new LogEntry(LogLevel.Information, "Polling migration 2"),
+            new LogEntry(LogLevel.Information, "Downloaded migration 2 to "),
+            new LogEntry(LogLevel.Information, "Downloading migration 1 to test"),
+            new LogEntry(LogLevel.Information, "Polling migration 1"),
+            new LogEntry(LogLevel.Information, "Downloaded migration 1 to ")
         );
 
         await Verify(_ansiConsole.Output);
     }
 
-    private DownloadRunner CreateRunner(bool quiet, bool latest, long[]? ids = null)
+    private DownloadRunner CreateRunner(bool quiet, bool latest, bool poll, long[]? ids = null)
     {
         var globalArgs = new GlobalArgs(LogLevel.Debug, quiet, new FileInfo("test"));
         var downloadArgs = new DownloadArgs(
             ids ?? Array.Empty<long>(),
             latest,
+            poll,
             new DirectoryInfo("test"),
             null,
             true,

GithubBackup/GithubBackup.Cli/Commands/Github/Backup/BackupCommand.cs

@@ -25,6 +25,7 @@ public static Command Create(string[] args, GlobalArguments globalArguments)
 
         var downloadOptions = downloadArguments.Options();
         downloadOptions.Remove(downloadArguments.MigrationsOption);
+        downloadOptions.Remove(downloadArguments.PollOption);
         command.AddOptions(downloadOptions);
 
         command.AddOptions(intervalArguments.Options());

GithubBackup/GithubBackup.Cli/Commands/Github/Backup/BackupRunner.cs

@@ -70,7 +70,11 @@ await _loginService.WithPersistentAsync(
             downloadOptions,
             update =>
             {
-                _ansiConsole.WriteLine($"Migration {update.Id} is {update.State}...");
+                if (!_globalArgs.Quiet)
+                {
+                    _ansiConsole.WriteLine($"Migration {update.Id} is {update.State}...");
+                }
+
                 return Task.CompletedTask;
             },
             ct

GithubBackup/GithubBackup.Cli/Commands/Github/Download/DowndloadArgsBinder.cs

@@ -27,6 +27,7 @@ protected override DownloadArgs GetBoundValue(BindingContext bindingContext)
     {
         var migrations = bindingContext.ParseResult.GetRequiredValueForOption(_downloadArguments.MigrationsOption);
         var latest = bindingContext.ParseResult.GetRequiredValueForOption(_downloadArguments.LatestOption);
+        var poll = bindingContext.ParseResult.GetRequiredValueForOption(_downloadArguments.PollOption);
         var destination = bindingContext.ParseResult.GetRequiredValueForOption(_downloadArguments.DestinationOption);
         var numberOfBackups = bindingContext.ParseResult.GetValueForOption(_downloadArguments.NumberOfBackupsOption);
         var overwrite = bindingContext.ParseResult.GetRequiredValueForOption(_downloadArguments.OverwriteOption);
@@ -37,6 +38,7 @@ protected override DownloadArgs GetBoundValue(BindingContext bindingContext)
         return new DownloadArgs(
             migrations,
             latest,
+            poll,
             destination,
             numberOfBackups,
             overwrite,

GithubBackup/GithubBackup.Cli/Commands/Github/Download/DownloadArgDescriptions.cs

@@ -19,6 +19,10 @@ Only download the latest migration.
         If id's are specified, this flag will be ignored.
         """
     );
+
+    public static readonly Description Poll = new("Poll", "Poll",
+        "Poll the latest migration until it is ready to be downloaded."
+    );
 
     public static readonly Description Destination = new("Destination", "Destination",
         "The destination directory to download the migrations to.");

GithubBackup/GithubBackup.Cli/Commands/Github/Download/DownloadArgs.cs

@@ -8,6 +8,7 @@ internal sealed class DownloadArgs : ICommandIntervalArgs
 {
     public long[] Migrations { get; }
     public bool Latest { get; }
+    public bool Poll { get; }
     public DirectoryInfo Destination { get; }
     public int? NumberOfBackups { get; }
     public bool Overwrite { get; }
@@ -17,6 +18,7 @@ internal sealed class DownloadArgs : ICommandIntervalArgs
     public DownloadArgs(
         long[] migrations,
         bool latest,
+        bool poll,
         DirectoryInfo destination,
         int? numberOfBackups,
         bool overwrite,
@@ -25,6 +27,7 @@ public DownloadArgs(
     {
         Migrations = migrations;
         Latest = latest;
+        Poll = poll;
         Destination = destination;
         NumberOfBackups = numberOfBackups;
         Overwrite = overwrite;

GithubBackup/GithubBackup.Cli/Commands/Github/Download/DownloadArguments.cs

@@ -7,6 +7,7 @@ public class DownloadArguments
 {
     public Option<long[]> MigrationsOption { get; }
     public Option<bool> LatestOption { get; }
+    public Option<bool> PollOption { get; }
     public Option<DirectoryInfo> DestinationOption { get; }
     public Option<int?> NumberOfBackupsOption { get; }
     public Option<bool> OverwriteOption { get; }
@@ -28,6 +29,12 @@ public DownloadArguments(bool piping)
             getDefaultValue: () => false,
             description: DownloadArgDescriptions.Latest.Long
         ) { IsRequired = false };
+
+        PollOption = new Option<bool>(
+            aliases: new[] { "-p", "--poll" },
+            getDefaultValue: () => false,
+            description: DownloadArgDescriptions.Poll.Long
+        ) { IsRequired = false };
 
         DestinationOption = new Option<DirectoryInfo>(
             aliases: new[] { "-d", "--destination" },
@@ -52,6 +59,7 @@ public List<Option> Options()
         {
             MigrationsOption,
             LatestOption,
+            PollOption,
             DestinationOption,
             NumberOfBackupsOption,
             OverwriteOption

GithubBackup/GithubBackup.Cli/Commands/Github/Download/DownloadRunner.cs

@@ -102,7 +102,11 @@ private async Task DownloadLatestAsync(CancellationToken ct)
 
     private async Task DownloadUsingIdsAsync(CancellationToken ct)
     {
-        foreach (var id in _downloadArgs.Migrations)
+        var migrations = (await _migrationService.GetMigrationsAsync(ct))
+            .OrderBy(m => m.CreatedAt)
+            .ToList();
+
+        foreach (var id in _downloadArgs.Migrations.OrderBy(m => migrations.FindIndex(e => e.Id == m)))
         {
             await DownloadMigrationUsingIdAsync(id, ct);
         }
@@ -133,8 +137,28 @@ private async Task DownloadMigrationAsync(long id, CancellationToken ct)
             _downloadArgs.Overwrite
         );
 
-        var path = await _migrationService.DownloadMigrationAsync(options, ct);
+        var path = await DownloadAsync(options, ct);
         _logger.LogInformation("Downloaded migration {Id} to {Path}", id, path);
         _ansiConsole.WriteLine(!_globalArgs.Quiet ? $"Downloaded migration {id} to {path}" : path);
     }
+
+    private Task<string> DownloadAsync(DownloadMigrationOptions options, CancellationToken ct)
+    {
+        if (_downloadArgs.Poll)
+        {
+            _logger.LogInformation("Polling migration {Id}", options.Id);
+            return _migrationService.PollAndDownloadMigrationAsync(options, update =>
+            {
+                if (!_globalArgs.Quiet)
+                {
+                    _ansiConsole.WriteLine($"Migration {update.Id} is {update.State}...");
+                }
+
+                return Task.CompletedTask;
+            }, ct);
+        }
+
+        _logger.LogInformation("Downloading migration {Id}", options.Id);
+        return _migrationService.DownloadMigrationAsync(options, ct);
+    }
 }

GithubBackup/GithubBackup.Cli/Commands/Github/Manual/ManualBackupRunner.cs

@@ -3,7 +3,6 @@
 using GithubBackup.Cli.Commands.Github.Login;
 using GithubBackup.Cli.Commands.Github.Migrate;
 using GithubBackup.Cli.Commands.Global;
-using GithubBackup.Cli.Utils;
 using GithubBackup.Core.Github.Migrations;
 using GithubBackup.Core.Github.Repositories;
 using GithubBackup.Core.Github.Users;
@@ -145,17 +144,13 @@ public async Task RunAsync(CancellationToken ct)
                 return;
             }
 
-            var migrationStatus = await migrations
-                .SelectAsync(m => _migrationService.GetMigrationAsync(m.Id, ct))
-                .ToListAsync(cancellationToken: ct);
-
-            _ansiConsole.WriteLine($"Found {migrationStatus.Count} migrations:");
-            foreach (var migration in migrationStatus)
+            _ansiConsole.WriteLine($"Found {migrations.Count} migrations:");
+            foreach (var migration in migrations)
             {
                 _ansiConsole.WriteLine($"- {migration.Id} {migration.State} {migration.CreatedAt} ({(_dateTimeProvider.Now - migration.CreatedAt).Days}d)");
             }
 
-            if (!migrationStatus.Any(m => m.State == MigrationState.Exported && m.CreatedAt > _dateTimeProvider.Now.AddDays(-7)))
+            if (!migrations.Any(m => m.State == MigrationState.Exported && m.CreatedAt > _dateTimeProvider.Now.AddDays(-7)))
             {
                 _ansiConsole.WriteLine("No exported migrations found in the last 7 days.");
                 return;
@@ -170,7 +165,7 @@ public async Task RunAsync(CancellationToken ct)
                     .InstructionsText(
                         "(Press [blue]<space>[/] to toggle a migration, " +
                         "[green]<enter>[/] to accept)")
-                    .AddChoices(migrationStatus.Where(m => m.State == MigrationState.Exported && m.CreatedAt > _dateTimeProvider.Now.AddDays(-7)))
+                    .AddChoices(migrations.Where(m => m.State == MigrationState.Exported && m.CreatedAt > _dateTimeProvider.Now.AddDays(-7)))
                     .UseConverter(m => $"{m.Id} {m.State} {m.CreatedAt} ({(_dateTimeProvider.Now - m.CreatedAt).Days}d)")
             );