diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 197667f09..ee778be6c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,4 +1,5 @@ - Include the migration ID in the default output filename when running `download-logs` - Include the date with the timestamp when writing to the log - When blob storage credentials are provided to the CLI but will not be used for a GHES migration, log a clear warning, not an info message -- Unhide the `--archive-download-host` argument in the documentation for `gh bbs2gh migrate-repo` and `gh bbs2gh generate-script` \ No newline at end of file +- Fix support for Azure Blob Storage in `bbs2gh` migrations when the archive is larger than 2GB +- Unhide the `--archive-download-host` argument in the documentation for `gh bbs2gh migrate-repo` and `gh bbs2gh generate-script` diff --git a/src/OctoshiftCLI.Tests/bbs2gh/Handlers/MigrateRepoCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/bbs2gh/Handlers/MigrateRepoCommandHandlerTests.cs index 53aa14029..c778419fc 100644 --- a/src/OctoshiftCLI.Tests/bbs2gh/Handlers/MigrateRepoCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/bbs2gh/Handlers/MigrateRepoCommandHandlerTests.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Text; using System.Threading.Tasks; using FluentAssertions; @@ -156,7 +157,7 @@ public async Task Happy_Path_Generate_Archive_Ssh_Download_Azure_Upload_And_Inge _mockBbsApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); _mockBbsArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), ARCHIVE_DATA)).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); _mockGithubApi.Setup(x => x.CreateBbsMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); @@ -240,8 +241,7 @@ public async Task Happy_Path_Full_Flow_Running_On_Bbs_Server() _mockBbsApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); _mockBbsApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(archivePath)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), ARCHIVE_DATA)).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); _mockGithubApi.Setup(x => x.CreateBbsMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); @@ -296,7 +296,7 @@ public async Task Happy_Path_Full_Flow_Bbs_Credentials_Via_Environment() _mockBbsApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); _mockBbsArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), ARCHIVE_DATA)).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); _mockGithubApi.Setup(x => x.CreateBbsMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); @@ -335,7 +335,7 @@ public async Task Happy_Path_Deletes_Downloaded_Archive() _mockBbsApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); _mockBbsArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), ARCHIVE_DATA)).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); // Act var args = new MigrateRepoCommandArgs @@ -367,7 +367,7 @@ public async Task It_Deletes_Downloaded_Archive_Even_If_Upload_Fails() _mockBbsApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); _mockBbsArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), ARCHIVE_DATA)).ThrowsAsync(new InvalidOperationException()); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ThrowsAsync(new InvalidOperationException()); // Act var args = new MigrateRepoCommandArgs @@ -398,7 +398,7 @@ public async Task Happy_Path_Does_Not_Throw_If_Fails_To_Delete_Downloaded_Archiv _mockBbsApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); _mockBbsArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), ARCHIVE_DATA)).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); _mockFileSystemProvider.Setup(x => x.DeleteIfExists(It.IsAny())).Throws(new UnauthorizedAccessException("Access Denied")); // Act @@ -616,7 +616,7 @@ public async Task Uses_Archive_Path_If_Provided() var archiveBytes = Encoding.ASCII.GetBytes("here are some bytes"); _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(archiveBytes); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), archiveBytes)).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); _mockGithubApi.Setup(x => x.CreateBbsMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); @@ -763,7 +763,7 @@ await _handler.Invoking(x => x.Handle(args)) public async Task It_Does_Not_Set_The_Archive_Path_When_Archive_Path_Is_Provided() { // Arrange - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); // Act var args = new MigrateRepoCommandArgs @@ -778,7 +778,7 @@ public async Task It_Does_Not_Set_The_Archive_Path_When_Archive_Path_Is_Provided // Assert args.ArchivePath.Should().Be(ARCHIVE_PATH); - _mockFileSystemProvider.Verify(m => m.ReadAllBytesAsync(ARCHIVE_PATH)); + _mockFileSystemProvider.Verify(m => m.OpenRead(ARCHIVE_PATH)); } [Fact] diff --git a/src/bbs2gh/Handlers/MigrateRepoCommandHandler.cs b/src/bbs2gh/Handlers/MigrateRepoCommandHandler.cs index 15a190d37..ee0b0b0be 100644 --- a/src/bbs2gh/Handlers/MigrateRepoCommandHandler.cs +++ b/src/bbs2gh/Handlers/MigrateRepoCommandHandler.cs @@ -188,11 +188,14 @@ private async Task UploadArchiveToAzure(string archivePath) { _log.LogInformation("Uploading Archive to Azure..."); - var archiveData = await _fileSystemProvider.ReadAllBytesAsync(archivePath); - var archiveName = GenerateArchiveName(); - var archiveBlobUrl = await _azureApi.UploadToBlob(archiveName, archiveData); - - return archiveBlobUrl.ToString(); +#pragma warning disable IDE0063 + await using (var archiveData = _fileSystemProvider.OpenRead(archivePath)) +#pragma warning restore IDE0063 + { + var archiveName = GenerateArchiveName(); + var archiveBlobUrl = await _azureApi.UploadToBlob(archiveName, archiveData); + return archiveBlobUrl.ToString(); + } } private string GenerateArchiveName() => $"{Guid.NewGuid()}.tar";