diff --git a/.azure/modules/containerApp/main.bicep b/.azure/modules/containerApp/main.bicep index 902cdfdd..3739b1ec 100644 --- a/.azure/modules/containerApp/main.bicep +++ b/.azure/modules/containerApp/main.bicep @@ -55,7 +55,7 @@ var containerAppEnvVars = [ } { name: 'MaskinportenSettings__EncodedJwk', secretRef: 'maskinporten-jwk' } { name: 'GeneralSettings__SlackUrl', secretRef: 'slack-url' } - { name: 'AzureStorageOptions__BlockSize', value: '4194304' } + { name: 'AzureStorageOptions__BlockSize', value: '33554432' } { name: 'AzureStorageOptions__ConcurrentUploadThreads', value: '3' } { name: 'AzureStorageOptions__BlocksBeforeCommit', value: '1000' } ] diff --git a/src/Altinn.Broker.API/appsettings.Development.json b/src/Altinn.Broker.API/appsettings.Development.json index ce7dafc0..56fff94f 100644 --- a/src/Altinn.Broker.API/appsettings.Development.json +++ b/src/Altinn.Broker.API/appsettings.Development.json @@ -30,11 +30,10 @@ "ExhangeToAltinnToken": true }, "GeneralSettings": { - "SlackUrl": "", - "CorrespondenceBaseUrl": "https://localhost:7241/" + "SlackUrl": "" }, "AzureStorageOptions": { - "BlockSize": 4194304, + "BlockSize": 33554432, "ConcurrentUploadThreads": 3, "BlocksBeforeCommit": 1000 } diff --git a/src/Altinn.Broker.Core/Options/GeneralSettings.cs b/src/Altinn.Broker.Core/Options/GeneralSettings.cs index aeceeaec..9cf7f701 100644 --- a/src/Altinn.Broker.Core/Options/GeneralSettings.cs +++ b/src/Altinn.Broker.Core/Options/GeneralSettings.cs @@ -3,6 +3,4 @@ namespace Altinn.Broker.Core.Options; public class GeneralSettings { public string SlackUrl { get; set; } = string.Empty; - - public string CorrespondenceBaseUrl { get; set; } = string.Empty; -} \ No newline at end of file +} diff --git a/src/Altinn.Broker.Integrations/Azure/AzureStorageService.cs b/src/Altinn.Broker.Integrations/Azure/AzureStorageService.cs index 60b0cab7..99c3b458 100644 --- a/src/Altinn.Broker.Integrations/Azure/AzureStorageService.cs +++ b/src/Altinn.Broker.Integrations/Azure/AzureStorageService.cs @@ -1,6 +1,7 @@ using System.Security.Cryptography; using Altinn.Broker.Core.Domain; +using Altinn.Broker.Core.Options; using Altinn.Broker.Core.Services; using Azure; @@ -9,17 +10,14 @@ using Azure.Storage.Blobs.Specialized; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Polly; namespace Altinn.Broker.Integrations.Azure; -public class AzureStorageService(IResourceManager resourceManager, ILogger logger) : IBrokerStorageService +public class AzureStorageService(IResourceManager resourceManager, IOptions azureStorageOptions, ILogger logger) : IBrokerStorageService { - private const int BLOCK_SIZE = 1024 * 1024 * 32; // 32MB - private const int BLOCKS_BEFORE_COMMIT = 1000; - private const int UPLOAD_THREADS = 3; - private async Task GetBlobContainerClient(FileTransferEntity fileTransferEntity, ServiceOwnerEntity serviceOwnerEntity) { var storageProvider = serviceOwnerEntity.GetStorageProvider(fileTransferEntity.UseVirusScan); @@ -69,7 +67,7 @@ public async Task DownloadFile(ServiceOwnerEntity serviceOwnerEntity, Fi int blocksInBatch = 0; var uploadTasks = new List(); - var semaphore = new SemaphoreSlim(UPLOAD_THREADS); // Limit concurrent operations + var semaphore = new SemaphoreSlim(azureStorageOptions.Value.ConcurrentUploadThreads); // Limit concurrent operations while (position < streamLength) { @@ -80,7 +78,7 @@ public async Task DownloadFile(ServiceOwnerEntity serviceOwnerEntity, Fi position += bytesRead; bool isLastBlock = position >= streamLength; - if (accumulationBuffer.Length >= BLOCK_SIZE || isLastBlock) + if (accumulationBuffer.Length >= azureStorageOptions.Value.BlockSize || isLastBlock) { accumulationBuffer.Position = 0; var blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); @@ -108,13 +106,13 @@ async Task UploadBlockAsync(BlockBlobClient client, string currentBlockId, byte[ } } - if (uploadTasks.Count >= BLOCKS_BEFORE_COMMIT) + if (uploadTasks.Count >= azureStorageOptions.Value.BlocksBeforeCommit) { await Task.WhenAll(uploadTasks); // Commit the blocks we have so far var blocksToCommit = blockList.ToList(); - var isFirstCommit = blockList.Count <= BLOCKS_BEFORE_COMMIT; + var isFirstCommit = blockList.Count <= azureStorageOptions.Value.BlocksBeforeCommit; await CommitBlocks(blockBlobClient, blocksToCommit, firstCommit: isFirstCommit, null, cancellationToken); uploadTasks.Clear(); @@ -129,7 +127,7 @@ async Task UploadBlockAsync(BlockBlobClient client, string currentBlockId, byte[ { throw new Exception("Failed to calculate MD5 hash of uploaded file"); } - await CommitBlocks(blockBlobClient, blockList, firstCommit: blockList.Count <= BLOCKS_BEFORE_COMMIT, blobMd5.Hash, cancellationToken); + await CommitBlocks(blockBlobClient, blockList, firstCommit: blockList.Count <= azureStorageOptions.Value.BlocksBeforeCommit, blobMd5.Hash, cancellationToken); double finalSpeedMBps = position / (1024.0 * 1024) / (stopwatch.ElapsedMilliseconds / 1000.0); logger.LogInformation($"Successfully uploaded {position / (1024.0 * 1024.0 * 1024.0):N2} GiB " +