Support for migration of Altinn1 and new index (#536)

- Support for Altinn 1 migration
- Test remove sas token
- Store mainaltinnversion
- New index instances_lastchanged_filtered
- Added exclude code coverage

Co-authored-by: Henning Normann <h.normann@accenture.com>

src/Storage/Controllers/DataController.cs

@@ -192,7 +192,7 @@ public async Task<ActionResult> Get(int instanceOwnerPartyId, Guid instanceGuid,
 
             string storageFileName = DataElementHelper.DataFileName(instance.AppId, instanceGuid.ToString(), dataGuid.ToString());
 
-            if (instance.AppId.Contains(@"/a2-") && _generalSettings.A2UseTtdAsServiceOwner)
+            if ((instance.AppId.Contains(@"/a1-") || instance.AppId.Contains(@"/a2-")) && _generalSettings.A2UseTtdAsServiceOwner)
             {
                 instance.Org = "ttd";
             }

src/Storage/Controllers/MigrationController.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -36,6 +37,7 @@ namespace Altinn.Platform.Storage.Controllers
     [Route("storage/api/v1/migration")]
     [ApiController]
     [ServiceFilter(typeof(ClientIpCheckActionFilterAttribute))]
+    [ExcludeFromCodeCoverage]
     public class MigrationController : ControllerBase
     {
         private readonly IInstanceRepository _instanceRepository;
@@ -109,16 +111,40 @@ public async Task<ActionResult<Instance>> CreateInstance([FromBody] Instance ins
             Instance storedInstance;
             try
             {
-                int a2ArchiveReference = int.Parse(instance.DataValues["A2ArchRef"]);
-                string instanceId = await _a2Repository.GetMigrationInstanceId(a2ArchiveReference);
+                int a1ArchiveReference = instance.DataValues.ContainsKey("A1ArchRef") ? int.Parse(instance.DataValues["A1ArchRef"]) : -1;
+                int a2ArchiveReference = instance.DataValues.ContainsKey("A2ArchRef") ? int.Parse(instance.DataValues["A2ArchRef"]) : -1;
+                bool isA1 = a1ArchiveReference > -1;
+                if (!isA1 && a2ArchiveReference == -1)
+                {
+                    throw new Exception($"Internal error - no archive reference found for {instance.Id}");
+                }
+
+                string instanceId = isA1 ? await _a2Repository.GetA1MigrationInstanceId(a1ArchiveReference) : await _a2Repository.GetA2MigrationInstanceId(a2ArchiveReference);
                 if (instanceId != null)
                 {
                     await CleanupOldMigrationInternal(instanceId);
                 }
 
-                await _a2Repository.CreateMigrationState(a2ArchiveReference);
-                storedInstance = await _instanceRepository.Create(instance);
-                await _a2Repository.UpdateStartMigrationState(a2ArchiveReference, storedInstance.Id.Split('/')[^1]);
+                if (isA1)
+                {
+                    await _a2Repository.CreateA1MigrationState(a1ArchiveReference);
+                }
+                else
+                {
+                    await _a2Repository.CreateA2MigrationState(a2ArchiveReference);
+                }
+
+                storedInstance = await _instanceRepository.Create(instance, isA1 ? 1 : 2);
+
+                if (isA1)
+                {
+                    await _a2Repository.UpdateStartA1MigrationState(a1ArchiveReference, storedInstance.Id.Split('/')[^1]);
+                }
+                else
+                {
+                    await _a2Repository.UpdateStartA2MigrationState(a2ArchiveReference, storedInstance.Id.Split('/')[^1]);
+                }
+
                 return Created((string)null, storedInstance);
             }
             catch (Exception storageException)
@@ -167,6 +193,11 @@ public async Task<ActionResult<DataElement>> CreateDataElement(
             }
 
             Application app = await _applicationRepository.FindOne(instance.AppId, instance.Org);
+            bool isA2 = app.Id.Contains("/a2-");
+            if (!isA2 && !app.Id.Contains("/a1-"))
+            {
+                throw new Exception($"Internal error. Can't determine app type for {app.Id}");
+            }
 
             DataElement storedDataElement;
             try
@@ -185,14 +216,14 @@ public async Task<ActionResult<DataElement>> CreateDataElement(
                     BlobStoragePath = DataElementHelper.DataFileName(instance.AppId, instanceGuid.ToString(), dataElementId),
                     Metadata = formid == null ? null : new()
                     {
-                        new() { Key = "formid", Value = formid },
-                        new() { Key = "lformid", Value = lformid }
+                        new() { Key = isA2 ? "formid" : "dataformid", Value = formid },
+                        lformid != null ? new() { Key = "lformid", Value = lformid } : null
                     }
                 };
 
                 if (presenationText != null)
                 {
-                    dataElement.Metadata.Add(new() { Key = "A2PresVal", Value = HttpUtility.UrlDecode(presenationText) });
+                    dataElement.Metadata.Add(new() { Key = isA2 ? "A2PresVal" : "A1PresVal", Value = HttpUtility.UrlDecode(presenationText) });
                 }
 
                 if (visiblePages != null)
@@ -476,7 +507,7 @@ public async Task<Stream> ProxyGeneratePdf([FromBody] PdfGeneratorRequest reques
         private async Task<bool> CleanupOldMigrationInternal(string instanceId)
         {
             (Instance instance, _) = await _instanceRepository.GetOne(new Guid(instanceId), false);
-            if (instance == null || string.IsNullOrEmpty(instance.DataValues?["A2ArchRef"]))
+            if (instance == null || (!instance.DataValues.ContainsKey("A1ArchRef") && !instance.DataValues.ContainsKey("A2ArchRef")))
             {
                 return false;
             }

src/Storage/Migration/FunctionsAndProcedures/deletemigrationstate.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE PROCEDURE storage.deletemigrationstate (_instanceguid UUID)
+    LANGUAGE 'plpgsql'
+AS $BODY$
+BEGIN
+	DELETE FROM storage.a1migrationstate WHERE instanceguid = _instanceguid;
+	DELETE FROM storage.a2migrationstate WHERE instanceguid = _instanceguid;
+END;
+$BODY$;

src/Storage/Migration/FunctionsAndProcedures/inserta1migrationstate.sql

@@ -0,0 +1,9 @@
+CREATE OR REPLACE PROCEDURE storage.inserta1migrationstate (_a1archiveReference BIGINT)
+    LANGUAGE 'plpgsql'
+AS $BODY$
+BEGIN
+	INSERT INTO storage.a1migrationstate (a1archivereference) VALUES
+		(_a1archiveReference)
+	ON CONFLICT (a1archivereference) DO NOTHING;
+END;
+$BODY$;

src/Storage/Migration/FunctionsAndProcedures/insertinstance.sql

@@ -1,7 +1,8 @@
-CREATE OR REPLACE PROCEDURE storage.insertinstance(_partyid BIGINT, _alternateid UUID, _instance JSONB, _created TIMESTAMPTZ, _lastchanged TIMESTAMPTZ, _org TEXT, _appid TEXT, _taskid TEXT)
+CREATE OR REPLACE PROCEDURE storage.insertinstance_v2(_partyid BIGINT, _alternateid UUID, _instance JSONB, _created TIMESTAMPTZ, _lastchanged TIMESTAMPTZ, _org TEXT, _appid TEXT, _taskid TEXT, _altinnmainversion INT)
     LANGUAGE 'plpgsql'	
 AS $BODY$
 BEGIN
-    INSERT INTO storage.instances(partyid, alternateid, instance, created, lastchanged, org, appid, taskid) VALUES (_partyid, _alternateid, jsonb_strip_nulls(_instance), _created, _lastchanged, _org, _appid, _taskid);
+    INSERT INTO storage.instances(partyid, alternateid, instance, created, lastchanged, org, appid, taskid, altinnmainversion)
+        VALUES (_partyid, _alternateid, jsonb_strip_nulls(_instance), _created, _lastchanged, _org, _appid, _taskid, _altinnmainversion);
 END;
 $BODY$;

src/Storage/Migration/FunctionsAndProcedures/reada1migrationstate.sql

@@ -0,0 +1,10 @@
+CREATE OR REPLACE FUNCTION storage.reada1migrationstate(_a1archivereference BIGINT)
+    RETURNS TABLE (instanceguid UUID)
+    LANGUAGE 'plpgsql'
+
+AS $BODY$
+BEGIN
+RETURN QUERY 
+    SELECT ms.instanceguid FROM storage.a1migrationstate ms WHERE ms.a1archivereference = _a1archivereference;
+END;
+$BODY$;

src/Storage/Migration/FunctionsAndProcedures/updatea1migrationstatestarted.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE PROCEDURE storage.updatea1migrationstatestarted (_a1archivereference BIGINT, _instanceguid UUID)
+    LANGUAGE 'plpgsql'
+AS $BODY$
+BEGIN
+	UPDATE storage.a1migrationstate SET instanceguid = _instanceguid, started = now()
+		WHERE a1archivereference = _a1archivereference;
+END;
+$BODY$;

src/Storage/Migration/FunctionsAndProcedures/updatemigrationstatecompleted.sql

@@ -0,0 +1,10 @@
+CREATE OR REPLACE PROCEDURE storage.updatemigrationstatecompleted (_instanceguid UUID)
+    LANGUAGE 'plpgsql'
+AS $BODY$
+BEGIN
+	UPDATE storage.a1migrationstate SET completed = now()
+		WHERE instanceguid = _instanceguid;
+	UPDATE storage.a2migrationstate SET completed = now()
+		WHERE instanceguid = _instanceguid;
+END;
+$BODY$;

src/Storage/Migration/v0.14/01-setup-tables.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS storage.a1migrationstate
+(
+	id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+	a1archivereference BIGINT UNIQUE NOT NULL,
+	instanceguid UUID UNIQUE NULL,
+	started TIMESTAMPTZ NULL,
+	completed TIMESTAMPTZ NULL
+)
+TABLESPACE pg_default;
+
+GRANT SELECT,INSERT,UPDATE,REFERENCES,DELETE,TRUNCATE,REFERENCES,TRIGGER ON ALL TABLES IN SCHEMA storage TO platform_storage;
+GRANT SELECT,INSERT,UPDATE,REFERENCES,DELETE,TRUNCATE,REFERENCES,TRIGGER ON ALL TABLES IN SCHEMA storage TO platform_storage_admin;

src/Storage/Migration/v0.14/02-functions-and-procedures.sql

@@ -0,0 +1 @@
+-- This script is autogenerated from the tool DbTools. Do not edit manually.

src/Storage/Migration/v0.14/03-new-index.sql

@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS instances_lastchanged_filtered ON storage.instances(lastChanged)
+	WHERE (instance -> 'Status' -> 'IsArchived')::BOOLEAN = false;

src/Storage/Repository/IA2Repository.cs

@@ -41,30 +41,46 @@ public interface IA2Repository
         /// <returns>Image</returns>
         Task<byte[]> GetImage(string name);
 
+        /// <summary>
+        /// Create an a1 migration state
+        /// </summary>
+        Task CreateA1MigrationState(int a1ArchiveReference);
+
         /// <summary>
         /// Create an a2 migration state
         /// </summary>
-        Task CreateMigrationState(int a2ArchiveReference);
+        Task CreateA2MigrationState(int a2ArchiveReference);
 
         /// <summary>
-        /// Update an a2 migration state
+        /// Update an a1 migration state
         /// </summary>
-        Task UpdateStartMigrationState(int a2ArchiveReference, string instanceGuid);
+        Task UpdateStartA1MigrationState(int a1ArchiveReference, string instanceGuid);
 
         /// <summary>
         /// Update an a2 migration state
         /// </summary>
+        Task UpdateStartA2MigrationState(int a2ArchiveReference, string instanceGuid);
+
+        /// <summary>
+        /// Update an a1/a2 migration state
+        /// </summary>
         Task UpdateCompleteMigrationState(string instanceGuid);
 
         /// <summary>
-        /// Delete an a2 migration state
+        /// Delete an a1/a2 migration state
         /// </summary>
         Task DeleteMigrationState(string instanceGuid);
 
         /// <summary>
         /// Get the instance id of the migration
         /// </summary>
         /// <returns>The instance id of the migration</returns>
-        Task<string> GetMigrationInstanceId(int a2ArchiveReference);
+        Task<string> GetA1MigrationInstanceId(int a1ArchiveReference);
+
+        /// <summary>
+        /// Get the instance id of the migration
+        /// </summary>
+        /// <returns>The instance id of the migration</returns>
+        Task<string> GetA2MigrationInstanceId(int a2ArchiveReference);
     }
 }

src/Storage/Repository/IInstanceRepository.cs

@@ -32,8 +32,9 @@ public interface IInstanceRepository
         /// insert new instance into collection
         /// </summary>
         /// <param name="instance">the instance to base the new one on</param>
+        /// <param name="altinnMainVersion">the altinn main version</param>
         /// <returns>The instance id</returns>
-        Task<Instance> Create(Instance instance);
+        Task<Instance> Create(Instance instance, int altinnMainVersion = 3);
 
         /// <summary>
         /// update existing instance

src/Storage/Repository/PgA2Repository.cs

@@ -31,11 +31,14 @@ public class PgA2Repository(
         private static readonly string _readCodelistSql = "select * from storage.reada2codelist (@_name, @_language)";
         private static readonly string _readImageSql = "select * from storage.reada2image (@_name)";
 
-        private static readonly string _readMigrationStateSql = "select * from storage.reada2migrationstate (@_a2archivereference)";
-        private static readonly string _insertMigrationStateSql = "call storage.inserta2migrationstate (@_a2archivereference)";
-        private static readonly string _updateMigrationStateStartedSql = "call storage.updatea2migrationstatestarted (@_a2archivereference, @_instanceguid)";
-        private static readonly string _updateMigrationStateCompletedSql = "call storage.updatea2migrationstatecompleted (@_instanceguid)";
-        private static readonly string _deleteMigrationStateSql = "call storage.deletea2migrationstate (@_instanceguid)";
+        private static readonly string _readA1MigrationStateSql = "select * from storage.reada1migrationstate (@_a1archivereference)";
+        private static readonly string _readA2MigrationStateSql = "select * from storage.reada2migrationstate (@_a2archivereference)";
+        private static readonly string _insertA1MigrationStateSql = "call storage.inserta1migrationstate (@_a1archivereference)";
+        private static readonly string _insertA2MigrationStateSql = "call storage.inserta2migrationstate (@_a2archivereference)";
+        private static readonly string _updateA1MigrationStateStartedSql = "call storage.updatea1migrationstatestarted (@_a1archivereference, @_instanceguid)";
+        private static readonly string _updateA2MigrationStateStartedSql = "call storage.updatea2migrationstatestarted (@_a2archivereference, @_instanceguid)";
+        private static readonly string _updateMigrationStateCompletedSql = "call storage.updatemigrationstatecompleted (@_instanceguid)";
+        private static readonly string _deleteMigrationStateSql = "call storage.deletemigrationstate (@_instanceguid)";
 
         private readonly NpgsqlDataSource _dataSource = dataSource;
         private readonly TelemetryClient _telemetryClient = telemetryClient;
@@ -167,9 +170,21 @@ public async Task<string> GetCodelist(string name, string preferredLanguage)
         }
 
         /// <inheritdoc/>
-        public async Task CreateMigrationState(int a2ArchiveReference)
+        public async Task CreateA1MigrationState(int a1ArchiveReference)
         {
-            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_insertMigrationStateSql);
+            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_insertA1MigrationStateSql);
+            pgcom.Parameters.AddWithValue("_a1archivereference", NpgsqlDbType.Bigint, a1ArchiveReference);
+            using TelemetryTracker tracker = new(_telemetryClient, pgcom);
+
+            await pgcom.ExecuteNonQueryAsync();
+
+            tracker.Track();
+        }
+
+        /// <inheritdoc/>
+        public async Task CreateA2MigrationState(int a2ArchiveReference)
+        {
+            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_insertA2MigrationStateSql);
             pgcom.Parameters.AddWithValue("_a2archivereference", NpgsqlDbType.Bigint, a2ArchiveReference);
             using TelemetryTracker tracker = new(_telemetryClient, pgcom);
 
@@ -179,9 +194,22 @@ public async Task CreateMigrationState(int a2ArchiveReference)
         }
 
         /// <inheritdoc/>
-        public async Task UpdateStartMigrationState(int a2ArchiveReference, string instanceGuid)
+        public async Task UpdateStartA1MigrationState(int a1ArchiveReference, string instanceGuid)
+        {
+            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_updateA1MigrationStateStartedSql);
+            pgcom.Parameters.AddWithValue("_a1archivereference", NpgsqlDbType.Bigint, a1ArchiveReference);
+            pgcom.Parameters.AddWithValue("_instanceGuid", NpgsqlDbType.Uuid, new Guid(instanceGuid));
+            using TelemetryTracker tracker = new(_telemetryClient, pgcom);
+
+            await pgcom.ExecuteNonQueryAsync();
+
+            tracker.Track();
+        }
+
+        /// <inheritdoc/>
+        public async Task UpdateStartA2MigrationState(int a2ArchiveReference, string instanceGuid)
         {
-            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_updateMigrationStateStartedSql);
+            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_updateA2MigrationStateStartedSql);
             pgcom.Parameters.AddWithValue("_a2archivereference", NpgsqlDbType.Bigint, a2ArchiveReference);
             pgcom.Parameters.AddWithValue("_instanceGuid", NpgsqlDbType.Uuid, new Guid(instanceGuid));
             using TelemetryTracker tracker = new(_telemetryClient, pgcom);
@@ -216,10 +244,28 @@ public async Task DeleteMigrationState(string instanceGuid)
         }
 
         /// <inheritdoc/>
-        public async Task<string> GetMigrationInstanceId(int a2ArchiveReference)
+        public async Task<string> GetA1MigrationInstanceId(int a1ArchiveReference)
+        {
+            string instanceId = null;
+            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_readA1MigrationStateSql);
+            pgcom.Parameters.AddWithValue("_a1archivereference", NpgsqlDbType.Bigint, a1ArchiveReference);
+            using TelemetryTracker tracker = new(_telemetryClient, pgcom);
+
+            await using NpgsqlDataReader reader = await pgcom.ExecuteReaderAsync();
+            if (await reader.ReadAsync())
+            {
+                instanceId = await reader.IsDBNullAsync("instanceguid") ? null : (await reader.GetFieldValueAsync<Guid>("instanceguid")).ToString();
+            }
+
+            tracker.Track();
+            return instanceId;
+        }
+
+        /// <inheritdoc/>
+        public async Task<string> GetA2MigrationInstanceId(int a2ArchiveReference)
         {
             string instanceId = null;
-            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_readMigrationStateSql);
+            await using NpgsqlCommand pgcom = _dataSource.CreateCommand(_readA2MigrationStateSql);
             pgcom.Parameters.AddWithValue("_a2archivereference", NpgsqlDbType.Bigint, a2ArchiveReference);
             using TelemetryTracker tracker = new(_telemetryClient, pgcom);