Skip to content

Commit

Permalink
Enable schema compare test locally and make normalization generic (#4845
Browse files Browse the repository at this point in the history 
)

* Reviving schema upgrade test for local

* Revive schema comparison test and make SQL normalization generic

* usings

* fixed var check

* return
  • Loading branch information
@SergeyGaluzo
SergeyGaluzo authored Feb 27, 2025
1 parent 3df8b19 commit d89eb58
Showing 1 changed file with 42 additions and 78 deletions.
120 changes: 42 additions & 78 deletions ...Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

using System;
using System.Data;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Text;
Expand Down Expand Up @@ -34,6 +35,7 @@
using Microsoft.Health.SqlServer.Features.Storage;
using Microsoft.Health.Test.Utilities;
using Microsoft.SqlServer.Dac.Compare;
using Microsoft.SqlServer.TransactSql.ScriptDom;
using NSubstitute;
using Xunit;

Expand All @@ -44,16 +46,21 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Persistence
[Trait(Traits.Category, Categories.Schema)]
public class SqlServerSchemaUpgradeTests
{
private const string LocalConnectionString = "server=(local);Integrated Security=true;TrustServerCertificate=True";
private const string MasterDatabaseName = "master";

public SqlServerSchemaUpgradeTests()
{
}

[Fact(Skip = "Issue connecting with SQL workload identity & custom auth provider. AB#122858")]
[Fact]
public async Task GivenTwoSchemaInitializationMethods_WhenCreatingTwoDatabases_BothSchemasShouldBeEquivalent()
{
// previously test was skipped with (Skip = "Issue connecting with SQL workload identity & custom auth provider. AB#122858")
if (!EnvironmentVariables.GetEnvironmentVariable(KnownEnvironmentVariableNames.SqlServerConnectionString).Contains("(local)", StringComparison.OrdinalIgnoreCase))
{
return;
}

var snapshotDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Snapshot");
var diffDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Diff");

Expand Down Expand Up @@ -263,45 +270,22 @@ private async Task<string> CompareDatabaseSchemas(string databaseName1, string d
var unexpectedDifference = new StringBuilder();
foreach (SchemaDifference schemaDifference in remainingDifferences)
{
if (schemaDifference.Name == "SqlTable" &&
(schemaDifference.SourceObject.Name.ToString() == "[dbo].[DateTimeSearchParam]" ||
schemaDifference.SourceObject.Name.ToString() == "[dbo].[StringSearchParam]"))
var objectsToSkip = new[] { "PartitionFunction_ResourceChangeData_Timestamp" }; // definition is not predictable as it has start time component
////var objectsToSkip = new[] { "GetResourceSearchParamStats", "MergeResourcesAdvanceTransactionVisibility", "DequeueJob", "DisableIndexes", "GetResourceVersions", "CleanupEventLog", "InitDefrag", "EnqueueJobs", "GetResourcesByTypeAndSurrogateIdRange", "GetResourceSurrogateIdRanges", "GetCommandsForRebuildIndexes", "GetIndexCommands", "SwitchPartitionsIn", "SwitchPartitionsOut" }.ToList();
if (schemaDifference.SourceObject != null && objectsToSkip.Any(_ => schemaDifference.SourceObject.Name.ToString().Contains(_)))
{
foreach (SchemaDifference child in schemaDifference.Children)
{
if (child.TargetObject == null && child.SourceObject == null && (child.Name == "PartitionColumn" || child.Name == "PartitionScheme"))
{
// The ParitionColumn and the PartitionScheme come up in the differences list even though
// when digging into the "difference" object the values being compared are equal.
continue;
}
else
{
unexpectedDifference.AppendLine($"source={child.SourceObject?.Name} target={child.TargetObject?.Name}");
}
}
continue;
}
else
{
//// Our home grown SQL schema generator does not understand that statements can be formatted differently but contain identical SQL
//// Skipping some objects
var objectsToSkip = new[] { "GetResourceSearchParamStats", "MergeResourcesAdvanceTransactionVisibility", "DequeueJob", "DisableIndexes", "GetResourceVersions", "CleanupEventLog", "InitDefrag", "EnqueueJobs", "GetResourcesByTypeAndSurrogateIdRange", "GetResourceSurrogateIdRanges", "GetCommandsForRebuildIndexes", "GetIndexCommands", "SwitchPartitionsIn", "SwitchPartitionsOut" }.ToList();
objectsToSkip.Add("PartitionFunction_ResourceChangeData_Timestamp"); // definition is not predictable as it has start time component
if (schemaDifference.SourceObject != null && objectsToSkip.Any(_ => schemaDifference.SourceObject.Name.ToString().Contains(_)))
{
continue;
}

if (schemaDifference.SourceObject != null
&& schemaDifference.TargetObject != null
&& schemaDifference.SourceObject.ObjectType.Name == "Procedure"
&& await IsStoredProcedureTextEqual(testConnectionString1, testConnectionString2, schemaDifference.SourceObject.Name.ToString()))
{
continue;
}

unexpectedDifference.AppendLine($"source={schemaDifference.SourceObject?.Name} target={schemaDifference.TargetObject?.Name}");
if (schemaDifference.SourceObject != null
&& schemaDifference.TargetObject != null
&& schemaDifference.SourceObject.ObjectType.Name == "Procedure"
&& await IsObjectTextEqual(testConnectionString1, testConnectionString2, schemaDifference.SourceObject.Name.ToString()))
{
continue;
}

unexpectedDifference.AppendLine($"source={schemaDifference.SourceObject?.Name} target={schemaDifference.TargetObject?.Name}");
}

// if TransactionCheckWithInitialiScript(which has current version as x-1) is not updated with the new x version then x.sql will have a wrong version inserted into SchemaVersion table
Expand All @@ -316,65 +300,45 @@ private async Task<string> CompareDatabaseSchemas(string databaseName1, string d
return unexpectedDifference.ToString();
}

private async Task<bool> IsStoredProcedureTextEqual(string connStr1, string connStr2, string storedProcedureName)
private async Task<bool> IsObjectTextEqual(string connStr1, string connStr2, string objectName)
{
var text1 = await GetStoredProcedureText(connStr1, storedProcedureName);
var text2 = await GetStoredProcedureText(connStr2, storedProcedureName);
var text1 = await GetObjectText(connStr1, objectName);
var text2 = await GetObjectText(connStr2, objectName);

text1 = Normalize(text1);
text2 = Normalize(text2);

Assert.Equal(text1, text2);
try
{
Assert.Equal(text1, text2);
}
catch (Xunit.Sdk.EqualException e)
{
throw new Exception($"object={objectName} message={e.Message}");
}

return true;
}

private string Normalize(string text)
{
// normalize newlines
text = text.Replace("\r\n", "\n").Replace("\r", "\n");

// remove inline comments
while (text.Contains("--"))
{
var indexStart = text.IndexOf("--");
var indexEnd = text.IndexOf("\n", indexStart);

if (indexEnd == -1)
{
text = text.Substring(0, indexStart);
}
else
{
text = text.Substring(0, indexStart) + text.Substring(indexEnd + 1);
}
}

return text.ToLowerInvariant()
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Replace(";", string.Empty)
.Replace(" output,", " out,")
.Replace(" output ", " out ")
.Replace(" inner join ", " join ")
.Replace(" delete from ", " delete ")
.Replace(" insert into ", " insert ")
.Replace(" rollback transaction ", " rollback ")
.Replace(" as ", string.Empty)
.Replace(" ", string.Empty)
.Replace("(index(", "(index=")
.Replace("),", ",")
.Replace("))on", ")on");
using var inputStream = new MemoryStream(Encoding.UTF8.GetBytes(text));
using var reader = new StreamReader(inputStream);
using var outputStream = new MemoryStream();
using var writer = new StreamWriter(outputStream);
new Sql150ScriptGenerator().GenerateScript(new TSql150Parser(true).Parse(reader, out var _), writer);
writer.Flush();
text = Encoding.UTF8.GetString(outputStream.ToArray());
return text;
}

private async Task<string> GetStoredProcedureText(string connStr, string storedProcedureName)
private async Task<string> GetObjectText(string connStr, string objectName)
{
using var conn = new SqlConnection(connStr);
await conn.OpenAsync();
using var cmd = new SqlCommand("dbo.sp_helptext", conn);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.AddWithValue("@objname", storedProcedureName);
cmd.Parameters.AddWithValue("@objname", objectName);
using var reader = cmd.ExecuteReader();
var text = string.Empty;
while (reader.Read())
Expand Down

0 comments on commit d89eb58

Please sign in to comment.