Merge pull request #613 from Particular/add-padding-to-outbox-data

Add padding to outbox data in SQL Server if outbox data size is between 4 and 8 KB

src/SqlPersistence/Outbox/OptimisticConcurrencyControlStrategy.cs

@@ -22,12 +22,15 @@ public override Task Begin(string messageId, DbConnection connection, DbTransact
 
     public override async Task Complete(OutboxMessage outboxMessage, DbConnection connection, DbTransaction transaction, ContextBag context)
     {
+        string json = Serializer.Serialize(outboxMessage.TransportOperations.ToSerializable());
+        json = sqlDialect.AddOutboxPadding(json);
+
         using (var command = sqlDialect.CreateCommand(connection))
         {
             command.CommandText = outboxCommands.OptimisticStore;
             command.Transaction = transaction;
             command.AddParameter("MessageId", outboxMessage.MessageId);
-            command.AddJsonParameter("Operations", Serializer.Serialize(outboxMessage.TransportOperations.ToSerializable()));
+            command.AddJsonParameter("Operations", json);
             command.AddParameter("PersistenceVersion", StaticVersions.PersistenceVersion);
             await command.ExecuteNonQueryEx().ConfigureAwait(false);
         }

src/SqlPersistence/Outbox/PessimisticConcurrencyControlStrategy.cs

@@ -29,12 +29,15 @@ public override async Task Begin(string messageId, DbConnection connection, DbTr
 
     public override async Task Complete(OutboxMessage outboxMessage, DbConnection connection, DbTransaction transaction, ContextBag context)
     {
+        string json = Serializer.Serialize(outboxMessage.TransportOperations.ToSerializable());
+        json = sqlDialect.AddOutboxPadding(json);
+
         using (var command = sqlDialect.CreateCommand(connection))
         {
             command.CommandText = outboxCommands.PessimisticComplete;
             command.Transaction = transaction;
             command.AddParameter("MessageId", outboxMessage.MessageId);
-            command.AddJsonParameter("Operations", Serializer.Serialize(outboxMessage.TransportOperations.ToSerializable()));
+            command.AddJsonParameter("Operations", json);
             await command.ExecuteNonQueryEx().ConfigureAwait(false);
         }
     }

src/SqlPersistence/Outbox/SqlDialect.cs

@@ -9,5 +9,6 @@ public partial class SqlDialect
         internal abstract string GetOutboxPessimisticBeginCommand(string tableName);
         internal abstract string GetOutboxPessimisticCompleteCommand(string tableName);
         internal abstract string GetOutboxCleanupCommand(string tableName);
+        internal virtual string AddOutboxPadding(string json) => json;
     }
 }

src/SqlPersistence/Outbox/SqlDialect_MsSqlServer.cs

@@ -80,6 +80,28 @@ delete top (@BatchSize) from {tableName}
 where Dispatched = 'true' and
       DispatchedAt < @DispatchedBefore";
             }
+
+            internal override string AddOutboxPadding(string json)
+            {
+                //We need to ensure the outbox content is at lest 8000 bytes long because otherwise SQL Server will attempt to
+                //store is inside the data page which will result in low space utilization.
+
+                //We tried using *varchar values out of the row* table option but while it did improve situation on on-premises 
+                //SQL Server it didn't work as expected in SQL Azure where it caused LOB pages to be allocated (one for each record) 
+                //but never deallocated after the messages data is supposed to be removed.
+
+                //We use 4000 instead of 8000 in the condition because the SQL Persistence uses nvarchar data type which encodes
+                //strings at UTF-16 (2 bytes per character)
+
+                //We allow content smaller than 1800 characters (3600 bytes) to not be padded because such content does not block
+                //SQL Server from re-using data pages.
+                if (json.Length >= 1800 && json.Length <= 4000)
+                {
+                    return json.PadRight(4000, ' ');
+                }
+
+                return json;
+            }
         }
     }
-}
+}