Merge pull request #623 from eventflow/fix-index-fragmentation-for-comb

Fix: Index fragmentation using COMB GUIDs in MSSQL

RELEASE_NOTES.md

@@ -18,6 +18,8 @@
   - `EventFlow.Sql`
   - `EventFlow.SQLite`
 * New: Added [SourceLink](https://github.com/dotnet/sourcelink) support
+* Fix: `Identity<T>.NewComb()` now produces string values that doesn't cause
+  too much index fragmentation in MSSQL string columns
 
 ### New in 0.69.3772 (released 2019-02-12)
 

Source/EventFlow.MsSql.Tests/Extensions/MsSqlDatabaseExtensions.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Linq;
+using Dapper;
+using EventFlow.TestHelpers.MsSql;
+
+namespace EventFlow.MsSql.Tests.Extensions
+{
+    public static class MsSqlDatabaseExtensions
+    {
+        public static IReadOnlyCollection<T> Query<T>(this IMsSqlDatabase database, string sql)
+        {
+            return database.WithConnection<IReadOnlyCollection<T>>(c => c.Query<T>(sql).ToList());
+        }
+
+        public static int Execute(this IMsSqlDatabase database, string sql, object param)
+        {
+            return database.WithConnection(c => c.Execute(sql, param));
+        }
+    }
+}

Source/EventFlow.MsSql.Tests/IntegrationTests/EventStores/MsSqlEventStoreTests.cs

@@ -21,7 +21,6 @@
 // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-using System.Threading.Tasks;
 using EventFlow.Configuration;
 using EventFlow.Extensions;
 using EventFlow.MsSql.EventStores;

Source/EventFlow.MsSql.Tests/IntegrationTests/IdentityIndexFragmentationTests.cs

@@ -0,0 +1,211 @@
+using System;
+using System.Linq;
+using System.Threading;
+using EventFlow.Core;
+using EventFlow.Extensions;
+using EventFlow.MsSql.Tests.Extensions;
+using EventFlow.TestHelpers;
+using EventFlow.TestHelpers.MsSql;
+using FluentAssertions;
+using NUnit.Framework;
+
+// ReSharper disable StringLiteralTypo
+
+namespace EventFlow.MsSql.Tests.IntegrationTests
+{
+    [Category(Categories.Integration)]
+    public class IdentityIndexFragmentationTests : Test
+    {
+        private const int ROWS = 10000;
+        private IMsSqlDatabase _testDatabase;
+
+        private class MagicId : Identity<MagicId>
+        {
+            public MagicId(string value) : base(value)
+            {
+            }
+        }
+
+        [Test]
+        public void VerifyIdentityHasThereLittleFragmentationUsingString()
+        {
+            // Act
+            InsertRows(() => MagicId.NewComb().Value, ROWS, "IndexFragmentationString");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationString");
+            fragmentation.Should().BeLessThan(10);
+        }
+
+
+        [Test]
+        public void SanityIntLowFragmentationStoredInGuid()
+        {
+            // Arrange
+            var i = 0;
+
+            // Act
+            InsertRows(() =>
+                {
+                    Interlocked.Increment(ref i);
+                    return $"{i,5}";
+                },
+                ROWS,
+                "IndexFragmentationString");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationString");
+            fragmentation.Should().BeLessThan(10);
+        }
+
+        [Test]
+        public void SanityIntAsHexLowFragmentationStoredInGuid()
+        {
+            // Arrange
+            var i = 0;
+
+            // Act
+            InsertRows(() =>
+                {
+                    Interlocked.Increment(ref i);
+                    return $"{i,5:X}";
+                },
+                ROWS,
+                "IndexFragmentationString");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationString");
+            fragmentation.Should().BeLessThan(10);
+        }
+
+
+        [Test]
+        public void SanityCombYieldsLowFragmentationStoredInGuid()
+        {
+            // Act
+            InsertRows(GuidFactories.Comb.Create, ROWS, "IndexFragmentationGuid");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationGuid");
+            fragmentation.Should().BeLessThan(10);
+        }
+
+        [Test]
+        public void SanityCombYieldsHighFragmentationStoredInString()
+        {
+            // Act
+            InsertRows(() => GuidFactories.Comb.Create().ToString("N"), ROWS, "IndexFragmentationString");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationString");
+            fragmentation.Should().BeGreaterThan(90);
+        }
+
+        [Test]
+        public void SanityGuidIdentityYieldsHighFragmentationStoredInString()
+        {
+            // Act
+            InsertRows(() => MagicId.New.Value, ROWS, "IndexFragmentationString");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationString");
+            fragmentation.Should().BeGreaterThan(30); // closer to 100 in reality
+        }
+
+        [Test]
+        public void SanityGuidIdentityYieldsHighFragmentationStoredInGuid()
+        {
+            // Act
+            InsertRows(() => MagicId.New.GetGuid(), ROWS, "IndexFragmentationGuid");
+
+            // Assert
+            var fragmentation = GetIndexFragmentation("IndexFragmentationGuid");
+            fragmentation.Should().BeGreaterThan(30); // closer to 100 in reality
+        }
+
+        public void InsertRows<T>(Func<T> generator, int count, string table)
+        {
+            var ids = Enumerable.Range(0, count)
+                .Select(_ => generator())
+                .ToList();
+
+            foreach (var id in ids.Take(20))
+            {
+                Console.WriteLine(id);
+            }
+
+            foreach (var id in ids)
+            {
+                _testDatabase.Execute($"INSERT INTO {table} (Id) VALUES (@Id)", new { Id = id });
+            }
+        }
+
+        private double GetIndexFragmentation(string table)
+        {
+            const string sql = @"
+                SELECT dbschemas.[name] as 'schema',
+                dbtables.[name] as 'table',
+                dbindexes.[name] as 'index',
+                indexstats.avg_fragmentation_in_percent AS 'fragmentation',
+                indexstats.page_count AS 'pageCount',
+                index_level AS 'level'
+                FROM sys.dm_db_index_physical_stats (DB_ID(), NULL, NULL, NULL, 'DETAILED') AS indexstats
+                INNER JOIN sys.tables dbtables on dbtables.[object_id] = indexstats.[object_id]
+                INNER JOIN sys.schemas dbschemas on dbtables.[schema_id] = dbschemas.[schema_id]
+                INNER JOIN sys.indexes AS dbindexes ON dbindexes.[object_id] = indexstats.[object_id]
+                AND indexstats.index_id = dbindexes.index_id
+                WHERE indexstats.database_id = DB_ID()
+                ORDER BY dbschemas.[name],dbtables.[name],dbindexes.[name],index_level  desc
+                ";
+
+            var rows = _testDatabase.Query<IndexFragmentationDetails>(sql)
+                .Where(r => string.Equals(table, r.Table, StringComparison.OrdinalIgnoreCase))
+                .OrderBy(r => r.Level)
+                .ToList();
+
+            return rows.First().Fragmentation;
+        }
+
+        [SetUp]
+        public void SetUp()
+        {
+            _testDatabase = MsSqlHelpz.CreateDatabase("index_fragmentation");
+            _testDatabase.Execute("CREATE TABLE IndexFragmentationString (Id nvarchar(250) PRIMARY KEY)");
+            _testDatabase.Execute("CREATE TABLE IndexFragmentationGuid (Id uniqueidentifier PRIMARY KEY)");
+        }
+
+        [TearDown]
+        public void TearDown()
+        {
+            _testDatabase.DisposeSafe("DROP test database");
+        }
+
+        private class IndexFragmentationDetails
+        {
+            public IndexFragmentationDetails(
+                string schema,
+                string table,
+                string index,
+                double fragmentation, 
+                long pageCount,
+                byte level)
+            {
+                Schema = schema;
+                Table = table;
+                Index = index;
+                Fragmentation = fragmentation;
+                PageCount = pageCount;
+                Level = level;
+            }
+
+            public string Schema { get; }
+            public string Table { get; }
+            public string Index { get; }
+
+            public double Fragmentation { get; }
+            public long PageCount { get; }
+
+            public byte Level { get; }
+        }
+    }
+}

Source/EventFlow.TestHelpers/MsSql/MsSqlConnectionString.cs

@@ -73,6 +73,7 @@ public void Execute(string sql)
                 }
             });
         }
+
         public void WithConnection(Action<SqlConnection> action)
         {
             WithConnection(c =>

Source/EventFlow/Core/GuidFactories.cs

@@ -24,33 +24,82 @@
 using System;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading;
 
 namespace EventFlow.Core
 {
     public static class GuidFactories
     {
-        /// <summary>
-        /// Creates a sequential Guid that can be used to avoid database fragmentation
-        /// http://stackoverflow.com/a/2187898
-        /// </summary>
         public static class Comb
         {
+            private static int _counter;
+
+            private static long GetTicks()
+            {
+                var i = Interlocked.Increment(ref _counter);
+                return DateTime.UtcNow.Ticks + i;
+            }
+
+            /// <summary>
+            /// Generates a GUID values that causes less index fragmentation when stored
+            /// in e.g. <c>uniqueidentifier</c> columns in MSSQL.
+            /// </summary>
+            /// <example>
+            /// 2825c1d8-4587-cc55-08c1-08d6bde2765b
+            /// 901337ba-c64b-c6d4-08c2-08d6bde2765b
+            /// 45d57ba2-acc5-ce80-08c3-08d6bde2765b
+            /// 36528acf-352a-c28c-08c4-08d6bde2765b
+            /// 6fc88b5e-3782-c8fd-08c5-08d6bde2765b
+            /// </example>
             public static Guid Create()
             {
-                var destinationArray = Guid.NewGuid().ToByteArray();
-                var time = new DateTime(0x76c, 1, 1);
-                var now = DateTime.Now;
-                var span = new TimeSpan(now.Ticks - time.Ticks);
-                var timeOfDay = now.TimeOfDay;
-                var bytes = BitConverter.GetBytes(span.Days);
-                var array = BitConverter.GetBytes((long)(timeOfDay.TotalMilliseconds / 3.333333));
-
-                Array.Reverse(bytes);
-                Array.Reverse(array);
-                Array.Copy(bytes, bytes.Length - 2, destinationArray, destinationArray.Length - 6, 2);
-                Array.Copy(array, array.Length - 4, destinationArray, destinationArray.Length - 4, 4);
-
-                return new Guid(destinationArray);
+                var uid = Guid.NewGuid().ToByteArray();
+                var binDate = BitConverter.GetBytes(GetTicks());
+
+                return new Guid(
+                    new[]
+                        {
+                            uid[0], uid[1], uid[2], uid[3],
+                            uid[4], uid[5],
+                            uid[6], (byte)(0xc0 | (0xf & uid[7])),
+                            binDate[1], binDate[0],
+                            binDate[7], binDate[6], binDate[5], binDate[4], binDate[3], binDate[2]
+                        });
+            }
+
+            /// <summary>
+            /// Generates a GUID values that causes less index fragmentation when stored
+            /// in e.g. <c>nvarchar(n)</c> columns in MSSQL.
+            /// </summary>
+            /// <example>
+            /// 899ee1b9-bde2-08d6-20d8-b7e20375c7c9
+            /// 899f09b9-bde2-08d6-fd1c-5ec8f3349bcf
+            /// 899f09ba-bde2-08d6-1521-51d781607ac4
+            /// 899f09bb-bde2-08d6-7e6a-fe84f5237dc4
+            /// 899f09bc-bde2-08d6-c2f0-276123e06fcf
+            /// </example>
+            public static Guid CreateForString()
+            {
+                /*
+                    From: https://docs.microsoft.com/en-us/dotnet/api/system.guid.tobytearray 
+                    Note that the order of bytes in the returned byte array is different from the string
+                    representation of a Guid value. The order of the beginning four-byte group and the
+                    next two two-byte groups is reversed, whereas the order of the last two-byte group
+                    and the closing six-byte group is the same.
+                */
+
+                var uid = Guid.NewGuid().ToByteArray();
+                var binDate = BitConverter.GetBytes(GetTicks());
+
+                return new Guid(
+                    new[]
+                        {
+                            binDate[0], binDate[1], binDate[2], binDate[3],
+                            binDate[4], binDate[5],
+                            binDate[6], binDate[7],
+                            uid[0], uid[1],
+                            uid[2], uid[3], uid[4], uid[5], uid[6], (byte)(0xc0 | (0xf & uid[7])),
+                        });
             }
         }
 

Source/EventFlow/Core/Identity.cs

@@ -64,7 +64,7 @@ public static T NewDeterministic(Guid namespaceId, byte[] nameBytes)
 
         public static T NewComb()
         {
-            var guid = GuidFactories.Comb.Create();
+            var guid = GuidFactories.Comb.CreateForString();
             return With(guid);
         }