Improve performance of deduplication (#47)

* Avoid filling dedup table

* cleanup

* Add benchmark

* tune

Author: ante012

Performance/Filter.cs

@@ -0,0 +1,32 @@
+using BenchmarkDotNet.Attributes;
+using Combination.StringPools;
+
+namespace Performance;
+
+#pragma warning disable IDE1006 // Naming Styles
+#pragma warning disable CS8618
+public class Filter
+{
+    internal static readonly Random Random = new();
+
+    [Params(10_000, 32_525, 100_000, 1_000_000)]
+    public int PoolSize;
+
+    private IUtf8DeduplicatedStringPool filledPool;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        filledPool = StringPool.DeduplicatedUtf8(4096, 1);
+        for (var i = 0; i < PoolSize; i++)
+        {
+            filledPool.Add(Guid.NewGuid().ToString());
+        }
+    }
+
+    [Benchmark]
+    public PooledUtf8String? DoTryGet()
+    {
+        return filledPool.TryGet("not-in-pool");
+    }
+}

Performance/Program.cs

@@ -2,6 +2,7 @@
 using Performance;
 
 var summary = BenchmarkRunner.Run<Deduplication>();
+// var summary = BenchmarkRunner.Run<Filter>();
 //var summary = BenchmarkRunner.Run<Hashing>();
 
 #if false

src/Combination.StringPools/Utf8StringPool.cs

@@ -13,6 +13,9 @@ private const int
         PoolIndexBits =
             24; // Number of bits to use for pool index in handle (more bits = more pools, but less strings per pool)
 
+    // Maximum fill factor for deduplication table. Performance degrades when it is close to 1.
+    private const float MaxDeduplicationTableFillFactor = 0.9f;
+
     private static readonly List<Utf8StringPool?> Pools = new();
 
 #pragma warning disable IDE1006 // Naming Styles
@@ -21,7 +24,7 @@ private const int
     internal static long totalAddedBytes;
 #pragma warning restore IDE1006 // Naming Styles
 
-    internal int overfillCount;
+    internal int deduplicationFillCount;
 
     private readonly List<nint> pages = new();
     private readonly int index;
@@ -309,12 +312,9 @@ private void AddToDeduplicationTable(ulong[]? currentTable, int currentTableBits
             var tableEntry = currentTable[(tableIndex + i) % tableSize];
             if (tableEntry == 0)
             {
-                if (i > 0)
-                {
-                    ++overfillCount;
-                }
+                ++deduplicationFillCount;
                 currentTable[(tableIndex + i) % tableSize] = handle + 1;
-                if (overfillCount > tableSize / 2)
+                if (deduplicationFillCount > tableSize * MaxDeduplicationTableFillFactor)
                 {
                     ResizeDeduplicationTable(currentTableBits + 1);
                 }
@@ -331,7 +331,7 @@ private void ResizeDeduplicationTable(int newBits)
             return;
         }
         var newDeduplicationTable = new ulong[1 << newBits];
-        overfillCount = 0;
+        deduplicationFillCount = 0;
         var tableSize = 1 << deduplicationTableBits;
         for (var i = 0; i < tableSize; ++i)
         {