Skip to content

Commit 243cb60

Browse files
rickardprickardp
authored
Use variable-size integer for size prefix (#30)
* Use var-int size * Update Program.cs * Migrated test * Migrated test
1 parent cf78b45 commit 243cb60

5 files changed

Lines changed: 90 additions & 23 deletions

File tree

‎src/Combination.StringPools.Tests/TestSupport.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using System.Numerics;
2+
3+
public static class TestSupport
4+
{
5+
public static int GetAllocationSize(int length) => length + 1 + (BitOperations.Log2((uint)length) / 7);
6+
public static int GetMaxStringSizeForAllocation(int allocationSize)
7+
{
8+
for (var i = allocationSize - 1; i >= 0; --i)
9+
{
10+
if (GetAllocationSize(i) == allocationSize)
11+
{
12+
return i;
13+
}
14+
}
15+
throw new ArgumentOutOfRangeException("allocationSize");
16+
}
17+
}

‎src/Combination.StringPools.Tests/Test_Allocation.cs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public void Add_String_Smaller_Than_Page_Succeeds(int pageSize)
5050
using var pool = StringPool.Utf8(pageSize, 1);
5151
Assert.Equal(0, pool.UsedBytes);
5252
Assert.Equal(pageSize, pool.AllocatedBytes);
53-
var someString = new string('c', pageSize - 2);
53+
var someString = new string('c', TestSupport.GetMaxStringSizeForAllocation(pageSize));
5454
var pooledString = pool.Add(someString);
5555
Assert.Equal(pageSize, pool.UsedBytes);
5656
Assert.Equal(pageSize, pool.AllocatedBytes);
@@ -69,7 +69,7 @@ public void Add_String_Larger_Than_Page_Fails(int pageSize)
6969
using var pool = StringPool.Utf8(pageSize, 1);
7070
Assert.Equal(0, pool.UsedBytes);
7171
Assert.Equal(pageSize, pool.AllocatedBytes);
72-
var someString = new string('c', pageSize - 1);
72+
var someString = new string('c', TestSupport.GetMaxStringSizeForAllocation(pageSize) + 1);
7373
var exception = Assert.Throws<ArgumentOutOfRangeException>(() => pool.Add(someString));
7474
Assert.Contains("String is too long to be pooled", exception.Message);
7575
}
@@ -86,13 +86,13 @@ public void Two_Strings_Fit_Same_Page(int pageSize)
8686
using var pool = StringPool.Utf8(pageSize, 1);
8787
Assert.Equal(0, pool.UsedBytes);
8888
Assert.Equal(pageSize, pool.AllocatedBytes);
89-
var string1 = new string('c', (pageSize / 2) - 2);
89+
var string1 = new string('c', TestSupport.GetMaxStringSizeForAllocation(pageSize / 2));
9090
var pooledString1 = pool.Add(string1);
9191
Assert.Equal(pageSize / 2, pool.UsedBytes);
9292
Assert.Equal(pageSize, pool.AllocatedBytes);
93-
var string2 = new string('d', pageSize - (pageSize / 2) - 2);
93+
var string2 = new string('d', TestSupport.GetMaxStringSizeForAllocation(pageSize / 2));
9494
var pooledString2 = pool.Add(string2);
95-
Assert.Equal(pageSize, pool.UsedBytes);
95+
Assert.InRange(pool.UsedBytes, pageSize - 1, pageSize);
9696
Assert.Equal(pageSize, pool.AllocatedBytes);
9797
Assert.Equal(string1, (string)pooledString1);
9898
Assert.Equal(string2, (string)pooledString2);
@@ -107,17 +107,17 @@ public void Two_Strings_Fit_Same_Page(int pageSize)
107107
[InlineData(65535)]
108108
public void Two_Strings_Dont_Fit_Need_More_Space(int pageSize)
109109
{
110-
var stringSize = (pageSize / 2) - 1;
110+
var stringSize = TestSupport.GetMaxStringSizeForAllocation(pageSize / 2) + 1;
111111
using var pool = StringPool.Utf8(pageSize, 1);
112112
Assert.Equal(0, pool.UsedBytes);
113113
Assert.Equal(pageSize, pool.AllocatedBytes);
114114
var string1 = new string('c', stringSize);
115115
var pooledString1 = pool.Add(string1);
116-
Assert.Equal(stringSize + 2, pool.UsedBytes);
116+
Assert.Equal(TestSupport.GetAllocationSize(stringSize), pool.UsedBytes);
117117
Assert.Equal(pageSize, pool.AllocatedBytes);
118118
var string2 = new string('d', stringSize);
119119
var pooledString2 = pool.Add(string2);
120-
Assert.Equal((2 * stringSize) + 4, pool.UsedBytes);
120+
Assert.Equal(2 * TestSupport.GetAllocationSize(stringSize), pool.UsedBytes);
121121
Assert.Equal(2 * pageSize, pool.AllocatedBytes);
122122
Assert.Equal(string1, (string)pooledString1);
123123
Assert.Equal(stringSize, pooledString1.Length);
@@ -269,7 +269,7 @@ public void Add_Deduplicated_Thread_Safe(int numThreads, int numPages)
269269
for (var i = 0; !stopped; ++i)
270270
{
271271
var str = pool.Add("foobar " + (i % 1000));
272-
Interlocked.Add(ref stringSum, 2 + str.ToString().Length);
272+
Interlocked.Add(ref stringSum, TestSupport.GetAllocationSize(str.ToString().Length));
273273
if (i == 10000)
274274
{
275275
Interlocked.Increment(ref numStarted);
@@ -300,7 +300,7 @@ public void Add_Deduplicated_Thread_Safe(int numThreads, int numPages)
300300
var sum = 0L;
301301
for (var i = 0; i < 1000; ++i)
302302
{
303-
var len = 2 + ("foobar " + i).Length;
303+
var len = TestSupport.GetAllocationSize(("foobar " + i).Length);
304304
sum += len;
305305
}
306306

‎src/Combination.StringPools.Tests/Test_Deduplication.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public void Equal_Strings_Deduplicated(int numStrings, int numUniqueStrings, int
1818
pool.Add(someString);
1919
}
2020

21-
Assert.Equal(((stringSize * 2) + 2) * numUniqueStrings, pool.UsedBytes);
21+
Assert.Equal(TestSupport.GetAllocationSize(stringSize * 2) * numUniqueStrings, pool.UsedBytes);
2222
for (var i = 0; i < numUniqueStrings * 2; ++i)
2323
{
2424
var someString = new string(Convert.ToChar('ä' + i), stringSize);
@@ -46,7 +46,7 @@ public void Equal_Strings_Deduplicated_Bytes(int numStrings, int numUniqueString
4646
pool.Add(bytes);
4747
}
4848

49-
Assert.Equal(((stringSize * 2) + 2) * numUniqueStrings, pool.UsedBytes);
49+
Assert.Equal(TestSupport.GetAllocationSize(stringSize * 2) * numUniqueStrings, pool.UsedBytes);
5050
for (var i = 0; i < numUniqueStrings * 2; ++i)
5151
{
5252
var someString = new string(Convert.ToChar('ä' + i), stringSize);
@@ -102,6 +102,6 @@ public void Equal_Strings_Deduplicated_Thread_Safe(int numThreads, int numString
102102
t.Join();
103103
}
104104

105-
Assert.Equal(10 * (2 + stringSize), pool.UsedBytes);
105+
Assert.Equal(10 * TestSupport.GetAllocationSize(stringSize), pool.UsedBytes);
106106
}
107107
}

‎src/Combination.StringPools.Tests/Test_Multiple.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public void Multiple_Pools_Kept_Separate(int numPools, int stringsPerPool, int s
6969

7070
foreach (var pool in pools)
7171
{
72-
Assert.Equal((stringSize + 2) * stringsPerPool, pool.UsedBytes);
72+
Assert.Equal(TestSupport.GetAllocationSize(stringSize) * stringsPerPool, pool.UsedBytes);
7373
pool.Dispose();
7474
Assert.Equal(0, pool.UsedBytes);
7575
}
@@ -94,7 +94,7 @@ public void Multiple_Pools_Kept_Separate_Thread_Safe(int numThreads, int strings
9494
Assert.Same(pool, str.StringPool);
9595
}
9696

97-
Assert.Equal(stringsPerPool * (2 + stringSize), pool.UsedBytes);
97+
Assert.Equal(stringsPerPool * TestSupport.GetAllocationSize(stringSize), pool.UsedBytes);
9898
});
9999
t.Start();
100100
threads.Add(t);
@@ -125,7 +125,7 @@ public void Multiple_Deduplicated_Pools_Kept_Separate_Thread_Safe(int numThreads
125125
Assert.Same(pool, str.StringPool);
126126
}
127127

128-
Assert.Equal(10 * (2 + stringSize), pool.UsedBytes);
128+
Assert.Equal(10 * TestSupport.GetAllocationSize(stringSize), pool.UsedBytes);
129129
});
130130
t.Start();
131131
threads.Add(t);

‎src/Combination.StringPools/Utf8StringPool.cs‎

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using System.Numerics;
12
using System.Runtime.CompilerServices;
23
using System.Runtime.InteropServices;
34
using System.Text;
@@ -97,6 +98,9 @@ PooledUtf8String IUtf8StringPool.Add(ReadOnlySpan<char> value)
9798
PooledUtf8String IUtf8StringPool.Add(ReadOnlySpan<byte> value)
9899
=> AddInternal(value);
99100

101+
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
102+
public static int GetAllocationSize(int length) => length + 1 + (BitOperations.Log2((uint)length) / 7);
103+
100104

101105
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
102106
private PooledUtf8String AddInternal(ReadOnlySpan<byte> value)
@@ -107,8 +111,8 @@ private PooledUtf8String AddInternal(ReadOnlySpan<byte> value)
107111
return PooledUtf8String.Empty;
108112
}
109113

110-
var structLength = length + 2;
111-
if (structLength > 0xffff || structLength > pageSize)
114+
var structLength = GetAllocationSize(length);
115+
if (structLength > pageSize)
112116
{
113117
throw new ArgumentOutOfRangeException(nameof(value), "String is too long to be pooled");
114118
}
@@ -168,8 +172,24 @@ private PooledUtf8String AddInternal(ReadOnlySpan<byte> value)
168172

169173
unsafe
170174
{
171-
*(ushort*)(writePtr + pageStartOffset) = checked((ushort)length);
172-
var stringWritePtr = new Span<byte>((byte*)(writePtr + pageStartOffset + 2), length);
175+
var ptr = (byte*)(writePtr + pageStartOffset);
176+
var write = length;
177+
while (true)
178+
{
179+
if (write > 0x7f)
180+
{
181+
*ptr++ = unchecked((byte)(0x80 | (write & 0x7f)));
182+
}
183+
else
184+
{
185+
*ptr++ = unchecked((byte)write);
186+
break;
187+
}
188+
189+
write >>= 7;
190+
}
191+
192+
var stringWritePtr = new Span<byte>(ptr, length);
173193
value.CopyTo(stringWritePtr);
174194
}
175195

@@ -388,8 +408,21 @@ private ReadOnlySpan<byte> GetStringBytes(ulong offset)
388408

389409
unsafe
390410
{
391-
var length = ((ushort*)(pages[page] + pageOffset))[0];
392-
return new ReadOnlySpan<byte>((byte*)(pages[page] + pageOffset + 2), length);
411+
var ptr = (byte*)(pages[page] + pageOffset);
412+
var length = 0;
413+
var shl = 0;
414+
while (true)
415+
{
416+
var t = *ptr++;
417+
length += (t & 0x7f) << shl;
418+
shl += 7;
419+
if ((t & 0x80) == 0)
420+
{
421+
break;
422+
}
423+
}
424+
425+
return new ReadOnlySpan<byte>(ptr, length);
393426
}
394427
}
395428

@@ -434,7 +467,24 @@ private int GetStringLength(ulong offset)
434467
throw new ArgumentOutOfRangeException(nameof(offset), $"Invalid handle value, page {page} is out of range 0..{pages.Count}");
435468
}
436469

437-
return unchecked((ushort)Marshal.ReadInt16(pages[page] + pageOffset));
470+
unsafe
471+
{
472+
var ptr = (byte*)(pages[page] + pageOffset);
473+
var length = 0;
474+
var shl = 0;
475+
while (true)
476+
{
477+
var t = *ptr++;
478+
length += (t & 0x7f) << shl;
479+
shl += 7;
480+
if ((t & 0x80) == 0)
481+
{
482+
break;
483+
}
484+
}
485+
486+
return length;
487+
}
438488
}
439489
}
440490

0 commit comments

Comments
 (0)