From 5e35323f3e5dc6a85b208c0a4fb4b8fefcb2dc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:40:44 +0200 Subject: [PATCH 1/8] perf(zstd): decompression's hot loops on x64 The decoder was tuned on Arm64, where its loops keep every value in one of 31 registers. On x64, with 15, they spilled, and some of Arm64's idioms cost more there. Measured on a Ryzen 9 7950X (Zen 4, Windows 11, Native AOT) against libzstd 1.5.7 built with clang, its assembly Huffman decoder and its BMI2 paths included: - Sequences run by a loop of x64's own (ExecuteSequencesFastX64): the bitstream's bounds proved once a batch of sequences, the repeat offsets and the count in the state rather than in registers it had not, tests as compares and branches rather than flags combined, and matches copied 32 bytes at a time when their offset allows it. The paired loop of two blocks, faster on Arm64, spilled on x64, where the fused one is 1.2 to 1.3x as fast: it is off there. - One Huffman stream decoded in locals (DecodeHuffmanSymbols), the reader's state out of memory. - Four Huffman streams: each stream's next entry read as soon as its container is shifted, as libzstd's assembly does, and the four outputs written a quarter apart from two cursors. Single symbols, 3.0 to 2.5 cycles a symbol (libzstd, 2.47); double symbols, 1.63 to 1.51 (1.38). - ReadNCount's Advance inlined: by reference, it kept the description's position, count and bits in memory. A table description reads 3.8x as fast. - OverlapCopy's shuffle as ShuffleNative, a bare vpshufb on x64. Over libzstd, from efc0ec6: silesia 0.82-0.89x to 1.02-1.12x, zstd's github records 0.90-1.01x to 1.08-1.14x, with their dictionary 0.94-0.97x to 1.12-1.20x. The Huffman loops, the single-stream decoder and ReadNCount are shared with Arm64; the rest is x64's alone. --- .../Internal/BackwardBitReader.cs | 68 ++++ src/Vorticity.Zstd/Internal/Fse.cs | 3 + src/Vorticity.Zstd/Internal/Huffman.cs | 189 +++++---- .../ZstdDecompressor.Execution.cs | 17 +- .../ZstdDecompressor.ExecutionX64.cs | 365 ++++++++++++++++++ src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs | 9 +- .../Fuzz/PairEquivalenceTests.cs | 4 +- 7 files changed, 571 insertions(+), 84 deletions(-) create mode 100644 src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs diff --git a/src/Vorticity.Zstd/Internal/BackwardBitReader.cs b/src/Vorticity.Zstd/Internal/BackwardBitReader.cs index df32cda87..b288b6541 100644 --- a/src/Vorticity.Zstd/Internal/BackwardBitReader.cs +++ b/src/Vorticity.Zstd/Internal/BackwardBitReader.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Vorticity.Zstd.Internal; @@ -173,6 +174,73 @@ public BitStreamStatus Reload() return status; } + /// + /// Decodes symbols of a single-symbol Huffman table (entries of + /// : the code length in the low byte, the symbol in the high byte), + /// each looked up with the next bits, the stream reloaded before any + /// symbol once more than 64 - of its bits are consumed: + /// the reloads of , at the same symbols, its statuses ignored. + /// + /// + /// + /// The reader's state is held in locals for the loop: through the reader, every symbol would read + /// and write it in memory. A reload eight bytes or more from the start takes the loop's own path; + /// any other goes through . + /// + /// + /// The symbols form one chain, each code's length deciding where the next starts. So the chain + /// runs on the container shifted past the consumed bits, which a symbol shifts by its entry (whose + /// low six bits are the code's length): a shift, the lookup, a shift. The count of consumed bits, + /// which only the reloads read, follows on the side. Until 64 bits are consumed the shifted + /// container is the reader's container shifted by that count; past that, the stream has + /// overflowed and fails its end check whatever the symbols read. + /// + /// + public void DecodeHuffmanSymbols(ref ushort table, int tableLog, ref byte output, nint length) + { + ulong container = _container; + int consumed = _bitsConsumed; + int position = _position; + ulong bits = container << (consumed & 63); + bits = consumed >= 64 ? 0 : bits; + ref byte source = ref MemoryMarshal.GetReference(_source); + int shift = (64 - tableLog) & 63; + for (nint i = 0; i < length; i++) + { + if (consumed > 64 - HuffmanTable.MaxTableLog) + { + if ((consumed <= 64) & (position >= sizeof(ulong))) + { + position -= consumed >> 3; + consumed &= 7; + container = Unsafe.ReadUnaligned(ref Unsafe.Add(ref source, position)); + } + else + { + _container = container; + _bitsConsumed = consumed; + _position = position; + Reload(); + container = _container; + consumed = _bitsConsumed; + position = _position; + } + + bits = container << (consumed & 63); + bits = consumed >= 64 ? 0 : bits; + } + + uint entry = Unsafe.Add(ref table, (nint)(bits >> shift)); + Unsafe.Add(ref output, i) = (byte)(entry >> 8); + bits <<= (int)entry; + consumed += (int)(entry & 0xFF); + } + + _container = container; + _bitsConsumed = consumed; + _position = position; + } + /// libzstd's BIT_reloadDStreamFast: refuses to come within eight bytes of the start. public BitStreamStatus ReloadFast() { diff --git a/src/Vorticity.Zstd/Internal/Fse.cs b/src/Vorticity.Zstd/Internal/Fse.cs index bc320eb14..fedd66c77 100644 --- a/src/Vorticity.Zstd/Internal/Fse.cs +++ b/src/Vorticity.Zstd/Internal/Fse.cs @@ -158,6 +158,9 @@ public static int ReadNCount( ip += (bitCount + 7) >> 3; return ip; + // Inlined: as a call, it took the reader's position, count and bits by reference, which kept + // them in memory for the whole description. + [MethodImpl(MethodImplOptions.AggressiveInlining)] static void Advance(ReadOnlySpan header, int end, ref int ip, ref int bitCount, out uint bitStream) { if (ip <= end - 7 || ip + (bitCount >> 3) <= end - 4) diff --git a/src/Vorticity.Zstd/Internal/Huffman.cs b/src/Vorticity.Zstd/Internal/Huffman.cs index dc010080d..80fce1bbb 100644 --- a/src/Vorticity.Zstd/Internal/Huffman.cs +++ b/src/Vorticity.Zstd/Internal/Huffman.cs @@ -457,8 +457,20 @@ private ref struct FourStreams /// /// The rounds of : a method of its own that calls nothing, so - /// that the twelve values of the four streams stay in registers, none of them living past it. + /// that the values of the four streams stay in registers, none of them living past it. /// + /// + /// + /// Each stream's next entry is read as soon as its container is shifted, as libzstd's assembly + /// does (): read where its symbol is written, after the three other streams' + /// work, the lookup started later, and the rounds took 3.0 cycles a symbol on x64 where they now + /// take 2.5, libzstd's. + /// + /// + /// The four outputs move together, a quarter apart: the second and the fourth are written at that + /// distance from the first and the third, a register less for the entries read ahead. + /// + /// [MethodImpl(MethodImplOptions.NoInlining)] private static void RunFourStreams(ref FourStreams s, ref ushort table, int shift, ref byte input, ref byte oend) { @@ -471,13 +483,13 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif ref byte ip2 = ref s.Ip2; ref byte ip3 = ref s.Ip3; ref byte op0 = ref s.Op0; - ref byte op1 = ref s.Op1; ref byte op2 = ref s.Op2; - ref byte op3 = ref s.Op3; + nint quarter = Unsafe.ByteOffset(ref s.Op0, ref s.Op1); + Debug.Assert(Unsafe.ByteOffset(ref s.Op2, ref s.Op3) == quarter && Unsafe.ByteOffset(ref s.Op1, ref s.Op2) == quarter); while (true) { - nint outputRounds = Unsafe.ByteOffset(ref op3, ref oend) / 5; + nint outputRounds = Unsafe.ByteOffset(ref Unsafe.Add(ref op2, quarter), ref oend) / 5; nint inputRounds = Unsafe.ByteOffset(ref input, ref ip0) / 7; nint rounds = Math.Min(outputRounds, inputRounds); if (rounds == 0 @@ -488,39 +500,45 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif break; } - ref byte olimit = ref Unsafe.Add(ref op3, rounds * 5); + ref byte olimit = ref Unsafe.Add(ref op0, rounds * 5); + nint entry0 = Unsafe.Add(ref table, (nint)(bits0 >> shift)); + nint entry1 = Unsafe.Add(ref table, (nint)(bits1 >> shift)); + nint entry2 = Unsafe.Add(ref table, (nint)(bits2 >> shift)); + nint entry3 = Unsafe.Add(ref table, (nint)(bits3 >> shift)); do { - Symbol(ref table, shift, ref bits0, ref op0, 0); - Symbol(ref table, shift, ref bits1, ref op1, 0); - Symbol(ref table, shift, ref bits2, ref op2, 0); - Symbol(ref table, shift, ref bits3, ref op3, 0); - Symbol(ref table, shift, ref bits0, ref op0, 1); - Symbol(ref table, shift, ref bits1, ref op1, 1); - Symbol(ref table, shift, ref bits2, ref op2, 1); - Symbol(ref table, shift, ref bits3, ref op3, 1); - Symbol(ref table, shift, ref bits0, ref op0, 2); - Symbol(ref table, shift, ref bits1, ref op1, 2); - Symbol(ref table, shift, ref bits2, ref op2, 2); - Symbol(ref table, shift, ref bits3, ref op3, 2); - Symbol(ref table, shift, ref bits0, ref op0, 3); - Symbol(ref table, shift, ref bits1, ref op1, 3); - Symbol(ref table, shift, ref bits2, ref op2, 3); - Symbol(ref table, shift, ref bits3, ref op3, 3); - Symbol(ref table, shift, ref bits0, ref op0, 4); - Symbol(ref table, shift, ref bits1, ref op1, 4); - Symbol(ref table, shift, ref bits2, ref op2, 4); - Symbol(ref table, shift, ref bits3, ref op3, 4); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 0); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 0); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 1); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter + 1); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 1); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter + 1); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 2); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter + 2); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 2); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter + 2); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 3); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter + 3); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 3); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter + 3); + Last(ref bits0, entry0, ref op0, 4); + Last(ref bits1, entry1, ref op0, quarter + 4); + Last(ref bits2, entry2, ref op2, 4); + Last(ref bits3, entry3, ref op2, quarter + 4); ip0 = ref ReloadFast(ref bits0, ref ip0); - op0 = ref Unsafe.Add(ref op0, 5); + entry0 = Unsafe.Add(ref table, (nint)(bits0 >> shift)); ip1 = ref ReloadFast(ref bits1, ref ip1); - op1 = ref Unsafe.Add(ref op1, 5); + entry1 = Unsafe.Add(ref table, (nint)(bits1 >> shift)); ip2 = ref ReloadFast(ref bits2, ref ip2); - op2 = ref Unsafe.Add(ref op2, 5); + entry2 = Unsafe.Add(ref table, (nint)(bits2 >> shift)); ip3 = ref ReloadFast(ref bits3, ref ip3); - op3 = ref Unsafe.Add(ref op3, 5); + entry3 = Unsafe.Add(ref table, (nint)(bits3 >> shift)); + op0 = ref Unsafe.Add(ref op0, 5); + op2 = ref Unsafe.Add(ref op2, 5); } - while (Unsafe.IsAddressLessThan(ref op3, ref olimit)); + while (Unsafe.IsAddressLessThan(ref op0, ref olimit)); } s.Bits0 = bits0; @@ -532,9 +550,9 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif s.Ip2 = ref ip2; s.Ip3 = ref ip3; s.Op0 = ref op0; - s.Op1 = ref op1; + s.Op1 = ref Unsafe.Add(ref op0, quarter); s.Op2 = ref op2; - s.Op3 = ref op3; + s.Op3 = ref Unsafe.Add(ref op2, quarter); } /// @@ -544,7 +562,8 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif /// /// A round takes at most 5 x 11 bits from each stream, as the single-symbol rounds, and writes at /// most ten bytes to each: the rounds are capped by what the most filled quarter has left, and - /// recounted when they run out. + /// recounted when they run out. The entries are read ahead as in + /// (1.63 cycles a symbol before, 1.51 after; libzstd's assembly, 1.38). /// [MethodImpl(MethodImplOptions.NoInlining)] private static void RunFourStreamsDouble( @@ -577,32 +596,40 @@ private static void RunFourStreamsDouble( break; } + ulong entry0 = Unsafe.Add(ref table, (nint)(bits0 >> (64 - DoubleLog))); + ulong entry1 = Unsafe.Add(ref table, (nint)(bits1 >> (64 - DoubleLog))); + ulong entry2 = Unsafe.Add(ref table, (nint)(bits2 >> (64 - DoubleLog))); + ulong entry3 = Unsafe.Add(ref table, (nint)(bits3 >> (64 - DoubleLog))); do { - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairLast(ref bits0, entry0, ref op0); + op1 = ref PairLast(ref bits1, entry1, ref op1); + op2 = ref PairLast(ref bits2, entry2, ref op2); + op3 = ref PairLast(ref bits3, entry3, ref op3); ip0 = ref ReloadFast(ref bits0, ref ip0); + entry0 = Unsafe.Add(ref table, (nint)(bits0 >> (64 - DoubleLog))); ip1 = ref ReloadFast(ref bits1, ref ip1); + entry1 = Unsafe.Add(ref table, (nint)(bits1 >> (64 - DoubleLog))); ip2 = ref ReloadFast(ref bits2, ref ip2); + entry2 = Unsafe.Add(ref table, (nint)(bits2 >> (64 - DoubleLog))); ip3 = ref ReloadFast(ref bits3, ref ip3); + entry3 = Unsafe.Add(ref table, (nint)(bits3 >> (64 - DoubleLog))); } while (--rounds > 0); } @@ -621,11 +648,24 @@ private static void RunFourStreamsDouble( s.Op3 = ref op3; } - /// One lookup of the double-symbol table: both bytes written, the output moved by the count. + /// + /// One lookup of the double-symbol table, its entry read ahead as : both bytes + /// written, the output moved by the count, the next entry read. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte Pair(scoped ref uint table, scoped ref ulong bits, ref byte op) + private static ref byte PairNext(scoped ref uint table, scoped ref ulong bits, scoped ref ulong entry, ref byte op) + { + Unsafe.WriteUnaligned(ref op, (ushort)(entry >> 8)); + bits <<= (int)entry; + ref byte next = ref Unsafe.Add(ref op, (nint)(entry >> 30)); + entry = Unsafe.Add(ref table, (nint)(bits >> (64 - DoubleLog))); + return ref next; + } + + /// The last lookup of a round, as : the stream reloads first. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref byte PairLast(scoped ref ulong bits, ulong entry, ref byte op) { - ulong entry = Unsafe.Add(ref table, (nint)(bits >> (64 - DoubleLog))); Unsafe.WriteUnaligned(ref op, (ushort)(entry >> 8)); bits <<= (int)entry; return ref Unsafe.Add(ref op, (nint)(entry >> 30)); @@ -745,11 +785,20 @@ private static ulong InitFastStream(ref byte ip) return (Unsafe.ReadUnaligned(ref ip) | 1) << consumed; } + /// The symbol of an entry read ahead: the bits it takes skipped, its byte written, the next entry read. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Next(ref ushort table, int shift, ref ulong bits, ref nint entry, ref byte op, nint k) + { + bits <<= (int)entry; + Unsafe.Add(ref op, k) = (byte)(entry >> 8); + entry = Unsafe.Add(ref table, (nint)(bits >> shift)); + } + + /// The last symbol of a round: no entry read ahead, the stream reloads first. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Symbol(ref ushort table, int shift, ref ulong bits, ref byte op, int k) + private static void Last(ref ulong bits, nint entry, ref byte op, nint k) { - int entry = Unsafe.Add(ref table, (nint)(bits >> shift)); - bits <<= entry; + bits <<= (int)entry; Unsafe.Add(ref op, k) = (byte)(entry >> 8); } @@ -767,21 +816,9 @@ private static ref byte ReloadFast(scoped ref ulong bits, ref byte ip) } /// Decodes one symbol per entry until is full. - private void DecodeStream(ref BackwardBitReader bits, Span output) - { - ushort[] entries = Entries; - int tableLog = TableLog; - for (int i = 0; i < output.Length; i++) - { - // A reload leaves at least 57 bits while the stream has them; a code takes at most 12. - if (bits.BitsConsumed > 64 - MaxTableLog) - { - bits.Reload(); - } - - ushort entry = entries[bits.PeekBitsFast(tableLog)]; - output[i] = (byte)(entry >> 8); - bits.SkipBits(entry & 0xFF); - } - } + private void DecodeStream(ref BackwardBitReader bits, Span output) => + // A reload leaves at least 57 bits while the stream has them; a code takes at most 12. An + // entry index is below 2^TableLog: no bounds to check. + bits.DecodeHuffmanSymbols( + ref MemoryMarshal.GetArrayDataReference(Entries), TableLog, ref MemoryMarshal.GetReference(output), output.Length); } diff --git a/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs b/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs index bcd6fa8b3..6ab226c1e 100644 --- a/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs +++ b/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using Vorticity.Zstd.Internal; namespace Vorticity.Zstd; @@ -190,9 +191,16 @@ private int RunSequences( state.HistoryLength = history.Length; ref byte frameStart = ref MemoryMarshal.GetReference(destination); nint fastRoom = Math.Min((nint)blockStart + blockSizeMax, destination.Length - WildCopyOverlength); - ExecuteSequencesFast( - ref state, ref frameStart, ref Unsafe.Add(ref frameStart, Math.Max(fastRoom, Unsafe.ByteOffset(ref frameStart, ref state.Dst))), - ref Unsafe.Add(ref MemoryMarshal.GetReference(literals), literalCount)); + ref byte fastLimit = ref Unsafe.Add(ref frameStart, Math.Max(fastRoom, Unsafe.ByteOffset(ref frameStart, ref state.Dst))); + ref byte litEnd = ref Unsafe.Add(ref MemoryMarshal.GetReference(literals), literalCount); + if (X86Base.IsSupported) + { + ExecuteSequencesFastX64(ref state, ref frameStart, ref fastLimit, ref litEnd); + } + else + { + ExecuteSequencesFast(ref state, ref frameStart, ref fastLimit, ref litEnd); + } } return ExecuteSequencesCareful(ref state, literals, literalCount, destination, blockStart, blockSizeMax, history); @@ -1010,7 +1018,8 @@ private static void OverlapCopy(ref byte dst, ref byte match, nuint offset, nint Debug.Assert(offset is > 0 and < 16); Vector128 source = Unsafe.ReadUnaligned>(ref match); Vector128 indices = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(PatternIndices), (nint)offset * 16)); - Vector128 pattern = Vector128.Shuffle(source, indices); + // The indices are all below 16: the native shuffle (tbl, pshufb) needs no masking of the others. + Vector128 pattern = Vector128.ShuffleNative(source, indices); nint step = PatternStep[(int)offset]; nint i = 0; do diff --git a/src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs b/src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs new file mode 100644 index 000000000..a43e5f66d --- /dev/null +++ b/src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs @@ -0,0 +1,365 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using Vorticity.Zstd.Internal; + +namespace Vorticity.Zstd; + +/// The fused sequence loop, as x64 runs it. +/// +/// +/// keeps its whole state in registers, which Arm64's 31 hold: +/// x64 has 15, and the JIT, short of them, spills the states it has just computed and reloads them +/// at the top of the next iteration, putting a store and a load on the chain every sequence waits +/// on. This loop decodes and executes the same sequences, in the same order, with the same results +/// and the same handoff to the careful path, from fewer live values: +/// +/// +/// The bitstream is one moving reference, as in , and its bound +/// is proven a batch of sequences at a time: a sequence reads at most 64 bits on the common paths, +/// so a batch of (position / 8) sequences never reloads below the start, and the sequences need no +/// test of their own. A sequence that reloads twice ends its batch. +/// What a sequence only reads or rarely writes stays in memory: the limits, the sequences left +/// outside the batch, the second and third repeat offsets. x64 compares with a memory operand at the +/// cost of a register compare. +/// Each test is a compare and a branch of its own, which x64 fuses into one operation: the +/// conditions the Arm64 loop combines with & become, without conditional compares, +/// chains of setcc. +/// +/// +public sealed partial class ZstdDecompressor +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ExecuteSequencesFastX64(ref SequenceState s, ref byte frameStart, ref byte fastLimit, ref byte litEnd) + { + ref byte stream = ref Unsafe.Add(ref s.Bits, s.Ptr); + nint bc = s.Consumed; + ulong container = s.Container; + ref SeqSymbol tables = ref s.Tables; + nint llState = s.LiteralLengthState; + nint mlState = s.MatchLengthState; + nint ofState = s.OffsetState; + nuint rep0 = s.Rep0; + ref byte dst = ref s.Dst; + ref byte lit = ref s.Lit; + + // The second and third repeat offsets stay in s, and so do the sequences before the last one + // (which the careful path takes) that are not in the batch: read and written there, in memory. + s.NbSeq--; + nint batch; + ulong llEntry; + ulong mlEntry; + ulong ofEntry; + nint litLength; + nint matchLength; + nuint offset; + + Batch: + batch = Math.Min(s.NbSeq, Unsafe.ByteOffset(ref s.Bits, ref stream) >> 3); + if (batch <= 0) + { + goto Edge; + } + + s.NbSeq -= batch; + + Loop: + llEntry = SeqEntry.Load(ref tables, llState); + mlEntry = SeqEntry.Load(ref tables, mlState); + ofEntry = SeqEntry.Load(ref tables, ofState); + if (((llEntry | mlEntry) & 0x1F00) != 0) + { + goto Extra; + } + + if ((ofEntry & 0x1E) == 0) + { + goto CommonRepeat; + } + + // ---- the common sequence with a new offset: lengths without extra bits, the states first + { + nint bcBefore = bc; + ulong containerBefore = container; + nint consumed = CommonNextStates(llEntry, mlEntry, ofEntry, bc, container, ref llState, ref mlState, ref ofState); + stream = ref Unsafe.Subtract(ref stream, (consumed >> 3) & 0x1F); + bc = consumed & 7; + container = Unsafe.ReadUnaligned(ref stream); + offset = SeqEntry.OffsetBase(ofEntry) + ReadBitsFast(containerBefore, bcBefore, (nint)ofEntry); + s.Rep2 = s.Rep1; + s.Rep1 = rep0; + rep0 = offset; + goto CommonCopy; + } + + CommonRepeat: + { + nint bcBefore = bc; + ulong containerBefore = container; + nint consumed = CommonNextStates(llEntry, mlEntry, ofEntry, bc, container, ref llState, ref mlState, ref ofState); + stream = ref Unsafe.Subtract(ref stream, (consumed >> 3) & 0x1F); + bc = consumed & 7; + container = Unsafe.ReadUnaligned(ref stream); + nuint raw = SeqEntry.OffsetBase(ofEntry) + ReadBits(containerBefore, bcBefore, SeqEntry.OffsetExtraBits(ofEntry)); + offset = ResolveOffsetX64(raw, SeqEntry.LengthBase(llEntry) == 0, ref rep0, ref s); + } + + CommonCopy: + { + matchLength = (nint)SeqEntry.LengthBase(mlEntry); + litLength = (nint)SeqEntry.LengthBase(llEntry); + ref byte matchStart = ref Unsafe.Add(ref dst, litLength); + + // A literal length without extra bits is its code, 0 to 15: one 16-byte copy. + if (Unsafe.IsAddressGreaterThan(ref Unsafe.Add(ref matchStart, matchLength), ref fastLimit)) + { + goto Execute; + } + + if (Unsafe.IsAddressGreaterThan(ref Unsafe.Add(ref lit, litLength), ref litEnd)) + { + goto Execute; + } + + if (offset > (nuint)Unsafe.ByteOffset(ref frameStart, ref matchStart)) + { + goto Execute; + } + + if ((offset < 16) | (matchLength > 16)) + { + goto Execute; + } + + Copy16(ref dst, ref lit); + Copy16(ref matchStart, ref Unsafe.Subtract(ref matchStart, offset)); + dst = ref Unsafe.Add(ref matchStart, matchLength); + lit = ref Unsafe.Add(ref lit, litLength); + if (--batch != 0) + { + goto Loop; + } + + goto Batch; + } + + // ---- lengths with extra bits + Extra: + { + nint ofBits = SeqEntry.OffsetExtraBits(ofEntry); + nint mlBits = SeqEntry.LengthExtraBits(mlEntry); + nint llBits = SeqEntry.LengthExtraBits(llEntry); + if (ofBits + mlBits + llBits >= 57 - (SequenceCodes.LiteralLengthMaxLog + SequenceCodes.MatchLengthMaxLog + SequenceCodes.OffsetMaxLog)) + { + // A reload inside the sequence, then another after it: up to 16 bytes back, which + // the batch did not count. It ends with this sequence. + s.NbSeq += batch - 1; + batch = 1; + goto General; + } + + // Every position from the entries, the states first, as the common path; one reload. + nint bcBefore = bc; + ulong containerBefore = container; + nint consumed = ExtraNextStates(llEntry, mlEntry, ofEntry, bc, container, ref llState, ref mlState, ref ofState, out nint mlAt); + stream = ref Unsafe.Subtract(ref stream, (consumed >> 3) & 0x1F); + bc = consumed & 7; + container = Unsafe.ReadUnaligned(ref stream); + nuint raw = SeqEntry.OffsetBase(ofEntry) + ReadBits(containerBefore, bcBefore, (nint)ofEntry); + matchLength = (nint)SeqEntry.LengthBase(mlEntry) + (nint)ReadBits(containerBefore, mlAt, (nint)(mlEntry >> 8)); + litLength = (nint)SeqEntry.LengthBase(llEntry) + (nint)ReadBits(containerBefore, mlAt + (nint)(mlEntry >> 8), (nint)(llEntry >> 8)); + if ((ofEntry & 0x1E) != 0) + { + offset = raw; + s.Rep2 = s.Rep1; + s.Rep1 = rep0; + rep0 = raw; + goto Execute; + } + + offset = ResolveOffsetX64(raw, SeqEntry.LengthBase(llEntry) == 0, ref rep0, ref s); + goto Execute; + } + + // ---- the stream within eight bytes of its start: one sequence at a time, with clamped reloads + Edge: + if (s.NbSeq <= 0) + { + batch = 0; + goto Stop; + } + + s.NbSeq--; + batch = 1; + llEntry = SeqEntry.Load(ref tables, llState); + mlEntry = SeqEntry.Load(ref tables, mlState); + ofEntry = SeqEntry.Load(ref tables, ofState); + + // ---- libzstd's ZSTD_decodeSequence in full + General: + { + nint ofBits = SeqEntry.OffsetExtraBits(ofEntry); + nint mlBits = SeqEntry.LengthExtraBits(mlEntry); + nint llBits = SeqEntry.LengthExtraBits(llEntry); + nint ptr = Unsafe.ByteOffset(ref s.Bits, ref stream); + nuint raw = SeqEntry.OffsetBase(ofEntry) + ReadBits(container, bc, ofBits); + bc += ofBits; + if (ofBits > 1) + { + offset = raw; + s.Rep2 = s.Rep1; + s.Rep1 = rep0; + rep0 = raw; + } + else + { + offset = ResolveOffsetX64(raw, SeqEntry.LengthBase(llEntry) == 0, ref rep0, ref s); + } + + matchLength = (nint)SeqEntry.LengthBase(mlEntry) + (nint)ReadBits(container, bc, mlBits); + bc += mlBits; + if (ofBits + mlBits + llBits >= 57 - (SequenceCodes.LiteralLengthMaxLog + SequenceCodes.MatchLengthMaxLog + SequenceCodes.OffsetMaxLog)) + { + ReloadClamped(ref s.Bits, ref ptr, ref bc, ref container); + } + + litLength = (nint)SeqEntry.LengthBase(llEntry) + (nint)ReadBits(container, bc, llBits); + bc += llBits; + NextStates(llEntry, mlEntry, ofEntry, container, ref bc, ref llState, ref mlState, ref ofState); + ReloadClamped(ref s.Bits, ref ptr, ref bc, ref container); + stream = ref Unsafe.Add(ref s.Bits, ptr); + } + + // ---- any sequence, decoded: libzstd's ZSTD_execSequence + Execute: + { + ref byte litAfter = ref Unsafe.Add(ref lit, litLength); + ref byte matchStart = ref Unsafe.Add(ref dst, litLength); + ref byte matchEnd = ref Unsafe.Add(ref matchStart, matchLength); + if (Unsafe.IsAddressGreaterThan(ref matchEnd, ref fastLimit)) + { + goto Pending; + } + + if (Unsafe.IsAddressGreaterThan(ref litAfter, ref litEnd)) + { + goto Pending; + } + + nuint prefix = (nuint)Unsafe.ByteOffset(ref frameStart, ref matchStart); + if (offset > prefix) + { + // A match that starts in the dictionary and ends there: see ExecuteSequencesFast. + nuint back = offset - prefix; + if (back > (nuint)s.HistoryLength) + { + goto Pending; + } + + if ((nint)back < matchLength) + { + goto Pending; + } + + WildCopyWide(ref dst, ref lit, litLength); + WildCopyWide(ref matchStart, ref Unsafe.Subtract(ref s.HistoryEnd, back), matchLength); + } + else + { + // The literals from their own buffer, then the match 32 bytes at a time when it is 32 + // or more back, else 16 at a time, else as a pattern: a copy of up to 32 bytes is one + // store, its loop not taken. + ref byte match = ref Unsafe.Subtract(ref matchStart, offset); + WildCopyWide(ref dst, ref lit, litLength); + if (offset >= 32) + { + WildCopyWide(ref matchStart, ref match, matchLength); + } + else if (offset >= 16) + { + WildCopy16(ref matchStart, ref match, matchLength); + } + else + { + OverlapCopy(ref matchStart, ref match, offset, matchLength); + } + } + + dst = ref matchEnd; + lit = ref litAfter; + if (--batch != 0) + { + goto Loop; + } + + goto Batch; + } + + Pending: + s.Pending = true; + s.LitLength = litLength; + s.MatchLength = matchLength; + s.Offset = offset; + + Stop: + s.Ptr = Unsafe.ByteOffset(ref s.Bits, ref stream); + s.Consumed = bc; + s.Container = container; + s.LiteralLengthState = llState; + s.MatchLengthState = mlState; + s.OffsetState = ofState; + s.Rep0 = rep0; + s.Dst = ref dst; + s.Lit = ref lit; + s.NbSeq += batch + 1; + } + + /// + /// Copies bytes 32 at a time, one block at least, between places 32 or + /// more apart: writing, and reading, up to 31 past the end, as , which it + /// is where 256-bit vectors are not accelerated. With AVX2, one load and one store a block. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void WildCopyWide(ref byte dst, ref byte src, nint length) + { + if (!Vector256.IsHardwareAccelerated) + { + WildCopy32(ref dst, ref src, length); + return; + } + + nint i = 0; + do + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dst, i), Unsafe.ReadUnaligned>(ref Unsafe.Add(ref src, i))); + i += 32; + } + while (i < length); + } + + /// + /// with the second and third repeat offsets in , + /// where the x64 loop keeps them. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static nuint ResolveOffsetX64(nuint raw, bool noLiteral, ref nuint rep0, ref SequenceState s) + { + nuint index = raw + (noLiteral ? 1u : 0u); + if (index == 0) + { + return rep0; + } + + nuint rep1 = s.Rep1; + nuint offset = index == 1 ? rep1 : index == 2 ? s.Rep2 : rep0 - 1; + offset = offset == 0 ? nuint.MaxValue : offset; + if (index != 1) + { + s.Rep2 = rep1; + } + + s.Rep1 = rep0; + rep0 = offset; + return offset; + } +} diff --git a/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs b/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs index 405682a07..0b56288c7 100644 --- a/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs +++ b/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs @@ -3,6 +3,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; using Vorticity.Zstd.Internal; namespace Vorticity.Zstd; @@ -55,8 +56,12 @@ public sealed partial class ZstdDecompressor /// The ulongs a step of the pair loop writes: A's record, then B's, each lengths then offset. private const int RecordStride = 4; - /// Whether blocks are decoded in pairs; tests turn it off to compare with one at a time. - internal bool PairsBlocks { get; set; } = true; + /// + /// Whether blocks are decoded in pairs; tests set it to compare with one at a time. Not on x64, + /// whose 15 registers cannot hold two chains: the JIT spills them, and the fused loop (see + /// ) decodes the same blocks one at a time 1.2 to 1.3 times as fast. + /// + internal bool PairsBlocks { get; set; } = !X86Base.IsSupported; /// The sequences decoded side by side so far, for the tests. internal long PairedSequences { get; private set; } diff --git a/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs b/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs index daf1b081b..ebe9f635c 100644 --- a/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs +++ b/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs @@ -37,8 +37,8 @@ public void Paired_blocks_decode_and_fail_as_blocks_one_at_a_time(int seed) if (!decoders.TryGetValue(dictionary ?? Array.Empty(), out var pair)) { pair = dictionary is null - ? (new ZstdDecompressor(), new ZstdDecompressor { PairsBlocks = false }) - : (new ZstdDecompressor(dictionary), new ZstdDecompressor(dictionary) { PairsBlocks = false }); + ? (new ZstdDecompressor { PairsBlocks = true }, new ZstdDecompressor { PairsBlocks = false }) + : (new ZstdDecompressor(dictionary) { PairsBlocks = true }, new ZstdDecompressor(dictionary) { PairsBlocks = false }); decoders.Add(dictionary ?? Array.Empty(), pair); } From cab3adb65a425002f99731ddc148d33196e84011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:40:59 +0200 Subject: [PATCH 2/8] perf(zstd): compression's hot paths on x64, its entropy tables by reference As the decoder, the match finders and the entropy coder were tuned on Arm64. On x64, where they spilled and Arm64's idioms cost more, the steps (same machine and reference as the last commit): - Match finders: conditions as compares and branches on x64, where Arm64 combines them with & into conditional compares and x64 made setcc chains (fast, double-fast, lazy, hash chains, optimal); the window as a clamp of a pointer; the rows' and the long-distance matcher's prefetches as prefetches, which .NET has on x64 only. - The double-fast search with an attached dictionary, of x64's own (CompressBlockAttachedX64): libzstd's form held some twenty values across calls and spilled them all, a 440-byte frame; the search now holds ten, the rest read as operands. github + dict L3 1.12x. - Hash chains: the insertion before the search's values, which were live across its loop and spilled it (L5 1.08x, L7 1.06x on github's records); the skipping test out of that loop. - Sequences stored as their lengths and offset on x64, their codes computed in one pass once the block is found (DefersCodes); the sequence encoder reads the three tables in place, without gathering them; LargestCount in a local and with vector maxima. - Entropy tables by reference: a table repeated, predefined or a dictionary's is that table, and a new one is built into a table the previous state does not use, where libzstd copies 2 to 5 KiB a block (small frames with a dictionary, 1.07 to 1.08x). - Huffman's sort with its totals and its symbols without count in registers (1.03 to 1.05x from L5 to L13), the histogram counted in two sets, the FSE writer of the tree's weights promoted to registers. Over libzstd, from efc0ec6: silesia 0.80-1.02x to 0.97-1.07x, zstd's github records 0.85-0.97x to 0.96-1.20x, with their dictionary 0.69-1.17x to 0.97-1.19x. The tables by reference, the hash chains' order, the sort, the histogram and the FSE writer are shared with Arm64; the rest is x64's alone. Every frame is still libzstd's, byte for byte. --- src/Vorticity.Zstd/Internal/BitWriter.cs | 2 + .../DoubleFastMatchFinder.AttachedX64.cs | 354 ++++++++++++++++++ .../DoubleFastMatchFinder.Dictionary.cs | 8 +- .../Internal/DoubleFastMatchFinder.cs | 26 +- .../Internal/FastMatchFinder.cs | 19 +- src/Vorticity.Zstd/Internal/FseEncoder.cs | 9 - src/Vorticity.Zstd/Internal/Histogram.cs | 65 +++- src/Vorticity.Zstd/Internal/HuffmanEncoder.cs | 62 +-- .../Internal/LazyMatchFinder.cs | 134 +++++-- .../Internal/LongDistanceMatcher.cs | 11 +- src/Vorticity.Zstd/Internal/MatchFinder.cs | 9 + .../Internal/OptimalMatchFinder.cs | 17 +- .../Internal/SequenceEncoder.cs | 217 ++++++++++- src/Vorticity.Zstd/Internal/SequenceStore.cs | 114 +++++- .../ZstdCompressor.Benchmarks.cs | 2 +- src/Vorticity.Zstd/ZstdCompressor.Block.cs | 16 +- src/Vorticity.Zstd/ZstdCompressor.Split.cs | 20 +- .../Compression/EncoderUnitTests.cs | 3 +- 18 files changed, 942 insertions(+), 146 deletions(-) create mode 100644 src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs diff --git a/src/Vorticity.Zstd/Internal/BitWriter.cs b/src/Vorticity.Zstd/Internal/BitWriter.cs index 3c7e725af..b7b450a38 100644 --- a/src/Vorticity.Zstd/Internal/BitWriter.cs +++ b/src/Vorticity.Zstd/Internal/BitWriter.cs @@ -22,6 +22,7 @@ internal unsafe struct BitWriter private readonly byte* _end; /// libzstd's BIT_initCStream; is false for a buffer of eight bytes or less. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public BitWriter(byte* start, nuint capacity) { _start = start; @@ -70,6 +71,7 @@ public void Flush() /// libzstd's BIT_closeCStream: the end marker, then the last bytes. /// The size of the stream, or 0 when it did not fit. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public nuint Close() { AddBitsFast(1, 1); diff --git a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs new file mode 100644 index 000000000..ae2eb7442 --- /dev/null +++ b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs @@ -0,0 +1,354 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using static Vorticity.Zstd.Internal.MatchFinder; + +namespace Vorticity.Zstd.Internal; + +/// The double-fast search with an attached dictionary, as x64 wants it. +internal static unsafe partial class DoubleFastMatchFinder +{ + /// + /// What the x64 search with an attached dictionary keeps out of registers, in a local whose address + /// is taken: read as the operands of its instructions, or once a match. The first fields are copies + /// of the search's own values, which it takes back after each match (see ). + /// + private struct AttachedCold + { + public byte* Base; + public uint* HashLong; + public uint* HashSmall; + public uint* DictHashLong; + public uint* DictHashSmall; + public int Shifts; + + public byte* Anchor; + public uint Offset1; + public uint Offset2; + public uint PrefixLowestIndex; + + /// The prefix's lowest index less one: as one subtraction. + public uint OverlapLimit; + public uint DictStartIndex; + public uint DictIndexDelta; + public uint Current; + public byte* Ilimit; + public byte* End; + public byte* PrefixLowest; + + /// The dictionary's base less the delta of its indices: a frame index's address there. + public byte* DictShifted; + public byte* DictBase; + public byte* DictStart; + public byte* DictEnd; + public byte* Lit; + public SequenceRecord* Sequence; + public uint* Counts; + } + + /// As , for . + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Expose(AttachedCold* cold) + { + } + + /// + /// on x64: the same search, the same sequences, with the + /// values its loop holds cut down to what x64 has registers for. + /// + /// + /// + /// libzstd's form, which Arm64 keeps, holds some twenty values across the loop, among them the + /// four tables, their four hash shifts, both segments' bounds, the repeat offsets and the cursors + /// of the store, live across the calls of the matches that cross the segments: x64, with 15 + /// registers and 8 kept across a call, spilled them all, the hashes and the entries read + /// included, a 440-byte frame whose stores and reloads made each position's probes twice as + /// long, the misses of the dictionary's tables less overlapped. + /// + /// + /// Here the search holds the cursor, the base and the four tables, and the four shifts in one + /// register, rotated to each. The rest is in , the anchor, the repeat + /// offset and the bounds read as operands. After a match the code reads only + /// , and the search's values are taken back from it before the next + /// position: none is live across a call. A dictionary entry is read with its tag's difference + /// from the hash's in its low byte: one value for the tag test and the index. + /// + /// + private static nuint CompressBlockAttachedX64(ref MatchState state, SequenceStore store, uint* rep, byte* source, nuint size) + where THash : IMatchHash + { + AttachedCold cold = default; + Expose(&cold); + + CompressionParameters parameters = state.Parameters; + MatchState* dictionary = state.Dictionary; + byte* @base = state.Base; + uint* hashLong = state.HashTable; + uint* hashSmall = state.ChainTable; + uint* dictHashLong = dictionary->HashTable; + uint* dictHashSmall = dictionary->ChainTable; + uint endIndex = (uint)(source - @base) + (uint)size; + uint prefixLowestIndex = state.LowestPrefixIndex(endIndex); + byte* dictBase = dictionary->Base; + + // The shifts of the frame's long hash, the dictionary's (tagged), the frame's short hash, the + // dictionary's, a byte each: a shift takes the low 6 bits of its count. + int shifts = (64 - parameters.HashLog) + | ((64 - (dictionary->Parameters.HashLog + TagBits)) << 8) + | ((64 - parameters.ChainLog) << 16) + | ((64 - (dictionary->Parameters.ChainLog + TagBits)) << 24); + + cold.Base = @base; + cold.HashLong = hashLong; + cold.HashSmall = hashSmall; + cold.DictHashLong = dictHashLong; + cold.DictHashSmall = dictHashSmall; + cold.Shifts = shifts; + cold.Offset1 = rep[0]; + cold.Offset2 = rep[1]; + cold.PrefixLowestIndex = prefixLowestIndex; + cold.OverlapLimit = prefixLowestIndex - 1; + cold.DictStartIndex = dictionary->DictLimit; + cold.DictIndexDelta = prefixLowestIndex - dictionary->End; + cold.End = source + size; + cold.Ilimit = cold.End - HashReadSize; + cold.PrefixLowest = @base + prefixLowestIndex; + cold.DictShifted = dictBase - (nuint)cold.DictIndexDelta; + cold.DictBase = dictBase; + cold.DictStart = dictBase + cold.DictStartIndex; + cold.DictEnd = dictBase + dictionary->End; + cold.Lit = store.Literals; + cold.Sequence = store.Sequences; + cold.Counts = store.Counts; + + byte* ip = source; + cold.Anchor = source; + uint dictAndPrefixLength = (uint)((ip - cold.PrefixLowest) + (cold.DictEnd - cold.DictStart)); + ip += dictAndPrefixLength == 0 ? 1 : 0; + + while (ip < cold.Ilimit) + { + nuint matchLength; + uint offset; + uint offBase; + byte* match; + byte* low; + + // The dictionary's entries first, the likeliest misses. + ulong bytes = Read64(ip); + ulong productLong = bytes * Hash8.Multiplier; + nuint dictHashAndTagLong = (nuint)(productLong >> (int)BitOperations.RotateRight((uint)shifts, 8)); + uint dictEntryLong = dictHashLong[dictHashAndTagLong >> TagBits] ^ ((uint)dictHashAndTagLong & TagMask); + nuint hashLongIndex = (nuint)(productLong >> shifts); + ulong productSmall = bytes * THash.Multiplier; + nuint dictHashAndTagSmall = (nuint)(productSmall >> (int)BitOperations.RotateRight((uint)shifts, 24)); + uint dictEntrySmall = dictHashSmall[dictHashAndTagSmall >> TagBits] ^ ((uint)dictHashAndTagSmall & TagMask); + nuint hashSmallIndex = (nuint)(productSmall >> (int)BitOperations.RotateRight((uint)shifts, 16)); + uint current = (uint)(ip - @base); + uint indexLong = hashLong[hashLongIndex]; + uint indexSmall = hashSmall[hashSmallIndex]; + hashLong[hashLongIndex] = hashSmall[hashSmallIndex] = current; + + // The first repeat offset at ip + 1. + uint repIndex = current + 1 - cold.Offset1; + if (cold.OverlapLimit - repIndex >= 3) + { + byte* repMatch = repIndex < cold.PrefixLowestIndex ? cold.DictShifted + repIndex : @base + repIndex; + if (Read32(repMatch) == Read32(ip + 1)) + { + cold.Current = current; + byte* repMatchEnd = repIndex < cold.PrefixLowestIndex ? cold.DictEnd : cold.End; + matchLength = CountAcross(ip + 1 + 4, repMatch + 4, cold.End, repMatchEnd, cold.PrefixLowest) + 4; + ip++; + offBase = RepeatCode1; + goto Store; + } + } + + if (indexLong >= cold.PrefixLowestIndex && Read64(@base + indexLong) == bytes) + { + // A long match in the prefix. + cold.Current = current; + match = @base + indexLong; + matchLength = Count(ip + 8, match + 8, cold.End) + 8; + offset = (uint)(ip - match); + low = cold.PrefixLowest; + goto ExtendBackward; + } + + if ((dictEntryLong & TagMask) == 0) + { + // A long match in the dictionary. + uint dictIndexLong = dictEntryLong >> TagBits; + match = cold.DictBase + dictIndexLong; + if (dictIndexLong > cold.DictStartIndex && Read64(match) == bytes) + { + cold.Current = current; + offset = current - dictIndexLong - cold.DictIndexDelta; + matchLength = CountAcross(ip + 8, match + 8, cold.End, cold.DictEnd, cold.PrefixLowest) + 8; + low = cold.DictStart; + goto ExtendBackward; + } + } + + if (indexSmall > cold.PrefixLowestIndex) + { + // A short match in the prefix. + match = @base + indexSmall; + if (Read32(match) == (uint)bytes) + { + goto SearchNextLong; + } + } + else if ((dictEntrySmall & TagMask) == 0) + { + // A short match in the dictionary. + uint dictIndexSmall = dictEntrySmall >> TagBits; + match = cold.DictBase + dictIndexSmall; + indexSmall = dictIndexSmall + cold.DictIndexDelta; + if (dictIndexSmall > cold.DictStartIndex && Read32(match) == (uint)bytes) + { + goto SearchNextLong; + } + } + + ip += ((nuint)(ip - cold.Anchor) >> SearchStrength) + 1; + continue; + + SearchNextLong: + cold.Current = current; + { + ulong nextBytes = Read64(ip + 1); + ulong nextProduct = nextBytes * Hash8.Multiplier; + nuint dictHashAndTagLong3 = (nuint)(nextProduct >> (int)BitOperations.RotateRight((uint)shifts, 8)); + uint dictEntryLong3 = dictHashLong[dictHashAndTagLong3 >> TagBits] ^ ((uint)dictHashAndTagLong3 & TagMask); + nuint hashLongIndex3 = (nuint)(nextProduct >> shifts); + uint indexLong3 = hashLong[hashLongIndex3]; + hashLong[hashLongIndex3] = current + 1; + + if (indexLong3 >= cold.PrefixLowestIndex && Read64(@base + indexLong3) == nextBytes) + { + // A long match at ip + 1 in the prefix. + match = @base + indexLong3; + matchLength = Count(ip + 9, match + 8, cold.End) + 8; + ip++; + offset = (uint)(ip - match); + low = cold.PrefixLowest; + goto ExtendBackward; + } + + if ((dictEntryLong3 & TagMask) == 0) + { + // A long match at ip + 1 in the dictionary. + uint dictIndexLong3 = dictEntryLong3 >> TagBits; + byte* dictMatchLong3 = cold.DictBase + dictIndexLong3; + if (dictIndexLong3 > cold.DictStartIndex && Read64(dictMatchLong3) == nextBytes) + { + offset = current + 1 - dictIndexLong3 - cold.DictIndexDelta; + matchLength = CountAcross(ip + 1 + 8, dictMatchLong3 + 8, cold.End, cold.DictEnd, cold.PrefixLowest) + 8; + ip++; + match = dictMatchLong3; + low = cold.DictStart; + goto ExtendBackward; + } + } + } + + // No long match at ip + 1: the short match found. + if (indexSmall < cold.PrefixLowestIndex) + { + offset = current - indexSmall; + matchLength = CountAcross(ip + 4, match + 4, cold.End, cold.DictEnd, cold.PrefixLowest) + 4; + low = cold.DictStart; + } + else + { + matchLength = Count(ip + 4, match + 4, cold.End) + 4; + offset = (uint)(ip - match); + low = cold.PrefixLowest; + } + + ExtendBackward: + { + byte* anchor = cold.Anchor; + while (ip > anchor && match > low && ip[-1] == match[-1]) + { + ip--; + match--; + matchLength++; + } + } + + cold.Offset2 = cold.Offset1; + cold.Offset1 = offset; + offBase = OffsetToOffBase(offset); + + Store: + { + byte* anchor = cold.Anchor; + byte* lit = cold.Lit; + SequenceRecord* sequence = cold.Sequence; + SequenceStore.Store(ref lit, ref sequence, cold.Counts, (nuint)(ip - anchor), anchor, cold.End, offBase, matchLength); + cold.Lit = lit; + cold.Sequence = sequence; + } + + ip += matchLength; + cold.Anchor = ip; + if (ip <= cold.Ilimit) + { + // Complementary insertion, after the limit test: the positions could pass the end - 8. + byte* b = cold.Base; + uint* tableLong = cold.HashLong; + uint* tableSmall = cold.HashSmall; + int shiftLong = cold.Shifts; + int shiftSmall = (int)BitOperations.RotateRight((uint)shiftLong, 16); + uint indexToInsert = cold.Current + 2; + ulong insertBytes = Read64(b + indexToInsert); + tableLong[(nuint)((insertBytes * Hash8.Multiplier) >> shiftLong)] = indexToInsert; + tableLong[(nuint)((Read64(ip - 2) * Hash8.Multiplier) >> shiftLong)] = (uint)(ip - 2 - b); + tableSmall[(nuint)((insertBytes * THash.Multiplier) >> shiftSmall)] = indexToInsert; + tableSmall[(nuint)((Read64(ip - 1) * THash.Multiplier) >> shiftSmall)] = (uint)(ip - 1 - b); + + // The second repeat offset, at once. + while (ip <= cold.Ilimit) + { + uint current2 = (uint)(ip - cold.Base); + uint offset2 = cold.Offset2; + uint repIndex2 = current2 - offset2; + bool inDictionary = repIndex2 < cold.PrefixLowestIndex; + byte* repMatch2 = (inDictionary ? cold.DictShifted : cold.Base) + repIndex2; + if (cold.OverlapLimit - repIndex2 < 3 || Read32(repMatch2) != Read32(ip)) + { + break; + } + + nuint repLength2 = CountAcross(ip + 4, repMatch2 + 4, cold.End, inDictionary ? cold.DictEnd : cold.End, cold.PrefixLowest) + 4; + cold.Offset2 = cold.Offset1; + cold.Offset1 = offset2; + SequenceRecord* sequence = cold.Sequence; + SequenceStore.StoreOnly(ref sequence, cold.Counts, 0, RepeatCode1, repLength2); + cold.Sequence = sequence; + ulong repBytes = Read64(ip); + cold.HashSmall[(nuint)((repBytes * THash.Multiplier) >> (int)BitOperations.RotateRight((uint)cold.Shifts, 16))] = current2; + cold.HashLong[(nuint)((repBytes * Hash8.Multiplier) >> cold.Shifts)] = current2; + ip += repLength2; + cold.Anchor = ip; + } + } + + // The search's values, taken back: none was live across the calls. + @base = cold.Base; + hashLong = cold.HashLong; + hashSmall = cold.HashSmall; + dictHashLong = cold.DictHashLong; + dictHashSmall = cold.DictHashSmall; + shifts = cold.Shifts; + } + + store.Literals = cold.Lit; + store.Sequences = cold.Sequence; + rep[0] = cold.Offset1; + rep[1] = cold.Offset2; + return (nuint)(cold.End - cold.Anchor); + } +} diff --git a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs index 2d74ee8d5..1b6816a3e 100644 --- a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs +++ b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -76,11 +77,16 @@ public static nuint CompressBlockAttached(ref MatchState state, SequenceStore st /// /// libzstd's ZSTD_compressBlock_doubleFast_dictMatchState_generic: the frame's tables first, /// the dictionary's (its own parameters, tagged indices) where they have nothing in the prefix; - /// the dictionary's indices sit just below the prefix's. + /// the dictionary's indices sit just below the prefix's. On x64, . /// private static nuint CompressBlockAttached(ref MatchState state, SequenceStore store, uint* rep, byte* source, nuint size) where THash : IMatchHash { + if (X86Base.IsSupported) + { + return CompressBlockAttachedX64(ref state, store, rep, source, size); + } + CompressionParameters parameters = state.Parameters; uint* hashLong = state.HashTable; int hashLogLong = parameters.HashLog; diff --git a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs index 0d65e7a5b..0744e1204 100644 --- a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -81,8 +82,10 @@ private struct Cold /// libzstd tests a candidate as read(safe) == read(ip) && safe == candidate, the /// safe address dummy bytes for a candidate below the window: the second test is the window test. /// Here a candidate below the window is read at the window's start instead, which is in the - /// source, and the two tests are one condition, combined with & into one chain of - /// compares: no select, no branch on the window. + /// source, and the two tests are one condition: on Arm64 combined with & into one chain + /// of compares, no select, no branch on the window; on x64, which has no conditional compare and + /// would combine them with setcc, two compares and branches that it fuses, the window's + /// taken only when the bytes match, as libzstd's. /// /// /// The values the search loop uses are locals; those it does not are in , read @@ -101,6 +104,7 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st uint endIndex = (uint)(istart - @base + (nint)size); nuint prefixLowestIndex = state.LowestPrefixIndex(endIndex); byte* ilimit = istart + size - HashReadSize; + byte* windowStart = @base + prefixLowestIndex; Cold cold = default; Expose(&cold); @@ -175,7 +179,9 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st // when the candidate is ip itself: compared with ip, it is not a test the JIT hoists and // keeps in a register. byte* repeat = ip - offset1; - if ((Read32(repeat + 1) == (uint)(ipBytes >> 8)) & (repeat != ip)) + if (X86Base.IsSupported + ? Read32(repeat + 1) == (uint)(ipBytes >> 8) && repeat != ip + : (Read32(repeat + 1) == (uint)(ipBytes >> 8)) & (repeat != ip)) { cold.Current = current; matchLength = Count(ip + 1 + 4, repeat + 1 + 4, cold.End) + 4; @@ -193,8 +199,11 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st hashLong1 = Hash8.Hash(ip1Bytes, hashLogLong); // A long match at ip. A candidate below the window is read at its start: see the remarks. - match = @base + ClampToWindow(indexLong0, prefixLowestIndex); - if ((Read64(match) == ipBytes) & (indexLong0 >= prefixLowestIndex)) + nint distanceLong0 = (nint)(indexLong0 - prefixLowestIndex); + match = X86Base.IsSupported ? InWindow(windowStart, distanceLong0) : @base + ClampToWindow(indexLong0, prefixLowestIndex); + if (X86Base.IsSupported + ? Read64(match) == ipBytes && distanceLong0 >= 0 + : (Read64(match) == ipBytes) & (indexLong0 >= prefixLowestIndex)) { cold.Current = current; matchLength = Count(ip + 8, match + 8, cold.End) + 8; @@ -205,8 +214,11 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st indexLong1 = hashLong[hashLong1]; // A short match at ip. - match = @base + ClampToWindow(indexShort0, prefixLowestIndex); - if ((Read32(match) == (uint)ipBytes) & (indexShort0 >= prefixLowestIndex)) + nint distanceShort0 = (nint)(indexShort0 - prefixLowestIndex); + match = X86Base.IsSupported ? InWindow(windowStart, distanceShort0) : @base + ClampToWindow(indexShort0, prefixLowestIndex); + if (X86Base.IsSupported + ? Read32(match) == (uint)ipBytes && distanceShort0 >= 0 + : (Read32(match) == (uint)ipBytes) & (indexShort0 >= prefixLowestIndex)) { cold.Current = current; goto SearchNextLong; diff --git a/src/Vorticity.Zstd/Internal/FastMatchFinder.cs b/src/Vorticity.Zstd/Internal/FastMatchFinder.cs index df0a016bb..f32b52ed4 100644 --- a/src/Vorticity.Zstd/Internal/FastMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/FastMatchFinder.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -161,7 +162,7 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st byte* repeat = ip2 - repOffset1; uint repValue = Read32(repeat); hashTable[hash0] = (uint)(ip0 - @base); - if ((Read32(ip2) == repValue) & (repeat != ip2)) + if (X86Base.IsSupported ? Read32(ip2) == repValue && repeat != ip2 : (Read32(ip2) == repValue) & (repeat != ip2)) { current0 = (uint)(ip0 - @base); @@ -295,9 +296,19 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st /// libzstd's ZSTD_match4Found_cmov: whether the candidate at , /// within the window, starts with the same four bytes. A candidate below the window is read at /// the window's start instead (see ), and the window test - /// joins the compare in one condition. + /// joins the compare in one condition: combined with & on Arm64, one chain of compares; + /// on x64, which would combine them with setcc, two compares and branches, which it fuses. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool MatchFound(byte* current, byte* @base, nuint matchIndex, nuint lowLimit) => - (Read32(current) == Read32(@base + ClampToWindow(matchIndex, lowLimit))) & (matchIndex >= lowLimit); + private static bool MatchFound(byte* current, byte* @base, nuint matchIndex, nuint lowLimit) + { + if (X86Base.IsSupported) + { + // The clamp as x64 takes it best: see MatchFinder.InWindow. + nint distance = (nint)(matchIndex - lowLimit); + return Read32(current) == Read32(InWindow(@base + lowLimit, distance)) && distance >= 0; + } + + return (Read32(current) == Read32(@base + ClampToWindow(matchIndex, lowLimit))) & (matchIndex >= lowLimit); + } } diff --git a/src/Vorticity.Zstd/Internal/FseEncoder.cs b/src/Vorticity.Zstd/Internal/FseEncoder.cs index e1318cd2b..08812efc2 100644 --- a/src/Vorticity.Zstd/Internal/FseEncoder.cs +++ b/src/Vorticity.Zstd/Internal/FseEncoder.cs @@ -39,15 +39,6 @@ public FseCTable(int maxTableLog, int maxSymbolValue) public ushort* StateTable { get; } public FseSymbolTransform* Symbols { get; } - - /// libzstd's memcpy of a table: the parts the table uses. - public void CopyFrom(FseCTable other) - { - TableLog = other.TableLog; - MaxSymbolValue = other.MaxSymbolValue; - Unsafe.CopyBlockUnaligned(StateTable, other.StateTable, (uint)((1 << other.TableLog) * sizeof(ushort))); - Unsafe.CopyBlockUnaligned(Symbols, other.Symbols, (uint)((other.MaxSymbolValue + 1) * sizeof(FseSymbolTransform))); - } } /// An encoding state: libzstd's FSE_CState_t. diff --git a/src/Vorticity.Zstd/Internal/Histogram.cs b/src/Vorticity.Zstd/Internal/Histogram.cs index e44cec218..50a46a98a 100644 --- a/src/Vorticity.Zstd/Internal/Histogram.cs +++ b/src/Vorticity.Zstd/Internal/Histogram.cs @@ -12,6 +12,12 @@ internal static unsafe class Histogram /// , whose first + 1 counters it clears. /// On return is the largest byte present (0 for no byte). /// + /// + /// From 64 bytes, the bytes at even and odd places are counted apart, then added: a byte's count + /// waits on its last increment, which a frequent byte made a chain of, half as long so. The + /// largest count is taken four at a time as they are added: a maximum in a loop is a branch on + /// each count, which the counts make unpredictable. + /// /// The largest count. public static uint CountSimple(uint* count, ref uint maxSymbolValue, byte* source, nuint size) { @@ -24,9 +30,64 @@ public static uint CountSimple(uint* count, ref uint maxSymbolValue, byte* sourc } byte* end = source + size; - for (byte* p = source; p < end; p++) + byte* p = source; + if (size < 64) { - count[*p]++; + for (; p < end; p++) + { + count[*p]++; + } + } + else + { + // Rounded up to whole vectors, which the sum reads. + uint* odd = stackalloc uint[256]; + uint rounded = (max + 4) & ~3u; + Unsafe.InitBlockUnaligned(odd, 0, rounded * sizeof(uint)); + for (; p + 1 < end; p += 2) + { + count[p[0]]++; + odd[p[1]]++; + } + + if (p < end) + { + count[*p]++; + } + + if (Vector128.IsHardwareAccelerated) + { + Vector128 largestLanes = Vector128.Zero; + uint s = 0; + for (; s + 4 <= max + 1; s += 4) + { + Vector128 total = Vector128.Load(count + s) + Vector128.Load(odd + s); + total.Store(count + s); + largestLanes = Vector128.Max(largestLanes, total); + } + + uint largestCount = Math.Max( + Math.Max(largestLanes.GetElement(0), largestLanes.GetElement(1)), + Math.Max(largestLanes.GetElement(2), largestLanes.GetElement(3))); + for (; s <= max; s++) + { + count[s] += odd[s]; + largestCount = Math.Max(largestCount, count[s]); + } + + while (count[max] == 0) + { + max--; + } + + maxSymbolValue = max; + return largestCount; + } + + for (uint s = 0; s <= max; s++) + { + count[s] += odd[s]; + } } while (count[max] == 0) diff --git a/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs b/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs index 6225499a5..6e1fc7b97 100644 --- a/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs +++ b/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs @@ -38,13 +38,6 @@ public HuffmanCTable() public int MaxSymbolValue; - public void CopyFrom(HuffmanCTable other) - { - TableLog = other.TableLog; - MaxSymbolValue = other.MaxSymbolValue; - Unsafe.CopyBlockUnaligned(Elements, other.Elements, HuffmanTable.MaxSymbols * sizeof(ulong)); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int NbBits(ulong element) => (int)(element & 0xFF); } @@ -58,7 +51,7 @@ internal sealed unsafe class HuffmanWorkspace /// public const int StreamCapacity = (((FrameFormat.MaxBlockSize / 4) * HuffmanTable.MaxTableLog) / 8) + 64; - /// libzstd's table->CTable: the new tree. + /// The trees the search of the optimal depth tries; the new tree is built into a block state's table. public readonly HuffmanCTable Table = new(); /// The encoding table of the tree's weights. @@ -123,30 +116,51 @@ private static uint BucketIndex(uint count) => /// libzstd's HUF_sort: the symbols by decreasing count, equal counts by increasing symbol /// except in the buckets the quicksort orders. /// + /// + /// The running totals and the symbols without count are in registers: in the table, each total + /// was a load of the entry just stored, and each symbol without count an increment of the same + /// entry, one store forwarded to the next load after another (clang keeps libzstd's totals in a + /// register; the JIT does not). + /// private static void Sort(Node* nodes, uint* count, uint maxSymbolValue, RankPosition* rankPosition) { uint maxSymbolValue1 = maxSymbolValue + 1; Unsafe.InitBlockUnaligned(rankPosition, 0, (uint)(sizeof(RankPosition) * RankPositionTableSize)); + uint zeros = 0; for (uint n = 0; n < maxSymbolValue1; n++) { - rankPosition[BucketIndex(count[n])].Base++; + uint c = count[n]; + if (c == 0) + { + zeros++; + } + else + { + rankPosition[BucketIndex(c)].Base++; + } } + rankPosition[0].Base = (ushort)zeros; + uint total = rankPosition[RankPositionTableSize - 1].Base; for (int n = RankPositionTableSize - 1; n > 0; n--) { - rankPosition[n - 1].Base += rankPosition[n].Base; - rankPosition[n - 1].Current = rankPosition[n - 1].Base; + total += rankPosition[n - 1].Base; + rankPosition[n - 1].Base = (ushort)total; + rankPosition[n - 1].Current = (ushort)total; } + // The symbols without count go to bucket 0's place, rankPosition[1], in order. + uint zeroPosition = rankPosition[1].Current; for (uint n = 0; n < maxSymbolValue1; n++) { uint c = count[n]; - uint r = BucketIndex(c) + 1; - uint position = rankPosition[r].Current++; + uint position = c == 0 ? zeroPosition++ : rankPosition[BucketIndex(c) + 1].Current++; nodes[position].Count = c; nodes[position].Symbol = (byte)n; } + rankPosition[1].Current = (ushort)zeroPosition; + for (int n = RankPositionDistinctCountCutoff; n < RankPositionTableSize - 1; n++) { int bucketSize = rankPosition[n].Current - rankPosition[n].Base; @@ -955,15 +969,15 @@ private static nuint CompressWithTable( /// Huffman-coded with the previous tree or a new one, whichever the heuristics prefer. /// /// - /// is nextHuf->CTable, a copy of the previous block's tree, - /// replaced by the new one when that one is used. says on entry whether - /// the previous tree may serve, and on return whether it did ( - /// otherwise). + /// is the previous block's tree. A new one is built into + /// , a table the previous state does not use, which the block's state + /// takes when the new tree is used. says on entry whether the previous + /// tree may serve, and on return whether it did ( otherwise). /// /// The size of the section after the literals header, 0 when not compressible, 1 for one repeated byte. public static nuint Compress( - byte* destination, nuint capacity, byte* source, nuint size, bool singleStream, HuffmanCTable previous, HuffmanWorkspace workspace, - ref HuffmanRepeat repeat, bool preferRepeat, bool optimalDepth, bool suspectUncompressible) + byte* destination, nuint capacity, byte* source, nuint size, bool singleStream, HuffmanCTable previous, HuffmanCTable fresh, + HuffmanWorkspace workspace, ref HuffmanRepeat repeat, bool preferRepeat, bool optimalDepth, bool suspectUncompressible) { const int SuspectIncompressibleSampleSize = 4096; const int SuspectIncompressibleSampleRatio = 10; @@ -1016,11 +1030,10 @@ public static nuint Compress( return CompressWithTable(destination, op, end, source, size, singleStream, previous, workspace); } - HuffmanCTable table = workspace.Table; uint huffLog = OptimalTableLog(LiteralsTableLog, size, maxSymbolValue, workspace, count, optimalDepth); - huffLog = BuildCTable(table, count, maxSymbolValue, huffLog); + huffLog = BuildCTable(fresh, count, maxSymbolValue, huffLog); - nuint headerSize = WriteCTable(op, capacity, table, maxSymbolValue, huffLog, workspace.Weights); + nuint headerSize = WriteCTable(op, capacity, fresh, maxSymbolValue, huffLog, workspace.Weights); if (headerSize == 0) { // libzstd's error, which its caller turns into raw literals. @@ -1031,7 +1044,7 @@ public static nuint Compress( { // The previous tree, unless the new one saves more than its description. nuint oldSize = EstimateCompressedSize(previous, count, maxSymbolValue); - nuint newSize = EstimateCompressedSize(table, count, maxSymbolValue); + nuint newSize = EstimateCompressedSize(fresh, count, maxSymbolValue); if (oldSize <= headerSize + newSize || headerSize + 12 >= size) { return CompressWithTable(destination, op, end, source, size, singleStream, previous, workspace); @@ -1045,7 +1058,6 @@ public static nuint Compress( op += headerSize; repeat = HuffmanRepeat.None; - previous.CopyFrom(table); - return CompressWithTable(destination, op, end, source, size, singleStream, table, workspace); + return CompressWithTable(destination, op, end, source, size, singleStream, fresh, workspace); } } diff --git a/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs b/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs index 92a77e869..aa910578c 100644 --- a/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -215,7 +216,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint repLength = ExtDictRepeatLength(ref state, ip, current, offset1, dictBase, dictEnd, iend); int gain2 = (int)(repLength * 3); int gain1 = (int)(((nint)matchLength * 3) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -228,7 +229,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 4); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -245,7 +246,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint repLength = ExtDictRepeatLength(ref state, ip, current, offset1, dictBase, dictEnd, iend); int gain2 = (int)(repLength * 4); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -258,7 +259,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 7); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -428,7 +429,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen } } } - else if ((offset1 > 0) & (Read32(ip + 1 - offset1) == Read32(ip + 1))) + else if (X86Base.IsSupported ? offset1 > 0 && Read32(ip + 1 - offset1) == Read32(ip + 1) : (offset1 > 0) & (Read32(ip + 1 - offset1) == Read32(ip + 1))) { matchLength = Count(ip + 1 + 4, ip + 1 + 4 - offset1, iend) + 4; if (TDepth.Depth == 0) @@ -470,19 +471,19 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint repLength = AttachedRepeatLength(ip, offset1, @base, prefixLowestIndex, dictBase, dictIndexDelta, dictEnd, iend); int gain2 = (int)(repLength * 3); int gain1 = (int)(((nint)matchLength * 3) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; start = ip; } } - else if ((offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) + else if (X86Base.IsSupported ? offset1 > 0 && Read32(ip) == Read32(ip - offset1) : (offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) { nuint repLength = Count(ip + 4, ip + 4 - offset1, iend) + 4; int gain2 = (int)(repLength * 3); int gain1 = (int)(((nint)matchLength * 3) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -495,7 +496,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 4); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -512,19 +513,19 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint repLength = AttachedRepeatLength(ip, offset1, @base, prefixLowestIndex, dictBase, dictIndexDelta, dictEnd, iend); int gain2 = (int)(repLength * 4); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; start = ip; } } - else if ((offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) + else if (X86Base.IsSupported ? offset1 > 0 && Read32(ip) == Read32(ip - offset1) : (offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) { nuint repLength = Count(ip + 4, ip + 4 - offset1, iend) + 4; int gain2 = (int)(repLength * 4); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -537,7 +538,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 7); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -570,7 +571,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen } else { - while (((start > anchor) & (start - offset > prefixLowest)) && start[-1] == (start - offset)[-1]) + while ((X86Base.IsSupported ? start > anchor && start - offset > prefixLowest : (start > anchor) & (start - offset > prefixLowest)) && start[-1] == (start - offset)[-1]) { start--; matchLength++; @@ -614,7 +615,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen } else { - while (((ip <= ilimit) & (offset2 > 0)) && Read32(ip) == Read32(ip - offset2)) + while ((X86Base.IsSupported ? ip <= ilimit && offset2 > 0 : (ip <= ilimit) & (offset2 > 0)) && Read32(ip) == Read32(ip - offset2)) { matchLength = Count(ip + 4, ip + 4 - offset2, iend) + 4; (offset1, offset2) = (offset2, offset1); @@ -733,6 +734,10 @@ public static void FillHashCache(ref MatchState state, uint index, byte* limit) [MethodImpl(MethodImplOptions.NoInlining)] public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref nuint offBase) { + // The insertion first, then the search's values: computed before it, they were live across + // its loop, which x64 has not the registers for, and spilled, the loop's own with them. + nuint matchIndex = InsertAndFindFirstIndex(ref state, ip); + uint* chainTable = state.ChainTable; uint chainSize = 1u << state.Parameters.ChainLog; nuint chainMask = chainSize - 1; @@ -750,9 +755,8 @@ public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref byte* dictBase = state.DictionaryBase; byte* dictEnd = dictBase + dictLimit; - nuint matchIndex = InsertAndFindFirstIndex(ref state, ip); nuint bestLength = 4 - 1; - for (; (matchIndex >= lowLimit) & (attempts > 0); attempts--) + for (; X86Base.IsSupported ? matchIndex >= lowLimit && attempts > 0 : (matchIndex >= lowLimit) & (attempts > 0); attempts--) { if (TDictionary.Mode != ExtDictionary.Value || matchIndex >= dictLimit) { @@ -870,16 +874,25 @@ internal static uint InsertAndFindFirstIndex(ref MatchState state, byte* ip) byte* @base = state.Base; uint target = (uint)(ip - @base); uint index = state.NextToUpdate; - bool skipping = state.LazySkipping; - while (index < target) + if (state.LazySkipping) { - nuint hash = THash.Hash(Read64(@base + index), hashLog); - chainTable[index & chainMask] = hashTable[hash]; - hashTable[hash] = index; - index++; - if (skipping) + // Only the first position while skipping. + if (index < target) { - break; + nuint hash = THash.Hash(Read64(@base + index), hashLog); + chainTable[index & chainMask] = hashTable[hash]; + hashTable[hash] = index; + } + } + else + { + // The skipping test out of the loop, which then holds one value less: on x64, short of + // registers in the search it is inlined into, the tables and the mask came from memory. + for (; index < target; index++) + { + nuint hash = THash.Hash(Read64(@base + index), hashLog); + chainTable[index & chainMask] = hashTable[hash]; + hashTable[hash] = index; } } @@ -1308,9 +1321,9 @@ internal interface IRowLog /// /// /// libzstd prefetches the rows of the position eight ahead, whose hash its cache computes, and the -/// candidates before it compares them. .NET has no prefetch on arm64. The row of entries is brought -/// in by a store instead: of a 0 to its place 0, which no entry takes and nothing reads, in the line -/// the position writes once inserted. A store retires as a prefetch does, before its line arrives, +/// candidates before it compares them. So does x64 (see PrefetchRow); .NET has no prefetch on +/// Arm64. There, the row of entries is brought in by a store instead: of a 0 to its place 0, which no +/// entry takes and nothing reads, in the line the position writes once inserted. A store retires as a prefetch does, before its line arrives, /// where a load holds up the retirement of everything after it until its line is there. The row of /// tags has no such place (its first byte is the head, its others are tags, all read by the search, /// which a store still on its way would hold up): it is read, as are the candidates, summed into @@ -1372,7 +1385,7 @@ public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref MatchState* dictionary = state.Dictionary; dictionaryHash = (nuint)((Read64(ip) * state.RowHashMultiplier) >> (64 - (dictionary->RowHashLog + LazyMatchFinder.RowHashTagBits))); nuint dictionaryRow = RowOf(dictionaryHash); - read += dictionary->TagTable[dictionaryRow] + dictionary->HashTable[dictionaryRow]; + PrefetchRow(dictionary->TagTable, dictionary->HashTable, dictionaryRow, ref read, write: false); } // libzstd's ZSTD_row_update_internal: the positions before ip into their rows, then ip's @@ -1437,7 +1450,7 @@ public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref candidates[count] = matchIndex; count++; - read += (TDictionary.Mode == ExtDictionary.Value && matchIndex < state.DictLimit ? state.DictionaryBase : @base)[matchIndex]; + PrefetchCandidate((TDictionary.Mode == ExtDictionary.Value && matchIndex < state.DictLimit ? state.DictionaryBase : @base) + matchIndex, ref read); if (--attempts == 0) { break; @@ -1539,7 +1552,7 @@ private static nuint SearchDictionary(ref MatchState state, byte* ip, byte* end, break; } - read += dictionaryBase[matchIndex]; + PrefetchCandidate(dictionaryBase + matchIndex, ref read); candidates[count++] = matchIndex; attempts--; } @@ -1584,8 +1597,7 @@ public static void FillHashCache(ref MatchState state, uint index, byte* limit) nuint hash = (nuint)(((Read64(@base + index) * multiplier) ^ salt) >> shift); cache[index & (LazyMatchFinder.RowHashCacheSize - 1)] = (uint)hash; nuint relativeRow = RowOf(hash); - read += state.TagTable[relativeRow]; - state.HashTable[relativeRow] = 0; + PrefetchRow(state.TagTable, state.HashTable, relativeRow, ref read, write: true); } state.Touched += read; @@ -1645,11 +1657,63 @@ private static nuint NextCachedHash( nuint next = (nuint)(((Read64(@base + index + LazyMatchFinder.RowHashCacheSize) * multiplier) ^ salt) >> shift); *slot = (uint)next; nuint nextRow = RowOf(next); - read += tagTable[nextRow]; - hashTable[nextRow] = 0; + PrefetchRow(tagTable, hashTable, nextRow, ref read, write: true); return hash; } + /// + /// Brings a row's entries and tags on their way. On x64, prefetches, as libzstd's + /// ZSTD_row_prefetch: they retire at once and wait on nothing. Without them (Arm64, see the + /// remarks), the row of tags is read, summed into , and the row of entries + /// written at its place 0 when it is the frame's own (), read otherwise. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void PrefetchRow(byte* tagTable, uint* hashTable, nuint relativeRow, ref uint read, bool write) + { + if (Sse.IsSupported) + { + Sse.Prefetch0(hashTable + relativeRow); + if (TRow.Log >= 5) + { + Sse.Prefetch0(hashTable + relativeRow + 16); + } + + Sse.Prefetch0(tagTable + relativeRow); + if (TRow.Log == 6) + { + Sse.Prefetch0(tagTable + relativeRow + 32); + } + + return; + } + + read += tagTable[relativeRow]; + if (write) + { + hashTable[relativeRow] = 0; + } + else + { + read += hashTable[relativeRow]; + } + } + + /// + /// Brings a candidate's first bytes on their way before it is compared: a prefetch on x64, as + /// libzstd's; a load summed into elsewhere. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void PrefetchCandidate(byte* candidate, ref uint read) + { + if (Sse.IsSupported) + { + Sse.Prefetch0(candidate); + return; + } + + read += *candidate; + } + /// /// The first entry of a hash's row: its bits above the tag, times the entries of a row, as one /// shift and one mask. diff --git a/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs b/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs index 6d38d2683..ad95193b9 100644 --- a/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs +++ b/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -470,7 +471,15 @@ private nuint GenerateBlock(byte* istart, nuint size, RawSequence* candidates[n].Split = split; candidates[n].Hash = hash; candidates[n].Checksum = (uint)(xxhash >> 32); - touched += table[(nuint)hash << bucketSizeLog]; + // The bucket on its way: a prefetch on x64, as libzstd's; a read kept in _touched elsewhere. + if (Sse.IsSupported) + { + Sse.Prefetch0(table + ((nuint)hash << bucketSizeLog)); + } + else + { + touched += table[(nuint)hash << bucketSizeLog]; + } } for (uint n = 0; n < splitCount; n++) diff --git a/src/Vorticity.Zstd/Internal/MatchFinder.cs b/src/Vorticity.Zstd/Internal/MatchFinder.cs index c7f9c9fce..9c4b0080a 100644 --- a/src/Vorticity.Zstd/Internal/MatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/MatchFinder.cs @@ -344,6 +344,15 @@ public static nuint ClampToWindow(nuint index, nuint low) return index - (nuint)(below & (below >> 63)); } + /// + /// x64's form of : a candidate's address from the + /// window's start and its signed distance past it, the start when the distance is negative. A + /// shift and a bit clear (andn), where the clamp takes eight instructions on x64, which has no + /// operand shifted in passing as Arm64's and; the distance's sign is then the window test. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte* InWindow(byte* windowStart, nint distance) => windowStart + (distance & ~(distance >> 63)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint Read32(byte* p) => Unsafe.ReadUnaligned(p); diff --git a/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs b/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs index dd39f0785..5b0582727 100644 --- a/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs @@ -2,6 +2,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -1161,7 +1162,9 @@ private static uint GetAllMatches( // current > repIndex >= dictLimit, by an intentional overflow that discards 0 and -1. if (repOffset - 1 < current - dictLimit) { - if ((repIndex >= windowLow) & (ReadMinMatch(ip, minMatch) == ReadMinMatch(ip - repOffset, minMatch))) + if (X86Base.IsSupported + ? repIndex >= windowLow && ReadMinMatch(ip, minMatch) == ReadMinMatch(ip - repOffset, minMatch) + : (repIndex >= windowLow) & (ReadMinMatch(ip, minMatch) == ReadMinMatch(ip - repOffset, minMatch))) { repLength = Count(ip + minMatch, ip + minMatch - repOffset, end) + minMatch; } @@ -1195,7 +1198,7 @@ private static uint GetAllMatches( matches[count].OffBase = repCode - ll0 + 1; matches[count].Length = (uint)repLength; count++; - if ((repLength > sufficientLength) | (ip + repLength == end)) + if (X86Base.IsSupported ? repLength > sufficientLength || ip + repLength == end : (repLength > sufficientLength) | (ip + repLength == end)) { return count; } @@ -1206,7 +1209,9 @@ private static uint GetAllMatches( if ((TLength.MinLength == 3) && (bestLength < 3)) { uint matchIndex3 = InsertAndFindFirstIndexHash3(ref state, ref nextToUpdate3, ip); - if ((matchIndex3 >= matchLow) & (current - matchIndex3 < (1 << 18))) + if (X86Base.IsSupported + ? matchIndex3 >= matchLow && current - matchIndex3 < (1 << 18) + : (matchIndex3 >= matchLow) & (current - matchIndex3 < (1 << 18))) { nuint length = TDictionary.Mode != ExtDictionary.Value || matchIndex3 >= dictLimit ? Count(ip, @base + matchIndex3, end) @@ -1217,7 +1222,7 @@ private static uint GetAllMatches( matches[0].OffBase = OffsetToOffBase(current - matchIndex3); matches[0].Length = (uint)length; count = 1; - if ((length > sufficientLength) | (ip + length == end)) + if (X86Base.IsSupported ? length > sufficientLength || ip + length == end : (length > sufficientLength) | (ip + length == end)) { // The longest possible: ip is not inserted. state.NextToUpdate = current + 1; @@ -1292,7 +1297,7 @@ private static uint WalkTree( matches[count].OffBase = OffsetToOffBase((uint)(current - matchIndex)); matches[count].Length = (uint)matchLength; count++; - if ((matchLength > OptNum) | (ip + matchLength == end)) + if (X86Base.IsSupported ? matchLength > OptNum || ip + matchLength == end : (matchLength > OptNum) | (ip + matchLength == end)) { // Dropped, to keep the tree consistent; the attached dictionary is not searched. if (TDictionary.Mode == AttachedDictionary.Value) @@ -1391,7 +1396,7 @@ private static uint WalkDictionaryTree( matches[count].OffBase = OffsetToOffBase(current - matchIndex); matches[count].Length = (uint)matchLength; count++; - if ((matchLength > OptNum) | (ip + matchLength == end)) + if (X86Base.IsSupported ? matchLength > OptNum || ip + matchLength == end : (matchLength > OptNum) | (ip + matchLength == end)) { break; } diff --git a/src/Vorticity.Zstd/Internal/SequenceEncoder.cs b/src/Vorticity.Zstd/Internal/SequenceEncoder.cs index d1493ff8f..4ad304ae8 100644 --- a/src/Vorticity.Zstd/Internal/SequenceEncoder.cs +++ b/src/Vorticity.Zstd/Internal/SequenceEncoder.cs @@ -3,6 +3,8 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace Vorticity.Zstd.Internal; @@ -32,21 +34,41 @@ internal enum SymbolEncodingType /// libzstd's ZSTD_compressedBlockState_t: what a block hands the next, once it is emitted /// compressed: its entropy tables, their repeat modes, and the repeat offsets. /// +/// +/// The tables are references: a table that carries over from the previous state, a predefined one or +/// a dictionary's is that table, where libzstd copies it into the state (the previous tree into each +/// block's state, each table repeated or predefined: 2 to 5 KiB a block, the largest cost of a small +/// frame after its match finding). A new table is built into one the previous state does not use +/// (). +/// internal sealed unsafe class BlockState { private readonly uint[] _rep = GC.AllocateArray(SequenceEncoder.RepeatOffsetCount, pinned: true); + // The tables this state owns, which this state or the next builds its new tables into. + private readonly HuffmanCTable _huffman = new(); + private readonly FseCTable _literalLengths = new(SequenceEncoder.LiteralLengthFseLog, SequenceCodes.MaxLiteralLength); + private readonly FseCTable _offsets = new(SequenceEncoder.OffsetFseLog, SequenceCodes.MaxOffset); + private readonly FseCTable _matchLengths = new(SequenceEncoder.MatchLengthFseLog, SequenceCodes.MaxMatchLength); + public BlockState() { Rep = (uint*)Unsafe.AsPointer(ref _rep[0]); + Huffman = _huffman; + LiteralLengths = _literalLengths; + Offsets = _offsets; + MatchLengths = _matchLengths; Reset(); } - public readonly HuffmanCTable Huffman = new(); + /// The tree in use: a table this state or the previous one owns, or a dictionary's. + public HuffmanCTable Huffman; public HuffmanRepeat HuffmanRepeat; - public readonly FseCTable LiteralLengths = new(SequenceEncoder.LiteralLengthFseLog, SequenceCodes.MaxLiteralLength); - public readonly FseCTable Offsets = new(SequenceEncoder.OffsetFseLog, SequenceCodes.MaxOffset); - public readonly FseCTable MatchLengths = new(SequenceEncoder.MatchLengthFseLog, SequenceCodes.MaxMatchLength); + + /// The tables in use, as , or predefined ones. + public FseCTable LiteralLengths; + public FseCTable Offsets; + public FseCTable MatchLengths; public FseRepeat LiteralLengthRepeat; public FseRepeat OffsetRepeat; public FseRepeat MatchLengthRepeat; @@ -54,6 +76,27 @@ public BlockState() /// The three repeat offsets, pinned: the match finders update them in place. public uint* Rep { get; } + /// + /// The table this state builds a new tree into: its own, unless uses + /// it, then the previous state's own, which that state does not use then. + /// + /// + /// Only the previous state's tables must outlive a block, which may still be emitted raw; this + /// state's are replaced, and neither is ever a dictionary's or a predefined table. + /// + public HuffmanCTable FreshHuffman(BlockState previous) => previous.Huffman != _huffman ? _huffman : previous._huffman; + + /// As , for the literal lengths' table. + public FseCTable FreshLiteralLengths(BlockState previous) => + previous.LiteralLengths != _literalLengths ? _literalLengths : previous._literalLengths; + + /// As , for the offsets' table. + public FseCTable FreshOffsets(BlockState previous) => previous.Offsets != _offsets ? _offsets : previous._offsets; + + /// As , for the match lengths' table. + public FseCTable FreshMatchLengths(BlockState previous) => + previous.MatchLengths != _matchLengths ? _matchLengths : previous._matchLengths; + /// libzstd's ZSTD_reset_compressedBlockState: the state every frame starts from. public void Reset() { @@ -69,7 +112,7 @@ public void Reset() /// A copy of : its tables, their repeat modes, its repeat offsets. public void CopyFrom(BlockState other) { - Huffman.CopyFrom(other.Huffman); + Huffman = other.Huffman; HuffmanRepeat = other.HuffmanRepeat; CopySequenceTablesFrom(other); Rep[0] = other.Rep[0]; @@ -80,9 +123,9 @@ public void CopyFrom(BlockState other) /// libzstd's copy of prevEntropy->fse into nextEntropy->fse, when a block has no sequence. public void CopySequenceTablesFrom(BlockState other) { - LiteralLengths.CopyFrom(other.LiteralLengths); - Offsets.CopyFrom(other.Offsets); - MatchLengths.CopyFrom(other.MatchLengths); + LiteralLengths = other.LiteralLengths; + Offsets = other.Offsets; + MatchLengths = other.MatchLengths; LiteralLengthRepeat = other.LiteralLengthRepeat; OffsetRepeat = other.OffsetRepeat; MatchLengthRepeat = other.MatchLengthRepeat; @@ -155,15 +198,35 @@ private static FseCTable BuildDefault(ReadOnlySpan norm, int maxSymbolVal ]; /// The largest symbol counted, and the largest count: what a histogram of codes returns. - private static uint LargestCount(uint* count, ref uint max) + /// + /// The largest symbol is searched in a local: through the reference, each step down was a + /// decrement in memory and a load of it, one waiting on the other. The largest count four at a + /// time: a maximum in a loop is a branch on each count, which the counts make unpredictable, + /// where a vector maximum has none. + /// + private static uint LargestCount(uint* count, ref uint largestSymbol) { + uint max = largestSymbol; while (count[max] == 0) { max--; } + largestSymbol = max; uint largest = 0; - for (uint s = 0; s <= max; s++) + uint s = 0; + if (Vector128.IsHardwareAccelerated && max >= 3) + { + Vector128 best = Vector128.Load(count); + for (s = 4; s + 3 <= max; s += 4) + { + best = Vector128.Max(best, Vector128.Load(count + s)); + } + + largest = Math.Max(Math.Max(best.GetElement(0), best.GetElement(1)), Math.Max(best.GetElement(2), best.GetElement(3))); + } + + for (; s <= max; s++) { largest = Math.Max(largest, count[s]); } @@ -320,26 +383,28 @@ public static SymbolEncodingType SelectEncodingType( /// libzstd's ZSTD_buildCTable: the table of a code in its mode, and its description when /// it has one. A new table leaves out the last sequence's symbol once, which the state that /// starts the bitstream encodes for free; libzstd takes it off its histogram, which it counts - /// again for its estimates, where it is put back here. + /// again for its estimates, where it is put back here. A table repeated or predefined is that + /// table, a new one is built into : is set to it. /// /// The size of the description. public static nuint BuildCTable( - byte* destination, FseCTable next, uint fseLog, SymbolEncodingType type, uint* count, uint max, uint firstCode, uint lastCode, - nuint sequenceCount, FseCTable defaultTable, FseCTable previous) + byte* destination, ref FseCTable next, FseCTable fresh, uint fseLog, SymbolEncodingType type, uint* count, uint max, uint firstCode, + uint lastCode, nuint sequenceCount, FseCTable defaultTable, FseCTable previous) { switch (type) { case SymbolEncodingType.Rle: - FseEncoder.BuildCTableRle(next, (byte)max); + FseEncoder.BuildCTableRle(fresh, (byte)max); + next = fresh; *destination = (byte)firstCode; return 1; case SymbolEncodingType.Repeat: - next.CopyFrom(previous); + next = previous; return 0; case SymbolEncodingType.Basic: - next.CopyFrom(defaultTable); + next = defaultTable; return 0; default: @@ -353,7 +418,8 @@ public static nuint BuildCTable( FseEncoder.NormalizeCount(normalized, tableLog, count, total, max, UseLowProbCount(total)); count[lastCode] += taken; nuint size = FseEncoder.WriteNCount(destination, normalized, max, tableLog); - FseEncoder.BuildCTable(next, normalized, max, tableLog); + FseEncoder.BuildCTable(fresh, normalized, max, tableLog); + next = fresh; return size; } } @@ -393,7 +459,7 @@ public static Statistics BuildStatistics(in SequenceSection store, BlockState pr ref next.LiteralLengthRepeat, llCount, max, mostFrequent, sequenceCount, LiteralLengthFseLog, previous.LiteralLengths, SequenceCodes.LiteralLengthDefaultNorm, LiteralLengthDefaultNormLog, isDefaultAllowed: true, strategy); nuint countSize = BuildCTable( - op, next.LiteralLengths, LiteralLengthFseLog, stats.LiteralLengths, llCount, max, first & 0x7F, last & 0x7F, sequenceCount, + op, ref next.LiteralLengths, next.FreshLiteralLengths(previous), LiteralLengthFseLog, stats.LiteralLengths, llCount, max, first & 0x7F, last & 0x7F, sequenceCount, DefaultLiteralLengths, previous.LiteralLengths); if (stats.LiteralLengths == SymbolEncodingType.Compressed) { @@ -414,7 +480,7 @@ public static Statistics BuildStatistics(in SequenceSection store, BlockState pr ref next.OffsetRepeat, ofCount, max, mostFrequent, sequenceCount, OffsetFseLog, previous.Offsets, SequenceCodes.OffsetDefaultNorm, OffsetDefaultNormLog, defaultAllowed, strategy); nuint countSize = BuildCTable( - op, next.Offsets, OffsetFseLog, stats.Offsets, ofCount, max, ((first >> 8) & 0x7F) - SequenceStore.OffsetCodes, ((last >> 8) & 0x7F) - SequenceStore.OffsetCodes, sequenceCount, + op, ref next.Offsets, next.FreshOffsets(previous), OffsetFseLog, stats.Offsets, ofCount, max, ((first >> 8) & 0x7F) - SequenceStore.OffsetCodes, ((last >> 8) & 0x7F) - SequenceStore.OffsetCodes, sequenceCount, DefaultOffsets, previous.Offsets); if (stats.Offsets == SymbolEncodingType.Compressed) { @@ -434,7 +500,7 @@ public static Statistics BuildStatistics(in SequenceSection store, BlockState pr ref next.MatchLengthRepeat, mlCount, max, mostFrequent, sequenceCount, MatchLengthFseLog, previous.MatchLengths, SequenceCodes.MatchLengthDefaultNorm, MatchLengthDefaultNormLog, isDefaultAllowed: true, strategy); nuint countSize = BuildCTable( - op, next.MatchLengths, MatchLengthFseLog, stats.MatchLengths, mlCount, max, ((first >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, ((last >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, + op, ref next.MatchLengths, next.FreshMatchLengths(previous), MatchLengthFseLog, stats.MatchLengths, mlCount, max, ((first >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, ((last >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, sequenceCount, DefaultMatchLengths, previous.MatchLengths); if (stats.MatchLengths == SymbolEncodingType.Compressed) @@ -492,6 +558,11 @@ public static nuint EncodeSequences( return 0; } + if (X86Base.IsSupported) + { + return EncodeSequencesInPlace(destination, capacity, literalLengthTable, offsetTable, matchLengthTable, sequences, sequenceCount); + } + // The three tables in two arrays: the transforms by code at the codes' places, the states one // table after the other, each transform's next-state offset moved by its table's place. The // loop then holds two table pointers instead of six. @@ -605,6 +676,96 @@ static void Gather(FseCTable table, FseSymbolTransform* transforms, ushort* stat } } + /// + /// as x64 runs it: the three tables read in place, as libzstd reads + /// them, and the three states encoded one after the other, in the stream's order, each one's bits + /// into the container as soon as they are known. + /// + /// + /// Gathering the tables costs a copy of their 2.5 KiB of states and a pass over their transforms + /// a block, which a small block (a record of a few hundred bytes, some fifty sequences) pays in + /// full; the gathered loop saves registers, which x64 has too few of to keep the three states' + /// parts anyway (the JIT spilled them), and which its six table pointers here only take in + /// memory, read without a store. The bits are those of the gathered loop: every state's low bits + /// in the stream's order, offset, match length, literal length, then the extra bits. + /// + private static nuint EncodeSequencesInPlace( + byte* destination, nuint capacity, FseCTable literalLengthTable, FseCTable offsetTable, FseCTable matchLengthTable, + SequenceRecord* sequences, nuint sequenceCount) + { + // Each table's transforms moved down by its codes' place, which the records index them by. + FseSymbolTransform* llTransforms = literalLengthTable.Symbols; + FseSymbolTransform* ofTransforms = offsetTable.Symbols - SequenceStore.OffsetCodes; + FseSymbolTransform* mlTransforms = matchLengthTable.Symbols - SequenceStore.MatchLengthCodes; + ushort* llStates = literalLengthTable.StateTable; + ushort* ofStates = offsetTable.StateTable; + ushort* mlStates = matchLengthTable.StateTable; + + byte* ptr = destination; + byte* end = destination + capacity - sizeof(ulong); + ulong container; + nint bitPosition; + + // ---- the last sequence: its states start the stream, then its extra bits + SequenceRecord* last = sequences + sequenceCount - 1; + nint lastCodes = (nint)last->Codes; + nint mlState = InitialState(mlStates, mlTransforms, (lastCodes >> 16) & 0x7F); + nint ofState = InitialState(ofStates, ofTransforms, (lastCodes >> 8) & 0x7F); + nint llState = InitialState(llStates, llTransforms, lastCodes & 0x7F); + container = last->Extras; + bitPosition = lastCodes >> 24; + Flush(ref container, ref bitPosition, ref ptr, end); + + for (SequenceRecord* sequence = last - 1; sequence >= sequences; sequence--) + { + nint codes = (nint)sequence->Codes; + EncodeState(ref ofState, ofTransforms[(codes >> 8) & 0x7F], ofStates, ref container, ref bitPosition); + EncodeState(ref mlState, mlTransforms[(codes >> 16) & 0x7F], mlStates, ref container, ref bitPosition); + EncodeState(ref llState, llTransforms[codes & 0x7F], llStates, ref container, ref bitPosition); + nint extraBits = codes >> 24; + if (extraBits >= 64 - 7 - (LiteralLengthFseLog + MatchLengthFseLog + OffsetFseLog)) + { + Flush(ref container, ref bitPosition, ref ptr, end); + } + + ulong extras = sequence->Extras; + if (extraBits <= 56) + { + container |= extras << (int)bitPosition; + bitPosition += extraBits; + } + else + { + // The lengths' bits, then a flush, then the offset's. + nint ofBits = (nint)BitOperations.Log2(sequence->OffBase); + nint lengthBits = extraBits - ofBits; + container |= LowBits(extras, lengthBits) << (int)bitPosition; + bitPosition += lengthBits; + Flush(ref container, ref bitPosition, ref ptr, end); + container |= (extras >> (int)lengthBits) << (int)bitPosition; + bitPosition += ofBits; + } + + Flush(ref container, ref bitPosition, ref ptr, end); + } + + // ---- the final states, which the decoder starts from, then the end marker + AddBits(ref container, ref bitPosition, (ulong)mlState, matchLengthTable.TableLog); + Flush(ref container, ref bitPosition, ref ptr, end); + AddBits(ref container, ref bitPosition, (ulong)ofState, offsetTable.TableLog); + Flush(ref container, ref bitPosition, ref ptr, end); + AddBits(ref container, ref bitPosition, (ulong)llState, literalLengthTable.TableLog); + Flush(ref container, ref bitPosition, ref ptr, end); + AddBits(ref container, ref bitPosition, 1, 1); + Flush(ref container, ref bitPosition, ref ptr, end); + if (ptr >= end) + { + return 0; + } + + return (nuint)(ptr - destination) + (bitPosition > 0 ? 1u : 0u); + } + /// libzstd's FSE_initCState2: see . [MethodImpl(MethodImplOptions.AggressiveInlining)] private static nint InitialState(ushort* states, FseSymbolTransform* symbols, nint symbol) @@ -615,6 +776,22 @@ private static nint InitialState(ushort* states, FseSymbolTransform* symbols, ni return states[(nint)(value >> (int)nbBitsOut) + transform.DeltaFindState]; } + /// + /// libzstd's FSE_encodeSymbol on a state of the gathered tables: its low bits into the + /// container, then its successor. As in the loop that gathers the three: the high part of the + /// state indexes the successor, the state less the high part shifted back is what it outputs. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EncodeState(ref nint state, FseSymbolTransform transform, ushort* states, ref ulong container, ref nint bitPosition) + { + nint nb = (state + (nint)transform.DeltaNbBits) >> 16; + nint high = state >> (int)nb; + ulong low = (ulong)(state - (high << (int)(nb + 64))); + container |= low << (int)bitPosition; + bitPosition += nb; + state = states[high + transform.DeltaFindState]; + } + /// The low bits (0 to 63) of : a shift and a bit clear. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static ulong LowBits(ulong value, nint count) => value & ~(ulong.MaxValue << (int)count); diff --git a/src/Vorticity.Zstd/Internal/SequenceStore.cs b/src/Vorticity.Zstd/Internal/SequenceStore.cs index a1492b15c..ae68d10a4 100644 --- a/src/Vorticity.Zstd/Internal/SequenceStore.cs +++ b/src/Vorticity.Zstd/Internal/SequenceStore.cs @@ -4,13 +4,15 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace Vorticity.Zstd.Internal; /// /// One sequence of a block, as its encoding wants it: libzstd's SeqDef and what /// ZSTD_seqToCodes derives from it, computed when the match finder stores the sequence (see -/// ). +/// ), or on x64 +/// once the block's sequences are found (). /// [StructLayout(LayoutKind.Sequential)] internal struct SequenceRecord @@ -35,8 +37,10 @@ internal struct SequenceRecord /// /// /// -/// The codes are computed as each sequence is stored, rather than in passes of their own as libzstd -/// does: the match finders wait on their loads and mispredictions, and have the issue slots to spare. +/// On Arm64 the codes are computed as each sequence is stored, rather than in passes of their own as +/// libzstd does: the match finders wait on their loads and mispredictions, and have the issue slots to +/// spare. On x64 they do not have the registers: the codes are computed in one pass once the block's +/// sequences are found (see ). /// Computed from the full lengths, a length over 16 bits gets the largest code and its low 16 bits /// as extra bits, which is what libzstd's long-length fix-up gives. /// @@ -139,23 +143,87 @@ public static void Store( StoreOnly(ref sequence, counts, litLength, offBase, matchLength); } + /// + /// Whether the match finders store a sequence as its two lengths and its offset alone, as libzstd + /// does, its codes computed once the block is found (): on x64. Computed + /// as each sequence is stored, the codes take some forty instructions and a dozen values the + /// match finders have no registers left for on x64, which spill them; in a pass of their own, + /// they take a loop that holds them all. Arm64, with its 31 registers, computes them as it stores. + /// + public static bool DefersCodes => X86Base.IsSupported; + + /// + /// libzstd's ZSTD_storeSeqOnly: with its codes (), or, where + /// , the lengths in the place of the extra bits and the offset in its + /// own, to complete them. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint litLength, uint offBase, nuint matchLength) + { + if (DefersCodes) + { + sequence->Extras = (ulong)litLength | ((ulong)matchLength << 32); + sequence->OffBase = offBase; + sequence++; + return; + } + + StoreCoded(ref sequence, counts, litLength, offBase, matchLength); + } + + /// + /// The codes of the sequences stored since the store was reset, where : + /// each record completed from its lengths and offset, its codes counted. Called once the block's + /// sequences are found, before anything reads them. + /// + public void ComputeCodes() + { + if (!DefersCodes) + { + return; + } + + uint* counts = Counts; + SequenceRecord* record = SequencesStart; + SequenceRecord* end = Sequences; + while (record < end) + { + ulong lengths = record->Extras; + StoreCoded(ref record, counts, (nuint)(uint)lengths, record->OffBase, (nuint)(lengths >> 32)); + } + } + /// /// libzstd's ZSTD_storeSeqOnly, and its codes: those of the two lengths from tables below 64 /// and 128, from their logarithms above (libzstd's ZSTD_LLcode, ZSTD_MLcode), the /// offset's from its logarithm; the extra bits gathered in the stream's order; the codes counted. /// + /// + /// On x64 an entry is read as its two bytes, the code and the bits (an entry split took four + /// instructions), the extra bits are kept with bzhi (a mask took three), and the offset's + /// logarithm, of a value never 0, without the guard against 0. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint litLength, uint offBase, nuint matchLength) + public static void StoreCoded(ref SequenceRecord* sequence, uint* counts, nuint litLength, uint offBase, nuint matchLength) { - Debug.Assert(matchLength >= 3 && litLength <= FrameFormat.MaxBlockSize); + Debug.Assert(matchLength >= 3 && litLength <= FrameFormat.MaxBlockSize && offBase != 0); nuint matchLengthBase = matchLength - 3; nuint llCode; nuint llBits; if (litLength < 64) { - nuint entry = ((ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(LiteralLengthCodeTable)))[litLength]; - llCode = entry & 0x3F; - llBits = entry >> 8; + ushort* table = (ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(LiteralLengthCodeTable)); + if (X86Base.IsSupported) + { + llCode = ((byte*)(table + litLength))[0]; + llBits = ((byte*)(table + litLength))[1]; + } + else + { + nuint entry = table[litLength]; + llCode = entry & 0x3F; + llBits = entry >> 8; + } } else { @@ -167,9 +235,18 @@ public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint l nuint mlBits; if (matchLengthBase < 128) { - nuint entry = ((ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(MatchLengthCodeTable)))[matchLengthBase]; - mlCode = entry & 0x3F; - mlBits = entry >> 8; + ushort* table = (ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(MatchLengthCodeTable)); + if (X86Base.IsSupported) + { + mlCode = ((byte*)(table + matchLengthBase))[0]; + mlBits = ((byte*)(table + matchLengthBase))[1]; + } + else + { + nuint entry = table[matchLengthBase]; + mlCode = entry & 0x3F; + mlBits = entry >> 8; + } } else { @@ -177,9 +254,11 @@ public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint l mlCode = mlBits + 36; } - nuint ofCode = (nuint)(uint)BitOperations.Log2(offBase); - ulong extras = ((ulong)litLength & ~(ulong.MaxValue << (int)llBits)) - | (((ulong)matchLengthBase & ~(ulong.MaxValue << (int)mlBits)) << (int)llBits) + nuint ofCode = X86Base.IsSupported + ? (nuint)(uint)(31 ^ BitOperations.LeadingZeroCount(offBase)) + : (nuint)(uint)BitOperations.Log2(offBase); + ulong extras = LowBits(litLength, llBits) + | (LowBits(matchLengthBase, mlBits) << (int)llBits) | ((ulong)(offBase ^ (1u << (int)ofCode)) << (int)(llBits + mlBits)); // The codes at their places in the common arrays, which index the counts as they are. @@ -194,6 +273,11 @@ public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint l sequence++; } + /// The low bits of , below 64. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong LowBits(nuint value, nuint count) => + Bmi2.X64.IsSupported ? Bmi2.X64.ZeroHighBits(value, count) : value & ~(ulong.MaxValue << (int)count); + /// The whole store, as a section the entropy coding takes. public SequenceSection Whole => new(LiteralsStart, LiteralCount, SequencesStart, SequenceCount, Counts); @@ -226,7 +310,7 @@ public static void ReplaceOffBase(SequenceRecord* record, uint offBase) nuint matchLength = MatchLengthOf(record); uint* scratch = stackalloc uint[AllCodes]; SequenceRecord* sequence = record; - StoreOnly(ref sequence, scratch, literalLength, offBase, matchLength); + StoreCoded(ref sequence, scratch, literalLength, offBase, matchLength); } /// The codes of records counted into , cleared first. diff --git a/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs b/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs index 61b581a53..92ed2745f 100644 --- a/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs +++ b/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs @@ -142,7 +142,7 @@ internal long EntropyOnly(List blocks, int level, bool literals, bo total += (long)CompressLiterals(destination, capacity, _store.LiteralsStart, _store.LiteralCount, disabled, suspect); if (block.Compressed) { - _previous.Huffman.CopyFrom(_next.Huffman); + _previous.Huffman = _next.Huffman; _previous.HuffmanRepeat = _next.HuffmanRepeat; } } diff --git a/src/Vorticity.Zstd/ZstdCompressor.Block.cs b/src/Vorticity.Zstd/ZstdCompressor.Block.cs index 5299f9287..60d1febf3 100644 --- a/src/Vorticity.Zstd/ZstdCompressor.Block.cs +++ b/src/Vorticity.Zstd/ZstdCompressor.Block.cs @@ -392,6 +392,7 @@ private void FindSequences(byte* source, int size) // attached dictionary. nuint dictionaryLiterals = FindSequencesWithDictionary(source, size, rep); _store.StoreLastLiterals(source + size - (nint)dictionaryLiterals, dictionaryLiterals); + _store.ComputeCodes(); SkipJobSequences(size); return; } @@ -405,6 +406,7 @@ private void FindSequences(byte* source, int size) }; _store.StoreLastLiterals(source + size - (nint)lastLiterals, lastLiterals); + _store.ComputeCodes(); SkipJobSequences(size); } @@ -508,8 +510,8 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, bool singleStream = size < 256; SymbolEncodingType type = SymbolEncodingType.Compressed; - // The previous tree, assumed reused. - _next.Huffman.CopyFrom(_previous.Huffman); + // The previous tree, assumed reused: the state takes a new one only when it is used. + _next.Huffman = _previous.Huffman; _next.HuffmanRepeat = _previous.HuffmanRepeat; if (disabled || size < MinLiteralsToCompress(strategy, _previous.HuffmanRepeat)) { @@ -525,9 +527,10 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, singleStream = true; } + HuffmanCTable fresh = _next.FreshHuffman(_previous); nuint compressed = HuffmanEncoder.Compress( - destination + headerSize, capacity - headerSize, source, size, singleStream, _next.Huffman, _huffmanWorkspace, - ref repeat, preferRepeat, optimalDepth, suspectUncompressible); + destination + headerSize, capacity - headerSize, source, size, singleStream, _previous.Huffman, fresh, + _huffmanWorkspace, ref repeat, preferRepeat, optimalDepth, suspectUncompressible); if (repeat != HuffmanRepeat.None) { type = SymbolEncodingType.Repeat; @@ -536,8 +539,6 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, nuint minGain = MinGain(size, strategy); if (compressed == 0 || compressed >= size - minGain) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return WriteRawLiterals(destination, source, size); } @@ -546,14 +547,13 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, // One symbol, or, for fewer than 8 literals, possibly one byte of stream: checked. if (size >= 8 || new ReadOnlySpan(source + 1, (int)size - 1).IndexOfAnyExcept(source[0]) < 0) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return WriteRleLiterals(destination, source, size); } } if (type == SymbolEncodingType.Compressed) { + _next.Huffman = fresh; _next.HuffmanRepeat = HuffmanRepeat.Check; } diff --git a/src/Vorticity.Zstd/ZstdCompressor.Split.cs b/src/Vorticity.Zstd/ZstdCompressor.Split.cs index 3796e5501..4b38dfc56 100644 --- a/src/Vorticity.Zstd/ZstdCompressor.Split.cs +++ b/src/Vorticity.Zstd/ZstdCompressor.Split.cs @@ -388,7 +388,7 @@ private nuint EstimateSize(in SequenceSection section) private SymbolEncodingType BuildLiteralStatistics(byte* literals, nuint size, out nuint descriptionSize) { descriptionSize = 0; - _next.Huffman.CopyFrom(_previous.Huffman); + _next.Huffman = _previous.Huffman; _next.HuffmanRepeat = _previous.HuffmanRepeat; nuint minimumSize = _previous.HuffmanRepeat == HuffmanRepeat.Valid ? 6u : 63u; @@ -415,16 +415,17 @@ private SymbolEncodingType BuildLiteralStatistics(byte* literals, nuint size, ou repeat = HuffmanRepeat.None; } - new Span(_next.Huffman.Elements, HuffmanTable.MaxSymbols).Clear(); - _next.Huffman.TableLog = 0; - _next.Huffman.MaxSymbolValue = 0; + HuffmanCTable fresh = _next.FreshHuffman(_previous); + new Span(fresh.Elements, HuffmanTable.MaxSymbols).Clear(); + fresh.TableLog = 0; + fresh.MaxSymbolValue = 0; uint huffLog = HuffmanEncoder.OptimalTableLog( HuffmanEncoder.LiteralsTableLog, size, maxSymbol, _huffmanWorkspace, count, _parameters.Strategy >= Strategy.BinaryTreeUltra); - huffLog = HuffmanEncoder.BuildCTable(_next.Huffman, count, maxSymbol, huffLog); + huffLog = HuffmanEncoder.BuildCTable(fresh, count, maxSymbol, huffLog); - nuint newSize = HuffmanEncoder.EstimateCompressedSize(_next.Huffman, count, maxSymbol); + nuint newSize = HuffmanEncoder.EstimateCompressedSize(fresh, count, maxSymbol); byte* description = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(_splitScratch)); - nuint headerSize = HuffmanEncoder.WriteCTable(description, MaxHuffmanHeaderSize, _next.Huffman, maxSymbol, huffLog, _huffmanWorkspace.Weights); + nuint headerSize = HuffmanEncoder.WriteCTable(description, MaxHuffmanHeaderSize, fresh, maxSymbol, huffLog, _huffmanWorkspace.Weights); if (headerSize == 0) { descriptionSize = nuint.MaxValue; @@ -436,19 +437,16 @@ private SymbolEncodingType BuildLiteralStatistics(byte* literals, nuint size, ou nuint oldSize = HuffmanEncoder.EstimateCompressedSize(_previous.Huffman, count, maxSymbol); if (oldSize < size && (oldSize <= headerSize + newSize || headerSize + 12 >= size)) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return SymbolEncodingType.Repeat; } } if (newSize + headerSize >= size) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return SymbolEncodingType.Basic; } + _next.Huffman = fresh; _next.HuffmanRepeat = HuffmanRepeat.Check; descriptionSize = headerSize; return SymbolEncodingType.Compressed; diff --git a/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs b/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs index 121f38c3c..159995c4a 100644 --- a/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs +++ b/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs @@ -75,6 +75,7 @@ public void Huffman_streams_decode_back(int seed) var random = new Random(seed); var workspace = new HuffmanWorkspace(); var previous = new HuffmanCTable(); + var fresh = new HuffmanCTable(); var table = new HuffmanTable(); byte[] output = new byte[1 << 17]; for (int round = 0; round < 100; round++) @@ -103,7 +104,7 @@ public void Huffman_streams_decode_back(int seed) fixed (byte* dst = destination) { compressed = HuffmanEncoder.Compress( - dst, (nuint)destination.Length, src, (nuint)size, singleStream, previous, workspace, ref repeat, + dst, (nuint)destination.Length, src, (nuint)size, singleStream, previous, fresh, workspace, ref repeat, preferRepeat: false, optimalDepth: random.Next(2) == 0, suspectUncompressible: false); } From ebbe5d4a3ce86451fcf9e8316b56f7b2a72ef46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:41:08 +0200 Subject: [PATCH 3/8] ci: run Vorticity.Zstd's suite with intrinsics disabled too Vorticity.Zstd now has paths of x64's own, which an x64 runner takes: its portable paths, Arm64's shape without its vectors, ran nowhere in CI. The pass with intrinsics disabled runs its suite whole, with zstd's test data written first. --- .github/workflows/build.yml | 9 +++++---- tests/scalar-pass.sh | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85329d22c..485eea157 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,10 +132,11 @@ jobs: # Every SIMD kernel has a scalar fallback, and the fallback is only real if it is # exercised: the classes tests/scalar-pass.txt names, which reach every fallback the # whole suite does, and the conformance corpus run again with intrinsics disabled. - # Vorticity.Zstd's suite does not: its only vector paths are Arm's, which an x64 runner - # never takes, so the run above already took every one of its fallbacks. - - name: Vorticity's fallbacks and the conformance corpus, intrinsics disabled - projects: tests/Vorticity.Tests tests/Vorticity.Conformance + # Vorticity.Zstd's suite runs whole: an x64 runner takes its x64 paths, and with + # intrinsics disabled the portable ones, Arm64's shape without its vectors. + - name: Vorticity's fallbacks, Vorticity.Zstd and the conformance corpus, intrinsics disabled + projects: tests/Vorticity.Tests tests/Vorticity.Zstd.Tests tests/Vorticity.Conformance + zstd-data: true scalar-pass: true steps: # Nothing here pushes to git: the token is not left on disk. diff --git a/tests/scalar-pass.sh b/tests/scalar-pass.sh index 56e3258c5..03facba20 100755 --- a/tests/scalar-pass.sh +++ b/tests/scalar-pass.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Runs what CI runs with intrinsics disabled: the classes of Vorticity.Tests that tests/scalar-pass.txt -# names, which reach every scalar fallback the whole suite does, and the conformance corpus, through -# the scalar decoders. Arguments go to each dotnet test, --no-build say. +# names, which reach every scalar fallback the whole suite does, Vorticity.Zstd's suite whole, through +# its portable paths rather than x64's, and the conformance corpus, through the scalar decoders. +# Arguments go to each dotnet test, --no-build say. # # bash tests/scalar-pass.sh set -euo pipefail @@ -15,5 +16,6 @@ done < <(grep -v -E '^[[:space:]]*(#|$)' "$root/tests/scalar-pass.txt") export DOTNET_EnableHWIntrinsic=0 status=0 dotnet test --project "$root/tests/Vorticity.Tests" -c Release "${filters[@]}" "$@" || status=1 +dotnet test --project "$root/tests/Vorticity.Zstd.Tests" -c Release "$@" || status=1 dotnet test --project "$root/tests/Vorticity.Conformance" -c Release "$@" || status=1 exit $status From a3dfc8ab9a9651ab139a507186fdc34b6fe2a309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:41:09 +0200 Subject: [PATCH 4/8] bench: profile Vorticity.Zstd on Windows x64 and compare it with libzstd's assembly - The native reference builds on Windows, from Git Bash, with a GNU-compatible clang such as llvm-mingw: zstd's assembly Huffman decoder and its BMI2 paths, which MSVC builds neither of, and a PDB. The tests and the benchmarks load the platform's library name. - The program-counter sampler runs on Windows, symbolized through DbgHelp: functions, lines, every instruction of one function (--pcfunc), and the callers of one (--pccaller). - --core N keeps the timed thread on one logical processor at a high priority; the corpus takes a 1 MiB sample of Silesia every 16 MiB (silesia-sample) and --directions; micro-benchmarks of whole frames and corpora (decode, ccorpus), of libzstd's Huffman decoders (nx1, nx2), and the clock measured. - zstd-ab-aot.sh compares two kept builds (--before, --after, --no-build) on a pinned core. --- bench/Vorticity.Zstd.Perf/Corpus.cs | 38 +- bench/Vorticity.Zstd.Perf/Micro.cs | 162 +++++++- bench/Vorticity.Zstd.Perf/PcSampler.cs | 349 +++++++++++++++++- bench/Vorticity.Zstd.Perf/Program.cs | 41 +- .../Vorticity.Zstd.Shared/NativeReference.cs | 68 +++- bench/zstd-ab-aot.sh | 74 ++-- src/Vorticity.Zstd/README.md | 17 +- .../Vorticity.Zstd.Tests/Support/LibzstdMt.cs | 4 +- .../Support/LibzstdOracle.cs | 2 +- tools/native-ref/build.sh | 67 ++-- tools/native-ref/testdata.sh | 3 +- 11 files changed, 746 insertions(+), 79 deletions(-) diff --git a/bench/Vorticity.Zstd.Perf/Corpus.cs b/bench/Vorticity.Zstd.Perf/Corpus.cs index a5f56b5ba..8c7a25276 100644 --- a/bench/Vorticity.Zstd.Perf/Corpus.cs +++ b/bench/Vorticity.Zstd.Perf/Corpus.cs @@ -23,7 +23,7 @@ namespace Vorticity.Zstd.Perf; /// frames byte for byte, and give the content back from them. /// /// -/// Usage: Vorticity.Zstd.Perf --corpus silesia,github,github-dict [--levels 1,3,19] [--seconds S] [--only zstd] +/// Usage: Vorticity.Zstd.Perf --corpus silesia,github,github-dict [--levels 1,3,19] [--seconds S] [--only zstd] [--directions decompression] /// [--results file.tsv] [--markdown file.md], or Vorticity.Zstd.Perf --corpus-report file.tsv [--markdown file.md] /// to render results already measured. silesia is silesia.tar, the twelve files in one frame, /// as lzbench reads it; github the 500 records, a frame each, on one context; @@ -63,8 +63,9 @@ public static Row FromTsv(string line) } } - public static int Run(string[] sets, string? levels, string? seconds, string? only, string? results, string? markdown) + public static int Run(string[] sets, string? levels, string? seconds, string? only, string? results, string? markdown, string? directions = null) { + Compresses = directions is null || directions.Split(',').Contains("compression"); int[] levelList = levels?.Split(',').Select(l => int.Parse(l, CultureInfo.InvariantCulture)).ToArray() ?? DefaultLevels; double budget = seconds is null ? 5 : double.Parse(seconds, CultureInfo.InvariantCulture); NativeReference? native = NativeReference.TryLoad(); @@ -179,6 +180,9 @@ private static (List Candidates, IDisposable[] Owned) Candidates(Nati /// every other candidate must write too; then the decoding of those frames, whose first round /// checks that every candidate gives the content back. /// + /// Whether compression is timed: --directions decompression leaves it out, the frames written once. + private static bool Compresses { get; set; } = true; + private static List Measure(DataSet set, int level, List candidates, double budget) { int records = set.Offsets.Length - 1; @@ -200,7 +204,13 @@ private static List Measure(DataSet set, int level, List candida int count = candidates.Count; bool[] compressionDiffers = new bool[count]; - double[] compression = Time(candidates, budget, (c, round) => + if (!Compresses) + { + // Decompression alone: libzstd's frames, written once, untimed. + Pass(candidates[0].Compress, set.Content, set.Offsets, reference, bounds, sizes); + } + + double[] compression = !Compresses ? new double[count] : Time(candidates, budget, (c, round) => { if (round > 0) { @@ -255,7 +265,7 @@ private static List Measure(DataSet set, int level, List candida double ratio = (double)set.Content.Length / frames.Length; var rows = new List(); - for (int c = 0; c < count; c++) + for (int c = 0; c < count && Compresses; c++) { rows.Add(new Row(set.Name, set.Label, set.Title, level, ratio, "compression", candidates[c].Name, candidates[c].Label, set.Content.Length / compression[c] / 1e6, compressionDiffers[c])); @@ -459,6 +469,24 @@ private static DataSet Load(string name) return new DataSet(name, "silesia.tar", $"the Silesia corpus as one tar, {tar.Length:N0} bytes, one frame", tar, [0, tar.Length], null); } + case "silesia-sample": + { + // 1 MiB every 16 MiB of silesia.tar, as one frame: its kinds of data, text, binaries, + // images and databases, in a frame that compresses in a fraction of the time, for the + // micro-benchmarks and the profiler. + byte[] tar = SilesiaTar(Path.Combine(directory, "silesia.zip")); + const int Slice = 1 << 20; + const int Stride = 16 << 20; + using var sample = new MemoryStream(); + for (int at = 0; at + Slice <= tar.Length; at += Stride) + { + sample.Write(tar, at, Slice); + } + + byte[] content = sample.ToArray(); + return new DataSet(name, "silesia sample", $"1 MiB every 16 MiB of silesia.tar, {content.Length:N0} bytes, one frame", content, [0, content.Length], null); + } + case "github": case "github-dict": { @@ -481,7 +509,7 @@ private static DataSet Load(string name) } default: - throw new ArgumentException("unknown corpus " + name + " (silesia, github, github-dict, github.tar, github.tar-dict)", nameof(name)); + throw new ArgumentException("unknown corpus " + name + " (silesia, silesia-sample, github, github-dict, github.tar, github.tar-dict)", nameof(name)); } } diff --git a/bench/Vorticity.Zstd.Perf/Micro.cs b/bench/Vorticity.Zstd.Perf/Micro.cs index 7a8dd5749..dcacf8b92 100644 --- a/bench/Vorticity.Zstd.Perf/Micro.cs +++ b/bench/Vorticity.Zstd.Perf/Micro.cs @@ -13,15 +13,14 @@ namespace Vorticity.Zstd.Perf; /// block are taken from the frame itself, then each step is timed alone, many times over. /// /// -/// Usage: Vorticity.Zstd.Perf --micro tables|ncount|tree|weights|x1|x2|x2build|compress|ncompress|cmatch|nmatch|compressab|cmatchab|centropy|cliterals|chist|csequences -/// [--frames a,b] [--repeat N] [--pcprofile file], the repeats for a profiler to attach; --pcprofile samples the +/// Usage: Vorticity.Zstd.Perf --micro decode|ndecode|ccorpus|nccorpus|nx1|nx2|tables|ncount|tree|weights|x1|x2|x2build|compress|ncompress|cmatch|nmatch|compressab|cmatchab|centropy|cliterals|chist|csequences +/// [--frames a,b] [--repeat N] [--pcprofile file] [--pcfunc name] [--pccaller name], the repeats for a profiler to attach; --pcprofile samples the /// timed loops' program counters (see ). Prints the median time -/// of one operation, and its cycles at the clock the M4 Pro's performance cores run (4.44 GHz); +/// of one operation, and its cycles at the clock the timed thread runs at, measured (see ); /// compressab and cmatchab time both sides of on one compressor. /// internal static class Micro { - private const double Ghz = 4.44; private sealed record Table(SequenceCode Code, short[] Norm, int TableLog, byte[] Description); @@ -48,6 +47,18 @@ public static int Run(string what, string[] frames, int repeat) continue; } + if (what is "ccorpus" or "nccorpus") + { + CompressCorpus(what, name); + continue; + } + + if (what is "decode" or "ndecode") + { + DecodeFrame(what, name); + continue; + } + Blocks blocks = Parse(BenchFrames.Load(frameName)); switch (what) { @@ -116,6 +127,38 @@ public static int Run(string what, string[] frames, int repeat) break; } + case "nx1": + case "nx2": + { + // The same four-stream sections, decoded by libzstd: HUF_readDTableX1 or X2 on + // each tree, then HUF_decompress4X_usingDTable (its assembly loops on x86-64). + NativeReference.HuffmanReference huffman = (NativeReference.TryLoad() ?? throw new InvalidOperationException("no " + NativeReference.LibraryPath)).Huffman + ?? throw new InvalidOperationException("the reference does not export HUF_decompress4X_usingDTable"); + bool x2 = what == "nx2"; + var output = new byte[128 << 10]; + int symbols = 0; + foreach (Literals l in blocks.Sections) + { + symbols += l.Size; + } + + Report(name, $"{blocks.Sections.Count} sections, {symbols} symbols (libzstd {what[1..]})", blocks.Sections.Count, symbols, () => + { + byte[]? current = null; + foreach (Literals l in blocks.Sections) + { + if (!ReferenceEquals(l.Tree, current)) + { + huffman.ReadTree(l.Tree, x2); + current = l.Tree; + } + + huffman.DecodeFourStreams(l.Streams, output.AsSpan(0, l.Size)); + } + }); + break; + } + case "ncount": { short[] norm = new short[SequenceCodes.MaxMatchLength + 1]; @@ -256,7 +299,7 @@ public static int Run(string what, string[] frames, int repeat) } default: - Console.WriteLine($"unknown micro-benchmark {what}: tables, ncount, tree, weights, x1, x2, x2build, compress, ncompress, cmatch, nmatch, compressab, cmatchab, centropy, cliterals, chist, csequences, dcorpus, ndcorpus, dcorpusab"); + Console.WriteLine($"unknown micro-benchmark {what}: decode, ndecode, ccorpus, nccorpus, nx1, nx2, tables, ncount, tree, weights, x1, x2, x2build, compress, ncompress, cmatch, nmatch, compressab, cmatchab, centropy, cliterals, chist, csequences, dcorpus, ndcorpus, dcorpusab"); return 1; } } @@ -324,6 +367,113 @@ void Body() } } + /// + /// The compression of a corpus of (--frames github-dict-L3: the set, then + /// the level), every record a frame of its own, on one compressor: Vorticity.Zstd (ccorpus) or + /// libzstd (nccorpus). Per frame, and per byte of content. + /// + private static void CompressCorpus(string what, string name) + { + int dash = name.LastIndexOf("-L", StringComparison.Ordinal); + int level = int.Parse(name.AsSpan(dash + 2), CultureInfo.InvariantCulture); + (byte[] content, int[] offsets, byte[]? dictionary) = Corpus.LoadSet(name[..dash]); + int records = offsets.Length - 1; + byte[] frames = new byte[ZstdCompressor.GetMaxCompressedLength(content.Length) + (records * 64)]; + SpanCodec compress; + if (what == "nccorpus") + { + NativeReference native = NativeReference.TryLoad() ?? throw new InvalidOperationException("no " + NativeReference.LibraryPath); + compress = dictionary is null ? (s, d) => native.Compress(s, d, level) : native.CompressorWith(dictionary, level); + } + else + { + var compressor = dictionary is null ? new ZstdCompressor(level) : new ZstdCompressor(level, dictionary); + compress = (s, d) => compressor.Compress(s, d, out _, out int written) == OperationStatus.Done ? written : -1; + } + + void Body() + { + int at = 0; + for (int i = 0; i < records; i++) + { + at += compress(content.AsSpan(offsets[i], offsets[i + 1] - offsets[i]), frames.AsSpan(at)); + } + } + + Body(); + string title = $"{(what == "nccorpus" ? "libzstd " : string.Empty)}compression, {records} frames, {content.Length} bytes, per frame"; + Report(name, title, records, content.Length, Body); + } + + /// + /// The decoding of one benchmark frame, whole, by Vorticity.Zstd (decode) or libzstd (ndecode): + /// per sequence, and per byte of content. + /// + private static void DecodeFrame(string what, string name) + { + byte[] frame = BenchFrames.Load(name); + byte[] output = new byte[BenchFrames.ContentSize(frame)]; + SpanCodec decode; + if (what == "ndecode") + { + NativeReference native = NativeReference.TryLoad() ?? throw new InvalidOperationException("no " + NativeReference.LibraryPath); + PcSampler.Libraries.Add((NativeReference.LibraryPath, "_ZSTD_versionNumber", native.VersionAddress)); + decode = native.Decompress; + } + else + { + var decompressor = new ZstdDecompressor(); + decode = (s, d) => decompressor.Decompress(s, d, out _, out int written) == OperationStatus.Done ? written : -1; + } + + if (decode(frame, output) != output.Length) + { + throw new InvalidOperationException(what + " failed on " + name); + } + + int sequences = Math.Max(1, CountSequences(frame)); + Report(name, $"{(what == "ndecode" ? "libzstd " : string.Empty)}decoding, {frame.Length} -> {output.Length} bytes, {sequences} sequences, per sequence", + sequences, output.Length, () => decode(frame, output)); + } + + /// + /// The clock the timed thread runs at, in GHz: a chain of dependent additions, one a cycle on + /// any core, timed. Measured once; the cycle columns use it. + /// + private static double Ghz => s_ghz ??= MeasureClock(); + + private static double? s_ghz; + + private static double MeasureClock() + { + const long Iterations = 20_000_000; + ulong step = (ulong)Environment.TickCount64 | 1; + double best = 0; + for (int round = 0; round < 5; round++) + { + long start = Stopwatch.GetTimestamp(); + ulong x = AddChain(step, step, Iterations); + double seconds = Stopwatch.GetElapsedTime(start).TotalSeconds; + GC.KeepAlive(x); + best = Math.Max(best, Iterations * 16 / seconds / 1e9); + } + + Console.WriteLine(string.Create(CultureInfo.InvariantCulture, $"clock: {best:F2} GHz (a chain of dependent additions)")); + return best; + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static ulong AddChain(ulong x, ulong y, long n) + { + for (long i = 0; i < n; i++) + { + x += y; x += y; x += y; x += y; x += y; x += y; x += y; x += y; + x += y; x += y; x += y; x += y; x += y; x += y; x += y; x += y; + } + + return x; + } + /// The histogram of each block's literals, as literals compression counts them. private static unsafe uint HistogramsOnly(List records) { @@ -362,6 +512,7 @@ private static void Report(string frame, string what, int operations, int cells, } var samples = new double[101]; + PcSampler.Label = $"{frame} {what}"; using (PcSampler.Start()) { for (int s = 0; s < samples.Length; s++) @@ -403,6 +554,7 @@ private static void ReportAb(string frame, string what, int operations, Action b var before = new double[101]; var after = new double[101]; + PcSampler.Label = $"{frame} {what}"; using (PcSampler.Start()) { for (int s = 0; s < before.Length; s++) diff --git a/bench/Vorticity.Zstd.Perf/PcSampler.cs b/bench/Vorticity.Zstd.Perf/PcSampler.cs index b0aa6b6f0..0710e18b0 100644 --- a/bench/Vorticity.Zstd.Perf/PcSampler.cs +++ b/bench/Vorticity.Zstd.Perf/PcSampler.cs @@ -10,12 +10,14 @@ namespace Vorticity.Zstd.Perf; /// /// A sampling profiler at the instruction, for the micro-benchmarks (--pcprofile file): a -/// thread suspends the measured one about every 100 microseconds, reads its program counter through -/// Mach's thread_get_state, and counts the addresses as they are, after a header that places -/// the images: the executable's slide, and where a symbol of each native library registered loaded -/// (bench/zstd-pcprofile.py resolves them, against the executable's dSYM or the library's symbols, -/// and annotates them). The address of a suspended thread is the next instruction to retire: one -/// that waits, or the one after it. +/// thread suspends the measured one about every 100 microseconds, reads its program counter, and +/// counts the addresses. On macOS through Mach's thread_get_state, the addresses as they are, +/// after a header that places the images: the executable's slide, and where a symbol of each native +/// library registered loaded (bench/zstd-pcprofile.py resolves them, against the executable's +/// dSYM or the library's symbols, and annotates them). On Windows through GetThreadContext, +/// resolved in the process by DbgHelp against the PDBs (see ), which +/// appends a report to the file. The address of a suspended thread is the next instruction to +/// retire: one that waits, or the one after it. /// internal static unsafe class PcSampler { @@ -26,6 +28,23 @@ internal static unsafe class PcSampler /// The file the samples are appended to, or null when not profiling. public static string? Path { get; set; } + /// + /// On Windows, the functions whose samples the report also gives instruction by instruction: + /// those whose name contains this (--pcfunc). + /// + public static string? Function { get; set; } + + /// + /// On Windows, the functions whose samples the report also attributes to their callers: those + /// whose name contains this (--pccaller). The caller is the first word of the stack, at + /// the sample, that follows a call instruction into another function: exact in a function that + /// keeps the return address within its first eight words, as a leaf such as memmove does. + /// + public static string? Callers { get; set; } + + /// What the next session measures, for the report's heading. + public static string Label { get; set; } = string.Empty; + /// Native libraries to place: their path, a symbol, and where it is loaded. public static List<(string Library, string Symbol, nint Address)> Libraries { get; } = []; @@ -45,7 +64,8 @@ internal static unsafe class PcSampler private static extern nint _dyld_get_image_vmaddr_slide(uint imageIndex); /// Samples the calling thread until disposed, when is set. - public static IDisposable? Start() => Path is null ? null : new Session(Path); + public static IDisposable? Start() => + Path is null ? null : OperatingSystem.IsWindows() ? new WindowsSession(Path) : new Session(Path); private sealed class Session : IDisposable { @@ -98,4 +118,319 @@ private void Run() } } } + + /// + /// The Windows sampler: the measured thread suspended, its CONTEXT's instruction pointer + /// read, the thread resumed, and only then the address counted, so that nothing allocates while + /// the thread may hold a lock. + /// + private sealed class WindowsSession : IDisposable + { + private const uint ContextControl = 0x0010_0001; // CONTEXT_AMD64 | CONTEXT_CONTROL + private const int ContextSize = 1232; + private const int ContextFlagsOffset = 0x30; + private const int RipOffset = 0xF8; + private const int RspOffset = 0x98; + private const int StackWords = 8; + private const uint DuplicateSameAccess = 2; + + private readonly string _path; + private readonly string _label = Label; + private readonly nint _target; + private readonly Dictionary _counts = []; + private readonly List<(ulong Rip, ulong[] Stack)> _stacks = []; + private readonly bool _callers = Callers is not null; + private readonly Thread _thread; + private volatile bool _stop; + + public WindowsSession(string path) + { + _path = path; + nint process = GetCurrentProcess(); + if (!DuplicateHandle(process, GetCurrentThread(), process, out _target, 0, false, DuplicateSameAccess)) + { + throw new InvalidOperationException("DuplicateHandle failed: " + Marshal.GetLastPInvokeError()); + } + + _thread = new Thread(Run) { IsBackground = true, Priority = ThreadPriority.Highest }; + _thread.Start(); + } + + public void Dispose() + { + _stop = true; + _thread.Join(); + _ = CloseHandle(_target); + string report = WindowsSymbols.Report(_counts, _label, Function) + WindowsSymbols.CallerReport(_stacks, Callers); + File.AppendAllText(_path, report); + Console.Write(report); + } + + private void Run() + { + byte* context = (byte*)NativeMemory.AlignedAlloc(ContextSize, 64); + ulong* window = stackalloc ulong[StackWords]; + long interval = Stopwatch.Frequency / 10_000; + try + { + while (!_stop) + { + long next = Stopwatch.GetTimestamp() + interval; + if (SuspendThread(_target) != uint.MaxValue) + { + *(uint*)(context + ContextFlagsOffset) = ContextControl; + bool read = GetThreadContext(_target, context); + if (read && _callers) + { + // The top of the stack, copied while the thread cannot change it. + ulong* top = (ulong*)*(ulong*)(context + RspOffset); + for (int k = 0; k < StackWords; k++) + { + window[k] = top[k]; + } + } + + _ = ResumeThread(_target); + if (read) + { + ulong rip = *(ulong*)(context + RipOffset); + _counts[rip] = _counts.GetValueOrDefault(rip) + 1; + if (_callers) + { + _stacks.Add((rip, new ReadOnlySpan(window, StackWords).ToArray())); + } + } + } + + while (Stopwatch.GetTimestamp() < next) + { + Thread.SpinWait(20); + } + } + } + finally + { + NativeMemory.AlignedFree(context); + } + } + + [DllImport("kernel32.dll")] + private static extern nint GetCurrentProcess(); + + [DllImport("kernel32.dll")] + private static extern nint GetCurrentThread(); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DuplicateHandle( + nint sourceProcess, nint sourceHandle, nint targetProcess, out nint targetHandle, uint desiredAccess, bool inheritHandle, uint options); + + [DllImport("kernel32.dll")] + private static extern bool CloseHandle(nint handle); + + [DllImport("kernel32.dll")] + private static extern uint SuspendThread(nint thread); + + [DllImport("kernel32.dll")] + private static extern uint ResumeThread(nint thread); + + [DllImport("kernel32.dll")] + private static extern bool GetThreadContext(nint thread, byte* context); + } +} + +/// +/// The samples of on Windows, resolved in the process by DbgHelp: the +/// executable's PDB, which Native AOT writes beside it, names its functions and their lines, and +/// a native library's names what it exports. The report gives the functions by samples, the lines +/// by samples, and for the functions --pcfunc names, every sampled instruction by its +/// relative virtual address, which dumpbin /disasm prints the code at. +/// +internal static unsafe class WindowsSymbols +{ + private const uint SymOptUndname = 0x2; + private const uint SymOptDeferredLoads = 0x4; + private const uint SymOptLoadLines = 0x10; + private const uint SymOptFailCriticalErrors = 0x200; + private const int MaxName = 1024; + private const int SymbolInfoSize = 88; + private const int LineSize = 40; + + private static bool s_initialized; + + public static string Report(Dictionary counts, string label, string? function) + { + nint process = -1; + if (!s_initialized) + { + SymSetOptions(SymOptUndname | SymOptDeferredLoads | SymOptLoadLines | SymOptFailCriticalErrors); + s_initialized = SymInitializeW(process, AppContext.BaseDirectory, true); + } + else + { + SymRefreshModuleList(process); + } + + long total = counts.Values.Sum(); + var functions = new Dictionary(); + var lines = new Dictionary<(string Function, string Line), long>(); + var instructions = new List<(string Function, ulong Rva, ulong Offset, string Line, int Samples)>(); + byte* symbol = stackalloc byte[SymbolInfoSize + (MaxName * 2)]; + byte* line = stackalloc byte[LineSize]; + foreach ((ulong address, int samples) in counts) + { + new Span(symbol, SymbolInfoSize).Clear(); + *(uint*)symbol = SymbolInfoSize; + *(uint*)(symbol + 80) = MaxName; + string name; + ulong displacement = 0; + ulong rva = 0; + if (SymFromAddrW(process, address, out displacement, symbol)) + { + int length = (int)*(uint*)(symbol + 76); + name = new string((char*)(symbol + 84), 0, Math.Min(length, MaxName - 1)); + rva = address - *(ulong*)(symbol + 32); + } + else + { + name = "?"; + } + + new Span(line, LineSize).Clear(); + *(uint*)line = LineSize; + string where = SymGetLineFromAddrW64(process, address, out _, line) + ? $"{System.IO.Path.GetFileName(new string(*(char**)(line + 24)))}:{*(uint*)(line + 16)}" + : "?"; + functions[name] = functions.GetValueOrDefault(name) + samples; + lines[(name, where)] = lines.GetValueOrDefault((name, where)) + samples; + if (function is not null && name.Contains(function, StringComparison.Ordinal)) + { + instructions.Add((name, rva, displacement, where, samples)); + } + } + + var report = new System.Text.StringBuilder(); + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $"== {label}: {total} samples"); + report.AppendLine(" samples % function"); + foreach ((string name, long samples) in functions.OrderByDescending(f => f.Value).Take(30)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" {samples,7} {100.0 * samples / total,6:F2} {name}"); + } + + report.AppendLine(" -- lines"); + foreach (((string name, string where), long samples) in lines.OrderByDescending(l => l.Value).Take(40)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" {samples,7} {100.0 * samples / total,6:F2} {where,-40} {name}"); + } + + foreach (var group in instructions.GroupBy(i => i.Function)) + { + long inFunction = group.Sum(i => (long)i.Samples); + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" -- {group.Key}: {inFunction} samples, by instruction (rva, offset)"); + foreach (var i in group.OrderBy(i => i.Rva)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, + $" {i.Rva:x8} +{i.Offset,-6:x} {i.Samples,7} {100.0 * i.Samples / inFunction,6:F2} {i.Line}"); + } + } + + return report.ToString(); + } + + /// + /// The samples of the functions names, by caller: for each, the first + /// stack word that resolves into another function and follows a call (a call rel32, or an + /// indirect call of two, three or six bytes), with the line of the call. + /// + public static string CallerReport(List<(ulong Rip, ulong[] Stack)> samples, string? function) + { + if (function is null || samples.Count == 0) + { + return string.Empty; + } + + nint process = -1; + byte* symbol = stackalloc byte[SymbolInfoSize + (MaxName * 2)]; + byte* line = stackalloc byte[LineSize]; + var names = new Dictionary(); + string? NameOf(ulong address) + { + if (names.TryGetValue(address, out string? known)) + { + return known; + } + + new Span(symbol, SymbolInfoSize).Clear(); + *(uint*)symbol = SymbolInfoSize; + *(uint*)(symbol + 80) = MaxName; + string? name = SymFromAddrW(process, address, out _, symbol) + ? new string((char*)(symbol + 84), 0, Math.Min((int)*(uint*)(symbol + 76), MaxName - 1)) + : null; + names[address] = name; + return name; + } + + var callers = new Dictionary(); + long total = 0; + foreach ((ulong rip, ulong[] stack) in samples) + { + string? name = NameOf(rip); + if (name is null || !name.Contains(function, StringComparison.Ordinal)) + { + continue; + } + + total++; + string caller = "?"; + foreach (ulong word in stack) + { + string? callerName = NameOf(word); + if (callerName is null || callerName == name || !FollowsCall(word)) + { + continue; + } + + new Span(line, LineSize).Clear(); + *(uint*)line = LineSize; + string where = SymGetLineFromAddrW64(process, word - 1, out _, line) + ? $"{System.IO.Path.GetFileName(new string(*(char**)(line + 24)))}:{*(uint*)(line + 16)}" + : "?"; + caller = $"{where,-40} {callerName}"; + break; + } + + callers[caller] = callers.GetValueOrDefault(caller) + 1; + } + + var report = new System.Text.StringBuilder(); + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" -- callers of {function}: {total} samples"); + foreach ((string caller, long count) in callers.OrderByDescending(c => c.Value).Take(30)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" {count,7} {100.0 * count / total,6:F2} {caller}"); + } + + return report.ToString(); + + // A return address follows the call that pushed it: E8 rel32, or FF /2 in its short forms. + static bool FollowsCall(ulong address) + { + byte* p = (byte*)address; + return p[-5] == 0xE8 || (p[-2] == 0xFF && (p[-1] & 0x38) == 0x10) + || (p[-3] == 0xFF && (p[-2] & 0x38) == 0x10) || (p[-6] == 0xFF && (p[-5] & 0x38) == 0x10); + } + } + + [DllImport("dbghelp.dll")] + private static extern uint SymSetOptions(uint options); + + [DllImport("dbghelp.dll", CharSet = CharSet.Unicode)] + private static extern bool SymInitializeW(nint process, string? searchPath, bool invadeProcess); + + [DllImport("dbghelp.dll")] + private static extern bool SymRefreshModuleList(nint process); + + [DllImport("dbghelp.dll")] + private static extern bool SymFromAddrW(nint process, ulong address, out ulong displacement, byte* symbol); + + [DllImport("dbghelp.dll")] + private static extern bool SymGetLineFromAddrW64(nint process, ulong address, out uint displacement, byte* line); } diff --git a/bench/Vorticity.Zstd.Perf/Program.cs b/bench/Vorticity.Zstd.Perf/Program.cs index cb6f8ad69..1c8c43e19 100644 --- a/bench/Vorticity.Zstd.Perf/Program.cs +++ b/bench/Vorticity.Zstd.Perf/Program.cs @@ -16,8 +16,10 @@ namespace Vorticity.Zstd.Perf; /// moment lands on all of them alike. Several passes show how stable the figures are. /// /// -/// Usage: Vorticity.Zstd.Perf [--frames a,b] [--passes N] [--reps N] [--only zstd,libzstd-ref] [--ab] [--dump dir] [--no-check] [--no-pair] -/// [--pmu default|EV1,EV2] [--compress]; --pmu counts hardware events per decode and sequence (under sudo); --ab +/// Usage: Vorticity.Zstd.Perf [--frames a,b] [--passes N] [--reps N] [--only zstd,libzstd-ref] [--ab] [--dump dir] [--no-check] [--no-pair | --pair] +/// [--pmu default|EV1,EV2] [--compress] [--core N]; --pmu counts hardware events per decode and sequence (macOS, under +/// sudo); --core N keeps the timed thread on logical processor N (Windows, where the process and +/// the thread also run at a high priority, as macOS's user-interactive QoS); --ab /// adds a "before" candidate that runs with the legacy switches of the point under work (see /// ). --compress times compression instead: each frame's content, at the /// level its name gives (3 for the reference). Before timing, @@ -30,9 +32,20 @@ public static class Program private sealed record Candidate(string Name, Func Decode); + private const int ThreadPriorityHighest = 2; + [DllImport("libSystem.dylib")] private static extern int pthread_set_qos_class_self_np(int qosClass, int relativePriority); + [DllImport("kernel32.dll")] + private static extern nint GetCurrentThread(); + + [DllImport("kernel32.dll")] + private static extern bool SetThreadPriority(nint thread, int priority); + + [DllImport("kernel32.dll")] + private static extern nuint SetThreadAffinityMask(nint thread, nuint mask); + public static int Main(string[] args) { // The timed thread at the user-interactive QoS: kept on the performance cores at their full @@ -41,6 +54,17 @@ public static int Main(string[] args) { _ = pthread_set_qos_class_self_np(QosClassUserInteractive, 0); } + else if (OperatingSystem.IsWindows()) + { + // The same on Windows: the process and the timed thread at a high priority, and with + // --core N the thread kept on logical processor N, which no migration then disturbs. + Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High; + _ = SetThreadPriority(GetCurrentThread(), ThreadPriorityHighest); + if (Option(args, "--core") is string core) + { + _ = SetThreadAffinityMask(GetCurrentThread(), (nuint)1 << int.Parse(core, CultureInfo.InvariantCulture)); + } + } string[] frames = Option(args, "--frames")?.Split(',') ?? BenchFrames.Names; if (Option(args, "--dump") is string directory) @@ -59,7 +83,7 @@ public static int Main(string[] args) { // zstd's own benchmark data, level by level: bench/zstd-corpus.sh downloads it. return Corpus.Run(corpora.Split(','), Option(args, "--levels"), Option(args, "--seconds"), Option(args, "--only"), - Option(args, "--results"), Option(args, "--markdown")); + Option(args, "--results"), Option(args, "--markdown"), Option(args, "--directions")); } if (Option(args, "--corpus-report") is string results) @@ -70,6 +94,8 @@ public static int Main(string[] args) if (Option(args, "--micro") is string micro) { PcSampler.Path = Option(args, "--pcprofile"); + PcSampler.Function = Option(args, "--pcfunc"); + PcSampler.Callers = Option(args, "--pccaller"); return Micro.Run(micro, frames, int.Parse(Option(args, "--repeat") ?? "1", CultureInfo.InvariantCulture)); } @@ -115,8 +141,13 @@ public static int Main(string[] args) return 0; } - // --no-pair: one block at a time, to measure what pairing them brings. - var zstd = new ZstdDecompressor { PairsBlocks = !args.Contains("--no-pair") }; + // --no-pair: one block at a time, --pair: two, to measure what pairing them brings; by + // default, as the decoder chooses for the machine. + var zstd = new ZstdDecompressor(); + if (args.Contains("--no-pair") || args.Contains("--pair")) + { + zstd.PairsBlocks = args.Contains("--pair"); + } var platform = new ZstandardDecoder(); NativeReference? native = NativeReference.TryLoad(); diff --git a/bench/Vorticity.Zstd.Shared/NativeReference.cs b/bench/Vorticity.Zstd.Shared/NativeReference.cs index 68840c662..dcb21ee2a 100644 --- a/bench/Vorticity.Zstd.Shared/NativeReference.cs +++ b/bench/Vorticity.Zstd.Shared/NativeReference.cs @@ -50,6 +50,11 @@ private NativeReference(nint library) _context = create(); _compressionContext = createCompression(); _sequencesContext = createCompression(); + if (NativeLibrary.TryGetExport(library, "HUF_decompress4X_usingDTable", out _) + && NativeLibrary.TryGetExport(library, "HUF_readDTableX2_wksp", out _)) + { + Huffman = new HuffmanReference(library, _isError); + } } /// Where ZSTD_versionNumber is loaded: the library's slide, for a profiler. @@ -59,10 +64,13 @@ private NativeReference(nint library) public static string LibraryPath => Environment.GetEnvironmentVariable("VORTICITY_ZSTD_LIBZSTD") is { Length: > 0 } path ? path - : Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref.dylib"); + : Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref" + LibrarySuffix); + + /// The extension of a shared library here: tools/native-ref builds dylibs on macOS, DLLs on Windows. + private static string LibrarySuffix => OperatingSystem.IsWindows() ? ".dll" : OperatingSystem.IsMacOS() ? ".dylib" : ".so"; /// The same library built with threads (build.sh mt): zstdmt. - public static string ThreadedLibraryPath => Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_mt.dylib"); + public static string ThreadedLibraryPath => Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_mt" + LibrarySuffix); /// The reference library, or null when tools/native-ref/build.sh has not been run. public static NativeReference? TryLoad() => TryLoad(LibraryPath); @@ -73,6 +81,62 @@ private NativeReference(nint library) private static NativeReference? TryLoad(string path) => File.Exists(path) && NativeLibrary.TryLoad(path, out nint library) ? new NativeReference(library) : null; + /// + /// libzstd's literals decoding alone, for the micro-benchmarks: HUF_readDTableX1_wksp or + /// HUF_readDTableX2_wksp on a tree description, then HUF_decompress4X_usingDTable on + /// four streams. Exported by the DLLs a GNU-compatible compiler builds (every symbol is); null + /// where the library does not export them. + /// + public HuffmanReference? Huffman { get; } + + /// See . + internal sealed class HuffmanReference + { + private const int TableLogMax = 12; + private const int Bmi2 = 1; + + private readonly delegate* unmanaged _readX1; + private readonly delegate* unmanaged _readX2; + private readonly delegate* unmanaged _decompress4X; + private readonly delegate* unmanaged _isError; + private readonly uint[] _table = new uint[1 + (1 << TableLogMax)]; + private readonly uint[] _workspace = new uint[4096]; + + public HuffmanReference(nint library, delegate* unmanaged isError) + { + _readX1 = (delegate* unmanaged)NativeLibrary.GetExport(library, "HUF_readDTableX1_wksp"); + _readX2 = (delegate* unmanaged)NativeLibrary.GetExport(library, "HUF_readDTableX2_wksp"); + _decompress4X = (delegate* unmanaged)NativeLibrary.GetExport(library, "HUF_decompress4X_usingDTable"); + _isError = isError; + } + + /// Reads a tree into the table, for X2 the double-symbol one: the tree's size, or -1. + public int ReadTree(ReadOnlySpan tree, bool x2) + { + // HUF_CREATE_STATIC_DTABLEX1 and X2: the table's largest log in the description word. + _table[0] = (uint)(x2 ? TableLogMax : TableLogMax - 1) * 0x01000001u; + fixed (uint* table = _table) + fixed (uint* workspace = _workspace) + fixed (byte* src = tree) + { + nuint result = (x2 ? _readX2 : _readX1)(table, src, (nuint)tree.Length, workspace, (nuint)(_workspace.Length * 4), Bmi2); + return _isError(result) != 0 ? -1 : (int)result; + } + } + + /// Decodes four streams with the table last read: the size written, or -1. + public int DecodeFourStreams(ReadOnlySpan streams, Span output) + { + fixed (uint* table = _table) + fixed (byte* src = streams) + fixed (byte* dst = output) + { + nuint result = _decompress4X(dst, (nuint)output.Length, src, (nuint)streams.Length, table, Bmi2); + return _isError(result) != 0 ? -1 : (int)result; + } + } + } + /// ZSTD_decompressDCtx: the decoded size, or -1 on an error. public int Decompress(ReadOnlySpan source, Span destination) { diff --git a/bench/zstd-ab-aot.sh b/bench/zstd-ab-aot.sh index fbe304aea..1a380a5ab 100755 --- a/bench/zstd-ab-aot.sh +++ b/bench/zstd-ab-aot.sh @@ -1,15 +1,23 @@ #!/usr/bin/env bash # A/B between two Native AOT builds of the perf tool, for changes that cannot be switched at run -# time (a table format, an assembly attribute). Builds HEAD (or the ref given) into a worktree as -# "before", the working tree as "after", then runs them alternately and compares their medians. +# time (a table format, an assembly attribute, a hot loop's code). Builds HEAD (or the ref given) +# into a worktree as "before", the working tree as "after", then runs them alternately and +# compares their medians. # -# bench/zstd-ab-aot.sh [--ref ] [--rounds N] [--frames a,b] [--compress | --micro ] +# bench/zstd-ab-aot.sh [--ref ] [--rounds N] [--frames a,b] [--compress | --micro ] [--core N] +# [--no-build | --before --after ] # # --compress times compression (Vorticity.Zstd.Perf --compress) instead of decompression; --micro times a # micro-benchmark (Vorticity.Zstd.Perf --micro , e.g. dcorpus with --frames github-dict-L3). +# --core N keeps the timed thread on logical processor N (Windows; see Program.cs). --no-build +# compares the two builds the last run left, for more rounds or other frames; --before X and +# --after Y compare two builds kept under artifacts/ab (implying --no-build). +# +# Both builds are published under artifacts/ab, in this tree: the tool finds the native reference +# and the data git does not keep (tools/native-ref/out, the test data) by walking up from itself. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ref=HEAD; rounds=8; frames=reference,text-L3,json-L19; mode=(); micro= +ref=HEAD; rounds=8; frames=reference,text-L3,json-L19; mode=(); micro=; pin=(); build=1; before=before; after=after while [[ $# -gt 0 ]]; do case "$1" in --ref) ref="$2"; shift 2 ;; @@ -17,36 +25,48 @@ while [[ $# -gt 0 ]]; do --frames) frames="$2"; shift 2 ;; --compress) mode=(--compress); shift ;; --micro) micro="$2"; shift 2 ;; + --core) pin=(--core "$2"); shift 2 ;; + --no-build) build=0; shift ;; + --before) before="$2"; build=0; shift 2 ;; + --after) after="$2"; build=0; shift 2 ;; *) echo "unknown $1" >&2; exit 2 ;; esac done work="$root/artifacts/ab" mkdir -p "$work" -rm -rf "$work/tree" -git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null 2>&1 || { git -C "$root" worktree prune; git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null; } -# The worktree has neither the native reference nor the data git does not keep (the test data, the -# benchmark corpora): point it at the main tree's. -ln -sfn "$root/tools/native-ref/out" "$work/tree/tools/native-ref/out" -[[ -e "$work/tree/tests/Vorticity.Zstd.Tests/testdata" ]] || ln -s "$root/tests/Vorticity.Zstd.Tests/testdata" "$work/tree/tests/Vorticity.Zstd.Tests/testdata" -dotnet publish "$work/tree/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/before" > /dev/null -dotnet publish "$root/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/after" > /dev/null -git -C "$root" worktree remove --force "$work/tree" +if [[ $build == 1 ]]; then + rm -rf "$work/tree" + git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null 2>&1 || { git -C "$root" worktree prune; git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null; } + dotnet publish "$work/tree/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/before" > /dev/null + dotnet publish "$root/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/after" > /dev/null + git -C "$root" worktree remove --force "$work/tree" +fi + +run() { + local side="$1" frame="$2" + if [[ -n "$micro" ]]; then + "$work/$side/Vorticity.Zstd.Perf" ${pin[@]+"${pin[@]}"} --micro "$micro" --frames "$frame" | sed -nE 's/.*: +([0-9.]+) ns.*/\1/p' + else + "$work/$side/Vorticity.Zstd.Perf" ${pin[@]+"${pin[@]}"} ${mode[@]+"${mode[@]}"} --frames "$frame" --only zstd --passes 1 --reps 300 | awk '/^ zstd/ {print $4}' + fi +} + for frame in ${frames//,/ }; do b=(); a=() for ((i = 0; i < rounds; i++)); do - if [[ -n "$micro" ]]; then - b+=("$("$work/before/Vorticity.Zstd.Perf" --micro "$micro" --frames "$frame" | sed -nE 's/.*: +([0-9.]+) ns.*/\1/p')") - a+=("$("$work/after/Vorticity.Zstd.Perf" --micro "$micro" --frames "$frame" | sed -nE 's/.*: +([0-9.]+) ns.*/\1/p')") - else - b+=("$("$work/before/Vorticity.Zstd.Perf" ${mode[@]+"${mode[@]}"} --frames "$frame" --only zstd --passes 1 --reps 300 | awk '/^ zstd/ {print $4}')") - a+=("$("$work/after/Vorticity.Zstd.Perf" ${mode[@]+"${mode[@]}"} --frames "$frame" --only zstd --passes 1 --reps 300 | awk '/^ zstd/ {print $4}')") - fi + b+=("$(run "$before" "$frame")") + a+=("$(run "$after" "$frame")") done - python3 - "$frame" "${b[*]}" "${a[*]}" <<'PY' -import sys, statistics -frame, b, a = sys.argv[1], list(map(float, sys.argv[2].split())), list(map(float, sys.argv[3].split())) -mb, ma = statistics.median(b), statistics.median(a) -pairs = " ".join(f"{x / y:.3f}" for x, y in zip(b, a)) -print(f"{frame:16} before {mb:9.1f} after {ma:9.1f} speedup {mb / ma:.3f}x (per round: {pairs})") -PY + awk -v frame="$frame" -v b="${b[*]}" -v a="${a[*]}" ' + function median(v, n, i, j, t, s) { + for (i = 1; i <= n; i++) s[i] = v[i] + for (i = 2; i <= n; i++) for (j = i; j > 1 && s[j - 1] > s[j]; j--) { t = s[j]; s[j] = s[j - 1]; s[j - 1] = t } + return n % 2 ? s[(n + 1) / 2] : (s[n / 2] + s[n / 2 + 1]) / 2 + } + BEGIN { + n = split(b, before, " "); split(a, after, " ") + for (i = 1; i <= n; i++) pairs = pairs sprintf(" %.3f", before[i] / after[i]) + mb = median(before, n); ma = median(after, n) + printf "%-16s before %9.1f after %9.1f speedup %.3fx (per round:%s)\n", frame, mb, ma, mb / ma, pairs + }' done diff --git a/src/Vorticity.Zstd/README.md b/src/Vorticity.Zstd/README.md index 95aa8f559..b36586fa6 100644 --- a/src/Vorticity.Zstd/README.md +++ b/src/Vorticity.Zstd/README.md @@ -227,6 +227,17 @@ dotnet publish bench/Vorticity.Zstd.Perf -c Release -o artifacts/perf artifacts/perf/Vorticity.Zstd.Perf --corpus silesia,github,github-dict --results corpus.tsv --markdown corpus.md ``` +On Windows x64, from Git Bash, the reference is built with a GNU-compatible clang such as +[llvm-mingw](https://github.com/mstorsjo/llvm-mingw): MSVC builds neither zstd's assembly Huffman +decoder nor its BMI2 dispatch, which libzstd has wherever a GNU-compatible compiler builds it. +`--core N` keeps the timed thread on one logical processor: + +```sh +CC=/bin/clang tools/native-ref/build.sh && bench/zstd-corpus.sh +dotnet publish bench/Vorticity.Zstd.Perf -c Release -o artifacts/perf +artifacts/perf/Vorticity.Zstd.Perf --core 2 --corpus silesia,github,github-dict --results corpus.tsv --markdown corpus.md +``` + ## Testing Correctness is checked against libzstd itself: @@ -256,8 +267,10 @@ dotnet test The SDK version is pinned in `global.json`. `testdata.sh` downloads the zstd 1.5.7 sources once, checks their SHA-256, builds zstd's `decodecorpus` with the system C compiler and writes the test data; it -runs on macOS and Linux. The native reference that `tools/native-ref/build.sh` builds is needed only -for the multithreaded-compression tests and for the benchmarks; that script targets macOS. +runs on macOS and Linux, and on Windows from Git Bash with a GNU-compatible clang as `CC`. The +native reference that `tools/native-ref/build.sh` builds is needed only for the +multithreaded-compression tests and for the benchmarks: dylibs on macOS, DLLs on Windows (with the +same `CC`), shared objects elsewhere. ## Repository layout diff --git a/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs b/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs index cc535095d..8bf038838 100644 --- a/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs +++ b/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs @@ -42,10 +42,10 @@ private LibzstdMt(nint library) _errorName = (delegate* unmanaged)NativeLibrary.GetExport(library, "ZSTD_getErrorName"); } - /// The library, loaded once; null when tools/native-ref/out/libzstd_mt.dylib is missing. + /// The library, loaded once; null when tools/native-ref/out/libzstd_mt (build.sh mt) is missing. public static LibzstdMt? Instance { get; } = Load(); - public static string LibraryPath => Path.Combine(RepositoryRoot(), "tools", "native-ref", "out", "libzstd_mt.dylib"); + public static string LibraryPath => Path.Combine(RepositoryRoot(), "tools", "native-ref", "out", "libzstd_mt" + (OperatingSystem.IsWindows() ? ".dll" : OperatingSystem.IsMacOS() ? ".dylib" : ".so")); private static LibzstdMt? Load() => File.Exists(LibraryPath) && NativeLibrary.TryLoad(LibraryPath, out nint library) ? new LibzstdMt(library) : null; diff --git a/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs b/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs index 6e605e621..a14b43d20 100644 --- a/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs +++ b/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs @@ -41,7 +41,7 @@ private LibzstdOracle(nint library) private static LibzstdOracle? Load() { - string path = Path.Combine(TestData.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref.dylib"); + string path = Path.Combine(TestData.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref" + (OperatingSystem.IsWindows() ? ".dll" : OperatingSystem.IsMacOS() ? ".dylib" : ".so")); return File.Exists(path) && NativeLibrary.TryLoad(path, out nint library) ? new LibzstdOracle(library) : null; } diff --git a/tools/native-ref/build.sh b/tools/native-ref/build.sh index 28906a356..74983c823 100755 --- a/tools/native-ref/build.sh +++ b/tools/native-ref/build.sh @@ -1,16 +1,20 @@ #!/usr/bin/env bash # Builds the native reference side of the comparisons, into tools/native-ref/out/: # -# libzstd_ref.dylib zstd 1.5.7 compiled with today's compiler at -O3: the libzstd a well-built +# libzstd_ref. zstd 1.5.7 compiled with today's compiler at -O3: the libzstd a well-built # C program would ship, and the speed this project aims for. -# zd the timing harness (zd.c): compresses the reference frame with the first -# library, writes it out, and times ZSTD_decompressDCtx in each library given. -# libzstd_mt.dylib the same library with ZSTD_MULTITHREAD: zstdmt, what libzstd writes with +# zd the timing harness (zd.c, macOS only): compresses the reference frame with the +# first library, writes it out, and times ZSTD_decompressDCtx in each library given. +# libzstd_mt. the same library with ZSTD_MULTITHREAD: zstdmt, what libzstd writes with # ZSTD_c_nbWorkers >= 1, the oracle of the parallel compressor. `build.sh mt` # builds it alone. # -# The sources are the zstd 1.5.7 release tarball (zstd-source.sh). macOS: the libraries are dylibs, -# and zd times with mach_absolute_time. The test data comes from testdata.sh, which runs anywhere. +# The sources are the zstd 1.5.7 release tarball (zstd-source.sh). The libraries are dylibs on macOS, +# where zd times with mach_absolute_time; DLLs on Windows, from Git Bash with a GNU-compatible clang +# (CC=/bin/clang: MSVC builds neither the assembly nor the BMI2 dispatch libzstd ships +# with); shared objects elsewhere. On x86-64 the library takes zstd's assembly Huffman decoder, as +# libzstd built by a GNU-compatible compiler does there. The test data comes from testdata.sh, which +# runs anywhere. set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -20,19 +24,39 @@ lib="$src/lib" cc="${CC:-cc}" echo "compiler: $($cc --version | head -1)" -# libzstd_mt.dylib: compression with worker threads (zstdmt), decompression for the round trips. +pic=() +pdb= +threads=(-pthread) +case "$(uname -s)" in + Darwin) shared=(-dynamiclib); ext=dylib ;; + # Windows threads for zstdmt (lib/common/threading.c), not winpthreads, which would be one more DLL. + # A PDB beside each DLL, for profilers: debug information leaves the code as it is. + MINGW* | MSYS* | CYGWIN*) shared=(-shared); ext=dll; threads=(); pdb=1 ;; + *) shared=(-shared); ext=so; pic=(-fPIC) ;; +esac + +asm=() +if [[ "$(uname -m)" == x86_64 ]]; then + asm=("$lib/decompress/huf_decompress_amd64.S") +fi + +sources=( + "$lib"/common/*.c "$lib"/compress/*.c + "$lib/decompress/zstd_ddict.c" "$lib/decompress/zstd_decompress.c" + "$lib/decompress/zstd_decompress_block.c" "$lib/decompress/huf_decompress.c" + ${asm[@]+"${asm[@]}"} +) + +# libzstd_mt: compression with worker threads (zstdmt), decompression for the round trips. build_mt() { local objs="$out/obj-mt" rm -rf "$objs" && mkdir -p "$objs" ( cd "$objs" - $cc -O3 -DZSTD_MULTITHREAD -pthread -c -I"$lib" -I"$lib/common" \ - "$lib"/common/*.c "$lib"/compress/*.c \ - "$lib/decompress/zstd_ddict.c" "$lib/decompress/zstd_decompress.c" \ - "$lib/decompress/zstd_decompress_block.c" "$lib/decompress/huf_decompress.c" - $cc -dynamiclib -pthread -o "$out/libzstd_mt.dylib" ./*.o + $cc -O3 ${pic[@]+"${pic[@]}"} -DZSTD_MULTITHREAD ${threads[@]+"${threads[@]}"} -c -I"$lib" -I"$lib/common" "${sources[@]}" + $cc "${shared[@]}" ${threads[@]+"${threads[@]}"} -o "$out/libzstd_mt.$ext" ./*.o ) - echo "built: $out/libzstd_mt.dylib" + echo "built: $out/libzstd_mt.$ext" } if [[ "${1:-}" == mt ]]; then @@ -40,20 +64,19 @@ if [[ "${1:-}" == mt ]]; then exit 0 fi -# libzstd_ref.dylib: the reference library, with the flags of the measurement it reproduces. +# libzstd_ref: the reference library, with the flags of the measurement it reproduces. objs="$out/obj-ref" rm -rf "$objs" && mkdir -p "$objs" ( cd "$objs" - $cc -O3 -c -I"$lib" -I"$lib/common" \ - "$lib"/common/*.c "$lib"/compress/*.c \ - "$lib/decompress/zstd_ddict.c" "$lib/decompress/zstd_decompress.c" \ - "$lib/decompress/zstd_decompress_block.c" "$lib/decompress/huf_decompress.c" - $cc -dynamiclib -o "$out/libzstd_ref.dylib" ./*.o + $cc -O3 ${pic[@]+"${pic[@]}"} ${pdb:+-g -gcodeview} -c -I"$lib" -I"$lib/common" "${sources[@]}" + $cc "${shared[@]}" ${pdb:+-Wl,--pdb=$out/libzstd_ref.pdb} -o "$out/libzstd_ref.$ext" ./*.o ) -# zd: the timing harness. -$cc -O2 -o "$out/zd" "$here/zd.c" +# zd: the timing harness, on macOS. +if [[ "$ext" == dylib ]]; then + $cc -O2 -o "$out/zd" "$here/zd.c" +fi build_mt -echo "built: $out/libzstd_ref.dylib $out/zd" +echo "built: $out/libzstd_ref.$ext" diff --git a/tools/native-ref/testdata.sh b/tools/native-ref/testdata.sh index e9ecdd04c..7d1b14794 100755 --- a/tools/native-ref/testdata.sh +++ b/tools/native-ref/testdata.sh @@ -10,7 +10,8 @@ # kept, with a manifest of each original's size and SHA-256: the originals would weigh # tens of megabytes. The seeds below make them the same bytes on every machine. # -# Needs curl, tar and a C compiler (CC, or cc); runs on macOS and Linux. CI runs it before the tests, +# Needs curl, tar and a C compiler (CC, or cc); runs on macOS and Linux, and on Windows from Git Bash +# with a GNU-compatible clang as CC (llvm-mingw). CI runs it before the tests, # which fail there without the data, and skip elsewhere. set -euo pipefail From 88925070e2898a4feb5380f8f3ee183dc039784a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:54:24 +0200 Subject: [PATCH 5/8] docs(guide): measure the x64 page's own sections again, on the corrected harness The encodings column by column, the advice and CompressorBenchmarks, measured again on the Ryzen 9 7950X at a3dfc8ab. The page had been measured before the harness held both sides to the same work (78cb0c4): its scans took over the mapping the last pass left, where every scan now maps its file anew, as on the arm64 page, and pays for its pages on Windows. The head says so. Zstd's columns read faster: a take of 1 000 rows 60.3 to 35.1 ms on jittered timestamps, 64.1 to 31.7 ms on integers in 0..999, both under the hint; their scans, which now count the mapping, stay near 80 ms. CompressorBenchmarks' zstd candidate takes 7.5 us where it took 26.8. Pco, newer than the last measure, is now Smallest's pick on several integer columns. --- docs/guide/benchmarks-x64.md | 390 ++++++++++++++++++----------------- 1 file changed, 196 insertions(+), 194 deletions(-) diff --git a/docs/guide/benchmarks-x64.md b/docs/guide/benchmarks-x64.md index 16a913b78..acfa44ac0 100644 --- a/docs/guide/benchmarks-x64.md +++ b/docs/guide/benchmarks-x64.md @@ -12,9 +12,9 @@ not affiliated with or endorsed by the Vortex project or LF Projects, LLC. **The sections that compare Vorticity with Rust are not measured on x64**: each says which arguments write it. The sections that compare Vorticity with itself are: the encodings column by -column, the advice, and the kernels against their baselines, measured with each scan pass -taking over the mapping the previous one left, where [the benchmark page](benchmarks.md) maps the -file anew at every call. The kernel tables' speedups are computed from the Mean column of their run. +column, the advice, and the kernels against their baselines, each scan mapping its file anew, as on +[the benchmark page](benchmarks.md). The kernel tables' speedups are computed from the Mean column +of their run. * **The two sides.** Vorticity as a Native AOT binary built for the machine's instruction set, with the workstation garbage collector, where a section says so, and on the JIT otherwise. Vortex @@ -85,7 +85,8 @@ on an x64 machine. One column of 10,000,000 rows per file, written under each compression profile and each hint that applies to it, then opened and read. A **scan** decodes every value to its plain form and reads it once; a **take** reads 1,000 rows spread over the file. Each figure is the median of 3 passes after -1 warm-up, the file in the page cache; a write is net of generating its rows. **Crosses at** is the +1 warm-up, the file in the page cache and mapped anew by each pass, so that a larger file pays for its +pages on every read; a write is net of generating its rows. **Crosses at** is the storage throughput at which a configuration and `Auto` read the column whole in the same time, counting its bytes at that throughput and then its scan: below it the smaller file reads faster end to end, above it the faster decode does. What the figures mean for a choice is in @@ -100,27 +101,27 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| a sequence | Sequence x77 | 0.00 | 26 | 2.8 | 1.19 | | -| sorted runs of 1 000 | RunEnd x77 | 0.01 | 55 | 2.3 | 1.15 | None: Canonical x77, 8.00 B/value, scan 1.4 ms, take 0.29 ms; reads faster above 88,339 MB/s | -| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 85 | 6.6 | 0.60 | Smallest: Zstd x77, 2.07 B/value, scan 74.5 ms, take 60.30 ms; reads faster below 193 MB/s; None: Canonical x77, 8.00 B/value, scan 1.3 ms, take 0.25 ms; reads faster above 8,691 MB/s | -| random in 0..999 | BitPacked x77 | 1.25 | 0 | 2.4 | 0.30 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 38 | 2.3 | 0.49 | None: Canonical x77, 8.00 B/value, scan 1.4 ms, take 0.24 ms; reads faster above 79,377 MB/s | -| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 32 | 2.6 | 0.26 | Auto, 16 MiB chunks: Dictionary x5, 2.36 B/value, scan 6.7 ms, take 0.47 ms; reads faster below 5,585 MB/s | -| uniform 64-bit | Canonical x77 | 8.00 | 31 | 1.4 | 0.22 | | -| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 15 | 3.6 | 0.31 | hint Dictionary: Dictionary x82, 1.27 B/value, scan 6.3 ms, take 0.53 ms; reads faster below 417 MB/s | +| a sequence | Sequence x77 | 0.00 | 13 | 1.8 | 0.26 | | +| sorted runs of 1 000 | RunEnd x77 | 0.01 | 14 | 2.7 | 0.36 | Smallest: RunEnd x76, Pco x1, 0.01 B/value, scan 3.0 ms, take 0.74 ms; reads faster below 96 MB/s | +| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 70 | 12.9 | 1.72 | Smallest: Pco x77, 1.35 B/value, scan 112.6 ms, take 118.47 ms; reads faster below 204 MB/s | +| random in 0..999 | BitPacked x77 | 1.25 | 22 | 5.5 | 1.54 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 55 | 3.3 | 1.14 | | +| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 38 | 12.9 | 1.51 | Smallest: Pco x77, 0.06 B/value, scan 72.5 ms, take 69.30 ms; reads faster below 767 MB/s; Auto, 16 MiB chunks: Dictionary x5, 2.36 B/value, scan 9.5 ms, take 1.32 ms; reads faster at any throughput | +| uniform 64-bit | Canonical x77 | 8.00 | 43 | 27.0 | 2.14 | | +| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 25 | 6.3 | 1.35 | Smallest: Pco x82, 1.26 B/value, scan 45.3 ms, take 44.57 ms; reads faster below 32 MB/s |
a sequence: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Sequence x77 | 38,836 | 0.00 | 26 | 2.8 | 1.19 | | -| Fastest | Sequence x77 | 38,836 | 0.00 | 27 | 2.7 | 1.22 | same | -| Smallest | Sequence x77 | 38,836 | 0.00 | 29 | 2.8 | 1.48 | same | -| None | Canonical x77 | 80,043,716 | 8.00 | 51 | 2.8 | 1.42 | reads slower at any throughput | -| hint Dictionary | Sequence x77 | 38,836 | 0.00 | 195 | 1.7 | 0.43 | reads faster at any throughput | -| hint BitPacked | Sequence x77 | 38,836 | 0.00 | 5 | 1.7 | 0.42 | reads faster at any throughput | -| hint RunEnd | Sequence x77 | 38,836 | 0.00 | 4 | 1.7 | 0.41 | reads faster at any throughput | -| hint Zstd | Sequence x77 | 38,836 | 0.00 | 4 | 1.7 | 0.43 | reads faster at any throughput | +| Auto | Sequence x77 | 38,844 | 0.00 | 13 | 1.8 | 0.26 | | +| Fastest | Sequence x77 | 38,844 | 0.00 | 22 | 1.9 | 0.28 | reads slower at any throughput | +| Smallest | Sequence x77 | 38,844 | 0.00 | 15 | 1.8 | 0.27 | same | +| None | Canonical x77 | 80,043,724 | 8.00 | 51 | 23.2 | 1.68 | reads slower at any throughput | +| hint Dictionary | Sequence x77 | 38,844 | 0.00 | 121 | 1.8 | 0.28 | same | +| hint BitPacked | Sequence x77 | 38,844 | 0.00 | 15 | 1.8 | 0.28 | same | +| hint RunEnd | Sequence x77 | 38,844 | 0.00 | 13 | 2.1 | 0.37 | reads slower at any throughput | +| hint Zstd | Sequence x77 | 38,844 | 0.00 | 11 | 1.8 | 0.27 | same |
@@ -128,14 +129,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x77 | 87,996 | 0.01 | 55 | 2.3 | 1.15 | | -| Fastest | RunEnd x77 | 87,996 | 0.01 | 67 | 2.3 | 1.14 | same | -| Smallest | RunEnd x76, Zstd x1 | 87,964 | 0.01 | 90 | 3.6 | 1.79 | reads slower at any throughput | -| None | Canonical x77 | 80,043,708 | 8.00 | 43 | 1.4 | 0.29 | reads faster above 88,339 MB/s | -| hint Dictionary | Dictionary x77 | 1,449,884 | 0.14 | 34 | 5.8 | 1.65 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 10,034,452 | 1.00 | 75 | 4.8 | 0.44 | reads slower at any throughput | -| hint RunEnd | RunEnd x77 | 87,996 | 0.01 | 23 | 2.4 | 0.65 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 136,620 | 0.01 | 13 | 6.3 | 4.86 | reads slower at any throughput | +| Auto | RunEnd x77 | 88,004 | 0.01 | 14 | 2.7 | 0.36 | | +| Fastest | RunEnd x77 | 88,004 | 0.01 | 10 | 2.6 | 0.38 | same | +| Smallest | RunEnd x76, Pco x1 | 53,748 | 0.01 | 110 | 3.0 | 0.74 | reads faster below 96 MB/s | +| None | Canonical x77 | 80,043,716 | 8.00 | 83 | 21.3 | 1.80 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 1,449,892 | 0.14 | 17 | 6.2 | 1.53 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 10,034,460 | 1.00 | 21 | 5.7 | 1.13 | reads slower at any throughput | +| hint RunEnd | RunEnd x77 | 88,004 | 0.01 | 10 | 2.6 | 0.36 | same | +| hint Zstd | Zstd x77 | 136,628 | 0.01 | 12 | 3.0 | 1.15 | reads slower at any throughput | @@ -143,14 +144,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 33,795,164 | 3.38 | 85 | 6.6 | 0.60 | | -| Fastest | BitPacked x77 | 33,795,164 | 3.38 | 16 | 3.6 | 0.29 | reads faster at any throughput | -| Smallest | Zstd x77 | 20,700,788 | 2.07 | 187 | 74.5 | 60.30 | reads faster below 193 MB/s | -| None | Canonical x77 | 80,043,716 | 8.00 | 28 | 1.3 | 0.25 | reads faster above 8,691 MB/s | -| hint Dictionary | BitPacked x77 | 33,795,164 | 3.38 | 93 | 3.6 | 0.26 | reads faster at any throughput | -| hint BitPacked | BitPacked x77 | 33,795,164 | 3.38 | 14 | 3.6 | 0.26 | reads faster at any throughput | -| hint RunEnd | BitPacked x77 | 33,795,164 | 3.38 | 26 | 3.6 | 0.28 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 20,700,788 | 2.07 | 150 | 74.1 | 60.66 | reads faster below 194 MB/s | +| Auto | BitPacked x77 | 33,795,172 | 3.38 | 70 | 12.9 | 1.72 | | +| Fastest | BitPacked x77 | 33,795,172 | 3.38 | 30 | 11.5 | 1.42 | reads faster at any throughput | +| Smallest | Pco x77 | 13,472,444 | 1.35 | 453 | 112.6 | 118.47 | reads faster below 204 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 43 | 19.3 | 1.57 | reads slower at any throughput | +| hint Dictionary | BitPacked x77 | 33,795,172 | 3.38 | 156 | 11.2 | 1.45 | reads faster at any throughput | +| hint BitPacked | BitPacked x77 | 33,795,172 | 3.38 | 29 | 11.3 | 1.39 | reads faster at any throughput | +| hint RunEnd | BitPacked x77 | 33,795,172 | 3.38 | 49 | 11.1 | 1.41 | reads faster at any throughput | +| hint Zstd | Zstd x77 | 20,700,796 | 2.07 | 182 | 80.8 | 35.05 | reads faster below 193 MB/s | @@ -158,14 +159,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.4 | 0.30 | | -| Fastest | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.4 | 0.25 | same | -| Smallest | BitPacked x77 | 12,544,276 | 1.25 | 151 | 2.4 | 0.25 | same | -| None | Canonical x77 | 80,043,708 | 8.00 | 5 | 1.3 | 0.23 | reads faster above 57,820 MB/s | -| hint Dictionary | Dictionary x77 | 12,652,716 | 1.27 | 28 | 5.7 | 0.56 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.7 | 0.41 | reads slower at any throughput | -| hint RunEnd | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.5 | 0.32 | same | -| hint Zstd | Zstd x77 | 20,141,164 | 2.01 | 143 | 78.7 | 64.11 | reads slower at any throughput | +| Auto | BitPacked x77 | 12,544,284 | 1.25 | 22 | 5.5 | 1.54 | | +| Fastest | BitPacked x77 | 12,544,284 | 1.25 | 32 | 6.9 | 1.67 | reads slower at any throughput | +| Smallest | BitPacked x76, Pco x1 | 12,543,924 | 1.25 | 344 | 5.1 | 1.34 | reads faster at any throughput | +| None | Canonical x77 | 80,043,716 | 8.00 | 43 | 21.5 | 1.82 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,652,724 | 1.27 | 70 | 6.5 | 1.25 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 12,544,284 | 1.25 | 23 | 4.9 | 1.15 | reads faster at any throughput | +| hint RunEnd | BitPacked x77 | 12,544,284 | 1.25 | 23 | 7.0 | 1.64 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 20,141,172 | 2.01 | 186 | 78.1 | 31.66 | reads slower at any throughput | @@ -173,15 +174,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,740 | 0.51 | 38 | 2.3 | 0.49 | | -| Fastest | Dictionary x77 | 5,063,740 | 0.51 | 37 | 2.0 | 0.30 | reads faster at any throughput | -| Smallest | Dictionary x77 | 5,063,740 | 0.51 | 169 | 2.0 | 0.31 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 31 | 1.4 | 0.24 | reads faster above 79,377 MB/s | -| hint Dictionary | Dictionary x77 | 5,063,740 | 0.51 | 37 | 2.0 | 0.34 | reads faster at any throughput | -| hint BitPacked | Dictionary x77 | 5,063,740 | 0.51 | 102 | 2.0 | 0.32 | reads faster at any throughput | -| hint RunEnd | Dictionary x77 | 5,063,740 | 0.51 | 93 | 2.0 | 0.31 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 10,539,388 | 1.05 | 125 | 61.4 | 49.82 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,924 | 0.50 | 51 | 2.9 | 0.50 | reads slower at any throughput | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 55 | 3.3 | 1.14 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 66 | 3.1 | 1.09 | reads faster at any throughput | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 614 | 3.4 | 1.37 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 55 | 19.7 | 1.47 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 54 | 3.1 | 1.08 | reads faster at any throughput | +| hint BitPacked | Dictionary x77 | 5,063,748 | 0.51 | 127 | 3.1 | 1.07 | reads faster at any throughput | +| hint RunEnd | Dictionary x77 | 5,063,748 | 0.51 | 105 | 3.2 | 1.08 | same | +| hint Zstd | Zstd x77 | 10,539,396 | 1.05 | 139 | 51.4 | 24.40 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 84 | 3.1 | 1.05 | reads faster at any throughput | @@ -189,15 +190,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 46,295,644 | 4.63 | 32 | 2.6 | 0.26 | | -| Fastest | BitPacked x77 | 46,295,644 | 4.63 | 37 | 2.6 | 0.29 | same | -| Smallest | BitPacked x77 | 46,295,644 | 4.63 | 162 | 2.6 | 0.29 | same | -| None | Canonical x77 | 80,043,716 | 8.00 | 40 | 1.3 | 0.22 | reads faster above 25,357 MB/s | -| hint Dictionary | BitPacked x77 | 46,295,644 | 4.63 | 105 | 2.7 | 0.34 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 46,295,644 | 4.63 | 33 | 2.6 | 0.32 | same | -| hint RunEnd | BitPacked x77 | 46,295,644 | 4.63 | 58 | 2.6 | 0.29 | same | -| hint Zstd | Zstd x77 | 51,815,860 | 5.18 | 132 | 60.3 | 49.04 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 23,603,540 | 2.36 | 117 | 6.7 | 0.47 | reads faster below 5,585 MB/s | +| Auto | BitPacked x77 | 46,295,652 | 4.63 | 38 | 12.9 | 1.51 | | +| Fastest | BitPacked x77 | 46,295,652 | 4.63 | 64 | 16.5 | 1.96 | reads slower at any throughput | +| Smallest | Pco x77 | 580,924 | 0.06 | 423 | 72.5 | 69.30 | reads faster below 767 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 42 | 19.6 | 1.56 | reads slower at any throughput | +| hint Dictionary | BitPacked x77 | 46,295,652 | 4.63 | 123 | 12.7 | 1.46 | same | +| hint BitPacked | BitPacked x77 | 46,295,652 | 4.63 | 37 | 13.2 | 1.55 | same | +| hint RunEnd | BitPacked x77 | 46,295,652 | 4.63 | 86 | 13.7 | 1.56 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 51,815,868 | 5.18 | 128 | 62.1 | 42.11 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 23,603,548 | 2.36 | 138 | 9.5 | 1.32 | reads faster at any throughput | @@ -205,14 +206,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x77 | 80,043,724 | 8.00 | 31 | 1.4 | 0.22 | | -| Fastest | Canonical x77 | 80,043,724 | 8.00 | 30 | 1.3 | 0.24 | reads faster at any throughput | -| Smallest | Canonical x77 | 80,043,724 | 8.00 | 95 | 1.4 | 0.28 | same | -| None | Canonical x77 | 80,043,724 | 8.00 | 30 | 1.3 | 0.22 | reads faster at any throughput | -| hint Dictionary | Canonical x77 | 80,043,724 | 8.00 | 130 | 1.3 | 0.24 | same | -| hint BitPacked | Canonical x77 | 80,043,724 | 8.00 | 97 | 1.3 | 0.22 | reads faster at any throughput | -| hint RunEnd | Canonical x77 | 80,043,724 | 8.00 | 101 | 1.5 | 0.27 | same | -| hint Zstd | Canonical x77 | 80,043,724 | 8.00 | 105 | 1.3 | 0.27 | reads faster at any throughput | +| Auto | Canonical x77 | 80,043,732 | 8.00 | 43 | 27.0 | 2.14 | | +| Fastest | Canonical x77 | 80,043,732 | 8.00 | 43 | 20.0 | 1.62 | reads faster at any throughput | +| Smallest | Canonical x77 | 80,043,732 | 8.00 | 350 | 20.1 | 1.70 | reads faster at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 43 | 23.3 | 1.64 | reads faster at any throughput | +| hint Dictionary | Canonical x77 | 80,043,732 | 8.00 | 152 | 20.0 | 1.58 | reads faster at any throughput | +| hint BitPacked | Canonical x77 | 80,043,732 | 8.00 | 117 | 19.9 | 1.68 | reads faster at any throughput | +| hint RunEnd | Canonical x77 | 80,043,732 | 8.00 | 114 | 24.1 | 2.27 | reads faster at any throughput | +| hint Zstd | Canonical x77 | 80,043,732 | 8.00 | 132 | 18.5 | 1.52 | reads faster at any throughput | @@ -220,14 +221,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x82 | 13,800,452 | 1.38 | 15 | 3.6 | 0.31 | | -| Fastest | BitPacked x82 | 13,800,452 | 1.38 | 15 | 3.5 | 0.30 | same | -| Smallest | BitPacked x82 | 13,800,452 | 1.38 | 164 | 3.5 | 0.31 | same | -| None | Canonical x82 | 81,294,660 | 8.13 | 34 | 1.4 | 0.33 | reads faster above 31,597 MB/s | -| hint Dictionary | Dictionary x82 | 12,676,388 | 1.27 | 68 | 6.3 | 0.53 | reads faster below 417 MB/s | -| hint BitPacked | BitPacked x82 | 13,800,452 | 1.38 | 14 | 3.5 | 0.32 | same | -| hint RunEnd | BitPacked x82 | 13,800,452 | 1.38 | 15 | 3.5 | 0.30 | same | -| hint Zstd | Zstd x82 | 19,283,932 | 1.93 | 156 | 74.0 | 58.66 | reads slower at any throughput | +| Auto | BitPacked x82 | 13,800,460 | 1.38 | 25 | 6.3 | 1.35 | | +| Fastest | BitPacked x82 | 13,800,460 | 1.38 | 23 | 7.6 | 1.97 | reads slower at any throughput | +| Smallest | Pco x82 | 12,567,268 | 1.26 | 311 | 45.3 | 44.57 | reads faster below 32 MB/s | +| None | Canonical x82 | 81,294,668 | 8.13 | 42 | 20.6 | 2.10 | reads slower at any throughput | +| hint Dictionary | Dictionary x82 | 12,676,396 | 1.27 | 89 | 10.0 | 1.52 | reads faster below 307 MB/s | +| hint BitPacked | BitPacked x82 | 13,800,460 | 1.38 | 27 | 6.3 | 1.34 | same | +| hint RunEnd | BitPacked x82 | 13,800,460 | 1.38 | 56 | 9.5 | 2.06 | reads slower at any throughput | +| hint Zstd | Zstd x82 | 19,283,940 | 1.93 | 169 | 65.0 | 30.96 | reads slower at any throughput | @@ -235,23 +236,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| prices (2 decimals) | Alp x77 | 2.50 | 59 | 6.9 | 0.55 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 30 | 2.1 | 0.34 | | -| 1 000 distinct prices | Dictionary x77 | 1.28 | 68 | 3.8 | 0.42 | | -| 100 003 distinct, repeating | Zstd x77 | 1.69 | 185 | 79.0 | 71.68 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 7.4 ms, take 3.33 ms; reads faster above 73 MB/s; None: Canonical x77, 8.00 B/value, scan 1.3 ms, take 0.23 ms; reads faster above 812 MB/s | -| uniform in [0, 1) | AlpRd x77 | 6.91 | 45 | 5.6 | 0.71 | None: Canonical x77, 8.00 B/value, scan 1.4 ms, take 0.27 ms; reads faster above 2,658 MB/s | +| prices (2 decimals) | Alp x77 | 2.50 | 47 | 10.5 | 1.33 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 47 | 3.1 | 1.08 | | +| 1 000 distinct prices | Dictionary x77 | 1.28 | 90 | 9.8 | 1.84 | hint Alp: Alp x77, 1.28 B/value, scan 7.5 ms, take 1.31 ms; reads faster at any throughput | +| 100 003 distinct, repeating | Zstd x77 | 1.69 | 161 | 48.8 | 23.24 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 11.0 ms, take 3.70 ms; reads faster above 138 MB/s; None: Canonical x77, 8.00 B/value, scan 19.8 ms, take 1.56 ms; reads faster above 2,178 MB/s | +| uniform in [0, 1) | AlpRd x77 | 6.91 | 60 | 22.9 | 2.46 | |
prices (2 decimals): every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Alp x77 | 25,049,724 | 2.50 | 59 | 6.9 | 0.55 | | -| Fastest | Alp x77 | 25,049,724 | 2.50 | 40 | 4.6 | 0.34 | reads faster at any throughput | -| Smallest | Alp x77 | 25,049,724 | 2.50 | 356 | 4.5 | 0.30 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 40 | 1.6 | 0.33 | reads faster above 10,473 MB/s | -| hint Dictionary | Alp x77 | 25,049,724 | 2.50 | 143 | 4.6 | 0.34 | reads faster at any throughput | -| hint Alp | Alp x77 | 25,049,724 | 2.50 | 39 | 4.6 | 0.35 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 41,240,252 | 4.12 | 286 | 77.1 | 62.95 | reads slower at any throughput | +| Auto | Alp x77 | 25,049,732 | 2.50 | 47 | 10.5 | 1.33 | | +| Fastest | Alp x77 | 25,049,732 | 2.50 | 50 | 9.6 | 1.20 | reads faster at any throughput | +| Smallest | Alp x77 | 25,048,868 | 2.50 | 880 | 9.9 | 1.38 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 43 | 19.3 | 1.59 | reads slower at any throughput | +| hint Dictionary | Alp x77 | 25,049,732 | 2.50 | 177 | 9.7 | 1.21 | reads faster at any throughput | +| hint Alp | Alp x77 | 25,049,732 | 2.50 | 48 | 9.7 | 1.23 | reads faster at any throughput | +| hint Zstd | Zstd x77 | 41,240,260 | 4.12 | 295 | 66.1 | 36.34 | reads slower at any throughput |
@@ -259,14 +260,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,740 | 0.51 | 30 | 2.1 | 0.34 | | -| Fastest | Dictionary x77 | 5,063,740 | 0.51 | 30 | 2.0 | 0.32 | same | -| Smallest | Dictionary x77 | 5,063,740 | 0.51 | 168 | 2.0 | 0.32 | same | -| None | Canonical x77 | 80,043,724 | 8.00 | 32 | 1.3 | 0.24 | reads faster above 99,549 MB/s | -| hint Dictionary | Dictionary x77 | 5,063,740 | 0.51 | 28 | 2.0 | 0.32 | same | -| hint Alp | Dictionary x77 | 5,063,740 | 0.51 | 100 | 2.0 | 0.32 | same | -| hint Zstd | Zstd x77 | 10,545,660 | 1.05 | 127 | 61.5 | 49.92 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,924 | 0.50 | 43 | 2.0 | 0.27 | same | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 47 | 3.1 | 1.08 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 41 | 3.1 | 1.07 | same | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 560 | 3.4 | 1.33 | reads slower at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 27 | 20.2 | 1.65 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 42 | 3.5 | 1.19 | reads slower at any throughput | +| hint Alp | Dictionary x77 | 5,063,748 | 0.51 | 100 | 3.1 | 1.08 | same | +| hint Zstd | Zstd x77 | 10,545,668 | 1.05 | 139 | 47.7 | 22.21 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 66 | 3.0 | 1.03 | same | @@ -274,14 +275,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 12,756,252 | 1.28 | 68 | 3.8 | 0.42 | | -| Fastest | Dictionary x77 | 12,756,252 | 1.28 | 87 | 5.1 | 0.68 | reads slower at any throughput | -| Smallest | Dictionary x77 | 12,756,252 | 1.28 | 334 | 5.3 | 0.65 | reads slower at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 46 | 1.6 | 0.34 | reads faster above 30,690 MB/s | -| hint Dictionary | Dictionary x77 | 12,756,252 | 1.28 | 99 | 5.4 | 0.67 | reads slower at any throughput | -| hint Alp | Alp x77 | 12,756,252 | 1.28 | 161 | 6.7 | 0.62 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 26,073,404 | 2.61 | 293 | 103.3 | 82.83 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 12,545,724 | 1.25 | 110 | 5.4 | 0.62 | reads faster below 130 MB/s | +| Auto | Dictionary x77 | 12,756,260 | 1.28 | 90 | 9.8 | 1.84 | | +| Fastest | Dictionary x77 | 12,756,260 | 1.28 | 75 | 7.8 | 1.42 | reads faster at any throughput | +| Smallest | Dictionary x77 | 12,751,356 | 1.28 | 499 | 6.8 | 1.61 | reads faster at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 46 | 23.2 | 1.73 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,756,260 | 1.28 | 68 | 6.3 | 1.28 | reads faster at any throughput | +| hint Alp | Alp x77 | 12,756,260 | 1.28 | 121 | 7.5 | 1.31 | reads faster at any throughput | +| hint Zstd | Zstd x77 | 26,073,412 | 2.61 | 213 | 61.7 | 31.41 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 12,545,732 | 1.25 | 100 | 7.0 | 1.22 | reads faster at any throughput | @@ -289,14 +290,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x77 | 16,909,884 | 1.69 | 185 | 79.0 | 71.68 | | -| Fastest | Zstd x77 | 16,909,884 | 1.69 | 203 | 87.1 | 62.72 | reads slower at any throughput | -| Smallest | Zstd x77 | 16,909,884 | 1.69 | 216 | 61.9 | 52.47 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 27 | 1.3 | 0.23 | reads faster above 812 MB/s | -| hint Dictionary | Zstd x77 | 16,909,884 | 1.69 | 247 | 77.3 | 50.25 | same | -| hint Alp | Zstd x77 | 16,909,884 | 1.69 | 224 | 62.2 | 50.60 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 16,909,884 | 1.69 | 148 | 61.8 | 50.31 | reads faster at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 22,129,980 | 2.21 | 98 | 7.4 | 3.33 | reads faster above 73 MB/s | +| Auto | Zstd x77 | 16,909,892 | 1.69 | 161 | 48.8 | 23.24 | | +| Fastest | Zstd x77 | 16,909,892 | 1.69 | 174 | 47.9 | 22.40 | same | +| Smallest | Zstd x77 | 16,909,892 | 1.69 | 296 | 61.3 | 28.88 | reads slower at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 43 | 19.8 | 1.56 | reads faster above 2,178 MB/s | +| hint Dictionary | Zstd x77 | 16,909,892 | 1.69 | 260 | 63.0 | 28.51 | reads slower at any throughput | +| hint Alp | Zstd x77 | 16,909,892 | 1.69 | 284 | 47.8 | 22.96 | same | +| hint Zstd | Zstd x77 | 16,909,892 | 1.69 | 163 | 48.3 | 24.00 | same | +| Auto, 16 MiB chunks | Dictionary x5 | 22,129,988 | 2.21 | 132 | 11.0 | 3.70 | reads faster above 138 MB/s | @@ -304,13 +305,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | AlpRd x77 | 69,054,372 | 6.91 | 45 | 5.6 | 0.71 | | -| Fastest | AlpRd x77 | 69,054,372 | 6.91 | 48 | 4.7 | 0.51 | reads faster at any throughput | -| Smallest | AlpRd x77 | 69,054,372 | 6.91 | 169 | 4.7 | 0.49 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 32 | 1.4 | 0.27 | reads faster above 2,658 MB/s | -| hint Dictionary | AlpRd x77 | 69,054,372 | 6.91 | 212 | 4.7 | 0.54 | reads faster at any throughput | -| hint Alp | AlpRd x77 | 69,054,372 | 6.91 | 175 | 4.7 | 0.59 | reads faster at any throughput | -| hint Zstd | AlpRd x77 | 69,054,372 | 6.91 | 231 | 4.7 | 0.51 | reads faster at any throughput | +| Auto | AlpRd x77 | 69,054,380 | 6.91 | 60 | 22.9 | 2.46 | | +| Fastest | AlpRd x77 | 69,054,380 | 6.91 | 82 | 24.1 | 3.54 | reads slower at any throughput | +| Smallest | Pco x77 | 67,609,668 | 6.76 | 548 | 118.0 | 117.47 | reads faster below 15 MB/s | +| None | Canonical x77 | 80,043,732 | 8.00 | 45 | 18.9 | 1.57 | reads faster above 2,761 MB/s | +| hint Dictionary | AlpRd x77 | 69,054,380 | 6.91 | 288 | 21.2 | 2.62 | reads faster at any throughput | +| hint Alp | AlpRd x77 | 69,054,380 | 6.91 | 235 | 23.3 | 3.00 | same | +| hint Zstd | AlpRd x77 | 69,054,380 | 6.91 | 269 | 21.8 | 2.56 | same | @@ -318,23 +319,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| 16 cities, random order | Dictionary x175 | 0.51 | 210 | 7.4 | 0.76 | | -| 10 000 distinct ids | Dictionary x306 | 2.54 | 622 | 35.5 | 29.36 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 8.7 ms, take 2.43 ms; reads faster at any throughput; hint Fsst: Fsst x306, 9.06 B/value, scan 32.1 ms, take 1.05 ms; reads faster above 19,417 MB/s | -| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 642 | 41.6 | 33.35 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 14.5 ms, take 3.70 ms; reads faster at any throughput; hint Fsst: Fsst x306, 8.80 B/value, scan 38.0 ms, take 0.98 ms; reads faster above 17,261 MB/s | -| unique UUIDs | Zstd x611 | 20.61 | 1520 | 293.3 | 290.51 | hint Fsst: Fsst x611, 24.91 B/value, scan 148.5 ms, take 2.13 ms; reads faster above 297 MB/s | -| log lines (~100 B) | Zstd x1221 | 13.71 | 1406 | 394.9 | 321.43 | hint Fsst: Fsst x1221, 23.96 B/value, scan 161.6 ms, take 3.69 ms; reads faster above 440 MB/s | +| 16 cities, random order | Dictionary x175 | 0.51 | 124 | 7.7 | 1.60 | | +| 10 000 distinct ids | Dictionary x306 | 2.54 | 1032 | 41.2 | 29.59 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 14.8 ms, take 3.78 ms; reads faster at any throughput | +| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 995 | 54.5 | 36.86 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 13.5 ms, take 3.38 ms; reads faster at any throughput | +| unique UUIDs | Zstd x611 | 20.61 | 1792 | 309.4 | 301.45 | hint Fsst: Fsst x611, 22.44 B/value, scan 175.1 ms, take 13.05 ms; reads faster above 136 MB/s | +| log lines (~100 B) | Zstd x1221 | 13.71 | 1468 | 406.8 | 321.21 | hint Fsst: Fsst x1221, 22.01 B/value, scan 177.4 ms, take 13.64 ms; reads faster above 362 MB/s |
16 cities, random order: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x175 | 5,117,276 | 0.51 | 210 | 7.4 | 0.76 | | -| Fastest | Dictionary x175 | 5,117,276 | 0.51 | 201 | 8.0 | 0.95 | reads slower at any throughput | -| Smallest | Dictionary x175 | 5,117,276 | 0.51 | 357 | 4.9 | 0.54 | reads faster at any throughput | -| None | Canonical x175 | 96,746,748 | 9.67 | 159 | 22.8 | 2.78 | reads slower at any throughput | -| hint Dictionary | Dictionary x175 | 5,117,276 | 0.51 | 204 | 4.8 | 0.54 | reads faster at any throughput | -| hint Fsst | Fsst x175 | 64,458,972 | 6.45 | 226 | 25.9 | 0.72 | reads slower at any throughput | -| hint Zstd | Zstd x175 | 17,427,092 | 1.74 | 233 | 91.6 | 64.44 | reads slower at any throughput | +| Auto | Dictionary x175 | 5,117,284 | 0.51 | 124 | 7.7 | 1.60 | | +| Fastest | Dictionary x175 | 5,117,284 | 0.51 | 141 | 6.1 | 1.39 | reads faster at any throughput | +| Smallest | Dictionary x175 | 5,117,284 | 0.51 | 323 | 6.4 | 1.37 | reads faster at any throughput | +| None | Canonical x175 | 96,746,756 | 9.67 | 183 | 46.6 | 20.78 | reads slower at any throughput | +| hint Dictionary | Dictionary x175 | 5,117,284 | 0.51 | 122 | 6.2 | 1.42 | reads faster at any throughput | +| hint Fsst | Fsst x175 | 41,142,948 | 4.11 | 315 | 42.9 | 8.95 | reads slower at any throughput | +| hint Zstd | Zstd x175 | 17,427,100 | 1.74 | 287 | 97.8 | 61.74 | reads slower at any throughput |
@@ -342,14 +343,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 25,433,780 | 2.54 | 622 | 35.5 | 29.36 | | -| Fastest | Dictionary x306 | 25,433,780 | 2.54 | 611 | 34.8 | 29.27 | same | -| Smallest | Dictionary x305, Zstd x1 | 25,427,444 | 2.54 | 1311 | 35.6 | 29.93 | same | -| None | Canonical x306 | 140,098,164 | 14.01 | 368 | 18.6 | 4.34 | reads faster above 6,808 MB/s | -| hint Dictionary | Dictionary x306 | 25,433,780 | 2.54 | 951 | 46.1 | 38.67 | reads slower at any throughput | -| hint Fsst | Fsst x306 | 90,615,708 | 9.06 | 374 | 32.1 | 1.05 | reads faster above 19,417 MB/s | -| hint Zstd | Zstd x306 | 27,097,620 | 2.71 | 368 | 90.7 | 68.49 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 18,014,004 | 1.80 | 398 | 8.7 | 2.43 | reads faster at any throughput | +| Auto | Dictionary x306 | 25,433,788 | 2.54 | 1032 | 41.2 | 29.59 | | +| Fastest | Dictionary x306 | 25,433,788 | 2.54 | 1009 | 58.0 | 32.81 | reads slower at any throughput | +| Smallest | Dictionary x305, Zstd x1 | 23,452,500 | 2.35 | 2062 | 138.8 | 134.32 | reads faster below 20 MB/s | +| None | Canonical x306 | 140,098,172 | 14.01 | 225 | 55.6 | 40.79 | reads slower at any throughput | +| hint Dictionary | Dictionary x306 | 25,433,788 | 2.54 | 713 | 38.1 | 28.72 | reads faster at any throughput | +| hint Fsst | Fsst x306 | 62,863,260 | 6.29 | 461 | 47.5 | 8.41 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 27,097,628 | 2.71 | 430 | 92.2 | 63.15 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 18,014,012 | 1.80 | 446 | 14.8 | 3.78 | reads faster at any throughput | @@ -357,14 +358,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 24,539,276 | 2.45 | 642 | 41.6 | 33.35 | | -| Fastest | Dictionary x306 | 24,539,276 | 2.45 | 655 | 41.8 | 33.52 | same | -| Smallest | Dictionary x305, Zstd x1 | 24,533,580 | 2.45 | 1384 | 41.8 | 33.98 | same | -| None | Canonical x306 | 151,883,724 | 15.19 | 312 | 26.1 | 3.88 | reads faster above 8,195 MB/s | -| hint Dictionary | Dictionary x306 | 24,539,276 | 2.45 | 646 | 42.1 | 33.53 | same | -| hint Fsst | Fsst x306 | 87,950,764 | 8.80 | 470 | 38.0 | 0.98 | reads faster above 17,261 MB/s | -| hint Zstd | Zstd x306 | 25,595,628 | 2.56 | 391 | 113.5 | 62.19 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 17,982,596 | 1.80 | 706 | 14.5 | 3.70 | reads faster at any throughput | +| Auto | Dictionary x306 | 24,539,284 | 2.45 | 995 | 54.5 | 36.86 | | +| Fastest | Dictionary x306 | 24,539,284 | 2.45 | 944 | 40.5 | 34.87 | reads faster at any throughput | +| Smallest | Dictionary x305, Zstd x1 | 22,854,628 | 2.29 | 1984 | 148.6 | 139.29 | reads faster below 18 MB/s | +| None | Canonical x306 | 151,883,732 | 15.19 | 293 | 69.6 | 39.43 | reads slower at any throughput | +| hint Dictionary | Dictionary x306 | 24,539,284 | 2.45 | 817 | 39.9 | 28.75 | reads faster at any throughput | +| hint Fsst | Fsst x306 | 64,408,284 | 6.44 | 533 | 60.9 | 11.45 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 25,595,636 | 2.56 | 400 | 99.0 | 63.07 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 17,982,604 | 1.80 | 504 | 13.5 | 3.38 | reads faster at any throughput | @@ -372,13 +373,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x611 | 206,058,580 | 20.61 | 1520 | 293.3 | 290.51 | | -| Fastest | Zstd x611 | 206,058,580 | 20.61 | 1509 | 293.2 | 288.92 | same | -| Smallest | Zstd x611 | 206,058,580 | 20.61 | 2820 | 290.7 | 288.95 | same | -| None | Canonical x611 | 360,155,508 | 36.02 | 196 | 21.0 | 7.27 | reads faster above 566 MB/s | -| hint Dictionary | Zstd x611 | 206,058,580 | 20.61 | 2884 | 289.7 | 288.12 | same | -| hint Fsst | Fsst x611 | 249,114,908 | 24.91 | 1497 | 148.5 | 2.13 | reads faster above 297 MB/s | -| hint Zstd | Zstd x611 | 206,058,580 | 20.61 | 1473 | 290.8 | 289.80 | same | +| Auto | Zstd x611 | 206,058,588 | 20.61 | 1792 | 309.4 | 301.45 | | +| Fastest | Zstd x611 | 206,058,588 | 20.61 | 1702 | 302.5 | 306.99 | same | +| Smallest | Zstd x611 | 206,058,588 | 20.61 | 3473 | 299.3 | 307.14 | same | +| None | Canonical x611 | 360,155,516 | 36.02 | 281 | 109.3 | 96.06 | reads faster above 770 MB/s | +| hint Dictionary | Zstd x610, Fsst x1 | 206,059,580 | 20.61 | 6414 | 301.3 | 323.16 | same | +| hint Fsst | Fsst x611 | 224,384,604 | 22.44 | 1798 | 175.1 | 13.05 | reads faster above 136 MB/s | +| hint Zstd | Zstd x611 | 206,058,588 | 20.61 | 1761 | 305.3 | 320.92 | same | @@ -386,13 +387,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x1221 | 137,107,180 | 13.71 | 1406 | 394.9 | 321.43 | | -| Fastest | Zstd x1221 | 137,107,180 | 13.71 | 1400 | 394.4 | 321.13 | same | -| Smallest | Zstd x1221 | 137,107,180 | 13.71 | 3392 | 400.1 | 321.98 | same | -| None | Canonical x1221 | 725,468,436 | 72.55 | 756 | 34.1 | 12.23 | reads faster above 1,631 MB/s | -| hint Dictionary | Zstd x1221 | 137,107,180 | 13.71 | 2815 | 391.9 | 320.55 | same | -| hint Fsst | Fsst x1221 | 239,634,612 | 23.96 | 1920 | 161.6 | 3.69 | reads faster above 440 MB/s | -| hint Zstd | Zstd x1221 | 137,107,180 | 13.71 | 1399 | 393.9 | 320.61 | same | +| Auto | Zstd x1221 | 137,107,188 | 13.71 | 1468 | 406.8 | 321.21 | | +| Fastest | Zstd x1221 | 137,107,188 | 13.71 | 1450 | 393.2 | 317.69 | same | +| Smallest | Zstd x1221 | 137,107,188 | 13.71 | 2932 | 387.2 | 335.15 | same | +| None | Canonical x1221 | 725,468,444 | 72.55 | 790 | 206.6 | 158.59 | reads faster above 2,939 MB/s | +| hint Dictionary | Zstd x1221 | 137,107,188 | 13.71 | 6070 | 404.3 | 347.13 | same | +| hint Fsst | Fsst x1221 | 220,058,684 | 22.01 | 1838 | 177.4 | 13.64 | reads faster above 362 MB/s | +| hint Zstd | Zstd x1221 | 137,107,188 | 13.71 | 1398 | 401.5 | 320.92 | same | @@ -400,18 +401,18 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| half true | Canonical x10 | 0.13 | 0 | 0.1 | 0.20 | | -| 1 % true | RunEnd x10 | 0.05 | 1 | 2.6 | 1.95 | None: Canonical x10, 0.13 B/value, scan 0.1 ms, take 0.17 ms; reads faster above 289 MB/s | +| half true | Canonical x10 | 0.13 | 10 | 0.4 | 0.46 | | +| 1 % true | RunEnd x10 | 0.05 | 5 | 2.5 | 0.49 | None: Canonical x10, 0.13 B/value, scan 0.6 ms, take 0.66 ms; reads faster above 390 MB/s |
half true: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.20 | | -| Fastest | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.18 | same | -| Smallest | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.21 | reads slower at any throughput | -| None | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.31 | reads slower at any throughput | -| hint RunEnd | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.17 | reads faster at any throughput | +| Auto | Canonical x10 | 1,262,556 | 0.13 | 10 | 0.4 | 0.46 | | +| Fastest | Canonical x10 | 1,262,556 | 0.13 | 0 | 0.5 | 0.46 | same | +| Smallest | Canonical x10 | 1,262,556 | 0.13 | 0 | 0.5 | 0.47 | same | +| None | Canonical x10 | 1,262,556 | 0.13 | 0 | 0.5 | 0.46 | same | +| hint RunEnd | Canonical x10 | 1,262,556 | 0.13 | 7 | 0.4 | 0.46 | same |
@@ -419,20 +420,20 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x10 | 548,084 | 0.05 | 1 | 2.6 | 1.95 | | -| Fastest | RunEnd x10 | 548,084 | 0.05 | 0 | 2.5 | 1.95 | same | -| Smallest | RunEnd x10 | 548,084 | 0.05 | 0 | 2.4 | 1.87 | same | -| None | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.17 | reads faster above 289 MB/s | -| hint RunEnd | RunEnd x10 | 548,084 | 0.05 | 0 | 2.5 | 1.88 | same | +| Auto | RunEnd x10 | 548,092 | 0.05 | 5 | 2.5 | 0.49 | | +| Fastest | RunEnd x10 | 548,092 | 0.05 | 5 | 2.5 | 0.50 | same | +| Smallest | RunEnd x10 | 164,692 | 0.02 | 27 | 4.0 | 3.67 | reads faster below 246 MB/s | +| None | Canonical x10 | 1,262,556 | 0.13 | 13 | 0.6 | 0.66 | reads faster above 390 MB/s | +| hint RunEnd | RunEnd x10 | 548,092 | 0.05 | 6 | 2.4 | 0.49 | same | ### The profiles side by side -`Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 10.8 times `Auto`'s write, 164 ms against 15 ms on *i64, random in 0..999, 10 % null*, since it tries every scheme on every chunk. +`Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *i64, random in 0..999*, *i64, 16 distinct, random order*, *i64, 100 003 distinct, repeating*, *i64, random in 0..999, 10 % null*, *f64, 16 distinct, random order*, *f64, uniform in [0, 1)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 18.8 times `Auto`'s write, 880 ms against 47 ms on *f64, prices (2 decimals)*, since it tries every scheme on every chunk. `None`, the plain form, made the largest file on 20 of 20 columns. -*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit b5104f2 with uncommitted changes, 2026-09-27 02:59 UTC.* +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit a3dfc8ab, 2026-10-09 16:49 UTC.* @@ -445,20 +446,21 @@ does not; a column it never departs on is left out. | column | scans at 2 GB/s | scans at 100 MB/s | scans at 10 GB/s | a row in 1 000 read by row | size | |---|---|---|---|---|---| -| i64, timestamps (ms, increasing, jittered) | | `Zstd` | | `Zstd` | | -| i64, 100 003 distinct, repeating | 16 MiB chunks | 16 MiB chunks | | | 16 MiB chunks | +| i64, sorted runs of 1 000 | | | `Zstd` | | | +| i64, timestamps (ms, increasing, jittered) | | `Pco` | | `Zstd` | | +| i64, 100 003 distinct, repeating | 16 MiB chunks | `Pco` | 16 MiB chunks | | `Dictionary`, 16 MiB chunks | | i64, random in 0..999, 10 % null | | `Dictionary` | | `Dictionary` | `Dictionary` | -| f64, 100 003 distinct, repeating | 16 MiB chunks | 16 MiB chunks | `Canonical` | | | +| f64, 100 003 distinct, repeating | 16 MiB chunks | | 16 MiB chunks | | `Dictionary`, 16 MiB chunks | | f64, uniform in [0, 1) | | | `Canonical` | | | -| utf8, 10 000 distinct ids | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `Zstd` | 16 MiB chunks | +| utf8, 10 000 distinct ids | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `OnPair` | 16 MiB chunks | | utf8, 10 000 distinct ids, 10 % null | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `Zstd` | 16 MiB chunks | | utf8, unique UUIDs | `Canonical` | | `Canonical` | `Fsst` | | -| utf8, log lines (~100 B) | `Fsst` | | `Canonical` | `Fsst` | | -| bool, 1 % true | `Canonical` | | `Canonical` | `Canonical` | | +| utf8, log lines (~100 B) | `OnPair` | | `Canonical` | `OnPair` | | +| bool, 1 % true | `Canonical` | | `Canonical` | | | -It keeps the writer's choice under every goal on the 10 other columns. +It keeps the writer's choice under every goal on the 9 other columns. -*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit b5104f2 with uncommitted changes, 2026-09-27 03:01 UTC.* +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit a3dfc8ab, 2026-10-09 16:52 UTC.* ## Kernels, against their baselines @@ -511,16 +513,16 @@ Benchmarks with issues: ### `CompressorBenchmarks` -| Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | -|----------------------------- |------- |--------------:|--------------:|------------:|--------:|---------------- |----------:|------------:| -| 'Choose, the whole decision' | i64 | 57,255.034 ns | 1,758.2423 ns | 456.6099 ns | 1.00x | Baseline | 72 B | 1.00 | -| 'candidate: sequence' | i64 | 13.330 ns | 3.0557 ns | 0.4729 ns | 4,295x | Faster | - | 0.00 | -| 'candidate: bit packing' | i64 | 21,888.618 ns | 3,518.0519 ns | 913.6269 ns | 2.62x | Faster | - | 0.00 | -| 'candidate: FSST' | i64 | 2.234 ns | 1.1546 ns | 0.2998 ns | 25,629x | Faster | - | 0.00 | -| 'candidate: zstd' | i64 | 26,766.801 ns | 2,571.6941 ns | 667.8608 ns | 2.14x | Faster | 72 B | 1.00 | -| 'candidate: ALP' | i64 | 2.614 ns | 0.0307 ns | 0.0047 ns | 21,903x | Faster | - | 0.00 | +| Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | +|----------------------------- |------- |--------------:|------------:|-----------:|--------:|---------------- |----------:|------------:| +| 'Choose, the whole decision' | i64 | 52,503.854 ns | 230.5989 ns | 35.6854 ns | 1.00x | Baseline | - | NA | +| 'candidate: sequence' | i64 | 10.728 ns | 0.1120 ns | 0.0291 ns | 4,894x | Faster | - | NA | +| 'candidate: bit packing' | i64 | 4,950.600 ns | 148.8102 ns | 23.0285 ns | 10.6x | Faster | - | NA | +| 'candidate: FSST' | i64 | 1.846 ns | 0.0144 ns | 0.0038 ns | 28,448x | Faster | - | NA | +| 'candidate: zstd' | i64 | 7,546.505 ns | 194.1759 ns | 30.0489 ns | 6.96x | Faster | - | NA | +| 'candidate: ALP' | i64 | 2.599 ns | 0.0136 ns | 0.0035 ns | 20,204x | Faster | - | NA | -*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.1, the fast profile; commit b5104f2 with uncommitted changes, 2026-09-27 03:15 UTC.* +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit a3dfc8ab, 2026-10-09 16:52 UTC.* From c2d4f3c9a9a2781278625ba3164c37212b9d9b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 19:08:14 +0200 Subject: [PATCH 6/8] test(scan): allow for Windows' reopen by name in the path ceilings On Windows the path ceilings failed by 16 bytes on the four axes that scan the main file. Windows 11 24H2 reopens a file the session keeps mapped by its name, with no handle (78bd3fc), and without a handle the tail is read as a slice of the mapping rather than positionally into a pooled buffer (29e0b37): one owner where the open of a handle had none, 16 bytes over that open, 40 over Linux and macOS, which open every file from a handle. A footer past the first read costs less by name, the handle's open then allocating the larger buffer: the map file's axis reads 8 bytes under Linux. The allowance is now asked of each file after its warm-ups, of the state the measured runs found: 40 bytes where the session keeps it mapped and the name answers, 24 for a handle's open, 0 elsewhere. Every axis then reads on Windows the headroom it reads on Linux, but the map file's, 48 bytes looser. --- .../Scan/PathAllocationTests.cs | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/tests/Vorticity.Tests/Scan/PathAllocationTests.cs b/tests/Vorticity.Tests/Scan/PathAllocationTests.cs index 93b7f86f2..dbcd6155a 100644 --- a/tests/Vorticity.Tests/Scan/PathAllocationTests.cs +++ b/tests/Vorticity.Tests/Scan/PathAllocationTests.cs @@ -45,6 +45,7 @@ using Vorticity.Columns; using Vorticity.Expressions; using Vorticity.File; +using Vorticity.IO; using Vorticity.Scanning; using Xunit; @@ -132,13 +133,43 @@ public sealed class PathAllocationTests private const int Runs = 10; /// - /// What the runtime's own file handle weighs on Windows beyond the platform the ceilings were - /// measured on: every axis opens one file, and every axis, the footer-only one included, reads - /// 24 bytes more there. The handle is the one allocation of an open that differs by platform: - /// Windows' SafeFileHandle carries its file type, its cached length and its thread-pool - /// binding, 80 bytes. The library allocates the same bytes on both. + /// What an open of allocates on Windows beyond the platforms the ceilings + /// were measured on, which open every file from a handle. /// - private static readonly long HandleAllowance = OperatingSystem.IsWindows() ? 24 : 0; + /// + /// + /// A file opened from a handle: 24 bytes. Windows' SafeFileHandle carries its file type, + /// its cached length and its thread-pool binding, 80 bytes; the library allocates the same bytes + /// around it. + /// + /// + /// A file the session keeps mapped, which Windows 11 24H2 reopens by its name, with no handle + /// (): at most 40 bytes. Without a handle the tail is read as + /// slices of the mapping rather than into pooled buffers: a tail read in one go, as the main + /// file's, is one owner where the other open had none, 16 bytes over a handle's open there; a + /// footer past the first read, as the map file's, costs less by name, the handle's open then + /// allocating the larger buffer. Every axis that scans the main file reopens it so once its + /// warm-ups have mapped it; the footer-only one does when an earlier test left it mapped; a file + /// the open reads whole is never mapped. So the allowance is asked after the warm-ups, of the + /// state the measured runs found. + /// + /// + private static long WindowsAllowance(string file) + { + if (!OperatingSystem.IsWindows()) + { + return 0; + } + + if (VortexSession.Default.Mappings is { } mappings && FileInode.TryGetByName(file, out FileInode identity, out long length) + && mappings.TryTake(identity, length) is { } kept) + { + kept.Release(); + return 40; + } + + return 24; + } /// Samples taken at each end of the retention check, of which the minimum counts. private const int Samples = 3; @@ -350,8 +381,9 @@ public void EveryReadPathAllocatesWithinItsCeiling() List over = []; foreach ((string axis, string file, long measured, Func> path) in Axes) { - long ceiling = measured + HandleAllowance; - long floor = Floor(path, Corpus.Path(file)); + string resolved = Corpus.Path(file); + long floor = Floor(path, resolved); + long ceiling = measured + WindowsAllowance(resolved); long headroom = ceiling - floor; report.Append(" ") .Append(axis.PadRight(32)) From 716b74b651472d58a66f1fe52a8de3da38c8c655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 19:31:58 +0200 Subject: [PATCH 7/8] bench: write another machine's in-process section under its own reference binary `--ratio-check` refused any reference binary but the one its references were calibrated against, `--out` included, so no machine but the M4 could write its page's in-process section: an x64 machine builds its own `vxbench`, whose ratios are its own. Under another binary, `--out` now measures every axis, writes the section and holds none to a ceiling; without `--out` the gate refuses as before. --- bench/README.md | 8 +++++--- bench/Vorticity.Benchmarks/Program.cs | 4 +++- bench/Vorticity.Benchmarks/RatioCheck.cs | 26 ++++++++++++++++++------ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/bench/README.md b/bench/README.md index 4b5da71ca..4bd07282f 100644 --- a/bench/README.md +++ b/bench/README.md @@ -279,9 +279,11 @@ and say which change in the commit message. **A new reference binary is carried, not recalibrated.** Both gates record the fingerprint of the `vxbench` build their tables were set under (`CalibratedShim`) and refuse to gate under another: a -ratio through a different denominator is a number about the rebuild. After rebuilding -`tools/vxbench-rs` — a new `vortex` pin, a new toolchain, a changed entry point — keep the previous -build and carry the tables over: +ratio through a different denominator is a number about the rebuild. Another machine builds another +binary, and its ratios are its own: there `--ratio-check --out` and `--throughput --take --out` (or +`--write`) measure every axis or file for that machine's page and hold none to a reference. After +rebuilding `tools/vxbench-rs` — a new `vortex` pin, a new toolchain, a changed entry point — keep +the previous build and carry the tables over: ``` dotnet run -c Release PROJ -- --ratio-check --recalibrate 3 --rebase-from diff --git a/bench/Vorticity.Benchmarks/Program.cs b/bench/Vorticity.Benchmarks/Program.cs index 60f6048cb..0e5f296e6 100644 --- a/bench/Vorticity.Benchmarks/Program.cs +++ b/bench/Vorticity.Benchmarks/Program.cs @@ -533,7 +533,9 @@ reports no change in our own time by what the binary moved the ratio --lanes N adds `full scan, N lanes`, threads pinned --out the axes as the page's section, after a - check of every axis + check of every axis; under a reference + binary the references do not name, every + axis measured and none held --throughput [family…] 57 encodings at a million rows, ~55 s --check hold each ratio to its ceiling. Refused with a family filter: a short run is +32% diff --git a/bench/Vorticity.Benchmarks/RatioCheck.cs b/bench/Vorticity.Benchmarks/RatioCheck.cs index 6a8da9e9e..33b480409 100644 --- a/bench/Vorticity.Benchmarks/RatioCheck.cs +++ b/bench/Vorticity.Benchmarks/RatioCheck.cs @@ -681,7 +681,15 @@ private static async Task CheckAsync(string path, Axis[] axes) : string.Equals(running, CalibratedShim, StringComparison.Ordinal) ? $" reference binary: {running}, the one the references were calibrated against" : $" reference binary: {running}, but the references were calibrated against {CalibratedShim}"); - if (CalibratedShim is not null && !string.Equals(running, CalibratedShim, StringComparison.Ordinal)) + + // The references are the calibrating machine's: another machine builds another binary, and + // its ratios are its own. Its page is still measured, every axis of it, and held to nothing. + bool held = CalibratedShim is null || string.Equals(running, CalibratedShim, StringComparison.Ordinal); + if (!held && Page is not null) + { + Console.Out.WriteLine(" for the page only: no axis is held to references set under another binary"); + } + else if (!held) { Console.Error.WriteLine( $"REFUSED: the references were calibrated against reference binary {CalibratedShim} and " + @@ -711,7 +719,11 @@ private static async Task CheckAsync(string path, Axis[] axes) Interval ratio = m.Ratio; string columns; string verdict = string.Empty; - if (!References.TryGetValue(axis.Name, out Reference entry)) + if (!held) + { + columns = " -- --"; + } + else if (!References.TryGetValue(axis.Name, out Reference entry)) { unreferenced.Add(string.Create( CultureInfo.InvariantCulture, @@ -798,10 +810,12 @@ private static async Task CheckAsync(string path, Axis[] axes) } Console.Out.WriteLine( - over == 0 - ? "Every axis is inside its ceiling." - : $"{over} axis/axes above ceiling. A ratio only moves when the code moves: " + - "find the change, do not raise the ceiling."); + !held + ? "No axis was held to a ceiling: this run writes the page and gates nothing." + : over == 0 + ? "Every axis is inside its ceiling." + : $"{over} axis/axes above ceiling. A ratio only moves when the code moves: " + + "find the change, do not raise the ceiling."); if (Page is not null) { await ResultsPage.WriteAsync(Page, [("in-process", Section(measured, running))]).ConfigureAwait(false); From ba2b86394d4b08f902f6c321e0174f3aee90ecae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 19:42:27 +0200 Subject: [PATCH 8/8] docs(guide): measure the x64 page against Rust, on a reference built there The comparisons with Rust on the x64 page were withdrawn in 9e94f340, until an x64 machine measured them on the corrected harness. They are measured again on the Ryzen 9 7950X, against a `vxbench` built on it: - a table, process against process: faster on 28 of 28 rows on one core, 26 of 28 on all 32, Rust ahead on a scan and a wide filter of its own file at ten million rows (0.75x, 0.74x); - decoding, per encoding: 43 of 61 files, a median of 1.12x; - in one process: 19 of 19 axes, 1.10x to 13.0x; - taking rows: 57 of 61 files, a median of 1.12x; - writing: 59 of 60 files, a median of 3.52x; - `LanesBenchmarks`: Rust at four workers takes 2.1 times our four lanes. The corpus is the generator's current one, four Pco widths more than the arm64 page's, and those are where Vorticity trails most: 0.32x to 0.49x to decode, 0.05x to 0.43x to take. The page's head says the ratios are this machine's, held to no ceiling, and the pages that pointed at it no longer say its Rust sections are missing. --- README.md | 4 +- docs/guide/benchmarks-x64.md | 459 +++++++++++++++++++++++++++++++++-- docs/guide/benchmarks.md | 5 +- 3 files changed, 446 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a3f56d58a..413adfe12 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ at the same version as the core. |---|---| | [docs/guide/](docs/guide/README.md) | **Using the library**: pages named by what you are trying to do, each one a program the samples project compiles and runs | | [docs/guide/benchmarks.md](docs/guide/benchmarks.md) | Every figure the bench publishes: what this costs against the Rust implementation, and each kernel against its baseline | -| [docs/guide/benchmarks-x64.md](docs/guide/benchmarks-x64.md) | The sections that compare Vorticity with itself, from an x64 machine, a Zen 4 processor under Windows | +| [docs/guide/benchmarks-x64.md](docs/guide/benchmarks-x64.md) | The same page from an x64 machine, a Zen 4 processor under Windows, against a Rust built there | | [docs/guide/benchmarks-duckdb.md](docs/guide/benchmarks-duckdb.md) | Our group bys against DuckDB's on the same files, through its Vortex reader and from its own table: where we lead and where we trail | | [docs/design/](docs/design/README.md) | **Why it is shaped this way**: the design documents, from the scope to the byte layout to the public API | @@ -176,6 +176,6 @@ prefix filter on FSST strings (0.93x) and a band over bit-packed integers (0.98x Every figure is on [the benchmark page](docs/guide/benchmarks.md), each section with the machine and the commit it was measured on, and [its x64 twin](docs/guide/benchmarks-x64.md) holds the -sections measured on x64. [docs/design/05-benchmarks.md](docs/design/05-benchmarks.md) says what is +same sections measured on x64. [docs/design/05-benchmarks.md](docs/design/05-benchmarks.md) says what is compared and how, and [bench/README.md](bench/README.md) how to run each instrument: `bench/gate.sh` runs everything that gates a commit. diff --git a/docs/guide/benchmarks-x64.md b/docs/guide/benchmarks-x64.md index acfa44ac0..7476d95e0 100644 --- a/docs/guide/benchmarks-x64.md +++ b/docs/guide/benchmarks-x64.md @@ -10,11 +10,12 @@ instrument measures; [bench/README.md](../../bench/README.md) how to run them. Vortex™ is a trademark of LF Projects, LLC. Vorticity is an independent implementation, not affiliated with or endorsed by the Vortex project or LF Projects, LLC. -**The sections that compare Vorticity with Rust are not measured on x64**: each says which -arguments write it. The sections that compare Vorticity with itself are: the encodings column by -column, the advice, and the kernels against their baselines, each scan mapping its file anew, as on -[the benchmark page](benchmarks.md). The kernel tables' speedups are computed from the Mean column -of their run. +**Rust's ratios here are this machine's.** Its `vxbench` is built here, and the ceilings that +`--ratio-check` and `--throughput --check` hold the ratios to are set on the arm64 machine, against +its own build: the in-process axes, the takes and the writes on this page are measured and held to +none. The per-encoding corpus is the generator's current one, 61 files where the arm64 page has 57: +`pco_f32`, `pco_i16`, `pco_u32` and `pco_nullable` are on this page alone. The kernel tables' +speedups are computed from the Mean column of their run. * **The two sides.** Vorticity as a Native AOT binary built for the machine's instruction set, with the workstation garbage collector, where a section says so, and on the JIT otherwise. Vortex @@ -23,6 +24,9 @@ of their run. gives its compiler this machine's instruction set, AVX-512 included, as .NET's JIT has it. Both map the file and read it where it lies, decode every value they return to its plain form, a constant column kept as one value on both, and must return the same rows or the run fails. +* **Rust at its faster setting.** On one core every Rust figure is the faster of its two ways of + splitting a scan, its default and one split per chunk, measured file by file; on all cores it + keeps its default, as on [the benchmark page](benchmarks.md). * **Speedup** is Rust's time over Vorticity's: above 1.00x, Vorticity took less. * **Throughput** is the plain size of the rows returned over the time, as on [the benchmark page](benchmarks.md). @@ -47,36 +51,453 @@ Release bench/Vorticity.Benchmarks.Runner`): an older one times older code. ## Reading and writing a table, process against process -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +* **The data.** A table of four columns: `monotone`, an increasing `i64` that the filters and + the projection use; `value`, an `f64`; `label`, one of five short strings; `flag`, a `bool` + with nulls. 2^20 rows, and ten times as many. +* **Two files of the same rows.** Each writer chooses its own encodings, and what a read costs + depends on them, so the rows are written twice: by Vorticity's writer (*Vorticity's file*) + and by Rust's (*Vortex Rust's file*). Both readers read both files; each size lists what the + two writers chose. +* **One core, or all 32.** On one core, Rust's single-threaded runtime and our scans at one + lane. On all of them, a Tokio worker per processor for Rust; for Vorticity, a lane per + processor for a scan and as many threads for the writer, the degree a session's + `MaxDegreeOfParallelism` gives both. +* **The same work on both sides.** Both map the file and read it where it lies, Rust through + `open_buffer` over a `memmap2` mapping; both decode every value they return to its plain form, + a constant column kept as one value on both; a write hands its bytes to a sink that keeps none, + on both. On one core Rust is timed under each of its two ways of splitting a scan, its default + and one split per chunk, and its figure is the faster; on all cores it keeps its default. +* **A figure** is the time a fresh process takes for its action, from opening the file to its + last row, by its own clock: 5 runs a side, the sides taking turns after one discarded run, the + median with the lowest and highest beside it. Both sides must return the same rows or the + run fails. +* **Throughput** is the plain size of the rows returned over the time: 25.5 bytes a row here. +* **Allocated** is what Vorticity allocates on its managed heap for the action: the median of 6 + calls in a process that has made 2 before them, the cost of a call once the pools are filled. + On all cores a call now and then allocates several times the median. Rust's allocator + reports no such figure. **Peak** is each process's peak resident memory. + +### 1,048,576 rows + +| column | Vorticity's, 2,448,644 bytes | Vortex Rust's, 7,903,964 bytes | +|---|---|---| +| `monotone` | 16 chunks of `vortex.sequence` | 8 chunks of `vortex.sequence` | +| `value` | 16 chunks of `vortex.zstd` | 8 chunks of `vortex.alprd` | +| `label` | 16 chunks of `vortex.dict` | a dictionary layout, its values in one chunk of `vortex.fsst` and its codes in 2 chunks of `fastlanes.bitpacked` | +| `flag` | 16 chunks of `vortex.bool` | one chunk of `vortex.bool` | + +#### One core + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.6-0.6) | 1.3 (1.1-1.4) | 2.29x | — | — | 5.4 KiB | 12 / 9 MiB | +| `open` | Vortex Rust's | 0.5 (0.5-0.6) | 1.1 (0.9-1.3) | 1.93x | — | — | 5.0 KiB | 12 / 9 MiB | +| `scan` | Vorticity's | 6.8 (6.6-7.1) | 17.3 (16.8-21.5) | 2.55x | 3.9 | 1.5 | 15.0 KiB | 15 / 15 MiB | +| `scan` | Vortex Rust's | 4.1 (4.0-5.0) | 5.2 (5.0-5.4) | 1.29x | 6.6 | 5.1 | 11.4 KiB | 23 / 22 MiB | +| `project` | Vorticity's | 1.0 (0.9-1.0) | 1.7 (1.6-1.9) | 1.81x | 8.8 | 4.9 | 17.2 KiB | 12 / 11 MiB | +| `project` | Vortex Rust's | 1.0 (1.0-1.4) | 1.8 (1.7-2.0) | 1.89x | 8.6 | 4.6 | 13.5 KiB | 13 / 11 MiB | +| `filter-narrow` | Vorticity's | 1.9 (1.9-2.1) | 2.6 (2.4-2.7) | 1.36x | 0.1 | 0.1 | 41.7 KiB | 15 / 13 MiB | +| `filter-narrow` | Vortex Rust's | 2.0 (2.0-2.2) | 2.4 (2.2-3.0) | 1.18x | 0.1 | 0.1 | 21.5 KiB | 16 / 13 MiB | +| `filter-wide` | Vorticity's | 4.4 (4.1-4.9) | 9.9 (9.6-10.4) | 2.24x | 3.0 | 1.4 | 24.1 KiB | 15 / 15 MiB | +| `filter-wide` | Vortex Rust's | 3.2 (3.0-3.7) | 4.4 (4.0-5.5) | 1.38x | 4.2 | 3.1 | 21.5 KiB | 20 / 17 MiB | +| `take` | Vorticity's | 6.1 (6.0-7.0) | 8.7 (8.6-8.8) | 1.42x | — | — | 38.7 KiB | 14 / 14 MiB | +| `take` | Vortex Rust's | 2.1 (2.0-2.2) | 3.3 (3.0-3.4) | 1.55x | — | — | 35.1 KiB | 15 / 16 MiB | +| `write` | Vorticity's | 31.4 (31.2-31.8) | 76.4 (75.6-96.0) | 2.43x | 0.8 | 0.3 | 31.1 KiB | 20 / 75 MiB | +| `write` | Vortex Rust's | 29.2 (28.8-36.9) | 75.8 (63.3-109.1) | 2.60x | 0.9 | 0.4 | 27.5 KiB | 29 / 80 MiB | +| `append` | Vorticity's | 11.6 (10.6-13.3) | not asked | n/a | — | — | 2.4 MiB | 21 / — MiB | + +#### All 32 cores + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.5-0.6) | 2.8 (2.7-3.2) | 5.16x | — | — | 5.4 KiB | 12 / 10 MiB | +| `open` | Vortex Rust's | 0.5 (0.5-0.7) | 3.0 (2.7-3.2) | 5.52x | — | — | 5.0 KiB | 11 / 10 MiB | +| `scan` | Vorticity's | 4.0 (3.9-5.7) | 7.3 (6.9-9.8) | 1.80x | 6.6 | 3.7 | 37.0 KiB | 47 / 40 MiB | +| `scan` | Vortex Rust's | 4.5 (4.3-4.5) | 5.8 (5.5-5.9) | 1.29x | 5.9 | 4.6 | 30.2 KiB | 63 / 40 MiB | +| `project` | Vorticity's | 1.8 (1.7-4.2) | 3.7 (3.1-3.9) | 2.02x | 4.6 | 2.3 | 34.3 KiB | 21 / 16 MiB | +| `project` | Vortex Rust's | 1.8 (1.6-1.9) | 3.7 (3.2-4.1) | 2.09x | 4.7 | 2.2 | 22.1 KiB | 21 / 16 MiB | +| `filter-narrow` | Vorticity's | 1.7 (1.7-1.7) | 4.4 (4.1-4.5) | 2.65x | 0.2 | 0.1 | 48.3 KiB | 14 / 15 MiB | +| `filter-narrow` | Vortex Rust's | 1.6 (1.6-2.0) | 4.2 (4.0-4.4) | 2.57x | 0.2 | 0.1 | 25.0 KiB | 14 / 15 MiB | +| `filter-wide` | Vorticity's | 3.5 (3.2-3.9) | 7.5 (6.7-8.7) | 2.13x | 3.8 | 1.8 | 38.0 KiB | 30 / 31 MiB | +| `filter-wide` | Vortex Rust's | 3.4 (3.3-3.5) | 5.4 (5.1-6.0) | 1.59x | 4.0 | 2.5 | 30.1 KiB | 37 / 33 MiB | +| `take` | Vorticity's | 5.0 (4.3-5.8) | 6.5 (5.3-6.9) | 1.30x | — | — | 88.5 KiB | 26 / 22 MiB | +| `take` | Vortex Rust's | 2.5 (2.4-2.9) | 4.3 (4.0-5.5) | 1.73x | — | — | 88.0 KiB | 18 / 19 MiB | +| `write` | Vorticity's | 14.7 (14.4-14.8) | 23.3 (23.1-30.8) | 1.59x | 1.8 | 1.1 | 2.5 MiB | 59 / 110 MiB | +| `write` | Vortex Rust's | 15.6 (14.6-17.9) | 23.8 (22.3-29.5) | 1.52x | 1.7 | 1.1 | 2.5 MiB | 74 / 117 MiB | +| `append` | Vorticity's | 11.0 (10.7-11.6) | not asked | n/a | — | — | 2.4 MiB | 22 / — MiB | + +### 10,485,760 rows + +| column | Vorticity's, 24,464,260 bytes | Vortex Rust's, 78,986,804 bytes | +|---|---|---| +| `monotone` | 160 chunks of `vortex.sequence` | 80 chunks of `vortex.sequence` | +| `value` | 160 chunks of `vortex.zstd` | 80 chunks of `vortex.alprd` | +| `label` | 160 chunks of `vortex.dict` | a dictionary layout, its values in one chunk of `vortex.fsst` and its codes in 20 chunks of `fastlanes.bitpacked` | +| `flag` | 160 chunks of `vortex.bool` | 3 chunks of `vortex.bool` | + +#### One core + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.6-0.6) | 1.1 (1.0-1.4) | 1.96x | — | — | 21.1 KiB | 11 / 9 MiB | +| `open` | Vortex Rust's | 0.6 (0.6-0.8) | 1.3 (1.1-1.4) | 1.99x | — | — | 9.5 KiB | 12 / 9 MiB | +| `scan` | Vorticity's | 59.1 (57.1-80.5) | 159.0 (151.8-212.6) | 2.69x | 4.5 | 1.7 | 74.6 KiB | 35 / 35 MiB | +| `scan` | Vortex Rust's | 33.1 (32.2-34.8) | 37.6 (37.0-45.6) | 1.14x | 8.1 | 7.1 | 28.7 KiB | 89 / 89 MiB | +| `project` | Vorticity's | 1.8 (1.8-2.0) | 3.0 (2.9-3.4) | 1.63x | 45.4 | 27.9 | 76.9 KiB | 13 / 12 MiB | +| `project` | Vortex Rust's | 2.0 (1.7-2.3) | 2.7 (2.5-3.5) | 1.37x | 42.5 | 31.0 | 30.9 KiB | 13 / 11 MiB | +| `filter-narrow` | Vorticity's | 2.7 (2.5-3.0) | 4.9 (4.7-5.8) | 1.78x | 1.0 | 0.5 | 107.4 KiB | 17 / 16 MiB | +| `filter-narrow` | Vortex Rust's | 2.9 (2.6-3.2) | 3.6 (3.4-3.8) | 1.21x | 0.9 | 0.7 | 62.5 KiB | 18 / 18 MiB | +| `filter-wide` | Vorticity's | 30.0 (28.9-36.0) | 78.9 (77.6-89.3) | 2.63x | 4.5 | 1.7 | 107.4 KiB | 24 / 25 MiB | +| `filter-wide` | Vortex Rust's | 19.4 (19.1-21.8) | 22.4 (21.0-30.3) | 1.15x | 6.9 | 6.0 | 62.5 KiB | 54 / 53 MiB | +| `take` | Vorticity's | 25.5 (25.4-26.3) | 69.8 (69.7-77.2) | 2.73x | — | — | 98.3 KiB | 30 / 37 MiB | +| `take` | Vortex Rust's | 5.6 (5.5-8.5) | 8.3 (8.0-8.9) | 1.47x | — | — | 52.4 KiB | 25 / 27 MiB | +| `write` | Vorticity's | 347.8 (280.3-398.4) | 705.1 (691.5-812.1) | 2.03x | 0.8 | 0.4 | 95.8 KiB | 42 / 377 MiB | +| `write` | Vortex Rust's | 267.9 (266.5-288.1) | 585.0 (572.0-652.1) | 2.18x | 1.0 | 0.5 | 49.9 KiB | 98 / 456 MiB | +| `append` | Vorticity's | 52.8 (51.2-69.3) | not asked | n/a | — | — | 3.3 MiB | 28 / — MiB | + +#### All 32 cores + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.6-0.6) | 3.3 (3.2-3.5) | 5.90x | — | — | 21.1 KiB | 12 / 10 MiB | +| `open` | Vortex Rust's | 0.6 (0.5-0.6) | 3.2 (3.0-3.4) | 5.82x | — | — | 9.5 KiB | 12 / 10 MiB | +| `scan` | Vorticity's | 16.5 (15.3-21.8) | 28.0 (24.5-39.6) | 1.70x | 16.1 | 9.5 | 148.0 KiB | 102 / 101 MiB | +| `scan` | Vortex Rust's | 27.3 (26.8-28.4) | 20.4 (18.0-23.8) | 0.75x | 9.8 | 13.1 | 78.6 KiB | 261 / 126 MiB | +| `project` | Vorticity's | 2.9 (2.9-3.0) | 4.2 (4.2-4.5) | 1.46x | 29.0 | 19.9 | 124.9 KiB | 30 / 36 MiB | +| `project` | Vortex Rust's | 3.3 (3.2-3.9) | 4.0 (3.9-5.3) | 1.19x | 25.1 | 21.1 | 69.2 KiB | 46 / 28 MiB | +| `filter-narrow` | Vorticity's | 2.4 (2.1-2.6) | 6.1 (5.7-7.0) | 2.54x | 1.1 | 0.4 | 109.7 KiB | 18 / 20 MiB | +| `filter-narrow` | Vortex Rust's | 2.6 (2.6-2.9) | 5.0 (4.9-5.6) | 1.91x | 1.0 | 0.5 | 65.5 KiB | 19 / 21 MiB | +| `filter-wide` | Vorticity's | 14.3 (9.8-19.3) | 21.1 (17.2-28.8) | 1.48x | 9.3 | 6.3 | 181.9 KiB | 92 / 78 MiB | +| `filter-wide` | Vortex Rust's | 17.6 (17.5-20.4) | 13.0 (12.3-16.6) | 0.74x | 7.6 | 10.3 | 620.9 KiB | 229 / 89 MiB | +| `take` | Vorticity's | 11.5 (10.9-12.2) | 15.7 (15.3-16.1) | 1.37x | — | — | 260.6 KiB | 62 / 56 MiB | +| `take` | Vortex Rust's | 5.0 (4.6-5.5) | 7.2 (7.2-7.9) | 1.46x | — | — | 150.1 KiB | 29 / 32 MiB | +| `write` | Vorticity's | 86.8 (86.3-88.0) | 155.4 (143.1-176.4) | 1.79x | 3.1 | 1.7 | 2.7 MiB | 116 / 465 MiB | +| `write` | Vortex Rust's | 98.3 (97.1-101.1) | 149.8 (145.8-157.7) | 1.52x | 2.7 | 1.8 | 2.7 MiB | 279 / 528 MiB | +| `append` | Vorticity's | 47.7 (46.8-54.7) | not asked | n/a | — | — | 2.5 MiB | 61 / — MiB | + +`open`: open the file and read no rows. `scan`: read every column of every row. `project`: read one column of four. `filter-narrow`: read the rows of a band holding about one in a hundred. `filter-wide`: read the rows of a band holding about half. `take`: take a thousand rows spread across the file. `write`: read the file and encode it back out. `append`: append a tenth of the rows to a copy of the file. + +**On one core.** Vorticity took less time than Rust on 28 of 28 compared rows. +The highest speedup is `take` at 10,485,760 rows on Vorticity's file (2.73x), the lowest +`scan` at 10,485,760 rows on Vortex Rust's file (1.14x). + +**On all 32 cores.** Vorticity took less time than Rust on 26 of 28 compared rows. +The highest speedup is `open` at 10,485,760 rows on Vorticity's file (5.90x), the lowest +`filter-wide` at 10,485,760 rows on Vortex Rust's file (0.74x). + +**The process start** is not in the figures: 46 ms for Vorticity's native binary and 32 ms for Rust's, +most of it the operating system starting a binary, and 113 ms for the managed runtime on the JIT. + +**Rust's splits.** On one core, one split per chunk was the faster of its two on 7 of 14 rows on Vorticity's file and 13 of 14 rows on Vortex Rust's file; +the two differ only where a chunk holds more than 100,000 rows. + +**Not asked of Rust**: `append`, which its harness has no entry point for; those figures +are Vorticity's alone. + +### A first call on the JIT + +The same scenarios in the framework-dependent build, under `dotnet`, on one core: what a +process that is not compiled ahead of time pays the first time, the code compiling as it runs. +Milliseconds, the median of the same runs. + +| scenario | 1,048,576 rows, Vorticity's file | 1,048,576 rows, Vortex Rust's file | 10,485,760 rows, Vorticity's file | 10,485,760 rows, Vortex Rust's file | +|---|---:|---:|---:|---:| +| `open` | 19.6 | 19.8 | 19.5 | 21.7 | +| `scan` | 114.7 | 102.2 | 220.3 | 139.3 | +| `project` | 70.3 | 70.7 | 76.5 | 81.0 | +| `filter-narrow` | 156.4 | 137.3 | 153.8 | 149.2 | +| `filter-wide` | 131.4 | 128.4 | 198.8 | 164.3 | +| `take` | 124.3 | 99.6 | 186.9 | 105.4 | +| `write` | 323.4 | 319.5 | 947.1 | 869.1 | +| `append` | 359.7 | — | 431.3 | — | + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01); commit 716b74b6, 2026-10-09 17:35 UTC.* ## Decoding, per encoding -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +One file per encoding, and per notable shape of one, written by Rust's writer +(`bench/gen-throughput.sh`): a million rows each, `table_wide` fifty columns of fifty thousand. +Each side scans the file, every value decoded to its plain form, 12 times in a process of its own on +one core; a figure is the median of the last 10 scans, and the median of 3 processes: what a +decoder costs once warm. Each scan opens the file and maps it anew, on both sides, and reads it +where it lies: a column stored in its plain form is a view of the mapping on both sides, nothing +decoded and nothing copied, and its figure, past what memory can move, measures the walk of the +layout. Throughput is over the plain size, as above. + +| encoding | rows | Vorticity, ns/row | Vortex Rust, ns/row | speedup | Vorticity, GB/s | Vortex Rust, GB/s | +|---|---:|---:|---:|---:|---:|---:| +| `alp` | 1,000,000 | 2.28 | 2.57 | 1.13x | 3.5 | 3.1 | +| `alp_no_patches` | 1,000,000 | 1.39 | 1.78 | 1.28x | 5.7 | 4.5 | +| `alp_patched_no_chunk_offsets` | 1,000,000 | 1.46 | 1.87 | 1.28x | 5.5 | 4.3 | +| `alprd` | 1,000,000 | 1.44 | 1.49 | 1.03x | 5.5 | 5.4 | +| `bool` | 1,000,000 | 0.17 | 0.18 | 1.05x | 0.7 | 0.7 | +| `bool_bit_offset3` | 1,000,000 | 0.18 | 0.17 | 0.93x | 0.7 | 0.8 | +| `bool_bit_offset7` | 1,000,000 | 0.17 | 0.17 | 0.99x | 0.7 | 0.7 | +| `bool_bit_offset_straddle` | 1,000,000 | 0.17 | 0.17 | 1.03x | 0.7 | 0.7 | +| `bytebool` | 1,000,000 | 0.32 | 0.34 | 1.08x | 0.4 | 0.4 | +| `chunked` | 1,000,000 | 0.19 | 0.72 | 3.73x | 41.7 | 11.2 | +| `chunked_bool` | 1,000,000 | 0.20 | 0.22 | 1.09x | 0.6 | 0.6 | +| `chunked_decimal` | 1,000,000 | 0.23 | 0.63 | 2.73x | 34.9 | 12.8 | +| `chunked_empty_chunks` | 1,000,000 | 0.20 | 0.52 | 2.68x | 41.0 | 15.3 | +| `chunked_mixed_validity` | 1,000,000 | 0.67 | 0.71 | 1.06x | 12.1 | 11.4 | +| `chunked_one_chunk` | 1,000,000 | 0.19 | 0.24 | 1.22x | 41.1 | 33.7 | +| `chunked_varbinview` | 1,000,000 | 4.35 | 10.8 | 2.49x | 3.1 | 1.2 | +| `constant` | 1,000,000 | 0.16 | 0.19 | 1.20x | 51.4 | 43.0 | +| `datetimeparts` | 1,000,000 | 2.82 | 3.19 | 1.13x | 2.8 | 2.5 | +| `decimal` | 1,000,000 | 0.19 | 0.18 | 0.97x | 42.9 | 44.3 | +| `decimal_byte_parts` | 1,000,000 | 0.22 | 0.18 | 0.81x | 36.0 | 44.3 | +| `dict` | 1,000,000 | 0.98 | 1.09 | 1.12x | 6.2 | 5.5 | +| `dict_nullable_codes` | 1,000,000 | 1.07 | 1.95 | 1.81x | 5.2 | 2.8 | +| `dict_nullable_values_nonnull_codes` | 1,000,000 | 0.94 | 1.02 | 1.09x | 6.1 | 5.6 | +| `dict_u64_codes` | 1,000,000 | 1.68 | 1.64 | 0.98x | 3.6 | 3.7 | +| `dict_u8_codes` | 1,000,000 | 0.57 | 0.64 | 1.12x | 13.9 | 12.4 | +| `ext` | 1,000,000 | 0.18 | 0.18 | 0.99x | 44.0 | 44.6 | +| `fastlanes_bitpacked` | 1,000,000 | 0.45 | 0.41 | 0.93x | 8.9 | 9.7 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,000,000 | 0.45 | 0.41 | 0.91x | 8.9 | 9.9 | +| `fastlanes_delta` | 1,000,000 | 2.15 | 1.90 | 0.88x | 3.7 | 4.2 | +| `fastlanes_for` | 1,000,000 | 1.38 | 1.38 | 1.00x | 5.8 | 5.8 | +| `fastlanes_rle` | 1,000,000 | 0.73 | 0.79 | 1.08x | 5.5 | 5.1 | +| `fixed_size_list` | 1,000,000 | 0.19 | 0.18 | 0.98x | 64.3 | 65.4 | +| `fsst` | 1,000,000 | 8.12 | 10.2 | 1.25x | 6.9 | 5.5 | +| `list` | 1,000,000 | 0.81 | 1.30 | 1.60x | 12.3 | 7.7 | +| `listview` | 1,000,000 | 2.55 | 3.04 | 1.19x | 3.9 | 3.3 | +| `map` | 1,000,000 | 5.42 | 11.0 | 2.02x | 3.7 | 1.8 | +| `masked` | 1,000,000 | 0.18 | 0.18 | 1.04x | 23.3 | 22.5 | +| `masked_all_invalid` | 1,000,000 | 0.21 | 0.18 | 0.90x | 19.4 | 21.7 | +| `masked_all_valid` | 1,000,000 | 0.20 | 0.18 | 0.89x | 19.5 | 22.0 | +| `null` | 1,000,000 | 0.14 | 0.16 | 1.18x | — | — | +| `onpair` | 1,000,000 | 3.95 | 6.79 | 1.72x | 5.5 | 3.2 | +| `parquet_variant` | 1,000,000 | 3.75 | 15.2 | 4.05x | 3.2 | 0.8 | +| `pco` | 1,000,000 | 0.42 | 1.19 | 2.86x | 19.2 | 6.7 | +| `pco_f32` | 1,000,000 | 7.86 | 2.99 | 0.38x | 0.5 | 1.3 | +| `pco_i16` | 1,000,000 | 6.96 | 2.61 | 0.37x | 0.3 | 0.8 | +| `pco_nullable` | 1,000,000 | 8.71 | 4.28 | 0.49x | 0.9 | 1.9 | +| `pco_u32` | 1,000,000 | 13.1 | 4.24 | 0.32x | 0.3 | 0.9 | +| `primitive` | 1,000,000 | 0.18 | 0.22 | 1.21x | 43.7 | 36.2 | +| `runend` | 1,000,000 | 0.27 | 0.30 | 1.13x | 15.0 | 13.2 | +| `sequence` | 1,000,000 | 0.20 | 0.23 | 1.17x | 40.2 | 34.5 | +| `sparse` | 1,000,000 | 0.24 | 0.24 | 1.00x | 16.4 | 16.4 | +| `struct` | 1,000,000 | 2.65 | 7.75 | 2.92x | 5.6 | 1.9 | +| `table_mixed` | 1,000,000 | 12.0 | 23.6 | 1.97x | 5.8 | 3.0 | +| `table_wide` | 50,000 | 5.19 | 4.78 | 0.92x | 77.1 | 83.7 | +| `varbin` | 1,000,000 | 3.65 | 14.7 | 4.04x | 4.4 | 1.1 | +| `varbinview` | 1,000,000 | 4.00 | 11.0 | 2.74x | 3.4 | 1.2 | +| `variant` | 1,000,000 | 0.14 | 0.17 | 1.21x | 113.1 | 93.3 | +| `zigzag` | 1,000,000 | 0.77 | 0.97 | 1.26x | 5.2 | 4.1 | +| `zstd` | 1,000,000 | 4.43 | 9.61 | 2.17x | 7.2 | 3.3 | +| `zstd_buffers` | 1,000,000 | 0.42 | 0.46 | 1.09x | 18.9 | 17.4 | +| `zstd_nullable` | 1,000,000 | 1.99 | 4.16 | 2.09x | 4.1 | 2.0 | + +Rust is timed under each of its two ways of splitting a scan, three processes each, and its +figure is the faster: its default cuts a chunk of more than 100,000 rows into splits of 100,000 and +decodes the whole chunk again for each, which on these files of one chunk of a million rows is ten +times; one split per chunk decodes it once, but builds a chunk made of smaller arrays in one +piece. One split per chunk was the faster on 49 of 61 files, its default on `bytebool`, `chunked`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `constant`, `datetimeparts`, `dict_u64_codes`, `fastlanes_delta`, `fsst`, `parquet_variant`. + +Vorticity decoded 43 of 61 files in less time than Rust; the median speedup is 1.12x. At 1.00x or below: `pco_u32` 0.32x, `pco_i16` 0.37x, `pco_f32` 0.38x, `pco_nullable` 0.49x, `decimal_byte_parts` 0.81x, `fastlanes_delta` 0.88x, `masked_all_valid` 0.89x, `masked_all_invalid` 0.90x, `fastlanes_bitpacked_patched_no_chunk_offsets` 0.91x, `table_wide` 0.92x, `fastlanes_bitpacked` 0.93x, `bool_bit_offset3` 0.93x, `decimal` 0.97x, `dict_u64_codes` 0.98x, `fixed_size_list` 0.98x, `ext` 0.99x, `bool_bit_offset7` 0.99x, `sparse` 1.00x. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01); commit 716b74b6, 2026-10-09 17:35 UTC.* ## In one process, after warm-up -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +Both readers called in turn in one process, Rust's through a C ABI, against one clock. A round times +both, and its speedup is Rust's time over Vorticity's: at least 21 rounds after a 2-second warm-up, +until the 95 % interval of their median is within 5 % of it. Our side runs on the JIT, warmed. Each +call opens its file and maps it anew, on both sides, and asks both the same question. Each axis is +one the `--ratio-check` gate holds to a ceiling; what each one reads and asks is in +[05-benchmarks.md](../design/05-benchmarks.md) §3. + +| axis | Vorticity, µs | Vortex Rust, µs | speedup | 95 % interval | +|---|---:|---:|---:|---| +| full scan | 927.6 | 3,312.2 | 3.51x | 3.38 to 3.61 | +| projected scan, 1 of 5 columns | 252.1 | 486.9 | 1.96x | 1.91 to 1.98 | +| open to first batch | 220.7 | 2,868.4 | 13.0x | 12.7 to 13.1 | +| open, footer only | 133.4 | 212.8 | 1.59x | 1.49 to 1.62 | +| read and write back | 5,966.6 | 19,356.3 | 3.24x | 3.14 to 3.31 | +| filtered scan, 1% band | 259.2 | 679.5 | 2.70x | 2.59 to 2.78 | +| filtered scan, half the rows | 331.5 | 1,096.9 | 3.28x | 3.25 to 3.39 | +| scattered take, 64 of 64 splits | 670.0 | 2,622.4 | 3.91x | 3.87 to 3.97 | +| key order, sorted column, 1% band | 267.4 | 361.7 | 1.36x | 1.35 to 1.37 | +| key order, uncorrelated, 64 rows | 369.2 | 491.6 | 1.38x | 1.29 to 1.41 | +| count, exact cover, 1% band | 276.7 | 386.1 | 1.53x | 1.44 to 1.59 | +| filtered scan, string equality, fsst | 645.9 | 831.2 | 1.28x | 1.27 to 1.28 | +| filtered scan, string prefix, fsst | 1,084.8 | 1,195.2 | 1.10x | 1.10 to 1.11 | +| filtered scan, string equality, dict | 189.5 | 312.5 | 1.66x | 1.58 to 1.66 | +| filtered scan, string prefix, dict | 201.3 | 406.8 | 2.02x | 2.00 to 2.04 | +| filtered scan, band, runend | 197.6 | 294.5 | 1.49x | 1.42 to 1.51 | +| filtered scan, band, bitpacked | 234.1 | 268.7 | 1.15x | 1.14 to 1.16 | +| full scan, 1M table | 25,271.0 | 41,306.8 | 1.62x | 1.58 to 1.75 | +| projected scan, 1 of 50 columns | 176.9 | 229.7 | 1.31x | 1.29 to 1.32 | + +Vorticity took less time on 19 of 19 axes. +Rust is timed under the faster of its two splits, axis by axis: one split per chunk on 13 of 19, its default on full scan; open, footer only; key order, uncorrelated, 64 rows; count, exact cover, 1% band; filtered scan, string equality, fsst; filtered scan, band, bitpacked. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 7527e2b5d861; commit 716b74b6, 2026-10-09 17:36 UTC.* ## Taking rows, per encoding -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +The files of the per-encoding corpus again, 64 rows spread evenly over each, one every 15,625: what a +decoder costs to reach a few rows rather than all of them. +Each call opens the file and maps it anew, on both sides. +Both implementations in one process, Rust's through a C ABI, taking turns against one clock: +at least 9 rounds after a 1-second warm-up per file, our side on the JIT, warmed. + +| encoding | Vorticity, µs | Vortex Rust, µs | speedup | +|---|---:|---:|---:| +| `alp` | 856 | 904 | 1.01x | +| `alp_no_patches` | 245 | 276 | 1.11x | +| `alp_patched_no_chunk_offsets` | 277 | 310 | 1.11x | +| `alprd` | 301 | 324 | 1.10x | +| `bool` | 183 | 206 | 1.12x | +| `bool_bit_offset3` | 185 | 205 | 1.12x | +| `bool_bit_offset7` | 191 | 212 | 1.10x | +| `bool_bit_offset_straddle` | 180 | 206 | 1.14x | +| `bytebool` | 224 | 432 | 1.91x | +| `chunked` | 258 | 294 | 1.15x | +| `chunked_bool` | 186 | 228 | 1.21x | +| `chunked_decimal` | 247 | 285 | 1.16x | +| `chunked_empty_chunks` | 248 | 286 | 1.15x | +| `chunked_mixed_validity` | 268 | 321 | 1.19x | +| `chunked_one_chunk` | 247 | 273 | 1.11x | +| `chunked_varbinview` | 408 | 13,592 | 34.0x | +| `constant` | 124 | 174 | 1.42x | +| `datetimeparts` | 422 | 462 | 1.10x | +| `decimal` | 245 | 268 | 1.10x | +| `decimal_byte_parts` | 248 | 277 | 1.14x | +| `dict` | 233 | 262 | 1.12x | +| `dict_nullable_codes` | 261 | 306 | 1.16x | +| `dict_nullable_values_nonnull_codes` | 232 | 259 | 1.11x | +| `dict_u64_codes` | 251 | 281 | 1.11x | +| `dict_u8_codes` | 319 | 340 | 1.07x | +| `ext` | 253 | 273 | 1.10x | +| `fastlanes_bitpacked` | 221 | 243 | 1.11x | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 239 | 266 | 1.12x | +| `fastlanes_delta` | 476 | 3,301 | 6.50x | +| `fastlanes_for` | 361 | 400 | 1.09x | +| `fastlanes_rle` | 953 | 1,016 | 1.03x | +| `fixed_size_list` | 259 | 299 | 1.17x | +| `fsst` | 423 | 455 | 1.09x | +| `list` | 253 | 1,197 | 4.75x | +| `listview` | 5,571 | 7,306 | 1.19x | +| `map` | 9,785 | 17,961 | 1.69x | +| `masked` | 201 | 230 | 1.15x | +| `masked_all_invalid` | 200 | 226 | 1.12x | +| `masked_all_valid` | 375 | 401 | 1.08x | +| `null` | 180 | 249 | 1.35x | +| `onpair` | 403 | 437 | 1.08x | +| `parquet_variant` | 465 | 502 | 1.05x | +| `pco` | 314 | 1,218 | 3.81x | +| `pco_f32` | 7,958 | 3,702 | 0.43x | +| `pco_i16` | 6,190 | 2,692 | 0.43x | +| `pco_nullable` | 6,446 | 348 | 0.05x | +| `pco_u32` | 11,965 | 4,353 | 0.36x | +| `primitive` | 361 | 384 | 1.08x | +| `runend` | 217 | 226 | 1.11x | +| `sequence` | 169 | 231 | 1.37x | +| `sparse` | 267 | 277 | 1.13x | +| `struct` | 5,918 | 15,734 | 2.52x | +| `table_mixed` | 19,179 | 32,506 | 1.76x | +| `table_wide` | 438 | 483 | 1.24x | +| `varbin` | 440 | 12,960 | 29.6x | +| `varbinview` | 7,504 | 14,748 | 2.04x | +| `variant` | 126 | 190 | 1.49x | +| `zigzag` | 364 | 395 | 1.11x | +| `zstd` | 701 | 10,564 | 15.5x | +| `zstd_buffers` | 514 | 555 | 1.13x | +| `zstd_nullable` | 521 | 4,496 | 7.60x | + +Vorticity took less time on 57 of 61 files; the median speedup is 1.12x. At 1.00x or below: `pco_nullable` 0.05x, `pco_u32` 0.36x, `pco_f32` 0.43x, `pco_i16` 0.43x. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 59 of 61 files, its default on `fastlanes_delta`, `table_wide`. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 7527e2b5d861; commit 716b74b6, 2026-10-09 17:37 UTC.* ## Writing, per encoding -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +The files of the per-encoding corpus again, each read and written back out to a sink that +keeps nothing, on both sides, both writers given the rows as our reader delivers them, decoded and a +constant kept as one value: what a writer costs, the read included on both sides. +Each call opens the file and maps it anew, on both sides. +Both implementations in one process, Rust's through a C ABI, taking turns against one clock: +at least 9 rounds after a 1-second warm-up per file, our side on the JIT, warmed. +A writer can be fast by compressing less, so each one's bytes are beside its time. + +| encoding | Vorticity, µs | Vortex Rust, µs | speedup | Vorticity, bytes | Vortex Rust, bytes | +|---|---:|---:|---:|---:|---:| +| `alp` | 8,314 | 33,734 | 3.97x | 2,740,820 | 2,976,924 | +| `alp_no_patches` | 6,988 | 24,835 | 3.72x | 1,955,668 | 1,956,060 | +| `alp_patched_no_chunk_offsets` | 6,125 | 25,662 | 4.15x | 2,155,956 | 2,141,772 | +| `alprd` | 8,436 | 27,115 | 3.24x | 6,508,892 | 6,509,316 | +| `bool` | 302 | 858 | 2.90x | 127,108 | 127,108 | +| `bool_bit_offset3` | 513 | 1,482 | 2.89x | 127,108 | 127,108 | +| `bool_bit_offset7` | 341 | 1,009 | 2.99x | 127,108 | 127,108 | +| `bool_bit_offset_straddle` | 350 | 1,034 | 2.97x | 127,108 | 127,108 | +| `bytebool` | 507 | 1,095 | 2.17x | 127,108 | 127,108 | +| `chunked` | 3,608 | 14,436 | 4.11x | 890,228 | 890,736 | +| `chunked_bool` | 379 | 1,053 | 2.85x | 127,236 | 127,248 | +| `chunked_decimal` | 5,560 | 18,287 | 3.35x | 1,822,388 | 1,825,056 | +| `chunked_empty_chunks` | 3,258 | 14,480 | 4.55x | 316,532 | 317,080 | +| `chunked_mixed_validity` | 4,128 | 17,561 | 4.26x | 1,431,516 | 1,374,016 | +| `chunked_one_chunk` | 3,240 | 14,569 | 4.35x | 5,044 | 4,640 | +| `chunked_varbinview` | 27,200 | 37,866 | 1.39x | 643,644 | 628,628 | +| `constant` | 239 | 4,271 | 18.0x | 4,212 | 3,300 | +| `datetimeparts` | 7,164 | 30,800 | 4.39x | 3,382,604 | 3,383,628 | +| `decimal` | 3,762 | 16,841 | 4.56x | 3,372 | 5,116 | +| `decimal_byte_parts` | 3,796 | 16,140 | 4.29x | 3,372 | 5,116 | +| `dict` | 9,259 | 21,212 | 2.24x | 383,892 | 378,288 | +| `dict_nullable_codes` | 18,159 | 20,804 | 1.29x | 384,628 | 378,772 | +| `dict_nullable_values_nonnull_codes` | 14,036 | 19,706 | 1.42x | 384,628 | 378,764 | +| `dict_u64_codes` | 10,514 | 23,074 | 2.18x | 383,892 | 378,288 | +| `dict_u8_codes` | 11,166 | 22,472 | 1.94x | 38,740 | 1,004,160 | +| `ext` | 3,153 | 24,750 | 7.90x | 5,548 | 4,988 | +| `fastlanes_bitpacked` | 1,467 | 4,569 | 3.12x | 1,254,340 | 1,253,668 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,490 | 5,240 | 3.52x | 1,270,452 | 1,272,740 | +| `fastlanes_delta` | 3,719 | 15,674 | 3.98x | 4,908 | 4,532 | +| `fastlanes_for` | 3,574 | 7,170 | 2.03x | 881,364 | 879,732 | +| `fastlanes_rle` | 1,781 | 8,660 | 4.82x | 3,964 | 3,812 | +| `fixed_size_list` | 4,256 | 26,980 | 6.38x | 4,268 | 4,724 | +| `fsst` | 50,186 | 312,191 | 6.05x | 933,668 | 8,574,500 | +| `list` | 8,889 | 28,170 | 3.06x | 4,782,492 | 5,122,712 | +| `listview` | 11,679 | 38,569 | 3.33x | 4,745,468 | 4,502,136 | +| `map` | 23,149 | 75,151 | 3.29x | 4,426,852 | 4,811,136 | +| `masked` | 3,366 | 19,212 | 6.13x | 2,380,060 | 2,381,156 | +| `masked_all_invalid` | 1,554 | 2,319 | 1.51x | 2,636 | 2,700 | +| `masked_all_valid` | 1,664 | 10,577 | 6.43x | 3,484 | 3,324 | +| `null` | 154 | 586 | 3.82x | 2,052 | 2,164 | +| `onpair` | 27,960 | 33,828 | 1.20x | 60,588 | 1,005,840 | +| `pco` | 1,195 | 14,063 | 12.0x | 4,908 | 4,508 | +| `pco_f32` | 9,774 | 21,230 | 2.19x | 2,130,020 | 2,130,108 | +| `pco_i16` | 6,927 | 8,154 | 1.18x | 1,878,788 | 1,878,460 | +| `pco_nullable` | 14,502 | 35,847 | 2.47x | 2,876,548 | 2,874,276 | +| `pco_u32` | 18,600 | 9,780 | 0.52x | 1,931,748 | 2,879,908 | +| `primitive` | 3,057 | 13,484 | 4.24x | 5,356 | 4,572 | +| `runend` | 813 | 9,510 | 11.7x | 4,028 | 3,500 | +| `sequence` | 1,076 | 14,202 | 13.3x | 4,908 | 4,516 | +| `sparse` | 823 | 4,224 | 5.07x | 131,172 | 3,884 | +| `struct` | 44,892 | 119,019 | 2.54x | 638,412 | 5,274,480 | +| `table_mixed` | 85,274 | 415,839 | 4.84x | 7,867,284 | 14,856,384 | +| `table_wide` | 11,150 | 40,954 | 3.52x | 5,163,108 | 5,168,320 | +| `varbin` | 55,308 | 263,222 | 4.75x | 637,012 | 5,066,652 | +| `varbinview` | 24,162 | 35,058 | 1.45x | 643,516 | 628,504 | +| `variant` | 193 | 1,058 | 5.50x | 2,220 | 52,520 | +| `zigzag` | 2,576 | 11,809 | 4.69x | 2,531,652 | 2,380,300 | +| `zstd` | 36,620 | 41,824 | 1.18x | 786,948 | 754,336 | +| `zstd_buffers` | 1,863 | 4,876 | 2.59x | 630,732 | 629,028 | +| `zstd_nullable` | 5,415 | 21,568 | 3.96x | 756,500 | 628,740 | + +Vorticity took less time on 59 of 60 files; the median speedup is 3.52x. At 1.00x or below: `pco_u32` 0.52x. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 45 of 60 files, its default on `alp`, `alprd`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_one_chunk`, `decimal`, `dict_nullable_values_nonnull_codes`, `fixed_size_list`, `fsst`, `list`, `pco_f32`, `runend`, `sparse`, `table_wide`, `zigzag`. +Vorticity's bytes were within 5 % of Rust's or fewer on 48 of 60 files; more than 5 % above them on `sparse` (33.8 times Rust's), `constant` (1.28 times), `zstd_nullable` (1.20 times), `primitive` (1.17 times), `runend` (1.15 times), `ext` (1.11 times), `pco` (1.09 times), `chunked_one_chunk` (1.09 times), `sequence` (1.09 times), `fastlanes_delta` (1.08 times), `zigzag` (1.06 times), `listview` (1.05 times). +Rust's writer declines `parquet_variant`, which is left out. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 7527e2b5d861; commit 716b74b6, 2026-10-09 17:40 UTC.* @@ -528,8 +949,12 @@ Benchmarks with issues: ### `LanesBenchmarks` -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +| Method | Lanes | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Completed Work Items | Lock Contentions | Allocated | Alloc Ratio | +|----------------------- |------ |-----------:|----------:|---------:|--------:|---------------- |---------------------:|-----------------:|----------:|------------:| +| 'ours, n lanes' | 4 | 622.1 μs | 46.33 μs | 7.17 μs | 1.00x | Baseline | 97.4875 | - | 46910 B | 1.00 | +| 'reference, n workers' | 4 | 1,311.8 μs | 184.25 μs | 47.85 μs | 0.47x | Slower | - | - | - | 0.00 | + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 716b74b6, 2026-10-09 17:41 UTC.* ## What this does not measure diff --git a/docs/guide/benchmarks.md b/docs/guide/benchmarks.md index d3aa44b05..9cb9ff677 100644 --- a/docs/guide/benchmarks.md +++ b/docs/guide/benchmarks.md @@ -44,8 +44,7 @@ and the arguments below, which rewrite that section of this page and leave the o ## On x64 [Benchmarks on x64](benchmarks-x64.md) is this page's twin on an AMD Ryzen 9 7950X (Zen 4, -AVX-512) under Windows 11: the sections that compare Vorticity with itself. Its sections that -compare Vorticity with Rust are not measured on x64. +AVX-512) under Windows 11, every section measured there, against a Rust built on that machine. ## Reading and writing a table, process against process @@ -950,7 +949,7 @@ reference profile on that class, the one to quote a small difference from. faults where the copy cost it a read, which on the per-encoding corpus makes its scans of `fsst` and `alp` 7 and 10 % slower, and moves the others by less. * **x64 on this page.** Every figure here is from one arm64 machine; [Benchmarks on - x64](benchmarks-x64.md) has the sections that compare Vorticity with itself from one x64 machine. + x64](benchmarks-x64.md) has the same sections from one x64 machine. CI builds and tests on x64 and arm64 and runs no benchmark. * **Your data and your machine.** A handful of tables and one machine: a column the compressor likes less, or a filter the zone maps cannot prune, moves these numbers more than either implementation