diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85329d22c..485eea157 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,10 +132,11 @@ jobs: # Every SIMD kernel has a scalar fallback, and the fallback is only real if it is # exercised: the classes tests/scalar-pass.txt names, which reach every fallback the # whole suite does, and the conformance corpus run again with intrinsics disabled. - # Vorticity.Zstd's suite does not: its only vector paths are Arm's, which an x64 runner - # never takes, so the run above already took every one of its fallbacks. - - name: Vorticity's fallbacks and the conformance corpus, intrinsics disabled - projects: tests/Vorticity.Tests tests/Vorticity.Conformance + # Vorticity.Zstd's suite runs whole: an x64 runner takes its x64 paths, and with + # intrinsics disabled the portable ones, Arm64's shape without its vectors. + - name: Vorticity's fallbacks, Vorticity.Zstd and the conformance corpus, intrinsics disabled + projects: tests/Vorticity.Tests tests/Vorticity.Zstd.Tests tests/Vorticity.Conformance + zstd-data: true scalar-pass: true steps: # Nothing here pushes to git: the token is not left on disk. diff --git a/README.md b/README.md index a3f56d58a..413adfe12 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ at the same version as the core. |---|---| | [docs/guide/](docs/guide/README.md) | **Using the library**: pages named by what you are trying to do, each one a program the samples project compiles and runs | | [docs/guide/benchmarks.md](docs/guide/benchmarks.md) | Every figure the bench publishes: what this costs against the Rust implementation, and each kernel against its baseline | -| [docs/guide/benchmarks-x64.md](docs/guide/benchmarks-x64.md) | The sections that compare Vorticity with itself, from an x64 machine, a Zen 4 processor under Windows | +| [docs/guide/benchmarks-x64.md](docs/guide/benchmarks-x64.md) | The same page from an x64 machine, a Zen 4 processor under Windows, against a Rust built there | | [docs/guide/benchmarks-duckdb.md](docs/guide/benchmarks-duckdb.md) | Our group bys against DuckDB's on the same files, through its Vortex reader and from its own table: where we lead and where we trail | | [docs/design/](docs/design/README.md) | **Why it is shaped this way**: the design documents, from the scope to the byte layout to the public API | @@ -176,6 +176,6 @@ prefix filter on FSST strings (0.93x) and a band over bit-packed integers (0.98x Every figure is on [the benchmark page](docs/guide/benchmarks.md), each section with the machine and the commit it was measured on, and [its x64 twin](docs/guide/benchmarks-x64.md) holds the -sections measured on x64. [docs/design/05-benchmarks.md](docs/design/05-benchmarks.md) says what is +same sections measured on x64. [docs/design/05-benchmarks.md](docs/design/05-benchmarks.md) says what is compared and how, and [bench/README.md](bench/README.md) how to run each instrument: `bench/gate.sh` runs everything that gates a commit. diff --git a/bench/README.md b/bench/README.md index 4b5da71ca..4bd07282f 100644 --- a/bench/README.md +++ b/bench/README.md @@ -279,9 +279,11 @@ and say which change in the commit message. **A new reference binary is carried, not recalibrated.** Both gates record the fingerprint of the `vxbench` build their tables were set under (`CalibratedShim`) and refuse to gate under another: a -ratio through a different denominator is a number about the rebuild. After rebuilding -`tools/vxbench-rs` — a new `vortex` pin, a new toolchain, a changed entry point — keep the previous -build and carry the tables over: +ratio through a different denominator is a number about the rebuild. Another machine builds another +binary, and its ratios are its own: there `--ratio-check --out` and `--throughput --take --out` (or +`--write`) measure every axis or file for that machine's page and hold none to a reference. After +rebuilding `tools/vxbench-rs` — a new `vortex` pin, a new toolchain, a changed entry point — keep +the previous build and carry the tables over: ``` dotnet run -c Release PROJ -- --ratio-check --recalibrate 3 --rebase-from diff --git a/bench/Vorticity.Benchmarks/Program.cs b/bench/Vorticity.Benchmarks/Program.cs index 60f6048cb..0e5f296e6 100644 --- a/bench/Vorticity.Benchmarks/Program.cs +++ b/bench/Vorticity.Benchmarks/Program.cs @@ -533,7 +533,9 @@ reports no change in our own time by what the binary moved the ratio --lanes N adds `full scan, N lanes`, threads pinned --out the axes as the page's section, after a - check of every axis + check of every axis; under a reference + binary the references do not name, every + axis measured and none held --throughput [family…] 57 encodings at a million rows, ~55 s --check hold each ratio to its ceiling. Refused with a family filter: a short run is +32% diff --git a/bench/Vorticity.Benchmarks/RatioCheck.cs b/bench/Vorticity.Benchmarks/RatioCheck.cs index 6a8da9e9e..33b480409 100644 --- a/bench/Vorticity.Benchmarks/RatioCheck.cs +++ b/bench/Vorticity.Benchmarks/RatioCheck.cs @@ -681,7 +681,15 @@ private static async Task CheckAsync(string path, Axis[] axes) : string.Equals(running, CalibratedShim, StringComparison.Ordinal) ? $" reference binary: {running}, the one the references were calibrated against" : $" reference binary: {running}, but the references were calibrated against {CalibratedShim}"); - if (CalibratedShim is not null && !string.Equals(running, CalibratedShim, StringComparison.Ordinal)) + + // The references are the calibrating machine's: another machine builds another binary, and + // its ratios are its own. Its page is still measured, every axis of it, and held to nothing. + bool held = CalibratedShim is null || string.Equals(running, CalibratedShim, StringComparison.Ordinal); + if (!held && Page is not null) + { + Console.Out.WriteLine(" for the page only: no axis is held to references set under another binary"); + } + else if (!held) { Console.Error.WriteLine( $"REFUSED: the references were calibrated against reference binary {CalibratedShim} and " + @@ -711,7 +719,11 @@ private static async Task CheckAsync(string path, Axis[] axes) Interval ratio = m.Ratio; string columns; string verdict = string.Empty; - if (!References.TryGetValue(axis.Name, out Reference entry)) + if (!held) + { + columns = " -- --"; + } + else if (!References.TryGetValue(axis.Name, out Reference entry)) { unreferenced.Add(string.Create( CultureInfo.InvariantCulture, @@ -798,10 +810,12 @@ private static async Task CheckAsync(string path, Axis[] axes) } Console.Out.WriteLine( - over == 0 - ? "Every axis is inside its ceiling." - : $"{over} axis/axes above ceiling. A ratio only moves when the code moves: " + - "find the change, do not raise the ceiling."); + !held + ? "No axis was held to a ceiling: this run writes the page and gates nothing." + : over == 0 + ? "Every axis is inside its ceiling." + : $"{over} axis/axes above ceiling. A ratio only moves when the code moves: " + + "find the change, do not raise the ceiling."); if (Page is not null) { await ResultsPage.WriteAsync(Page, [("in-process", Section(measured, running))]).ConfigureAwait(false); diff --git a/bench/Vorticity.Zstd.Perf/Corpus.cs b/bench/Vorticity.Zstd.Perf/Corpus.cs index a5f56b5ba..8c7a25276 100644 --- a/bench/Vorticity.Zstd.Perf/Corpus.cs +++ b/bench/Vorticity.Zstd.Perf/Corpus.cs @@ -23,7 +23,7 @@ namespace Vorticity.Zstd.Perf; /// frames byte for byte, and give the content back from them. /// /// -/// Usage: Vorticity.Zstd.Perf --corpus silesia,github,github-dict [--levels 1,3,19] [--seconds S] [--only zstd] +/// Usage: Vorticity.Zstd.Perf --corpus silesia,github,github-dict [--levels 1,3,19] [--seconds S] [--only zstd] [--directions decompression] /// [--results file.tsv] [--markdown file.md], or Vorticity.Zstd.Perf --corpus-report file.tsv [--markdown file.md] /// to render results already measured. silesia is silesia.tar, the twelve files in one frame, /// as lzbench reads it; github the 500 records, a frame each, on one context; @@ -63,8 +63,9 @@ public static Row FromTsv(string line) } } - public static int Run(string[] sets, string? levels, string? seconds, string? only, string? results, string? markdown) + public static int Run(string[] sets, string? levels, string? seconds, string? only, string? results, string? markdown, string? directions = null) { + Compresses = directions is null || directions.Split(',').Contains("compression"); int[] levelList = levels?.Split(',').Select(l => int.Parse(l, CultureInfo.InvariantCulture)).ToArray() ?? DefaultLevels; double budget = seconds is null ? 5 : double.Parse(seconds, CultureInfo.InvariantCulture); NativeReference? native = NativeReference.TryLoad(); @@ -179,6 +180,9 @@ private static (List Candidates, IDisposable[] Owned) Candidates(Nati /// every other candidate must write too; then the decoding of those frames, whose first round /// checks that every candidate gives the content back. /// + /// Whether compression is timed: --directions decompression leaves it out, the frames written once. + private static bool Compresses { get; set; } = true; + private static List Measure(DataSet set, int level, List candidates, double budget) { int records = set.Offsets.Length - 1; @@ -200,7 +204,13 @@ private static List Measure(DataSet set, int level, List candida int count = candidates.Count; bool[] compressionDiffers = new bool[count]; - double[] compression = Time(candidates, budget, (c, round) => + if (!Compresses) + { + // Decompression alone: libzstd's frames, written once, untimed. + Pass(candidates[0].Compress, set.Content, set.Offsets, reference, bounds, sizes); + } + + double[] compression = !Compresses ? new double[count] : Time(candidates, budget, (c, round) => { if (round > 0) { @@ -255,7 +265,7 @@ private static List Measure(DataSet set, int level, List candida double ratio = (double)set.Content.Length / frames.Length; var rows = new List(); - for (int c = 0; c < count; c++) + for (int c = 0; c < count && Compresses; c++) { rows.Add(new Row(set.Name, set.Label, set.Title, level, ratio, "compression", candidates[c].Name, candidates[c].Label, set.Content.Length / compression[c] / 1e6, compressionDiffers[c])); @@ -459,6 +469,24 @@ private static DataSet Load(string name) return new DataSet(name, "silesia.tar", $"the Silesia corpus as one tar, {tar.Length:N0} bytes, one frame", tar, [0, tar.Length], null); } + case "silesia-sample": + { + // 1 MiB every 16 MiB of silesia.tar, as one frame: its kinds of data, text, binaries, + // images and databases, in a frame that compresses in a fraction of the time, for the + // micro-benchmarks and the profiler. + byte[] tar = SilesiaTar(Path.Combine(directory, "silesia.zip")); + const int Slice = 1 << 20; + const int Stride = 16 << 20; + using var sample = new MemoryStream(); + for (int at = 0; at + Slice <= tar.Length; at += Stride) + { + sample.Write(tar, at, Slice); + } + + byte[] content = sample.ToArray(); + return new DataSet(name, "silesia sample", $"1 MiB every 16 MiB of silesia.tar, {content.Length:N0} bytes, one frame", content, [0, content.Length], null); + } + case "github": case "github-dict": { @@ -481,7 +509,7 @@ private static DataSet Load(string name) } default: - throw new ArgumentException("unknown corpus " + name + " (silesia, github, github-dict, github.tar, github.tar-dict)", nameof(name)); + throw new ArgumentException("unknown corpus " + name + " (silesia, silesia-sample, github, github-dict, github.tar, github.tar-dict)", nameof(name)); } } diff --git a/bench/Vorticity.Zstd.Perf/Micro.cs b/bench/Vorticity.Zstd.Perf/Micro.cs index 7a8dd5749..dcacf8b92 100644 --- a/bench/Vorticity.Zstd.Perf/Micro.cs +++ b/bench/Vorticity.Zstd.Perf/Micro.cs @@ -13,15 +13,14 @@ namespace Vorticity.Zstd.Perf; /// block are taken from the frame itself, then each step is timed alone, many times over. /// /// -/// Usage: Vorticity.Zstd.Perf --micro tables|ncount|tree|weights|x1|x2|x2build|compress|ncompress|cmatch|nmatch|compressab|cmatchab|centropy|cliterals|chist|csequences -/// [--frames a,b] [--repeat N] [--pcprofile file], the repeats for a profiler to attach; --pcprofile samples the +/// Usage: Vorticity.Zstd.Perf --micro decode|ndecode|ccorpus|nccorpus|nx1|nx2|tables|ncount|tree|weights|x1|x2|x2build|compress|ncompress|cmatch|nmatch|compressab|cmatchab|centropy|cliterals|chist|csequences +/// [--frames a,b] [--repeat N] [--pcprofile file] [--pcfunc name] [--pccaller name], the repeats for a profiler to attach; --pcprofile samples the /// timed loops' program counters (see ). Prints the median time -/// of one operation, and its cycles at the clock the M4 Pro's performance cores run (4.44 GHz); +/// of one operation, and its cycles at the clock the timed thread runs at, measured (see ); /// compressab and cmatchab time both sides of on one compressor. /// internal static class Micro { - private const double Ghz = 4.44; private sealed record Table(SequenceCode Code, short[] Norm, int TableLog, byte[] Description); @@ -48,6 +47,18 @@ public static int Run(string what, string[] frames, int repeat) continue; } + if (what is "ccorpus" or "nccorpus") + { + CompressCorpus(what, name); + continue; + } + + if (what is "decode" or "ndecode") + { + DecodeFrame(what, name); + continue; + } + Blocks blocks = Parse(BenchFrames.Load(frameName)); switch (what) { @@ -116,6 +127,38 @@ public static int Run(string what, string[] frames, int repeat) break; } + case "nx1": + case "nx2": + { + // The same four-stream sections, decoded by libzstd: HUF_readDTableX1 or X2 on + // each tree, then HUF_decompress4X_usingDTable (its assembly loops on x86-64). + NativeReference.HuffmanReference huffman = (NativeReference.TryLoad() ?? throw new InvalidOperationException("no " + NativeReference.LibraryPath)).Huffman + ?? throw new InvalidOperationException("the reference does not export HUF_decompress4X_usingDTable"); + bool x2 = what == "nx2"; + var output = new byte[128 << 10]; + int symbols = 0; + foreach (Literals l in blocks.Sections) + { + symbols += l.Size; + } + + Report(name, $"{blocks.Sections.Count} sections, {symbols} symbols (libzstd {what[1..]})", blocks.Sections.Count, symbols, () => + { + byte[]? current = null; + foreach (Literals l in blocks.Sections) + { + if (!ReferenceEquals(l.Tree, current)) + { + huffman.ReadTree(l.Tree, x2); + current = l.Tree; + } + + huffman.DecodeFourStreams(l.Streams, output.AsSpan(0, l.Size)); + } + }); + break; + } + case "ncount": { short[] norm = new short[SequenceCodes.MaxMatchLength + 1]; @@ -256,7 +299,7 @@ public static int Run(string what, string[] frames, int repeat) } default: - Console.WriteLine($"unknown micro-benchmark {what}: tables, ncount, tree, weights, x1, x2, x2build, compress, ncompress, cmatch, nmatch, compressab, cmatchab, centropy, cliterals, chist, csequences, dcorpus, ndcorpus, dcorpusab"); + Console.WriteLine($"unknown micro-benchmark {what}: decode, ndecode, ccorpus, nccorpus, nx1, nx2, tables, ncount, tree, weights, x1, x2, x2build, compress, ncompress, cmatch, nmatch, compressab, cmatchab, centropy, cliterals, chist, csequences, dcorpus, ndcorpus, dcorpusab"); return 1; } } @@ -324,6 +367,113 @@ void Body() } } + /// + /// The compression of a corpus of (--frames github-dict-L3: the set, then + /// the level), every record a frame of its own, on one compressor: Vorticity.Zstd (ccorpus) or + /// libzstd (nccorpus). Per frame, and per byte of content. + /// + private static void CompressCorpus(string what, string name) + { + int dash = name.LastIndexOf("-L", StringComparison.Ordinal); + int level = int.Parse(name.AsSpan(dash + 2), CultureInfo.InvariantCulture); + (byte[] content, int[] offsets, byte[]? dictionary) = Corpus.LoadSet(name[..dash]); + int records = offsets.Length - 1; + byte[] frames = new byte[ZstdCompressor.GetMaxCompressedLength(content.Length) + (records * 64)]; + SpanCodec compress; + if (what == "nccorpus") + { + NativeReference native = NativeReference.TryLoad() ?? throw new InvalidOperationException("no " + NativeReference.LibraryPath); + compress = dictionary is null ? (s, d) => native.Compress(s, d, level) : native.CompressorWith(dictionary, level); + } + else + { + var compressor = dictionary is null ? new ZstdCompressor(level) : new ZstdCompressor(level, dictionary); + compress = (s, d) => compressor.Compress(s, d, out _, out int written) == OperationStatus.Done ? written : -1; + } + + void Body() + { + int at = 0; + for (int i = 0; i < records; i++) + { + at += compress(content.AsSpan(offsets[i], offsets[i + 1] - offsets[i]), frames.AsSpan(at)); + } + } + + Body(); + string title = $"{(what == "nccorpus" ? "libzstd " : string.Empty)}compression, {records} frames, {content.Length} bytes, per frame"; + Report(name, title, records, content.Length, Body); + } + + /// + /// The decoding of one benchmark frame, whole, by Vorticity.Zstd (decode) or libzstd (ndecode): + /// per sequence, and per byte of content. + /// + private static void DecodeFrame(string what, string name) + { + byte[] frame = BenchFrames.Load(name); + byte[] output = new byte[BenchFrames.ContentSize(frame)]; + SpanCodec decode; + if (what == "ndecode") + { + NativeReference native = NativeReference.TryLoad() ?? throw new InvalidOperationException("no " + NativeReference.LibraryPath); + PcSampler.Libraries.Add((NativeReference.LibraryPath, "_ZSTD_versionNumber", native.VersionAddress)); + decode = native.Decompress; + } + else + { + var decompressor = new ZstdDecompressor(); + decode = (s, d) => decompressor.Decompress(s, d, out _, out int written) == OperationStatus.Done ? written : -1; + } + + if (decode(frame, output) != output.Length) + { + throw new InvalidOperationException(what + " failed on " + name); + } + + int sequences = Math.Max(1, CountSequences(frame)); + Report(name, $"{(what == "ndecode" ? "libzstd " : string.Empty)}decoding, {frame.Length} -> {output.Length} bytes, {sequences} sequences, per sequence", + sequences, output.Length, () => decode(frame, output)); + } + + /// + /// The clock the timed thread runs at, in GHz: a chain of dependent additions, one a cycle on + /// any core, timed. Measured once; the cycle columns use it. + /// + private static double Ghz => s_ghz ??= MeasureClock(); + + private static double? s_ghz; + + private static double MeasureClock() + { + const long Iterations = 20_000_000; + ulong step = (ulong)Environment.TickCount64 | 1; + double best = 0; + for (int round = 0; round < 5; round++) + { + long start = Stopwatch.GetTimestamp(); + ulong x = AddChain(step, step, Iterations); + double seconds = Stopwatch.GetElapsedTime(start).TotalSeconds; + GC.KeepAlive(x); + best = Math.Max(best, Iterations * 16 / seconds / 1e9); + } + + Console.WriteLine(string.Create(CultureInfo.InvariantCulture, $"clock: {best:F2} GHz (a chain of dependent additions)")); + return best; + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static ulong AddChain(ulong x, ulong y, long n) + { + for (long i = 0; i < n; i++) + { + x += y; x += y; x += y; x += y; x += y; x += y; x += y; x += y; + x += y; x += y; x += y; x += y; x += y; x += y; x += y; x += y; + } + + return x; + } + /// The histogram of each block's literals, as literals compression counts them. private static unsafe uint HistogramsOnly(List records) { @@ -362,6 +512,7 @@ private static void Report(string frame, string what, int operations, int cells, } var samples = new double[101]; + PcSampler.Label = $"{frame} {what}"; using (PcSampler.Start()) { for (int s = 0; s < samples.Length; s++) @@ -403,6 +554,7 @@ private static void ReportAb(string frame, string what, int operations, Action b var before = new double[101]; var after = new double[101]; + PcSampler.Label = $"{frame} {what}"; using (PcSampler.Start()) { for (int s = 0; s < before.Length; s++) diff --git a/bench/Vorticity.Zstd.Perf/PcSampler.cs b/bench/Vorticity.Zstd.Perf/PcSampler.cs index b0aa6b6f0..0710e18b0 100644 --- a/bench/Vorticity.Zstd.Perf/PcSampler.cs +++ b/bench/Vorticity.Zstd.Perf/PcSampler.cs @@ -10,12 +10,14 @@ namespace Vorticity.Zstd.Perf; /// /// A sampling profiler at the instruction, for the micro-benchmarks (--pcprofile file): a -/// thread suspends the measured one about every 100 microseconds, reads its program counter through -/// Mach's thread_get_state, and counts the addresses as they are, after a header that places -/// the images: the executable's slide, and where a symbol of each native library registered loaded -/// (bench/zstd-pcprofile.py resolves them, against the executable's dSYM or the library's symbols, -/// and annotates them). The address of a suspended thread is the next instruction to retire: one -/// that waits, or the one after it. +/// thread suspends the measured one about every 100 microseconds, reads its program counter, and +/// counts the addresses. On macOS through Mach's thread_get_state, the addresses as they are, +/// after a header that places the images: the executable's slide, and where a symbol of each native +/// library registered loaded (bench/zstd-pcprofile.py resolves them, against the executable's +/// dSYM or the library's symbols, and annotates them). On Windows through GetThreadContext, +/// resolved in the process by DbgHelp against the PDBs (see ), which +/// appends a report to the file. The address of a suspended thread is the next instruction to +/// retire: one that waits, or the one after it. /// internal static unsafe class PcSampler { @@ -26,6 +28,23 @@ internal static unsafe class PcSampler /// The file the samples are appended to, or null when not profiling. public static string? Path { get; set; } + /// + /// On Windows, the functions whose samples the report also gives instruction by instruction: + /// those whose name contains this (--pcfunc). + /// + public static string? Function { get; set; } + + /// + /// On Windows, the functions whose samples the report also attributes to their callers: those + /// whose name contains this (--pccaller). The caller is the first word of the stack, at + /// the sample, that follows a call instruction into another function: exact in a function that + /// keeps the return address within its first eight words, as a leaf such as memmove does. + /// + public static string? Callers { get; set; } + + /// What the next session measures, for the report's heading. + public static string Label { get; set; } = string.Empty; + /// Native libraries to place: their path, a symbol, and where it is loaded. public static List<(string Library, string Symbol, nint Address)> Libraries { get; } = []; @@ -45,7 +64,8 @@ internal static unsafe class PcSampler private static extern nint _dyld_get_image_vmaddr_slide(uint imageIndex); /// Samples the calling thread until disposed, when is set. - public static IDisposable? Start() => Path is null ? null : new Session(Path); + public static IDisposable? Start() => + Path is null ? null : OperatingSystem.IsWindows() ? new WindowsSession(Path) : new Session(Path); private sealed class Session : IDisposable { @@ -98,4 +118,319 @@ private void Run() } } } + + /// + /// The Windows sampler: the measured thread suspended, its CONTEXT's instruction pointer + /// read, the thread resumed, and only then the address counted, so that nothing allocates while + /// the thread may hold a lock. + /// + private sealed class WindowsSession : IDisposable + { + private const uint ContextControl = 0x0010_0001; // CONTEXT_AMD64 | CONTEXT_CONTROL + private const int ContextSize = 1232; + private const int ContextFlagsOffset = 0x30; + private const int RipOffset = 0xF8; + private const int RspOffset = 0x98; + private const int StackWords = 8; + private const uint DuplicateSameAccess = 2; + + private readonly string _path; + private readonly string _label = Label; + private readonly nint _target; + private readonly Dictionary _counts = []; + private readonly List<(ulong Rip, ulong[] Stack)> _stacks = []; + private readonly bool _callers = Callers is not null; + private readonly Thread _thread; + private volatile bool _stop; + + public WindowsSession(string path) + { + _path = path; + nint process = GetCurrentProcess(); + if (!DuplicateHandle(process, GetCurrentThread(), process, out _target, 0, false, DuplicateSameAccess)) + { + throw new InvalidOperationException("DuplicateHandle failed: " + Marshal.GetLastPInvokeError()); + } + + _thread = new Thread(Run) { IsBackground = true, Priority = ThreadPriority.Highest }; + _thread.Start(); + } + + public void Dispose() + { + _stop = true; + _thread.Join(); + _ = CloseHandle(_target); + string report = WindowsSymbols.Report(_counts, _label, Function) + WindowsSymbols.CallerReport(_stacks, Callers); + File.AppendAllText(_path, report); + Console.Write(report); + } + + private void Run() + { + byte* context = (byte*)NativeMemory.AlignedAlloc(ContextSize, 64); + ulong* window = stackalloc ulong[StackWords]; + long interval = Stopwatch.Frequency / 10_000; + try + { + while (!_stop) + { + long next = Stopwatch.GetTimestamp() + interval; + if (SuspendThread(_target) != uint.MaxValue) + { + *(uint*)(context + ContextFlagsOffset) = ContextControl; + bool read = GetThreadContext(_target, context); + if (read && _callers) + { + // The top of the stack, copied while the thread cannot change it. + ulong* top = (ulong*)*(ulong*)(context + RspOffset); + for (int k = 0; k < StackWords; k++) + { + window[k] = top[k]; + } + } + + _ = ResumeThread(_target); + if (read) + { + ulong rip = *(ulong*)(context + RipOffset); + _counts[rip] = _counts.GetValueOrDefault(rip) + 1; + if (_callers) + { + _stacks.Add((rip, new ReadOnlySpan(window, StackWords).ToArray())); + } + } + } + + while (Stopwatch.GetTimestamp() < next) + { + Thread.SpinWait(20); + } + } + } + finally + { + NativeMemory.AlignedFree(context); + } + } + + [DllImport("kernel32.dll")] + private static extern nint GetCurrentProcess(); + + [DllImport("kernel32.dll")] + private static extern nint GetCurrentThread(); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DuplicateHandle( + nint sourceProcess, nint sourceHandle, nint targetProcess, out nint targetHandle, uint desiredAccess, bool inheritHandle, uint options); + + [DllImport("kernel32.dll")] + private static extern bool CloseHandle(nint handle); + + [DllImport("kernel32.dll")] + private static extern uint SuspendThread(nint thread); + + [DllImport("kernel32.dll")] + private static extern uint ResumeThread(nint thread); + + [DllImport("kernel32.dll")] + private static extern bool GetThreadContext(nint thread, byte* context); + } +} + +/// +/// The samples of on Windows, resolved in the process by DbgHelp: the +/// executable's PDB, which Native AOT writes beside it, names its functions and their lines, and +/// a native library's names what it exports. The report gives the functions by samples, the lines +/// by samples, and for the functions --pcfunc names, every sampled instruction by its +/// relative virtual address, which dumpbin /disasm prints the code at. +/// +internal static unsafe class WindowsSymbols +{ + private const uint SymOptUndname = 0x2; + private const uint SymOptDeferredLoads = 0x4; + private const uint SymOptLoadLines = 0x10; + private const uint SymOptFailCriticalErrors = 0x200; + private const int MaxName = 1024; + private const int SymbolInfoSize = 88; + private const int LineSize = 40; + + private static bool s_initialized; + + public static string Report(Dictionary counts, string label, string? function) + { + nint process = -1; + if (!s_initialized) + { + SymSetOptions(SymOptUndname | SymOptDeferredLoads | SymOptLoadLines | SymOptFailCriticalErrors); + s_initialized = SymInitializeW(process, AppContext.BaseDirectory, true); + } + else + { + SymRefreshModuleList(process); + } + + long total = counts.Values.Sum(); + var functions = new Dictionary(); + var lines = new Dictionary<(string Function, string Line), long>(); + var instructions = new List<(string Function, ulong Rva, ulong Offset, string Line, int Samples)>(); + byte* symbol = stackalloc byte[SymbolInfoSize + (MaxName * 2)]; + byte* line = stackalloc byte[LineSize]; + foreach ((ulong address, int samples) in counts) + { + new Span(symbol, SymbolInfoSize).Clear(); + *(uint*)symbol = SymbolInfoSize; + *(uint*)(symbol + 80) = MaxName; + string name; + ulong displacement = 0; + ulong rva = 0; + if (SymFromAddrW(process, address, out displacement, symbol)) + { + int length = (int)*(uint*)(symbol + 76); + name = new string((char*)(symbol + 84), 0, Math.Min(length, MaxName - 1)); + rva = address - *(ulong*)(symbol + 32); + } + else + { + name = "?"; + } + + new Span(line, LineSize).Clear(); + *(uint*)line = LineSize; + string where = SymGetLineFromAddrW64(process, address, out _, line) + ? $"{System.IO.Path.GetFileName(new string(*(char**)(line + 24)))}:{*(uint*)(line + 16)}" + : "?"; + functions[name] = functions.GetValueOrDefault(name) + samples; + lines[(name, where)] = lines.GetValueOrDefault((name, where)) + samples; + if (function is not null && name.Contains(function, StringComparison.Ordinal)) + { + instructions.Add((name, rva, displacement, where, samples)); + } + } + + var report = new System.Text.StringBuilder(); + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $"== {label}: {total} samples"); + report.AppendLine(" samples % function"); + foreach ((string name, long samples) in functions.OrderByDescending(f => f.Value).Take(30)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" {samples,7} {100.0 * samples / total,6:F2} {name}"); + } + + report.AppendLine(" -- lines"); + foreach (((string name, string where), long samples) in lines.OrderByDescending(l => l.Value).Take(40)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" {samples,7} {100.0 * samples / total,6:F2} {where,-40} {name}"); + } + + foreach (var group in instructions.GroupBy(i => i.Function)) + { + long inFunction = group.Sum(i => (long)i.Samples); + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" -- {group.Key}: {inFunction} samples, by instruction (rva, offset)"); + foreach (var i in group.OrderBy(i => i.Rva)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, + $" {i.Rva:x8} +{i.Offset,-6:x} {i.Samples,7} {100.0 * i.Samples / inFunction,6:F2} {i.Line}"); + } + } + + return report.ToString(); + } + + /// + /// The samples of the functions names, by caller: for each, the first + /// stack word that resolves into another function and follows a call (a call rel32, or an + /// indirect call of two, three or six bytes), with the line of the call. + /// + public static string CallerReport(List<(ulong Rip, ulong[] Stack)> samples, string? function) + { + if (function is null || samples.Count == 0) + { + return string.Empty; + } + + nint process = -1; + byte* symbol = stackalloc byte[SymbolInfoSize + (MaxName * 2)]; + byte* line = stackalloc byte[LineSize]; + var names = new Dictionary(); + string? NameOf(ulong address) + { + if (names.TryGetValue(address, out string? known)) + { + return known; + } + + new Span(symbol, SymbolInfoSize).Clear(); + *(uint*)symbol = SymbolInfoSize; + *(uint*)(symbol + 80) = MaxName; + string? name = SymFromAddrW(process, address, out _, symbol) + ? new string((char*)(symbol + 84), 0, Math.Min((int)*(uint*)(symbol + 76), MaxName - 1)) + : null; + names[address] = name; + return name; + } + + var callers = new Dictionary(); + long total = 0; + foreach ((ulong rip, ulong[] stack) in samples) + { + string? name = NameOf(rip); + if (name is null || !name.Contains(function, StringComparison.Ordinal)) + { + continue; + } + + total++; + string caller = "?"; + foreach (ulong word in stack) + { + string? callerName = NameOf(word); + if (callerName is null || callerName == name || !FollowsCall(word)) + { + continue; + } + + new Span(line, LineSize).Clear(); + *(uint*)line = LineSize; + string where = SymGetLineFromAddrW64(process, word - 1, out _, line) + ? $"{System.IO.Path.GetFileName(new string(*(char**)(line + 24)))}:{*(uint*)(line + 16)}" + : "?"; + caller = $"{where,-40} {callerName}"; + break; + } + + callers[caller] = callers.GetValueOrDefault(caller) + 1; + } + + var report = new System.Text.StringBuilder(); + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" -- callers of {function}: {total} samples"); + foreach ((string caller, long count) in callers.OrderByDescending(c => c.Value).Take(30)) + { + report.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $" {count,7} {100.0 * count / total,6:F2} {caller}"); + } + + return report.ToString(); + + // A return address follows the call that pushed it: E8 rel32, or FF /2 in its short forms. + static bool FollowsCall(ulong address) + { + byte* p = (byte*)address; + return p[-5] == 0xE8 || (p[-2] == 0xFF && (p[-1] & 0x38) == 0x10) + || (p[-3] == 0xFF && (p[-2] & 0x38) == 0x10) || (p[-6] == 0xFF && (p[-5] & 0x38) == 0x10); + } + } + + [DllImport("dbghelp.dll")] + private static extern uint SymSetOptions(uint options); + + [DllImport("dbghelp.dll", CharSet = CharSet.Unicode)] + private static extern bool SymInitializeW(nint process, string? searchPath, bool invadeProcess); + + [DllImport("dbghelp.dll")] + private static extern bool SymRefreshModuleList(nint process); + + [DllImport("dbghelp.dll")] + private static extern bool SymFromAddrW(nint process, ulong address, out ulong displacement, byte* symbol); + + [DllImport("dbghelp.dll")] + private static extern bool SymGetLineFromAddrW64(nint process, ulong address, out uint displacement, byte* line); } diff --git a/bench/Vorticity.Zstd.Perf/Program.cs b/bench/Vorticity.Zstd.Perf/Program.cs index cb6f8ad69..1c8c43e19 100644 --- a/bench/Vorticity.Zstd.Perf/Program.cs +++ b/bench/Vorticity.Zstd.Perf/Program.cs @@ -16,8 +16,10 @@ namespace Vorticity.Zstd.Perf; /// moment lands on all of them alike. Several passes show how stable the figures are. /// /// -/// Usage: Vorticity.Zstd.Perf [--frames a,b] [--passes N] [--reps N] [--only zstd,libzstd-ref] [--ab] [--dump dir] [--no-check] [--no-pair] -/// [--pmu default|EV1,EV2] [--compress]; --pmu counts hardware events per decode and sequence (under sudo); --ab +/// Usage: Vorticity.Zstd.Perf [--frames a,b] [--passes N] [--reps N] [--only zstd,libzstd-ref] [--ab] [--dump dir] [--no-check] [--no-pair | --pair] +/// [--pmu default|EV1,EV2] [--compress] [--core N]; --pmu counts hardware events per decode and sequence (macOS, under +/// sudo); --core N keeps the timed thread on logical processor N (Windows, where the process and +/// the thread also run at a high priority, as macOS's user-interactive QoS); --ab /// adds a "before" candidate that runs with the legacy switches of the point under work (see /// ). --compress times compression instead: each frame's content, at the /// level its name gives (3 for the reference). Before timing, @@ -30,9 +32,20 @@ public static class Program private sealed record Candidate(string Name, Func Decode); + private const int ThreadPriorityHighest = 2; + [DllImport("libSystem.dylib")] private static extern int pthread_set_qos_class_self_np(int qosClass, int relativePriority); + [DllImport("kernel32.dll")] + private static extern nint GetCurrentThread(); + + [DllImport("kernel32.dll")] + private static extern bool SetThreadPriority(nint thread, int priority); + + [DllImport("kernel32.dll")] + private static extern nuint SetThreadAffinityMask(nint thread, nuint mask); + public static int Main(string[] args) { // The timed thread at the user-interactive QoS: kept on the performance cores at their full @@ -41,6 +54,17 @@ public static int Main(string[] args) { _ = pthread_set_qos_class_self_np(QosClassUserInteractive, 0); } + else if (OperatingSystem.IsWindows()) + { + // The same on Windows: the process and the timed thread at a high priority, and with + // --core N the thread kept on logical processor N, which no migration then disturbs. + Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High; + _ = SetThreadPriority(GetCurrentThread(), ThreadPriorityHighest); + if (Option(args, "--core") is string core) + { + _ = SetThreadAffinityMask(GetCurrentThread(), (nuint)1 << int.Parse(core, CultureInfo.InvariantCulture)); + } + } string[] frames = Option(args, "--frames")?.Split(',') ?? BenchFrames.Names; if (Option(args, "--dump") is string directory) @@ -59,7 +83,7 @@ public static int Main(string[] args) { // zstd's own benchmark data, level by level: bench/zstd-corpus.sh downloads it. return Corpus.Run(corpora.Split(','), Option(args, "--levels"), Option(args, "--seconds"), Option(args, "--only"), - Option(args, "--results"), Option(args, "--markdown")); + Option(args, "--results"), Option(args, "--markdown"), Option(args, "--directions")); } if (Option(args, "--corpus-report") is string results) @@ -70,6 +94,8 @@ public static int Main(string[] args) if (Option(args, "--micro") is string micro) { PcSampler.Path = Option(args, "--pcprofile"); + PcSampler.Function = Option(args, "--pcfunc"); + PcSampler.Callers = Option(args, "--pccaller"); return Micro.Run(micro, frames, int.Parse(Option(args, "--repeat") ?? "1", CultureInfo.InvariantCulture)); } @@ -115,8 +141,13 @@ public static int Main(string[] args) return 0; } - // --no-pair: one block at a time, to measure what pairing them brings. - var zstd = new ZstdDecompressor { PairsBlocks = !args.Contains("--no-pair") }; + // --no-pair: one block at a time, --pair: two, to measure what pairing them brings; by + // default, as the decoder chooses for the machine. + var zstd = new ZstdDecompressor(); + if (args.Contains("--no-pair") || args.Contains("--pair")) + { + zstd.PairsBlocks = args.Contains("--pair"); + } var platform = new ZstandardDecoder(); NativeReference? native = NativeReference.TryLoad(); diff --git a/bench/Vorticity.Zstd.Shared/NativeReference.cs b/bench/Vorticity.Zstd.Shared/NativeReference.cs index 68840c662..dcb21ee2a 100644 --- a/bench/Vorticity.Zstd.Shared/NativeReference.cs +++ b/bench/Vorticity.Zstd.Shared/NativeReference.cs @@ -50,6 +50,11 @@ private NativeReference(nint library) _context = create(); _compressionContext = createCompression(); _sequencesContext = createCompression(); + if (NativeLibrary.TryGetExport(library, "HUF_decompress4X_usingDTable", out _) + && NativeLibrary.TryGetExport(library, "HUF_readDTableX2_wksp", out _)) + { + Huffman = new HuffmanReference(library, _isError); + } } /// Where ZSTD_versionNumber is loaded: the library's slide, for a profiler. @@ -59,10 +64,13 @@ private NativeReference(nint library) public static string LibraryPath => Environment.GetEnvironmentVariable("VORTICITY_ZSTD_LIBZSTD") is { Length: > 0 } path ? path - : Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref.dylib"); + : Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref" + LibrarySuffix); + + /// The extension of a shared library here: tools/native-ref builds dylibs on macOS, DLLs on Windows. + private static string LibrarySuffix => OperatingSystem.IsWindows() ? ".dll" : OperatingSystem.IsMacOS() ? ".dylib" : ".so"; /// The same library built with threads (build.sh mt): zstdmt. - public static string ThreadedLibraryPath => Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_mt.dylib"); + public static string ThreadedLibraryPath => Path.Combine(BenchFrames.RepositoryRoot, "tools", "native-ref", "out", "libzstd_mt" + LibrarySuffix); /// The reference library, or null when tools/native-ref/build.sh has not been run. public static NativeReference? TryLoad() => TryLoad(LibraryPath); @@ -73,6 +81,62 @@ private NativeReference(nint library) private static NativeReference? TryLoad(string path) => File.Exists(path) && NativeLibrary.TryLoad(path, out nint library) ? new NativeReference(library) : null; + /// + /// libzstd's literals decoding alone, for the micro-benchmarks: HUF_readDTableX1_wksp or + /// HUF_readDTableX2_wksp on a tree description, then HUF_decompress4X_usingDTable on + /// four streams. Exported by the DLLs a GNU-compatible compiler builds (every symbol is); null + /// where the library does not export them. + /// + public HuffmanReference? Huffman { get; } + + /// See . + internal sealed class HuffmanReference + { + private const int TableLogMax = 12; + private const int Bmi2 = 1; + + private readonly delegate* unmanaged _readX1; + private readonly delegate* unmanaged _readX2; + private readonly delegate* unmanaged _decompress4X; + private readonly delegate* unmanaged _isError; + private readonly uint[] _table = new uint[1 + (1 << TableLogMax)]; + private readonly uint[] _workspace = new uint[4096]; + + public HuffmanReference(nint library, delegate* unmanaged isError) + { + _readX1 = (delegate* unmanaged)NativeLibrary.GetExport(library, "HUF_readDTableX1_wksp"); + _readX2 = (delegate* unmanaged)NativeLibrary.GetExport(library, "HUF_readDTableX2_wksp"); + _decompress4X = (delegate* unmanaged)NativeLibrary.GetExport(library, "HUF_decompress4X_usingDTable"); + _isError = isError; + } + + /// Reads a tree into the table, for X2 the double-symbol one: the tree's size, or -1. + public int ReadTree(ReadOnlySpan tree, bool x2) + { + // HUF_CREATE_STATIC_DTABLEX1 and X2: the table's largest log in the description word. + _table[0] = (uint)(x2 ? TableLogMax : TableLogMax - 1) * 0x01000001u; + fixed (uint* table = _table) + fixed (uint* workspace = _workspace) + fixed (byte* src = tree) + { + nuint result = (x2 ? _readX2 : _readX1)(table, src, (nuint)tree.Length, workspace, (nuint)(_workspace.Length * 4), Bmi2); + return _isError(result) != 0 ? -1 : (int)result; + } + } + + /// Decodes four streams with the table last read: the size written, or -1. + public int DecodeFourStreams(ReadOnlySpan streams, Span output) + { + fixed (uint* table = _table) + fixed (byte* src = streams) + fixed (byte* dst = output) + { + nuint result = _decompress4X(dst, (nuint)output.Length, src, (nuint)streams.Length, table, Bmi2); + return _isError(result) != 0 ? -1 : (int)result; + } + } + } + /// ZSTD_decompressDCtx: the decoded size, or -1 on an error. public int Decompress(ReadOnlySpan source, Span destination) { diff --git a/bench/zstd-ab-aot.sh b/bench/zstd-ab-aot.sh index fbe304aea..1a380a5ab 100755 --- a/bench/zstd-ab-aot.sh +++ b/bench/zstd-ab-aot.sh @@ -1,15 +1,23 @@ #!/usr/bin/env bash # A/B between two Native AOT builds of the perf tool, for changes that cannot be switched at run -# time (a table format, an assembly attribute). Builds HEAD (or the ref given) into a worktree as -# "before", the working tree as "after", then runs them alternately and compares their medians. +# time (a table format, an assembly attribute, a hot loop's code). Builds HEAD (or the ref given) +# into a worktree as "before", the working tree as "after", then runs them alternately and +# compares their medians. # -# bench/zstd-ab-aot.sh [--ref ] [--rounds N] [--frames a,b] [--compress | --micro ] +# bench/zstd-ab-aot.sh [--ref ] [--rounds N] [--frames a,b] [--compress | --micro ] [--core N] +# [--no-build | --before --after ] # # --compress times compression (Vorticity.Zstd.Perf --compress) instead of decompression; --micro times a # micro-benchmark (Vorticity.Zstd.Perf --micro , e.g. dcorpus with --frames github-dict-L3). +# --core N keeps the timed thread on logical processor N (Windows; see Program.cs). --no-build +# compares the two builds the last run left, for more rounds or other frames; --before X and +# --after Y compare two builds kept under artifacts/ab (implying --no-build). +# +# Both builds are published under artifacts/ab, in this tree: the tool finds the native reference +# and the data git does not keep (tools/native-ref/out, the test data) by walking up from itself. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ref=HEAD; rounds=8; frames=reference,text-L3,json-L19; mode=(); micro= +ref=HEAD; rounds=8; frames=reference,text-L3,json-L19; mode=(); micro=; pin=(); build=1; before=before; after=after while [[ $# -gt 0 ]]; do case "$1" in --ref) ref="$2"; shift 2 ;; @@ -17,36 +25,48 @@ while [[ $# -gt 0 ]]; do --frames) frames="$2"; shift 2 ;; --compress) mode=(--compress); shift ;; --micro) micro="$2"; shift 2 ;; + --core) pin=(--core "$2"); shift 2 ;; + --no-build) build=0; shift ;; + --before) before="$2"; build=0; shift 2 ;; + --after) after="$2"; build=0; shift 2 ;; *) echo "unknown $1" >&2; exit 2 ;; esac done work="$root/artifacts/ab" mkdir -p "$work" -rm -rf "$work/tree" -git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null 2>&1 || { git -C "$root" worktree prune; git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null; } -# The worktree has neither the native reference nor the data git does not keep (the test data, the -# benchmark corpora): point it at the main tree's. -ln -sfn "$root/tools/native-ref/out" "$work/tree/tools/native-ref/out" -[[ -e "$work/tree/tests/Vorticity.Zstd.Tests/testdata" ]] || ln -s "$root/tests/Vorticity.Zstd.Tests/testdata" "$work/tree/tests/Vorticity.Zstd.Tests/testdata" -dotnet publish "$work/tree/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/before" > /dev/null -dotnet publish "$root/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/after" > /dev/null -git -C "$root" worktree remove --force "$work/tree" +if [[ $build == 1 ]]; then + rm -rf "$work/tree" + git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null 2>&1 || { git -C "$root" worktree prune; git -C "$root" worktree add --detach "$work/tree" "$ref" > /dev/null; } + dotnet publish "$work/tree/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/before" > /dev/null + dotnet publish "$root/bench/Vorticity.Zstd.Perf/Vorticity.Zstd.Perf.csproj" -c Release -o "$work/after" > /dev/null + git -C "$root" worktree remove --force "$work/tree" +fi + +run() { + local side="$1" frame="$2" + if [[ -n "$micro" ]]; then + "$work/$side/Vorticity.Zstd.Perf" ${pin[@]+"${pin[@]}"} --micro "$micro" --frames "$frame" | sed -nE 's/.*: +([0-9.]+) ns.*/\1/p' + else + "$work/$side/Vorticity.Zstd.Perf" ${pin[@]+"${pin[@]}"} ${mode[@]+"${mode[@]}"} --frames "$frame" --only zstd --passes 1 --reps 300 | awk '/^ zstd/ {print $4}' + fi +} + for frame in ${frames//,/ }; do b=(); a=() for ((i = 0; i < rounds; i++)); do - if [[ -n "$micro" ]]; then - b+=("$("$work/before/Vorticity.Zstd.Perf" --micro "$micro" --frames "$frame" | sed -nE 's/.*: +([0-9.]+) ns.*/\1/p')") - a+=("$("$work/after/Vorticity.Zstd.Perf" --micro "$micro" --frames "$frame" | sed -nE 's/.*: +([0-9.]+) ns.*/\1/p')") - else - b+=("$("$work/before/Vorticity.Zstd.Perf" ${mode[@]+"${mode[@]}"} --frames "$frame" --only zstd --passes 1 --reps 300 | awk '/^ zstd/ {print $4}')") - a+=("$("$work/after/Vorticity.Zstd.Perf" ${mode[@]+"${mode[@]}"} --frames "$frame" --only zstd --passes 1 --reps 300 | awk '/^ zstd/ {print $4}')") - fi + b+=("$(run "$before" "$frame")") + a+=("$(run "$after" "$frame")") done - python3 - "$frame" "${b[*]}" "${a[*]}" <<'PY' -import sys, statistics -frame, b, a = sys.argv[1], list(map(float, sys.argv[2].split())), list(map(float, sys.argv[3].split())) -mb, ma = statistics.median(b), statistics.median(a) -pairs = " ".join(f"{x / y:.3f}" for x, y in zip(b, a)) -print(f"{frame:16} before {mb:9.1f} after {ma:9.1f} speedup {mb / ma:.3f}x (per round: {pairs})") -PY + awk -v frame="$frame" -v b="${b[*]}" -v a="${a[*]}" ' + function median(v, n, i, j, t, s) { + for (i = 1; i <= n; i++) s[i] = v[i] + for (i = 2; i <= n; i++) for (j = i; j > 1 && s[j - 1] > s[j]; j--) { t = s[j]; s[j] = s[j - 1]; s[j - 1] = t } + return n % 2 ? s[(n + 1) / 2] : (s[n / 2] + s[n / 2 + 1]) / 2 + } + BEGIN { + n = split(b, before, " "); split(a, after, " ") + for (i = 1; i <= n; i++) pairs = pairs sprintf(" %.3f", before[i] / after[i]) + mb = median(before, n); ma = median(after, n) + printf "%-16s before %9.1f after %9.1f speedup %.3fx (per round:%s)\n", frame, mb, ma, mb / ma, pairs + }' done diff --git a/docs/guide/benchmarks-x64.md b/docs/guide/benchmarks-x64.md index 16a913b78..7476d95e0 100644 --- a/docs/guide/benchmarks-x64.md +++ b/docs/guide/benchmarks-x64.md @@ -10,11 +10,12 @@ instrument measures; [bench/README.md](../../bench/README.md) how to run them. Vortex™ is a trademark of LF Projects, LLC. Vorticity is an independent implementation, not affiliated with or endorsed by the Vortex project or LF Projects, LLC. -**The sections that compare Vorticity with Rust are not measured on x64**: each says which -arguments write it. The sections that compare Vorticity with itself are: the encodings column by -column, the advice, and the kernels against their baselines, measured with each scan pass -taking over the mapping the previous one left, where [the benchmark page](benchmarks.md) maps the -file anew at every call. The kernel tables' speedups are computed from the Mean column of their run. +**Rust's ratios here are this machine's.** Its `vxbench` is built here, and the ceilings that +`--ratio-check` and `--throughput --check` hold the ratios to are set on the arm64 machine, against +its own build: the in-process axes, the takes and the writes on this page are measured and held to +none. The per-encoding corpus is the generator's current one, 61 files where the arm64 page has 57: +`pco_f32`, `pco_i16`, `pco_u32` and `pco_nullable` are on this page alone. The kernel tables' +speedups are computed from the Mean column of their run. * **The two sides.** Vorticity as a Native AOT binary built for the machine's instruction set, with the workstation garbage collector, where a section says so, and on the JIT otherwise. Vortex @@ -23,6 +24,9 @@ file anew at every call. The kernel tables' speedups are computed from the Mean gives its compiler this machine's instruction set, AVX-512 included, as .NET's JIT has it. Both map the file and read it where it lies, decode every value they return to its plain form, a constant column kept as one value on both, and must return the same rows or the run fails. +* **Rust at its faster setting.** On one core every Rust figure is the faster of its two ways of + splitting a scan, its default and one split per chunk, measured file by file; on all cores it + keeps its default, as on [the benchmark page](benchmarks.md). * **Speedup** is Rust's time over Vorticity's: above 1.00x, Vorticity took less. * **Throughput** is the plain size of the rows returned over the time, as on [the benchmark page](benchmarks.md). @@ -47,36 +51,453 @@ Release bench/Vorticity.Benchmarks.Runner`): an older one times older code. ## Reading and writing a table, process against process -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +* **The data.** A table of four columns: `monotone`, an increasing `i64` that the filters and + the projection use; `value`, an `f64`; `label`, one of five short strings; `flag`, a `bool` + with nulls. 2^20 rows, and ten times as many. +* **Two files of the same rows.** Each writer chooses its own encodings, and what a read costs + depends on them, so the rows are written twice: by Vorticity's writer (*Vorticity's file*) + and by Rust's (*Vortex Rust's file*). Both readers read both files; each size lists what the + two writers chose. +* **One core, or all 32.** On one core, Rust's single-threaded runtime and our scans at one + lane. On all of them, a Tokio worker per processor for Rust; for Vorticity, a lane per + processor for a scan and as many threads for the writer, the degree a session's + `MaxDegreeOfParallelism` gives both. +* **The same work on both sides.** Both map the file and read it where it lies, Rust through + `open_buffer` over a `memmap2` mapping; both decode every value they return to its plain form, + a constant column kept as one value on both; a write hands its bytes to a sink that keeps none, + on both. On one core Rust is timed under each of its two ways of splitting a scan, its default + and one split per chunk, and its figure is the faster; on all cores it keeps its default. +* **A figure** is the time a fresh process takes for its action, from opening the file to its + last row, by its own clock: 5 runs a side, the sides taking turns after one discarded run, the + median with the lowest and highest beside it. Both sides must return the same rows or the + run fails. +* **Throughput** is the plain size of the rows returned over the time: 25.5 bytes a row here. +* **Allocated** is what Vorticity allocates on its managed heap for the action: the median of 6 + calls in a process that has made 2 before them, the cost of a call once the pools are filled. + On all cores a call now and then allocates several times the median. Rust's allocator + reports no such figure. **Peak** is each process's peak resident memory. + +### 1,048,576 rows + +| column | Vorticity's, 2,448,644 bytes | Vortex Rust's, 7,903,964 bytes | +|---|---|---| +| `monotone` | 16 chunks of `vortex.sequence` | 8 chunks of `vortex.sequence` | +| `value` | 16 chunks of `vortex.zstd` | 8 chunks of `vortex.alprd` | +| `label` | 16 chunks of `vortex.dict` | a dictionary layout, its values in one chunk of `vortex.fsst` and its codes in 2 chunks of `fastlanes.bitpacked` | +| `flag` | 16 chunks of `vortex.bool` | one chunk of `vortex.bool` | + +#### One core + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.6-0.6) | 1.3 (1.1-1.4) | 2.29x | — | — | 5.4 KiB | 12 / 9 MiB | +| `open` | Vortex Rust's | 0.5 (0.5-0.6) | 1.1 (0.9-1.3) | 1.93x | — | — | 5.0 KiB | 12 / 9 MiB | +| `scan` | Vorticity's | 6.8 (6.6-7.1) | 17.3 (16.8-21.5) | 2.55x | 3.9 | 1.5 | 15.0 KiB | 15 / 15 MiB | +| `scan` | Vortex Rust's | 4.1 (4.0-5.0) | 5.2 (5.0-5.4) | 1.29x | 6.6 | 5.1 | 11.4 KiB | 23 / 22 MiB | +| `project` | Vorticity's | 1.0 (0.9-1.0) | 1.7 (1.6-1.9) | 1.81x | 8.8 | 4.9 | 17.2 KiB | 12 / 11 MiB | +| `project` | Vortex Rust's | 1.0 (1.0-1.4) | 1.8 (1.7-2.0) | 1.89x | 8.6 | 4.6 | 13.5 KiB | 13 / 11 MiB | +| `filter-narrow` | Vorticity's | 1.9 (1.9-2.1) | 2.6 (2.4-2.7) | 1.36x | 0.1 | 0.1 | 41.7 KiB | 15 / 13 MiB | +| `filter-narrow` | Vortex Rust's | 2.0 (2.0-2.2) | 2.4 (2.2-3.0) | 1.18x | 0.1 | 0.1 | 21.5 KiB | 16 / 13 MiB | +| `filter-wide` | Vorticity's | 4.4 (4.1-4.9) | 9.9 (9.6-10.4) | 2.24x | 3.0 | 1.4 | 24.1 KiB | 15 / 15 MiB | +| `filter-wide` | Vortex Rust's | 3.2 (3.0-3.7) | 4.4 (4.0-5.5) | 1.38x | 4.2 | 3.1 | 21.5 KiB | 20 / 17 MiB | +| `take` | Vorticity's | 6.1 (6.0-7.0) | 8.7 (8.6-8.8) | 1.42x | — | — | 38.7 KiB | 14 / 14 MiB | +| `take` | Vortex Rust's | 2.1 (2.0-2.2) | 3.3 (3.0-3.4) | 1.55x | — | — | 35.1 KiB | 15 / 16 MiB | +| `write` | Vorticity's | 31.4 (31.2-31.8) | 76.4 (75.6-96.0) | 2.43x | 0.8 | 0.3 | 31.1 KiB | 20 / 75 MiB | +| `write` | Vortex Rust's | 29.2 (28.8-36.9) | 75.8 (63.3-109.1) | 2.60x | 0.9 | 0.4 | 27.5 KiB | 29 / 80 MiB | +| `append` | Vorticity's | 11.6 (10.6-13.3) | not asked | n/a | — | — | 2.4 MiB | 21 / — MiB | + +#### All 32 cores + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.5-0.6) | 2.8 (2.7-3.2) | 5.16x | — | — | 5.4 KiB | 12 / 10 MiB | +| `open` | Vortex Rust's | 0.5 (0.5-0.7) | 3.0 (2.7-3.2) | 5.52x | — | — | 5.0 KiB | 11 / 10 MiB | +| `scan` | Vorticity's | 4.0 (3.9-5.7) | 7.3 (6.9-9.8) | 1.80x | 6.6 | 3.7 | 37.0 KiB | 47 / 40 MiB | +| `scan` | Vortex Rust's | 4.5 (4.3-4.5) | 5.8 (5.5-5.9) | 1.29x | 5.9 | 4.6 | 30.2 KiB | 63 / 40 MiB | +| `project` | Vorticity's | 1.8 (1.7-4.2) | 3.7 (3.1-3.9) | 2.02x | 4.6 | 2.3 | 34.3 KiB | 21 / 16 MiB | +| `project` | Vortex Rust's | 1.8 (1.6-1.9) | 3.7 (3.2-4.1) | 2.09x | 4.7 | 2.2 | 22.1 KiB | 21 / 16 MiB | +| `filter-narrow` | Vorticity's | 1.7 (1.7-1.7) | 4.4 (4.1-4.5) | 2.65x | 0.2 | 0.1 | 48.3 KiB | 14 / 15 MiB | +| `filter-narrow` | Vortex Rust's | 1.6 (1.6-2.0) | 4.2 (4.0-4.4) | 2.57x | 0.2 | 0.1 | 25.0 KiB | 14 / 15 MiB | +| `filter-wide` | Vorticity's | 3.5 (3.2-3.9) | 7.5 (6.7-8.7) | 2.13x | 3.8 | 1.8 | 38.0 KiB | 30 / 31 MiB | +| `filter-wide` | Vortex Rust's | 3.4 (3.3-3.5) | 5.4 (5.1-6.0) | 1.59x | 4.0 | 2.5 | 30.1 KiB | 37 / 33 MiB | +| `take` | Vorticity's | 5.0 (4.3-5.8) | 6.5 (5.3-6.9) | 1.30x | — | — | 88.5 KiB | 26 / 22 MiB | +| `take` | Vortex Rust's | 2.5 (2.4-2.9) | 4.3 (4.0-5.5) | 1.73x | — | — | 88.0 KiB | 18 / 19 MiB | +| `write` | Vorticity's | 14.7 (14.4-14.8) | 23.3 (23.1-30.8) | 1.59x | 1.8 | 1.1 | 2.5 MiB | 59 / 110 MiB | +| `write` | Vortex Rust's | 15.6 (14.6-17.9) | 23.8 (22.3-29.5) | 1.52x | 1.7 | 1.1 | 2.5 MiB | 74 / 117 MiB | +| `append` | Vorticity's | 11.0 (10.7-11.6) | not asked | n/a | — | — | 2.4 MiB | 22 / — MiB | + +### 10,485,760 rows + +| column | Vorticity's, 24,464,260 bytes | Vortex Rust's, 78,986,804 bytes | +|---|---|---| +| `monotone` | 160 chunks of `vortex.sequence` | 80 chunks of `vortex.sequence` | +| `value` | 160 chunks of `vortex.zstd` | 80 chunks of `vortex.alprd` | +| `label` | 160 chunks of `vortex.dict` | a dictionary layout, its values in one chunk of `vortex.fsst` and its codes in 20 chunks of `fastlanes.bitpacked` | +| `flag` | 160 chunks of `vortex.bool` | 3 chunks of `vortex.bool` | + +#### One core + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.6-0.6) | 1.1 (1.0-1.4) | 1.96x | — | — | 21.1 KiB | 11 / 9 MiB | +| `open` | Vortex Rust's | 0.6 (0.6-0.8) | 1.3 (1.1-1.4) | 1.99x | — | — | 9.5 KiB | 12 / 9 MiB | +| `scan` | Vorticity's | 59.1 (57.1-80.5) | 159.0 (151.8-212.6) | 2.69x | 4.5 | 1.7 | 74.6 KiB | 35 / 35 MiB | +| `scan` | Vortex Rust's | 33.1 (32.2-34.8) | 37.6 (37.0-45.6) | 1.14x | 8.1 | 7.1 | 28.7 KiB | 89 / 89 MiB | +| `project` | Vorticity's | 1.8 (1.8-2.0) | 3.0 (2.9-3.4) | 1.63x | 45.4 | 27.9 | 76.9 KiB | 13 / 12 MiB | +| `project` | Vortex Rust's | 2.0 (1.7-2.3) | 2.7 (2.5-3.5) | 1.37x | 42.5 | 31.0 | 30.9 KiB | 13 / 11 MiB | +| `filter-narrow` | Vorticity's | 2.7 (2.5-3.0) | 4.9 (4.7-5.8) | 1.78x | 1.0 | 0.5 | 107.4 KiB | 17 / 16 MiB | +| `filter-narrow` | Vortex Rust's | 2.9 (2.6-3.2) | 3.6 (3.4-3.8) | 1.21x | 0.9 | 0.7 | 62.5 KiB | 18 / 18 MiB | +| `filter-wide` | Vorticity's | 30.0 (28.9-36.0) | 78.9 (77.6-89.3) | 2.63x | 4.5 | 1.7 | 107.4 KiB | 24 / 25 MiB | +| `filter-wide` | Vortex Rust's | 19.4 (19.1-21.8) | 22.4 (21.0-30.3) | 1.15x | 6.9 | 6.0 | 62.5 KiB | 54 / 53 MiB | +| `take` | Vorticity's | 25.5 (25.4-26.3) | 69.8 (69.7-77.2) | 2.73x | — | — | 98.3 KiB | 30 / 37 MiB | +| `take` | Vortex Rust's | 5.6 (5.5-8.5) | 8.3 (8.0-8.9) | 1.47x | — | — | 52.4 KiB | 25 / 27 MiB | +| `write` | Vorticity's | 347.8 (280.3-398.4) | 705.1 (691.5-812.1) | 2.03x | 0.8 | 0.4 | 95.8 KiB | 42 / 377 MiB | +| `write` | Vortex Rust's | 267.9 (266.5-288.1) | 585.0 (572.0-652.1) | 2.18x | 1.0 | 0.5 | 49.9 KiB | 98 / 456 MiB | +| `append` | Vorticity's | 52.8 (51.2-69.3) | not asked | n/a | — | — | 3.3 MiB | 28 / — MiB | + +#### All 32 cores + +| scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `open` | Vorticity's | 0.6 (0.6-0.6) | 3.3 (3.2-3.5) | 5.90x | — | — | 21.1 KiB | 12 / 10 MiB | +| `open` | Vortex Rust's | 0.6 (0.5-0.6) | 3.2 (3.0-3.4) | 5.82x | — | — | 9.5 KiB | 12 / 10 MiB | +| `scan` | Vorticity's | 16.5 (15.3-21.8) | 28.0 (24.5-39.6) | 1.70x | 16.1 | 9.5 | 148.0 KiB | 102 / 101 MiB | +| `scan` | Vortex Rust's | 27.3 (26.8-28.4) | 20.4 (18.0-23.8) | 0.75x | 9.8 | 13.1 | 78.6 KiB | 261 / 126 MiB | +| `project` | Vorticity's | 2.9 (2.9-3.0) | 4.2 (4.2-4.5) | 1.46x | 29.0 | 19.9 | 124.9 KiB | 30 / 36 MiB | +| `project` | Vortex Rust's | 3.3 (3.2-3.9) | 4.0 (3.9-5.3) | 1.19x | 25.1 | 21.1 | 69.2 KiB | 46 / 28 MiB | +| `filter-narrow` | Vorticity's | 2.4 (2.1-2.6) | 6.1 (5.7-7.0) | 2.54x | 1.1 | 0.4 | 109.7 KiB | 18 / 20 MiB | +| `filter-narrow` | Vortex Rust's | 2.6 (2.6-2.9) | 5.0 (4.9-5.6) | 1.91x | 1.0 | 0.5 | 65.5 KiB | 19 / 21 MiB | +| `filter-wide` | Vorticity's | 14.3 (9.8-19.3) | 21.1 (17.2-28.8) | 1.48x | 9.3 | 6.3 | 181.9 KiB | 92 / 78 MiB | +| `filter-wide` | Vortex Rust's | 17.6 (17.5-20.4) | 13.0 (12.3-16.6) | 0.74x | 7.6 | 10.3 | 620.9 KiB | 229 / 89 MiB | +| `take` | Vorticity's | 11.5 (10.9-12.2) | 15.7 (15.3-16.1) | 1.37x | — | — | 260.6 KiB | 62 / 56 MiB | +| `take` | Vortex Rust's | 5.0 (4.6-5.5) | 7.2 (7.2-7.9) | 1.46x | — | — | 150.1 KiB | 29 / 32 MiB | +| `write` | Vorticity's | 86.8 (86.3-88.0) | 155.4 (143.1-176.4) | 1.79x | 3.1 | 1.7 | 2.7 MiB | 116 / 465 MiB | +| `write` | Vortex Rust's | 98.3 (97.1-101.1) | 149.8 (145.8-157.7) | 1.52x | 2.7 | 1.8 | 2.7 MiB | 279 / 528 MiB | +| `append` | Vorticity's | 47.7 (46.8-54.7) | not asked | n/a | — | — | 2.5 MiB | 61 / — MiB | + +`open`: open the file and read no rows. `scan`: read every column of every row. `project`: read one column of four. `filter-narrow`: read the rows of a band holding about one in a hundred. `filter-wide`: read the rows of a band holding about half. `take`: take a thousand rows spread across the file. `write`: read the file and encode it back out. `append`: append a tenth of the rows to a copy of the file. + +**On one core.** Vorticity took less time than Rust on 28 of 28 compared rows. +The highest speedup is `take` at 10,485,760 rows on Vorticity's file (2.73x), the lowest +`scan` at 10,485,760 rows on Vortex Rust's file (1.14x). + +**On all 32 cores.** Vorticity took less time than Rust on 26 of 28 compared rows. +The highest speedup is `open` at 10,485,760 rows on Vorticity's file (5.90x), the lowest +`filter-wide` at 10,485,760 rows on Vortex Rust's file (0.74x). + +**The process start** is not in the figures: 46 ms for Vorticity's native binary and 32 ms for Rust's, +most of it the operating system starting a binary, and 113 ms for the managed runtime on the JIT. + +**Rust's splits.** On one core, one split per chunk was the faster of its two on 7 of 14 rows on Vorticity's file and 13 of 14 rows on Vortex Rust's file; +the two differ only where a chunk holds more than 100,000 rows. + +**Not asked of Rust**: `append`, which its harness has no entry point for; those figures +are Vorticity's alone. + +### A first call on the JIT + +The same scenarios in the framework-dependent build, under `dotnet`, on one core: what a +process that is not compiled ahead of time pays the first time, the code compiling as it runs. +Milliseconds, the median of the same runs. + +| scenario | 1,048,576 rows, Vorticity's file | 1,048,576 rows, Vortex Rust's file | 10,485,760 rows, Vorticity's file | 10,485,760 rows, Vortex Rust's file | +|---|---:|---:|---:|---:| +| `open` | 19.6 | 19.8 | 19.5 | 21.7 | +| `scan` | 114.7 | 102.2 | 220.3 | 139.3 | +| `project` | 70.3 | 70.7 | 76.5 | 81.0 | +| `filter-narrow` | 156.4 | 137.3 | 153.8 | 149.2 | +| `filter-wide` | 131.4 | 128.4 | 198.8 | 164.3 | +| `take` | 124.3 | 99.6 | 186.9 | 105.4 | +| `write` | 323.4 | 319.5 | 947.1 | 869.1 | +| `append` | 359.7 | — | 431.3 | — | + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01); commit 716b74b6, 2026-10-09 17:35 UTC.* ## Decoding, per encoding -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +One file per encoding, and per notable shape of one, written by Rust's writer +(`bench/gen-throughput.sh`): a million rows each, `table_wide` fifty columns of fifty thousand. +Each side scans the file, every value decoded to its plain form, 12 times in a process of its own on +one core; a figure is the median of the last 10 scans, and the median of 3 processes: what a +decoder costs once warm. Each scan opens the file and maps it anew, on both sides, and reads it +where it lies: a column stored in its plain form is a view of the mapping on both sides, nothing +decoded and nothing copied, and its figure, past what memory can move, measures the walk of the +layout. Throughput is over the plain size, as above. + +| encoding | rows | Vorticity, ns/row | Vortex Rust, ns/row | speedup | Vorticity, GB/s | Vortex Rust, GB/s | +|---|---:|---:|---:|---:|---:|---:| +| `alp` | 1,000,000 | 2.28 | 2.57 | 1.13x | 3.5 | 3.1 | +| `alp_no_patches` | 1,000,000 | 1.39 | 1.78 | 1.28x | 5.7 | 4.5 | +| `alp_patched_no_chunk_offsets` | 1,000,000 | 1.46 | 1.87 | 1.28x | 5.5 | 4.3 | +| `alprd` | 1,000,000 | 1.44 | 1.49 | 1.03x | 5.5 | 5.4 | +| `bool` | 1,000,000 | 0.17 | 0.18 | 1.05x | 0.7 | 0.7 | +| `bool_bit_offset3` | 1,000,000 | 0.18 | 0.17 | 0.93x | 0.7 | 0.8 | +| `bool_bit_offset7` | 1,000,000 | 0.17 | 0.17 | 0.99x | 0.7 | 0.7 | +| `bool_bit_offset_straddle` | 1,000,000 | 0.17 | 0.17 | 1.03x | 0.7 | 0.7 | +| `bytebool` | 1,000,000 | 0.32 | 0.34 | 1.08x | 0.4 | 0.4 | +| `chunked` | 1,000,000 | 0.19 | 0.72 | 3.73x | 41.7 | 11.2 | +| `chunked_bool` | 1,000,000 | 0.20 | 0.22 | 1.09x | 0.6 | 0.6 | +| `chunked_decimal` | 1,000,000 | 0.23 | 0.63 | 2.73x | 34.9 | 12.8 | +| `chunked_empty_chunks` | 1,000,000 | 0.20 | 0.52 | 2.68x | 41.0 | 15.3 | +| `chunked_mixed_validity` | 1,000,000 | 0.67 | 0.71 | 1.06x | 12.1 | 11.4 | +| `chunked_one_chunk` | 1,000,000 | 0.19 | 0.24 | 1.22x | 41.1 | 33.7 | +| `chunked_varbinview` | 1,000,000 | 4.35 | 10.8 | 2.49x | 3.1 | 1.2 | +| `constant` | 1,000,000 | 0.16 | 0.19 | 1.20x | 51.4 | 43.0 | +| `datetimeparts` | 1,000,000 | 2.82 | 3.19 | 1.13x | 2.8 | 2.5 | +| `decimal` | 1,000,000 | 0.19 | 0.18 | 0.97x | 42.9 | 44.3 | +| `decimal_byte_parts` | 1,000,000 | 0.22 | 0.18 | 0.81x | 36.0 | 44.3 | +| `dict` | 1,000,000 | 0.98 | 1.09 | 1.12x | 6.2 | 5.5 | +| `dict_nullable_codes` | 1,000,000 | 1.07 | 1.95 | 1.81x | 5.2 | 2.8 | +| `dict_nullable_values_nonnull_codes` | 1,000,000 | 0.94 | 1.02 | 1.09x | 6.1 | 5.6 | +| `dict_u64_codes` | 1,000,000 | 1.68 | 1.64 | 0.98x | 3.6 | 3.7 | +| `dict_u8_codes` | 1,000,000 | 0.57 | 0.64 | 1.12x | 13.9 | 12.4 | +| `ext` | 1,000,000 | 0.18 | 0.18 | 0.99x | 44.0 | 44.6 | +| `fastlanes_bitpacked` | 1,000,000 | 0.45 | 0.41 | 0.93x | 8.9 | 9.7 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,000,000 | 0.45 | 0.41 | 0.91x | 8.9 | 9.9 | +| `fastlanes_delta` | 1,000,000 | 2.15 | 1.90 | 0.88x | 3.7 | 4.2 | +| `fastlanes_for` | 1,000,000 | 1.38 | 1.38 | 1.00x | 5.8 | 5.8 | +| `fastlanes_rle` | 1,000,000 | 0.73 | 0.79 | 1.08x | 5.5 | 5.1 | +| `fixed_size_list` | 1,000,000 | 0.19 | 0.18 | 0.98x | 64.3 | 65.4 | +| `fsst` | 1,000,000 | 8.12 | 10.2 | 1.25x | 6.9 | 5.5 | +| `list` | 1,000,000 | 0.81 | 1.30 | 1.60x | 12.3 | 7.7 | +| `listview` | 1,000,000 | 2.55 | 3.04 | 1.19x | 3.9 | 3.3 | +| `map` | 1,000,000 | 5.42 | 11.0 | 2.02x | 3.7 | 1.8 | +| `masked` | 1,000,000 | 0.18 | 0.18 | 1.04x | 23.3 | 22.5 | +| `masked_all_invalid` | 1,000,000 | 0.21 | 0.18 | 0.90x | 19.4 | 21.7 | +| `masked_all_valid` | 1,000,000 | 0.20 | 0.18 | 0.89x | 19.5 | 22.0 | +| `null` | 1,000,000 | 0.14 | 0.16 | 1.18x | — | — | +| `onpair` | 1,000,000 | 3.95 | 6.79 | 1.72x | 5.5 | 3.2 | +| `parquet_variant` | 1,000,000 | 3.75 | 15.2 | 4.05x | 3.2 | 0.8 | +| `pco` | 1,000,000 | 0.42 | 1.19 | 2.86x | 19.2 | 6.7 | +| `pco_f32` | 1,000,000 | 7.86 | 2.99 | 0.38x | 0.5 | 1.3 | +| `pco_i16` | 1,000,000 | 6.96 | 2.61 | 0.37x | 0.3 | 0.8 | +| `pco_nullable` | 1,000,000 | 8.71 | 4.28 | 0.49x | 0.9 | 1.9 | +| `pco_u32` | 1,000,000 | 13.1 | 4.24 | 0.32x | 0.3 | 0.9 | +| `primitive` | 1,000,000 | 0.18 | 0.22 | 1.21x | 43.7 | 36.2 | +| `runend` | 1,000,000 | 0.27 | 0.30 | 1.13x | 15.0 | 13.2 | +| `sequence` | 1,000,000 | 0.20 | 0.23 | 1.17x | 40.2 | 34.5 | +| `sparse` | 1,000,000 | 0.24 | 0.24 | 1.00x | 16.4 | 16.4 | +| `struct` | 1,000,000 | 2.65 | 7.75 | 2.92x | 5.6 | 1.9 | +| `table_mixed` | 1,000,000 | 12.0 | 23.6 | 1.97x | 5.8 | 3.0 | +| `table_wide` | 50,000 | 5.19 | 4.78 | 0.92x | 77.1 | 83.7 | +| `varbin` | 1,000,000 | 3.65 | 14.7 | 4.04x | 4.4 | 1.1 | +| `varbinview` | 1,000,000 | 4.00 | 11.0 | 2.74x | 3.4 | 1.2 | +| `variant` | 1,000,000 | 0.14 | 0.17 | 1.21x | 113.1 | 93.3 | +| `zigzag` | 1,000,000 | 0.77 | 0.97 | 1.26x | 5.2 | 4.1 | +| `zstd` | 1,000,000 | 4.43 | 9.61 | 2.17x | 7.2 | 3.3 | +| `zstd_buffers` | 1,000,000 | 0.42 | 0.46 | 1.09x | 18.9 | 17.4 | +| `zstd_nullable` | 1,000,000 | 1.99 | 4.16 | 2.09x | 4.1 | 2.0 | + +Rust is timed under each of its two ways of splitting a scan, three processes each, and its +figure is the faster: its default cuts a chunk of more than 100,000 rows into splits of 100,000 and +decodes the whole chunk again for each, which on these files of one chunk of a million rows is ten +times; one split per chunk decodes it once, but builds a chunk made of smaller arrays in one +piece. One split per chunk was the faster on 49 of 61 files, its default on `bytebool`, `chunked`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `constant`, `datetimeparts`, `dict_u64_codes`, `fastlanes_delta`, `fsst`, `parquet_variant`. + +Vorticity decoded 43 of 61 files in less time than Rust; the median speedup is 1.12x. At 1.00x or below: `pco_u32` 0.32x, `pco_i16` 0.37x, `pco_f32` 0.38x, `pco_nullable` 0.49x, `decimal_byte_parts` 0.81x, `fastlanes_delta` 0.88x, `masked_all_valid` 0.89x, `masked_all_invalid` 0.90x, `fastlanes_bitpacked_patched_no_chunk_offsets` 0.91x, `table_wide` 0.92x, `fastlanes_bitpacked` 0.93x, `bool_bit_offset3` 0.93x, `decimal` 0.97x, `dict_u64_codes` 0.98x, `fixed_size_list` 0.98x, `ext` 0.99x, `bool_bit_offset7` 0.99x, `sparse` 1.00x. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01); commit 716b74b6, 2026-10-09 17:35 UTC.* ## In one process, after warm-up -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +Both readers called in turn in one process, Rust's through a C ABI, against one clock. A round times +both, and its speedup is Rust's time over Vorticity's: at least 21 rounds after a 2-second warm-up, +until the 95 % interval of their median is within 5 % of it. Our side runs on the JIT, warmed. Each +call opens its file and maps it anew, on both sides, and asks both the same question. Each axis is +one the `--ratio-check` gate holds to a ceiling; what each one reads and asks is in +[05-benchmarks.md](../design/05-benchmarks.md) §3. + +| axis | Vorticity, µs | Vortex Rust, µs | speedup | 95 % interval | +|---|---:|---:|---:|---| +| full scan | 927.6 | 3,312.2 | 3.51x | 3.38 to 3.61 | +| projected scan, 1 of 5 columns | 252.1 | 486.9 | 1.96x | 1.91 to 1.98 | +| open to first batch | 220.7 | 2,868.4 | 13.0x | 12.7 to 13.1 | +| open, footer only | 133.4 | 212.8 | 1.59x | 1.49 to 1.62 | +| read and write back | 5,966.6 | 19,356.3 | 3.24x | 3.14 to 3.31 | +| filtered scan, 1% band | 259.2 | 679.5 | 2.70x | 2.59 to 2.78 | +| filtered scan, half the rows | 331.5 | 1,096.9 | 3.28x | 3.25 to 3.39 | +| scattered take, 64 of 64 splits | 670.0 | 2,622.4 | 3.91x | 3.87 to 3.97 | +| key order, sorted column, 1% band | 267.4 | 361.7 | 1.36x | 1.35 to 1.37 | +| key order, uncorrelated, 64 rows | 369.2 | 491.6 | 1.38x | 1.29 to 1.41 | +| count, exact cover, 1% band | 276.7 | 386.1 | 1.53x | 1.44 to 1.59 | +| filtered scan, string equality, fsst | 645.9 | 831.2 | 1.28x | 1.27 to 1.28 | +| filtered scan, string prefix, fsst | 1,084.8 | 1,195.2 | 1.10x | 1.10 to 1.11 | +| filtered scan, string equality, dict | 189.5 | 312.5 | 1.66x | 1.58 to 1.66 | +| filtered scan, string prefix, dict | 201.3 | 406.8 | 2.02x | 2.00 to 2.04 | +| filtered scan, band, runend | 197.6 | 294.5 | 1.49x | 1.42 to 1.51 | +| filtered scan, band, bitpacked | 234.1 | 268.7 | 1.15x | 1.14 to 1.16 | +| full scan, 1M table | 25,271.0 | 41,306.8 | 1.62x | 1.58 to 1.75 | +| projected scan, 1 of 50 columns | 176.9 | 229.7 | 1.31x | 1.29 to 1.32 | + +Vorticity took less time on 19 of 19 axes. +Rust is timed under the faster of its two splits, axis by axis: one split per chunk on 13 of 19, its default on full scan; open, footer only; key order, uncorrelated, 64 rows; count, exact cover, 1% band; filtered scan, string equality, fsst; filtered scan, band, bitpacked. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 7527e2b5d861; commit 716b74b6, 2026-10-09 17:36 UTC.* ## Taking rows, per encoding -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +The files of the per-encoding corpus again, 64 rows spread evenly over each, one every 15,625: what a +decoder costs to reach a few rows rather than all of them. +Each call opens the file and maps it anew, on both sides. +Both implementations in one process, Rust's through a C ABI, taking turns against one clock: +at least 9 rounds after a 1-second warm-up per file, our side on the JIT, warmed. + +| encoding | Vorticity, µs | Vortex Rust, µs | speedup | +|---|---:|---:|---:| +| `alp` | 856 | 904 | 1.01x | +| `alp_no_patches` | 245 | 276 | 1.11x | +| `alp_patched_no_chunk_offsets` | 277 | 310 | 1.11x | +| `alprd` | 301 | 324 | 1.10x | +| `bool` | 183 | 206 | 1.12x | +| `bool_bit_offset3` | 185 | 205 | 1.12x | +| `bool_bit_offset7` | 191 | 212 | 1.10x | +| `bool_bit_offset_straddle` | 180 | 206 | 1.14x | +| `bytebool` | 224 | 432 | 1.91x | +| `chunked` | 258 | 294 | 1.15x | +| `chunked_bool` | 186 | 228 | 1.21x | +| `chunked_decimal` | 247 | 285 | 1.16x | +| `chunked_empty_chunks` | 248 | 286 | 1.15x | +| `chunked_mixed_validity` | 268 | 321 | 1.19x | +| `chunked_one_chunk` | 247 | 273 | 1.11x | +| `chunked_varbinview` | 408 | 13,592 | 34.0x | +| `constant` | 124 | 174 | 1.42x | +| `datetimeparts` | 422 | 462 | 1.10x | +| `decimal` | 245 | 268 | 1.10x | +| `decimal_byte_parts` | 248 | 277 | 1.14x | +| `dict` | 233 | 262 | 1.12x | +| `dict_nullable_codes` | 261 | 306 | 1.16x | +| `dict_nullable_values_nonnull_codes` | 232 | 259 | 1.11x | +| `dict_u64_codes` | 251 | 281 | 1.11x | +| `dict_u8_codes` | 319 | 340 | 1.07x | +| `ext` | 253 | 273 | 1.10x | +| `fastlanes_bitpacked` | 221 | 243 | 1.11x | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 239 | 266 | 1.12x | +| `fastlanes_delta` | 476 | 3,301 | 6.50x | +| `fastlanes_for` | 361 | 400 | 1.09x | +| `fastlanes_rle` | 953 | 1,016 | 1.03x | +| `fixed_size_list` | 259 | 299 | 1.17x | +| `fsst` | 423 | 455 | 1.09x | +| `list` | 253 | 1,197 | 4.75x | +| `listview` | 5,571 | 7,306 | 1.19x | +| `map` | 9,785 | 17,961 | 1.69x | +| `masked` | 201 | 230 | 1.15x | +| `masked_all_invalid` | 200 | 226 | 1.12x | +| `masked_all_valid` | 375 | 401 | 1.08x | +| `null` | 180 | 249 | 1.35x | +| `onpair` | 403 | 437 | 1.08x | +| `parquet_variant` | 465 | 502 | 1.05x | +| `pco` | 314 | 1,218 | 3.81x | +| `pco_f32` | 7,958 | 3,702 | 0.43x | +| `pco_i16` | 6,190 | 2,692 | 0.43x | +| `pco_nullable` | 6,446 | 348 | 0.05x | +| `pco_u32` | 11,965 | 4,353 | 0.36x | +| `primitive` | 361 | 384 | 1.08x | +| `runend` | 217 | 226 | 1.11x | +| `sequence` | 169 | 231 | 1.37x | +| `sparse` | 267 | 277 | 1.13x | +| `struct` | 5,918 | 15,734 | 2.52x | +| `table_mixed` | 19,179 | 32,506 | 1.76x | +| `table_wide` | 438 | 483 | 1.24x | +| `varbin` | 440 | 12,960 | 29.6x | +| `varbinview` | 7,504 | 14,748 | 2.04x | +| `variant` | 126 | 190 | 1.49x | +| `zigzag` | 364 | 395 | 1.11x | +| `zstd` | 701 | 10,564 | 15.5x | +| `zstd_buffers` | 514 | 555 | 1.13x | +| `zstd_nullable` | 521 | 4,496 | 7.60x | + +Vorticity took less time on 57 of 61 files; the median speedup is 1.12x. At 1.00x or below: `pco_nullable` 0.05x, `pco_u32` 0.36x, `pco_f32` 0.43x, `pco_i16` 0.43x. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 59 of 61 files, its default on `fastlanes_delta`, `table_wide`. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 7527e2b5d861; commit 716b74b6, 2026-10-09 17:37 UTC.* ## Writing, per encoding -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +The files of the per-encoding corpus again, each read and written back out to a sink that +keeps nothing, on both sides, both writers given the rows as our reader delivers them, decoded and a +constant kept as one value: what a writer costs, the read included on both sides. +Each call opens the file and maps it anew, on both sides. +Both implementations in one process, Rust's through a C ABI, taking turns against one clock: +at least 9 rounds after a 1-second warm-up per file, our side on the JIT, warmed. +A writer can be fast by compressing less, so each one's bytes are beside its time. + +| encoding | Vorticity, µs | Vortex Rust, µs | speedup | Vorticity, bytes | Vortex Rust, bytes | +|---|---:|---:|---:|---:|---:| +| `alp` | 8,314 | 33,734 | 3.97x | 2,740,820 | 2,976,924 | +| `alp_no_patches` | 6,988 | 24,835 | 3.72x | 1,955,668 | 1,956,060 | +| `alp_patched_no_chunk_offsets` | 6,125 | 25,662 | 4.15x | 2,155,956 | 2,141,772 | +| `alprd` | 8,436 | 27,115 | 3.24x | 6,508,892 | 6,509,316 | +| `bool` | 302 | 858 | 2.90x | 127,108 | 127,108 | +| `bool_bit_offset3` | 513 | 1,482 | 2.89x | 127,108 | 127,108 | +| `bool_bit_offset7` | 341 | 1,009 | 2.99x | 127,108 | 127,108 | +| `bool_bit_offset_straddle` | 350 | 1,034 | 2.97x | 127,108 | 127,108 | +| `bytebool` | 507 | 1,095 | 2.17x | 127,108 | 127,108 | +| `chunked` | 3,608 | 14,436 | 4.11x | 890,228 | 890,736 | +| `chunked_bool` | 379 | 1,053 | 2.85x | 127,236 | 127,248 | +| `chunked_decimal` | 5,560 | 18,287 | 3.35x | 1,822,388 | 1,825,056 | +| `chunked_empty_chunks` | 3,258 | 14,480 | 4.55x | 316,532 | 317,080 | +| `chunked_mixed_validity` | 4,128 | 17,561 | 4.26x | 1,431,516 | 1,374,016 | +| `chunked_one_chunk` | 3,240 | 14,569 | 4.35x | 5,044 | 4,640 | +| `chunked_varbinview` | 27,200 | 37,866 | 1.39x | 643,644 | 628,628 | +| `constant` | 239 | 4,271 | 18.0x | 4,212 | 3,300 | +| `datetimeparts` | 7,164 | 30,800 | 4.39x | 3,382,604 | 3,383,628 | +| `decimal` | 3,762 | 16,841 | 4.56x | 3,372 | 5,116 | +| `decimal_byte_parts` | 3,796 | 16,140 | 4.29x | 3,372 | 5,116 | +| `dict` | 9,259 | 21,212 | 2.24x | 383,892 | 378,288 | +| `dict_nullable_codes` | 18,159 | 20,804 | 1.29x | 384,628 | 378,772 | +| `dict_nullable_values_nonnull_codes` | 14,036 | 19,706 | 1.42x | 384,628 | 378,764 | +| `dict_u64_codes` | 10,514 | 23,074 | 2.18x | 383,892 | 378,288 | +| `dict_u8_codes` | 11,166 | 22,472 | 1.94x | 38,740 | 1,004,160 | +| `ext` | 3,153 | 24,750 | 7.90x | 5,548 | 4,988 | +| `fastlanes_bitpacked` | 1,467 | 4,569 | 3.12x | 1,254,340 | 1,253,668 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,490 | 5,240 | 3.52x | 1,270,452 | 1,272,740 | +| `fastlanes_delta` | 3,719 | 15,674 | 3.98x | 4,908 | 4,532 | +| `fastlanes_for` | 3,574 | 7,170 | 2.03x | 881,364 | 879,732 | +| `fastlanes_rle` | 1,781 | 8,660 | 4.82x | 3,964 | 3,812 | +| `fixed_size_list` | 4,256 | 26,980 | 6.38x | 4,268 | 4,724 | +| `fsst` | 50,186 | 312,191 | 6.05x | 933,668 | 8,574,500 | +| `list` | 8,889 | 28,170 | 3.06x | 4,782,492 | 5,122,712 | +| `listview` | 11,679 | 38,569 | 3.33x | 4,745,468 | 4,502,136 | +| `map` | 23,149 | 75,151 | 3.29x | 4,426,852 | 4,811,136 | +| `masked` | 3,366 | 19,212 | 6.13x | 2,380,060 | 2,381,156 | +| `masked_all_invalid` | 1,554 | 2,319 | 1.51x | 2,636 | 2,700 | +| `masked_all_valid` | 1,664 | 10,577 | 6.43x | 3,484 | 3,324 | +| `null` | 154 | 586 | 3.82x | 2,052 | 2,164 | +| `onpair` | 27,960 | 33,828 | 1.20x | 60,588 | 1,005,840 | +| `pco` | 1,195 | 14,063 | 12.0x | 4,908 | 4,508 | +| `pco_f32` | 9,774 | 21,230 | 2.19x | 2,130,020 | 2,130,108 | +| `pco_i16` | 6,927 | 8,154 | 1.18x | 1,878,788 | 1,878,460 | +| `pco_nullable` | 14,502 | 35,847 | 2.47x | 2,876,548 | 2,874,276 | +| `pco_u32` | 18,600 | 9,780 | 0.52x | 1,931,748 | 2,879,908 | +| `primitive` | 3,057 | 13,484 | 4.24x | 5,356 | 4,572 | +| `runend` | 813 | 9,510 | 11.7x | 4,028 | 3,500 | +| `sequence` | 1,076 | 14,202 | 13.3x | 4,908 | 4,516 | +| `sparse` | 823 | 4,224 | 5.07x | 131,172 | 3,884 | +| `struct` | 44,892 | 119,019 | 2.54x | 638,412 | 5,274,480 | +| `table_mixed` | 85,274 | 415,839 | 4.84x | 7,867,284 | 14,856,384 | +| `table_wide` | 11,150 | 40,954 | 3.52x | 5,163,108 | 5,168,320 | +| `varbin` | 55,308 | 263,222 | 4.75x | 637,012 | 5,066,652 | +| `varbinview` | 24,162 | 35,058 | 1.45x | 643,516 | 628,504 | +| `variant` | 193 | 1,058 | 5.50x | 2,220 | 52,520 | +| `zigzag` | 2,576 | 11,809 | 4.69x | 2,531,652 | 2,380,300 | +| `zstd` | 36,620 | 41,824 | 1.18x | 786,948 | 754,336 | +| `zstd_buffers` | 1,863 | 4,876 | 2.59x | 630,732 | 629,028 | +| `zstd_nullable` | 5,415 | 21,568 | 3.96x | 756,500 | 628,740 | + +Vorticity took less time on 59 of 60 files; the median speedup is 3.52x. At 1.00x or below: `pco_u32` 0.52x. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 45 of 60 files, its default on `alp`, `alprd`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_one_chunk`, `decimal`, `dict_nullable_values_nonnull_codes`, `fixed_size_list`, `fsst`, `list`, `pco_f32`, `runend`, `sparse`, `table_wide`, `zigzag`. +Vorticity's bytes were within 5 % of Rust's or fewer on 48 of 60 files; more than 5 % above them on `sparse` (33.8 times Rust's), `constant` (1.28 times), `zstd_nullable` (1.20 times), `primitive` (1.17 times), `runend` (1.15 times), `ext` (1.11 times), `pco` (1.09 times), `chunked_one_chunk` (1.09 times), `sequence` (1.09 times), `fastlanes_delta` (1.08 times), `zigzag` (1.06 times), `listview` (1.05 times). +Rust's writer declines `parquet_variant`, which is left out. + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 7527e2b5d861; commit 716b74b6, 2026-10-09 17:40 UTC.* @@ -85,7 +506,8 @@ on an x64 machine. One column of 10,000,000 rows per file, written under each compression profile and each hint that applies to it, then opened and read. A **scan** decodes every value to its plain form and reads it once; a **take** reads 1,000 rows spread over the file. Each figure is the median of 3 passes after -1 warm-up, the file in the page cache; a write is net of generating its rows. **Crosses at** is the +1 warm-up, the file in the page cache and mapped anew by each pass, so that a larger file pays for its +pages on every read; a write is net of generating its rows. **Crosses at** is the storage throughput at which a configuration and `Auto` read the column whole in the same time, counting its bytes at that throughput and then its scan: below it the smaller file reads faster end to end, above it the faster decode does. What the figures mean for a choice is in @@ -100,27 +522,27 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| a sequence | Sequence x77 | 0.00 | 26 | 2.8 | 1.19 | | -| sorted runs of 1 000 | RunEnd x77 | 0.01 | 55 | 2.3 | 1.15 | None: Canonical x77, 8.00 B/value, scan 1.4 ms, take 0.29 ms; reads faster above 88,339 MB/s | -| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 85 | 6.6 | 0.60 | Smallest: Zstd x77, 2.07 B/value, scan 74.5 ms, take 60.30 ms; reads faster below 193 MB/s; None: Canonical x77, 8.00 B/value, scan 1.3 ms, take 0.25 ms; reads faster above 8,691 MB/s | -| random in 0..999 | BitPacked x77 | 1.25 | 0 | 2.4 | 0.30 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 38 | 2.3 | 0.49 | None: Canonical x77, 8.00 B/value, scan 1.4 ms, take 0.24 ms; reads faster above 79,377 MB/s | -| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 32 | 2.6 | 0.26 | Auto, 16 MiB chunks: Dictionary x5, 2.36 B/value, scan 6.7 ms, take 0.47 ms; reads faster below 5,585 MB/s | -| uniform 64-bit | Canonical x77 | 8.00 | 31 | 1.4 | 0.22 | | -| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 15 | 3.6 | 0.31 | hint Dictionary: Dictionary x82, 1.27 B/value, scan 6.3 ms, take 0.53 ms; reads faster below 417 MB/s | +| a sequence | Sequence x77 | 0.00 | 13 | 1.8 | 0.26 | | +| sorted runs of 1 000 | RunEnd x77 | 0.01 | 14 | 2.7 | 0.36 | Smallest: RunEnd x76, Pco x1, 0.01 B/value, scan 3.0 ms, take 0.74 ms; reads faster below 96 MB/s | +| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 70 | 12.9 | 1.72 | Smallest: Pco x77, 1.35 B/value, scan 112.6 ms, take 118.47 ms; reads faster below 204 MB/s | +| random in 0..999 | BitPacked x77 | 1.25 | 22 | 5.5 | 1.54 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 55 | 3.3 | 1.14 | | +| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 38 | 12.9 | 1.51 | Smallest: Pco x77, 0.06 B/value, scan 72.5 ms, take 69.30 ms; reads faster below 767 MB/s; Auto, 16 MiB chunks: Dictionary x5, 2.36 B/value, scan 9.5 ms, take 1.32 ms; reads faster at any throughput | +| uniform 64-bit | Canonical x77 | 8.00 | 43 | 27.0 | 2.14 | | +| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 25 | 6.3 | 1.35 | Smallest: Pco x82, 1.26 B/value, scan 45.3 ms, take 44.57 ms; reads faster below 32 MB/s |
a sequence: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Sequence x77 | 38,836 | 0.00 | 26 | 2.8 | 1.19 | | -| Fastest | Sequence x77 | 38,836 | 0.00 | 27 | 2.7 | 1.22 | same | -| Smallest | Sequence x77 | 38,836 | 0.00 | 29 | 2.8 | 1.48 | same | -| None | Canonical x77 | 80,043,716 | 8.00 | 51 | 2.8 | 1.42 | reads slower at any throughput | -| hint Dictionary | Sequence x77 | 38,836 | 0.00 | 195 | 1.7 | 0.43 | reads faster at any throughput | -| hint BitPacked | Sequence x77 | 38,836 | 0.00 | 5 | 1.7 | 0.42 | reads faster at any throughput | -| hint RunEnd | Sequence x77 | 38,836 | 0.00 | 4 | 1.7 | 0.41 | reads faster at any throughput | -| hint Zstd | Sequence x77 | 38,836 | 0.00 | 4 | 1.7 | 0.43 | reads faster at any throughput | +| Auto | Sequence x77 | 38,844 | 0.00 | 13 | 1.8 | 0.26 | | +| Fastest | Sequence x77 | 38,844 | 0.00 | 22 | 1.9 | 0.28 | reads slower at any throughput | +| Smallest | Sequence x77 | 38,844 | 0.00 | 15 | 1.8 | 0.27 | same | +| None | Canonical x77 | 80,043,724 | 8.00 | 51 | 23.2 | 1.68 | reads slower at any throughput | +| hint Dictionary | Sequence x77 | 38,844 | 0.00 | 121 | 1.8 | 0.28 | same | +| hint BitPacked | Sequence x77 | 38,844 | 0.00 | 15 | 1.8 | 0.28 | same | +| hint RunEnd | Sequence x77 | 38,844 | 0.00 | 13 | 2.1 | 0.37 | reads slower at any throughput | +| hint Zstd | Sequence x77 | 38,844 | 0.00 | 11 | 1.8 | 0.27 | same |
@@ -128,14 +550,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x77 | 87,996 | 0.01 | 55 | 2.3 | 1.15 | | -| Fastest | RunEnd x77 | 87,996 | 0.01 | 67 | 2.3 | 1.14 | same | -| Smallest | RunEnd x76, Zstd x1 | 87,964 | 0.01 | 90 | 3.6 | 1.79 | reads slower at any throughput | -| None | Canonical x77 | 80,043,708 | 8.00 | 43 | 1.4 | 0.29 | reads faster above 88,339 MB/s | -| hint Dictionary | Dictionary x77 | 1,449,884 | 0.14 | 34 | 5.8 | 1.65 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 10,034,452 | 1.00 | 75 | 4.8 | 0.44 | reads slower at any throughput | -| hint RunEnd | RunEnd x77 | 87,996 | 0.01 | 23 | 2.4 | 0.65 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 136,620 | 0.01 | 13 | 6.3 | 4.86 | reads slower at any throughput | +| Auto | RunEnd x77 | 88,004 | 0.01 | 14 | 2.7 | 0.36 | | +| Fastest | RunEnd x77 | 88,004 | 0.01 | 10 | 2.6 | 0.38 | same | +| Smallest | RunEnd x76, Pco x1 | 53,748 | 0.01 | 110 | 3.0 | 0.74 | reads faster below 96 MB/s | +| None | Canonical x77 | 80,043,716 | 8.00 | 83 | 21.3 | 1.80 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 1,449,892 | 0.14 | 17 | 6.2 | 1.53 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 10,034,460 | 1.00 | 21 | 5.7 | 1.13 | reads slower at any throughput | +| hint RunEnd | RunEnd x77 | 88,004 | 0.01 | 10 | 2.6 | 0.36 | same | +| hint Zstd | Zstd x77 | 136,628 | 0.01 | 12 | 3.0 | 1.15 | reads slower at any throughput | @@ -143,14 +565,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 33,795,164 | 3.38 | 85 | 6.6 | 0.60 | | -| Fastest | BitPacked x77 | 33,795,164 | 3.38 | 16 | 3.6 | 0.29 | reads faster at any throughput | -| Smallest | Zstd x77 | 20,700,788 | 2.07 | 187 | 74.5 | 60.30 | reads faster below 193 MB/s | -| None | Canonical x77 | 80,043,716 | 8.00 | 28 | 1.3 | 0.25 | reads faster above 8,691 MB/s | -| hint Dictionary | BitPacked x77 | 33,795,164 | 3.38 | 93 | 3.6 | 0.26 | reads faster at any throughput | -| hint BitPacked | BitPacked x77 | 33,795,164 | 3.38 | 14 | 3.6 | 0.26 | reads faster at any throughput | -| hint RunEnd | BitPacked x77 | 33,795,164 | 3.38 | 26 | 3.6 | 0.28 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 20,700,788 | 2.07 | 150 | 74.1 | 60.66 | reads faster below 194 MB/s | +| Auto | BitPacked x77 | 33,795,172 | 3.38 | 70 | 12.9 | 1.72 | | +| Fastest | BitPacked x77 | 33,795,172 | 3.38 | 30 | 11.5 | 1.42 | reads faster at any throughput | +| Smallest | Pco x77 | 13,472,444 | 1.35 | 453 | 112.6 | 118.47 | reads faster below 204 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 43 | 19.3 | 1.57 | reads slower at any throughput | +| hint Dictionary | BitPacked x77 | 33,795,172 | 3.38 | 156 | 11.2 | 1.45 | reads faster at any throughput | +| hint BitPacked | BitPacked x77 | 33,795,172 | 3.38 | 29 | 11.3 | 1.39 | reads faster at any throughput | +| hint RunEnd | BitPacked x77 | 33,795,172 | 3.38 | 49 | 11.1 | 1.41 | reads faster at any throughput | +| hint Zstd | Zstd x77 | 20,700,796 | 2.07 | 182 | 80.8 | 35.05 | reads faster below 193 MB/s | @@ -158,14 +580,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.4 | 0.30 | | -| Fastest | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.4 | 0.25 | same | -| Smallest | BitPacked x77 | 12,544,276 | 1.25 | 151 | 2.4 | 0.25 | same | -| None | Canonical x77 | 80,043,708 | 8.00 | 5 | 1.3 | 0.23 | reads faster above 57,820 MB/s | -| hint Dictionary | Dictionary x77 | 12,652,716 | 1.27 | 28 | 5.7 | 0.56 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.7 | 0.41 | reads slower at any throughput | -| hint RunEnd | BitPacked x77 | 12,544,276 | 1.25 | 0 | 2.5 | 0.32 | same | -| hint Zstd | Zstd x77 | 20,141,164 | 2.01 | 143 | 78.7 | 64.11 | reads slower at any throughput | +| Auto | BitPacked x77 | 12,544,284 | 1.25 | 22 | 5.5 | 1.54 | | +| Fastest | BitPacked x77 | 12,544,284 | 1.25 | 32 | 6.9 | 1.67 | reads slower at any throughput | +| Smallest | BitPacked x76, Pco x1 | 12,543,924 | 1.25 | 344 | 5.1 | 1.34 | reads faster at any throughput | +| None | Canonical x77 | 80,043,716 | 8.00 | 43 | 21.5 | 1.82 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,652,724 | 1.27 | 70 | 6.5 | 1.25 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 12,544,284 | 1.25 | 23 | 4.9 | 1.15 | reads faster at any throughput | +| hint RunEnd | BitPacked x77 | 12,544,284 | 1.25 | 23 | 7.0 | 1.64 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 20,141,172 | 2.01 | 186 | 78.1 | 31.66 | reads slower at any throughput | @@ -173,15 +595,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,740 | 0.51 | 38 | 2.3 | 0.49 | | -| Fastest | Dictionary x77 | 5,063,740 | 0.51 | 37 | 2.0 | 0.30 | reads faster at any throughput | -| Smallest | Dictionary x77 | 5,063,740 | 0.51 | 169 | 2.0 | 0.31 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 31 | 1.4 | 0.24 | reads faster above 79,377 MB/s | -| hint Dictionary | Dictionary x77 | 5,063,740 | 0.51 | 37 | 2.0 | 0.34 | reads faster at any throughput | -| hint BitPacked | Dictionary x77 | 5,063,740 | 0.51 | 102 | 2.0 | 0.32 | reads faster at any throughput | -| hint RunEnd | Dictionary x77 | 5,063,740 | 0.51 | 93 | 2.0 | 0.31 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 10,539,388 | 1.05 | 125 | 61.4 | 49.82 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,924 | 0.50 | 51 | 2.9 | 0.50 | reads slower at any throughput | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 55 | 3.3 | 1.14 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 66 | 3.1 | 1.09 | reads faster at any throughput | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 614 | 3.4 | 1.37 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 55 | 19.7 | 1.47 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 54 | 3.1 | 1.08 | reads faster at any throughput | +| hint BitPacked | Dictionary x77 | 5,063,748 | 0.51 | 127 | 3.1 | 1.07 | reads faster at any throughput | +| hint RunEnd | Dictionary x77 | 5,063,748 | 0.51 | 105 | 3.2 | 1.08 | same | +| hint Zstd | Zstd x77 | 10,539,396 | 1.05 | 139 | 51.4 | 24.40 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 84 | 3.1 | 1.05 | reads faster at any throughput | @@ -189,15 +611,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 46,295,644 | 4.63 | 32 | 2.6 | 0.26 | | -| Fastest | BitPacked x77 | 46,295,644 | 4.63 | 37 | 2.6 | 0.29 | same | -| Smallest | BitPacked x77 | 46,295,644 | 4.63 | 162 | 2.6 | 0.29 | same | -| None | Canonical x77 | 80,043,716 | 8.00 | 40 | 1.3 | 0.22 | reads faster above 25,357 MB/s | -| hint Dictionary | BitPacked x77 | 46,295,644 | 4.63 | 105 | 2.7 | 0.34 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 46,295,644 | 4.63 | 33 | 2.6 | 0.32 | same | -| hint RunEnd | BitPacked x77 | 46,295,644 | 4.63 | 58 | 2.6 | 0.29 | same | -| hint Zstd | Zstd x77 | 51,815,860 | 5.18 | 132 | 60.3 | 49.04 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 23,603,540 | 2.36 | 117 | 6.7 | 0.47 | reads faster below 5,585 MB/s | +| Auto | BitPacked x77 | 46,295,652 | 4.63 | 38 | 12.9 | 1.51 | | +| Fastest | BitPacked x77 | 46,295,652 | 4.63 | 64 | 16.5 | 1.96 | reads slower at any throughput | +| Smallest | Pco x77 | 580,924 | 0.06 | 423 | 72.5 | 69.30 | reads faster below 767 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 42 | 19.6 | 1.56 | reads slower at any throughput | +| hint Dictionary | BitPacked x77 | 46,295,652 | 4.63 | 123 | 12.7 | 1.46 | same | +| hint BitPacked | BitPacked x77 | 46,295,652 | 4.63 | 37 | 13.2 | 1.55 | same | +| hint RunEnd | BitPacked x77 | 46,295,652 | 4.63 | 86 | 13.7 | 1.56 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 51,815,868 | 5.18 | 128 | 62.1 | 42.11 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 23,603,548 | 2.36 | 138 | 9.5 | 1.32 | reads faster at any throughput | @@ -205,14 +627,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x77 | 80,043,724 | 8.00 | 31 | 1.4 | 0.22 | | -| Fastest | Canonical x77 | 80,043,724 | 8.00 | 30 | 1.3 | 0.24 | reads faster at any throughput | -| Smallest | Canonical x77 | 80,043,724 | 8.00 | 95 | 1.4 | 0.28 | same | -| None | Canonical x77 | 80,043,724 | 8.00 | 30 | 1.3 | 0.22 | reads faster at any throughput | -| hint Dictionary | Canonical x77 | 80,043,724 | 8.00 | 130 | 1.3 | 0.24 | same | -| hint BitPacked | Canonical x77 | 80,043,724 | 8.00 | 97 | 1.3 | 0.22 | reads faster at any throughput | -| hint RunEnd | Canonical x77 | 80,043,724 | 8.00 | 101 | 1.5 | 0.27 | same | -| hint Zstd | Canonical x77 | 80,043,724 | 8.00 | 105 | 1.3 | 0.27 | reads faster at any throughput | +| Auto | Canonical x77 | 80,043,732 | 8.00 | 43 | 27.0 | 2.14 | | +| Fastest | Canonical x77 | 80,043,732 | 8.00 | 43 | 20.0 | 1.62 | reads faster at any throughput | +| Smallest | Canonical x77 | 80,043,732 | 8.00 | 350 | 20.1 | 1.70 | reads faster at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 43 | 23.3 | 1.64 | reads faster at any throughput | +| hint Dictionary | Canonical x77 | 80,043,732 | 8.00 | 152 | 20.0 | 1.58 | reads faster at any throughput | +| hint BitPacked | Canonical x77 | 80,043,732 | 8.00 | 117 | 19.9 | 1.68 | reads faster at any throughput | +| hint RunEnd | Canonical x77 | 80,043,732 | 8.00 | 114 | 24.1 | 2.27 | reads faster at any throughput | +| hint Zstd | Canonical x77 | 80,043,732 | 8.00 | 132 | 18.5 | 1.52 | reads faster at any throughput | @@ -220,14 +642,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x82 | 13,800,452 | 1.38 | 15 | 3.6 | 0.31 | | -| Fastest | BitPacked x82 | 13,800,452 | 1.38 | 15 | 3.5 | 0.30 | same | -| Smallest | BitPacked x82 | 13,800,452 | 1.38 | 164 | 3.5 | 0.31 | same | -| None | Canonical x82 | 81,294,660 | 8.13 | 34 | 1.4 | 0.33 | reads faster above 31,597 MB/s | -| hint Dictionary | Dictionary x82 | 12,676,388 | 1.27 | 68 | 6.3 | 0.53 | reads faster below 417 MB/s | -| hint BitPacked | BitPacked x82 | 13,800,452 | 1.38 | 14 | 3.5 | 0.32 | same | -| hint RunEnd | BitPacked x82 | 13,800,452 | 1.38 | 15 | 3.5 | 0.30 | same | -| hint Zstd | Zstd x82 | 19,283,932 | 1.93 | 156 | 74.0 | 58.66 | reads slower at any throughput | +| Auto | BitPacked x82 | 13,800,460 | 1.38 | 25 | 6.3 | 1.35 | | +| Fastest | BitPacked x82 | 13,800,460 | 1.38 | 23 | 7.6 | 1.97 | reads slower at any throughput | +| Smallest | Pco x82 | 12,567,268 | 1.26 | 311 | 45.3 | 44.57 | reads faster below 32 MB/s | +| None | Canonical x82 | 81,294,668 | 8.13 | 42 | 20.6 | 2.10 | reads slower at any throughput | +| hint Dictionary | Dictionary x82 | 12,676,396 | 1.27 | 89 | 10.0 | 1.52 | reads faster below 307 MB/s | +| hint BitPacked | BitPacked x82 | 13,800,460 | 1.38 | 27 | 6.3 | 1.34 | same | +| hint RunEnd | BitPacked x82 | 13,800,460 | 1.38 | 56 | 9.5 | 2.06 | reads slower at any throughput | +| hint Zstd | Zstd x82 | 19,283,940 | 1.93 | 169 | 65.0 | 30.96 | reads slower at any throughput | @@ -235,23 +657,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| prices (2 decimals) | Alp x77 | 2.50 | 59 | 6.9 | 0.55 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 30 | 2.1 | 0.34 | | -| 1 000 distinct prices | Dictionary x77 | 1.28 | 68 | 3.8 | 0.42 | | -| 100 003 distinct, repeating | Zstd x77 | 1.69 | 185 | 79.0 | 71.68 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 7.4 ms, take 3.33 ms; reads faster above 73 MB/s; None: Canonical x77, 8.00 B/value, scan 1.3 ms, take 0.23 ms; reads faster above 812 MB/s | -| uniform in [0, 1) | AlpRd x77 | 6.91 | 45 | 5.6 | 0.71 | None: Canonical x77, 8.00 B/value, scan 1.4 ms, take 0.27 ms; reads faster above 2,658 MB/s | +| prices (2 decimals) | Alp x77 | 2.50 | 47 | 10.5 | 1.33 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 47 | 3.1 | 1.08 | | +| 1 000 distinct prices | Dictionary x77 | 1.28 | 90 | 9.8 | 1.84 | hint Alp: Alp x77, 1.28 B/value, scan 7.5 ms, take 1.31 ms; reads faster at any throughput | +| 100 003 distinct, repeating | Zstd x77 | 1.69 | 161 | 48.8 | 23.24 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 11.0 ms, take 3.70 ms; reads faster above 138 MB/s; None: Canonical x77, 8.00 B/value, scan 19.8 ms, take 1.56 ms; reads faster above 2,178 MB/s | +| uniform in [0, 1) | AlpRd x77 | 6.91 | 60 | 22.9 | 2.46 | |
prices (2 decimals): every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Alp x77 | 25,049,724 | 2.50 | 59 | 6.9 | 0.55 | | -| Fastest | Alp x77 | 25,049,724 | 2.50 | 40 | 4.6 | 0.34 | reads faster at any throughput | -| Smallest | Alp x77 | 25,049,724 | 2.50 | 356 | 4.5 | 0.30 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 40 | 1.6 | 0.33 | reads faster above 10,473 MB/s | -| hint Dictionary | Alp x77 | 25,049,724 | 2.50 | 143 | 4.6 | 0.34 | reads faster at any throughput | -| hint Alp | Alp x77 | 25,049,724 | 2.50 | 39 | 4.6 | 0.35 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 41,240,252 | 4.12 | 286 | 77.1 | 62.95 | reads slower at any throughput | +| Auto | Alp x77 | 25,049,732 | 2.50 | 47 | 10.5 | 1.33 | | +| Fastest | Alp x77 | 25,049,732 | 2.50 | 50 | 9.6 | 1.20 | reads faster at any throughput | +| Smallest | Alp x77 | 25,048,868 | 2.50 | 880 | 9.9 | 1.38 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 43 | 19.3 | 1.59 | reads slower at any throughput | +| hint Dictionary | Alp x77 | 25,049,732 | 2.50 | 177 | 9.7 | 1.21 | reads faster at any throughput | +| hint Alp | Alp x77 | 25,049,732 | 2.50 | 48 | 9.7 | 1.23 | reads faster at any throughput | +| hint Zstd | Zstd x77 | 41,240,260 | 4.12 | 295 | 66.1 | 36.34 | reads slower at any throughput |
@@ -259,14 +681,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,740 | 0.51 | 30 | 2.1 | 0.34 | | -| Fastest | Dictionary x77 | 5,063,740 | 0.51 | 30 | 2.0 | 0.32 | same | -| Smallest | Dictionary x77 | 5,063,740 | 0.51 | 168 | 2.0 | 0.32 | same | -| None | Canonical x77 | 80,043,724 | 8.00 | 32 | 1.3 | 0.24 | reads faster above 99,549 MB/s | -| hint Dictionary | Dictionary x77 | 5,063,740 | 0.51 | 28 | 2.0 | 0.32 | same | -| hint Alp | Dictionary x77 | 5,063,740 | 0.51 | 100 | 2.0 | 0.32 | same | -| hint Zstd | Zstd x77 | 10,545,660 | 1.05 | 127 | 61.5 | 49.92 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,924 | 0.50 | 43 | 2.0 | 0.27 | same | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 47 | 3.1 | 1.08 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 41 | 3.1 | 1.07 | same | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 560 | 3.4 | 1.33 | reads slower at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 27 | 20.2 | 1.65 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 42 | 3.5 | 1.19 | reads slower at any throughput | +| hint Alp | Dictionary x77 | 5,063,748 | 0.51 | 100 | 3.1 | 1.08 | same | +| hint Zstd | Zstd x77 | 10,545,668 | 1.05 | 139 | 47.7 | 22.21 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 66 | 3.0 | 1.03 | same | @@ -274,14 +696,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 12,756,252 | 1.28 | 68 | 3.8 | 0.42 | | -| Fastest | Dictionary x77 | 12,756,252 | 1.28 | 87 | 5.1 | 0.68 | reads slower at any throughput | -| Smallest | Dictionary x77 | 12,756,252 | 1.28 | 334 | 5.3 | 0.65 | reads slower at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 46 | 1.6 | 0.34 | reads faster above 30,690 MB/s | -| hint Dictionary | Dictionary x77 | 12,756,252 | 1.28 | 99 | 5.4 | 0.67 | reads slower at any throughput | -| hint Alp | Alp x77 | 12,756,252 | 1.28 | 161 | 6.7 | 0.62 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 26,073,404 | 2.61 | 293 | 103.3 | 82.83 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 12,545,724 | 1.25 | 110 | 5.4 | 0.62 | reads faster below 130 MB/s | +| Auto | Dictionary x77 | 12,756,260 | 1.28 | 90 | 9.8 | 1.84 | | +| Fastest | Dictionary x77 | 12,756,260 | 1.28 | 75 | 7.8 | 1.42 | reads faster at any throughput | +| Smallest | Dictionary x77 | 12,751,356 | 1.28 | 499 | 6.8 | 1.61 | reads faster at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 46 | 23.2 | 1.73 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,756,260 | 1.28 | 68 | 6.3 | 1.28 | reads faster at any throughput | +| hint Alp | Alp x77 | 12,756,260 | 1.28 | 121 | 7.5 | 1.31 | reads faster at any throughput | +| hint Zstd | Zstd x77 | 26,073,412 | 2.61 | 213 | 61.7 | 31.41 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 12,545,732 | 1.25 | 100 | 7.0 | 1.22 | reads faster at any throughput | @@ -289,14 +711,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x77 | 16,909,884 | 1.69 | 185 | 79.0 | 71.68 | | -| Fastest | Zstd x77 | 16,909,884 | 1.69 | 203 | 87.1 | 62.72 | reads slower at any throughput | -| Smallest | Zstd x77 | 16,909,884 | 1.69 | 216 | 61.9 | 52.47 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 27 | 1.3 | 0.23 | reads faster above 812 MB/s | -| hint Dictionary | Zstd x77 | 16,909,884 | 1.69 | 247 | 77.3 | 50.25 | same | -| hint Alp | Zstd x77 | 16,909,884 | 1.69 | 224 | 62.2 | 50.60 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 16,909,884 | 1.69 | 148 | 61.8 | 50.31 | reads faster at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 22,129,980 | 2.21 | 98 | 7.4 | 3.33 | reads faster above 73 MB/s | +| Auto | Zstd x77 | 16,909,892 | 1.69 | 161 | 48.8 | 23.24 | | +| Fastest | Zstd x77 | 16,909,892 | 1.69 | 174 | 47.9 | 22.40 | same | +| Smallest | Zstd x77 | 16,909,892 | 1.69 | 296 | 61.3 | 28.88 | reads slower at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 43 | 19.8 | 1.56 | reads faster above 2,178 MB/s | +| hint Dictionary | Zstd x77 | 16,909,892 | 1.69 | 260 | 63.0 | 28.51 | reads slower at any throughput | +| hint Alp | Zstd x77 | 16,909,892 | 1.69 | 284 | 47.8 | 22.96 | same | +| hint Zstd | Zstd x77 | 16,909,892 | 1.69 | 163 | 48.3 | 24.00 | same | +| Auto, 16 MiB chunks | Dictionary x5 | 22,129,988 | 2.21 | 132 | 11.0 | 3.70 | reads faster above 138 MB/s | @@ -304,13 +726,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | AlpRd x77 | 69,054,372 | 6.91 | 45 | 5.6 | 0.71 | | -| Fastest | AlpRd x77 | 69,054,372 | 6.91 | 48 | 4.7 | 0.51 | reads faster at any throughput | -| Smallest | AlpRd x77 | 69,054,372 | 6.91 | 169 | 4.7 | 0.49 | reads faster at any throughput | -| None | Canonical x77 | 80,043,724 | 8.00 | 32 | 1.4 | 0.27 | reads faster above 2,658 MB/s | -| hint Dictionary | AlpRd x77 | 69,054,372 | 6.91 | 212 | 4.7 | 0.54 | reads faster at any throughput | -| hint Alp | AlpRd x77 | 69,054,372 | 6.91 | 175 | 4.7 | 0.59 | reads faster at any throughput | -| hint Zstd | AlpRd x77 | 69,054,372 | 6.91 | 231 | 4.7 | 0.51 | reads faster at any throughput | +| Auto | AlpRd x77 | 69,054,380 | 6.91 | 60 | 22.9 | 2.46 | | +| Fastest | AlpRd x77 | 69,054,380 | 6.91 | 82 | 24.1 | 3.54 | reads slower at any throughput | +| Smallest | Pco x77 | 67,609,668 | 6.76 | 548 | 118.0 | 117.47 | reads faster below 15 MB/s | +| None | Canonical x77 | 80,043,732 | 8.00 | 45 | 18.9 | 1.57 | reads faster above 2,761 MB/s | +| hint Dictionary | AlpRd x77 | 69,054,380 | 6.91 | 288 | 21.2 | 2.62 | reads faster at any throughput | +| hint Alp | AlpRd x77 | 69,054,380 | 6.91 | 235 | 23.3 | 3.00 | same | +| hint Zstd | AlpRd x77 | 69,054,380 | 6.91 | 269 | 21.8 | 2.56 | same | @@ -318,23 +740,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| 16 cities, random order | Dictionary x175 | 0.51 | 210 | 7.4 | 0.76 | | -| 10 000 distinct ids | Dictionary x306 | 2.54 | 622 | 35.5 | 29.36 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 8.7 ms, take 2.43 ms; reads faster at any throughput; hint Fsst: Fsst x306, 9.06 B/value, scan 32.1 ms, take 1.05 ms; reads faster above 19,417 MB/s | -| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 642 | 41.6 | 33.35 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 14.5 ms, take 3.70 ms; reads faster at any throughput; hint Fsst: Fsst x306, 8.80 B/value, scan 38.0 ms, take 0.98 ms; reads faster above 17,261 MB/s | -| unique UUIDs | Zstd x611 | 20.61 | 1520 | 293.3 | 290.51 | hint Fsst: Fsst x611, 24.91 B/value, scan 148.5 ms, take 2.13 ms; reads faster above 297 MB/s | -| log lines (~100 B) | Zstd x1221 | 13.71 | 1406 | 394.9 | 321.43 | hint Fsst: Fsst x1221, 23.96 B/value, scan 161.6 ms, take 3.69 ms; reads faster above 440 MB/s | +| 16 cities, random order | Dictionary x175 | 0.51 | 124 | 7.7 | 1.60 | | +| 10 000 distinct ids | Dictionary x306 | 2.54 | 1032 | 41.2 | 29.59 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 14.8 ms, take 3.78 ms; reads faster at any throughput | +| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 995 | 54.5 | 36.86 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 13.5 ms, take 3.38 ms; reads faster at any throughput | +| unique UUIDs | Zstd x611 | 20.61 | 1792 | 309.4 | 301.45 | hint Fsst: Fsst x611, 22.44 B/value, scan 175.1 ms, take 13.05 ms; reads faster above 136 MB/s | +| log lines (~100 B) | Zstd x1221 | 13.71 | 1468 | 406.8 | 321.21 | hint Fsst: Fsst x1221, 22.01 B/value, scan 177.4 ms, take 13.64 ms; reads faster above 362 MB/s |
16 cities, random order: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x175 | 5,117,276 | 0.51 | 210 | 7.4 | 0.76 | | -| Fastest | Dictionary x175 | 5,117,276 | 0.51 | 201 | 8.0 | 0.95 | reads slower at any throughput | -| Smallest | Dictionary x175 | 5,117,276 | 0.51 | 357 | 4.9 | 0.54 | reads faster at any throughput | -| None | Canonical x175 | 96,746,748 | 9.67 | 159 | 22.8 | 2.78 | reads slower at any throughput | -| hint Dictionary | Dictionary x175 | 5,117,276 | 0.51 | 204 | 4.8 | 0.54 | reads faster at any throughput | -| hint Fsst | Fsst x175 | 64,458,972 | 6.45 | 226 | 25.9 | 0.72 | reads slower at any throughput | -| hint Zstd | Zstd x175 | 17,427,092 | 1.74 | 233 | 91.6 | 64.44 | reads slower at any throughput | +| Auto | Dictionary x175 | 5,117,284 | 0.51 | 124 | 7.7 | 1.60 | | +| Fastest | Dictionary x175 | 5,117,284 | 0.51 | 141 | 6.1 | 1.39 | reads faster at any throughput | +| Smallest | Dictionary x175 | 5,117,284 | 0.51 | 323 | 6.4 | 1.37 | reads faster at any throughput | +| None | Canonical x175 | 96,746,756 | 9.67 | 183 | 46.6 | 20.78 | reads slower at any throughput | +| hint Dictionary | Dictionary x175 | 5,117,284 | 0.51 | 122 | 6.2 | 1.42 | reads faster at any throughput | +| hint Fsst | Fsst x175 | 41,142,948 | 4.11 | 315 | 42.9 | 8.95 | reads slower at any throughput | +| hint Zstd | Zstd x175 | 17,427,100 | 1.74 | 287 | 97.8 | 61.74 | reads slower at any throughput |
@@ -342,14 +764,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 25,433,780 | 2.54 | 622 | 35.5 | 29.36 | | -| Fastest | Dictionary x306 | 25,433,780 | 2.54 | 611 | 34.8 | 29.27 | same | -| Smallest | Dictionary x305, Zstd x1 | 25,427,444 | 2.54 | 1311 | 35.6 | 29.93 | same | -| None | Canonical x306 | 140,098,164 | 14.01 | 368 | 18.6 | 4.34 | reads faster above 6,808 MB/s | -| hint Dictionary | Dictionary x306 | 25,433,780 | 2.54 | 951 | 46.1 | 38.67 | reads slower at any throughput | -| hint Fsst | Fsst x306 | 90,615,708 | 9.06 | 374 | 32.1 | 1.05 | reads faster above 19,417 MB/s | -| hint Zstd | Zstd x306 | 27,097,620 | 2.71 | 368 | 90.7 | 68.49 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 18,014,004 | 1.80 | 398 | 8.7 | 2.43 | reads faster at any throughput | +| Auto | Dictionary x306 | 25,433,788 | 2.54 | 1032 | 41.2 | 29.59 | | +| Fastest | Dictionary x306 | 25,433,788 | 2.54 | 1009 | 58.0 | 32.81 | reads slower at any throughput | +| Smallest | Dictionary x305, Zstd x1 | 23,452,500 | 2.35 | 2062 | 138.8 | 134.32 | reads faster below 20 MB/s | +| None | Canonical x306 | 140,098,172 | 14.01 | 225 | 55.6 | 40.79 | reads slower at any throughput | +| hint Dictionary | Dictionary x306 | 25,433,788 | 2.54 | 713 | 38.1 | 28.72 | reads faster at any throughput | +| hint Fsst | Fsst x306 | 62,863,260 | 6.29 | 461 | 47.5 | 8.41 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 27,097,628 | 2.71 | 430 | 92.2 | 63.15 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 18,014,012 | 1.80 | 446 | 14.8 | 3.78 | reads faster at any throughput | @@ -357,14 +779,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 24,539,276 | 2.45 | 642 | 41.6 | 33.35 | | -| Fastest | Dictionary x306 | 24,539,276 | 2.45 | 655 | 41.8 | 33.52 | same | -| Smallest | Dictionary x305, Zstd x1 | 24,533,580 | 2.45 | 1384 | 41.8 | 33.98 | same | -| None | Canonical x306 | 151,883,724 | 15.19 | 312 | 26.1 | 3.88 | reads faster above 8,195 MB/s | -| hint Dictionary | Dictionary x306 | 24,539,276 | 2.45 | 646 | 42.1 | 33.53 | same | -| hint Fsst | Fsst x306 | 87,950,764 | 8.80 | 470 | 38.0 | 0.98 | reads faster above 17,261 MB/s | -| hint Zstd | Zstd x306 | 25,595,628 | 2.56 | 391 | 113.5 | 62.19 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 17,982,596 | 1.80 | 706 | 14.5 | 3.70 | reads faster at any throughput | +| Auto | Dictionary x306 | 24,539,284 | 2.45 | 995 | 54.5 | 36.86 | | +| Fastest | Dictionary x306 | 24,539,284 | 2.45 | 944 | 40.5 | 34.87 | reads faster at any throughput | +| Smallest | Dictionary x305, Zstd x1 | 22,854,628 | 2.29 | 1984 | 148.6 | 139.29 | reads faster below 18 MB/s | +| None | Canonical x306 | 151,883,732 | 15.19 | 293 | 69.6 | 39.43 | reads slower at any throughput | +| hint Dictionary | Dictionary x306 | 24,539,284 | 2.45 | 817 | 39.9 | 28.75 | reads faster at any throughput | +| hint Fsst | Fsst x306 | 64,408,284 | 6.44 | 533 | 60.9 | 11.45 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 25,595,636 | 2.56 | 400 | 99.0 | 63.07 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 17,982,604 | 1.80 | 504 | 13.5 | 3.38 | reads faster at any throughput | @@ -372,13 +794,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x611 | 206,058,580 | 20.61 | 1520 | 293.3 | 290.51 | | -| Fastest | Zstd x611 | 206,058,580 | 20.61 | 1509 | 293.2 | 288.92 | same | -| Smallest | Zstd x611 | 206,058,580 | 20.61 | 2820 | 290.7 | 288.95 | same | -| None | Canonical x611 | 360,155,508 | 36.02 | 196 | 21.0 | 7.27 | reads faster above 566 MB/s | -| hint Dictionary | Zstd x611 | 206,058,580 | 20.61 | 2884 | 289.7 | 288.12 | same | -| hint Fsst | Fsst x611 | 249,114,908 | 24.91 | 1497 | 148.5 | 2.13 | reads faster above 297 MB/s | -| hint Zstd | Zstd x611 | 206,058,580 | 20.61 | 1473 | 290.8 | 289.80 | same | +| Auto | Zstd x611 | 206,058,588 | 20.61 | 1792 | 309.4 | 301.45 | | +| Fastest | Zstd x611 | 206,058,588 | 20.61 | 1702 | 302.5 | 306.99 | same | +| Smallest | Zstd x611 | 206,058,588 | 20.61 | 3473 | 299.3 | 307.14 | same | +| None | Canonical x611 | 360,155,516 | 36.02 | 281 | 109.3 | 96.06 | reads faster above 770 MB/s | +| hint Dictionary | Zstd x610, Fsst x1 | 206,059,580 | 20.61 | 6414 | 301.3 | 323.16 | same | +| hint Fsst | Fsst x611 | 224,384,604 | 22.44 | 1798 | 175.1 | 13.05 | reads faster above 136 MB/s | +| hint Zstd | Zstd x611 | 206,058,588 | 20.61 | 1761 | 305.3 | 320.92 | same | @@ -386,13 +808,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x1221 | 137,107,180 | 13.71 | 1406 | 394.9 | 321.43 | | -| Fastest | Zstd x1221 | 137,107,180 | 13.71 | 1400 | 394.4 | 321.13 | same | -| Smallest | Zstd x1221 | 137,107,180 | 13.71 | 3392 | 400.1 | 321.98 | same | -| None | Canonical x1221 | 725,468,436 | 72.55 | 756 | 34.1 | 12.23 | reads faster above 1,631 MB/s | -| hint Dictionary | Zstd x1221 | 137,107,180 | 13.71 | 2815 | 391.9 | 320.55 | same | -| hint Fsst | Fsst x1221 | 239,634,612 | 23.96 | 1920 | 161.6 | 3.69 | reads faster above 440 MB/s | -| hint Zstd | Zstd x1221 | 137,107,180 | 13.71 | 1399 | 393.9 | 320.61 | same | +| Auto | Zstd x1221 | 137,107,188 | 13.71 | 1468 | 406.8 | 321.21 | | +| Fastest | Zstd x1221 | 137,107,188 | 13.71 | 1450 | 393.2 | 317.69 | same | +| Smallest | Zstd x1221 | 137,107,188 | 13.71 | 2932 | 387.2 | 335.15 | same | +| None | Canonical x1221 | 725,468,444 | 72.55 | 790 | 206.6 | 158.59 | reads faster above 2,939 MB/s | +| hint Dictionary | Zstd x1221 | 137,107,188 | 13.71 | 6070 | 404.3 | 347.13 | same | +| hint Fsst | Fsst x1221 | 220,058,684 | 22.01 | 1838 | 177.4 | 13.64 | reads faster above 362 MB/s | +| hint Zstd | Zstd x1221 | 137,107,188 | 13.71 | 1398 | 401.5 | 320.92 | same | @@ -400,18 +822,18 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| half true | Canonical x10 | 0.13 | 0 | 0.1 | 0.20 | | -| 1 % true | RunEnd x10 | 0.05 | 1 | 2.6 | 1.95 | None: Canonical x10, 0.13 B/value, scan 0.1 ms, take 0.17 ms; reads faster above 289 MB/s | +| half true | Canonical x10 | 0.13 | 10 | 0.4 | 0.46 | | +| 1 % true | RunEnd x10 | 0.05 | 5 | 2.5 | 0.49 | None: Canonical x10, 0.13 B/value, scan 0.6 ms, take 0.66 ms; reads faster above 390 MB/s |
half true: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.20 | | -| Fastest | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.18 | same | -| Smallest | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.21 | reads slower at any throughput | -| None | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.31 | reads slower at any throughput | -| hint RunEnd | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.17 | reads faster at any throughput | +| Auto | Canonical x10 | 1,262,556 | 0.13 | 10 | 0.4 | 0.46 | | +| Fastest | Canonical x10 | 1,262,556 | 0.13 | 0 | 0.5 | 0.46 | same | +| Smallest | Canonical x10 | 1,262,556 | 0.13 | 0 | 0.5 | 0.47 | same | +| None | Canonical x10 | 1,262,556 | 0.13 | 0 | 0.5 | 0.46 | same | +| hint RunEnd | Canonical x10 | 1,262,556 | 0.13 | 7 | 0.4 | 0.46 | same |
@@ -419,20 +841,20 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x10 | 548,084 | 0.05 | 1 | 2.6 | 1.95 | | -| Fastest | RunEnd x10 | 548,084 | 0.05 | 0 | 2.5 | 1.95 | same | -| Smallest | RunEnd x10 | 548,084 | 0.05 | 0 | 2.4 | 1.87 | same | -| None | Canonical x10 | 1,262,548 | 0.13 | 0 | 0.1 | 0.17 | reads faster above 289 MB/s | -| hint RunEnd | RunEnd x10 | 548,084 | 0.05 | 0 | 2.5 | 1.88 | same | +| Auto | RunEnd x10 | 548,092 | 0.05 | 5 | 2.5 | 0.49 | | +| Fastest | RunEnd x10 | 548,092 | 0.05 | 5 | 2.5 | 0.50 | same | +| Smallest | RunEnd x10 | 164,692 | 0.02 | 27 | 4.0 | 3.67 | reads faster below 246 MB/s | +| None | Canonical x10 | 1,262,556 | 0.13 | 13 | 0.6 | 0.66 | reads faster above 390 MB/s | +| hint RunEnd | RunEnd x10 | 548,092 | 0.05 | 6 | 2.4 | 0.49 | same | ### The profiles side by side -`Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 10.8 times `Auto`'s write, 164 ms against 15 ms on *i64, random in 0..999, 10 % null*, since it tries every scheme on every chunk. +`Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *i64, random in 0..999*, *i64, 16 distinct, random order*, *i64, 100 003 distinct, repeating*, *i64, random in 0..999, 10 % null*, *f64, 16 distinct, random order*, *f64, uniform in [0, 1)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 18.8 times `Auto`'s write, 880 ms against 47 ms on *f64, prices (2 decimals)*, since it tries every scheme on every chunk. `None`, the plain form, made the largest file on 20 of 20 columns. -*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit b5104f2 with uncommitted changes, 2026-09-27 02:59 UTC.* +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit a3dfc8ab, 2026-10-09 16:49 UTC.* @@ -445,20 +867,21 @@ does not; a column it never departs on is left out. | column | scans at 2 GB/s | scans at 100 MB/s | scans at 10 GB/s | a row in 1 000 read by row | size | |---|---|---|---|---|---| -| i64, timestamps (ms, increasing, jittered) | | `Zstd` | | `Zstd` | | -| i64, 100 003 distinct, repeating | 16 MiB chunks | 16 MiB chunks | | | 16 MiB chunks | +| i64, sorted runs of 1 000 | | | `Zstd` | | | +| i64, timestamps (ms, increasing, jittered) | | `Pco` | | `Zstd` | | +| i64, 100 003 distinct, repeating | 16 MiB chunks | `Pco` | 16 MiB chunks | | `Dictionary`, 16 MiB chunks | | i64, random in 0..999, 10 % null | | `Dictionary` | | `Dictionary` | `Dictionary` | -| f64, 100 003 distinct, repeating | 16 MiB chunks | 16 MiB chunks | `Canonical` | | | +| f64, 100 003 distinct, repeating | 16 MiB chunks | | 16 MiB chunks | | `Dictionary`, 16 MiB chunks | | f64, uniform in [0, 1) | | | `Canonical` | | | -| utf8, 10 000 distinct ids | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `Zstd` | 16 MiB chunks | +| utf8, 10 000 distinct ids | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `OnPair` | 16 MiB chunks | | utf8, 10 000 distinct ids, 10 % null | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `Zstd` | 16 MiB chunks | | utf8, unique UUIDs | `Canonical` | | `Canonical` | `Fsst` | | -| utf8, log lines (~100 B) | `Fsst` | | `Canonical` | `Fsst` | | -| bool, 1 % true | `Canonical` | | `Canonical` | `Canonical` | | +| utf8, log lines (~100 B) | `OnPair` | | `Canonical` | `OnPair` | | +| bool, 1 % true | `Canonical` | | `Canonical` | | | -It keeps the writer's choice under every goal on the 10 other columns. +It keeps the writer's choice under every goal on the 9 other columns. -*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit b5104f2 with uncommitted changes, 2026-09-27 03:01 UTC.* +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; commit a3dfc8ab, 2026-10-09 16:52 UTC.* ## Kernels, against their baselines @@ -511,23 +934,27 @@ Benchmarks with issues: ### `CompressorBenchmarks` -| Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | -|----------------------------- |------- |--------------:|--------------:|------------:|--------:|---------------- |----------:|------------:| -| 'Choose, the whole decision' | i64 | 57,255.034 ns | 1,758.2423 ns | 456.6099 ns | 1.00x | Baseline | 72 B | 1.00 | -| 'candidate: sequence' | i64 | 13.330 ns | 3.0557 ns | 0.4729 ns | 4,295x | Faster | - | 0.00 | -| 'candidate: bit packing' | i64 | 21,888.618 ns | 3,518.0519 ns | 913.6269 ns | 2.62x | Faster | - | 0.00 | -| 'candidate: FSST' | i64 | 2.234 ns | 1.1546 ns | 0.2998 ns | 25,629x | Faster | - | 0.00 | -| 'candidate: zstd' | i64 | 26,766.801 ns | 2,571.6941 ns | 667.8608 ns | 2.14x | Faster | 72 B | 1.00 | -| 'candidate: ALP' | i64 | 2.614 ns | 0.0307 ns | 0.0047 ns | 21,903x | Faster | - | 0.00 | +| Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | +|----------------------------- |------- |--------------:|------------:|-----------:|--------:|---------------- |----------:|------------:| +| 'Choose, the whole decision' | i64 | 52,503.854 ns | 230.5989 ns | 35.6854 ns | 1.00x | Baseline | - | NA | +| 'candidate: sequence' | i64 | 10.728 ns | 0.1120 ns | 0.0291 ns | 4,894x | Faster | - | NA | +| 'candidate: bit packing' | i64 | 4,950.600 ns | 148.8102 ns | 23.0285 ns | 10.6x | Faster | - | NA | +| 'candidate: FSST' | i64 | 1.846 ns | 0.0144 ns | 0.0038 ns | 28,448x | Faster | - | NA | +| 'candidate: zstd' | i64 | 7,546.505 ns | 194.1759 ns | 30.0489 ns | 6.96x | Faster | - | NA | +| 'candidate: ALP' | i64 | 2.599 ns | 0.0136 ns | 0.0035 ns | 20,204x | Faster | - | NA | -*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.1, the fast profile; commit b5104f2 with uncommitted changes, 2026-09-27 03:15 UTC.* +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit a3dfc8ab, 2026-10-09 16:52 UTC.* ### `LanesBenchmarks` -*Not measured on x64.* This section compares Vorticity with Rust; the arguments above write it -on an x64 machine. +| Method | Lanes | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Completed Work Items | Lock Contentions | Allocated | Alloc Ratio | +|----------------------- |------ |-----------:|----------:|---------:|--------:|---------------- |---------------------:|-----------------:|----------:|------------:| +| 'ours, n lanes' | 4 | 622.1 μs | 46.33 μs | 7.17 μs | 1.00x | Baseline | 97.4875 | - | 46910 B | 1.00 | +| 'reference, n workers' | 4 | 1,311.8 μs | 184.25 μs | 47.85 μs | 0.47x | Slower | - | - | - | 0.00 | + +*Measured on AMD Ryzen 9 7950X 16-Core Processor (X64), 32 processors, Microsoft Windows 10.0.26200; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 716b74b6, 2026-10-09 17:41 UTC.* ## What this does not measure diff --git a/docs/guide/benchmarks.md b/docs/guide/benchmarks.md index d3aa44b05..9cb9ff677 100644 --- a/docs/guide/benchmarks.md +++ b/docs/guide/benchmarks.md @@ -44,8 +44,7 @@ and the arguments below, which rewrite that section of this page and leave the o ## On x64 [Benchmarks on x64](benchmarks-x64.md) is this page's twin on an AMD Ryzen 9 7950X (Zen 4, -AVX-512) under Windows 11: the sections that compare Vorticity with itself. Its sections that -compare Vorticity with Rust are not measured on x64. +AVX-512) under Windows 11, every section measured there, against a Rust built on that machine. ## Reading and writing a table, process against process @@ -950,7 +949,7 @@ reference profile on that class, the one to quote a small difference from. faults where the copy cost it a read, which on the per-encoding corpus makes its scans of `fsst` and `alp` 7 and 10 % slower, and moves the others by less. * **x64 on this page.** Every figure here is from one arm64 machine; [Benchmarks on - x64](benchmarks-x64.md) has the sections that compare Vorticity with itself from one x64 machine. + x64](benchmarks-x64.md) has the same sections from one x64 machine. CI builds and tests on x64 and arm64 and runs no benchmark. * **Your data and your machine.** A handful of tables and one machine: a column the compressor likes less, or a filter the zone maps cannot prune, moves these numbers more than either implementation diff --git a/src/Vorticity.Zstd/Internal/BackwardBitReader.cs b/src/Vorticity.Zstd/Internal/BackwardBitReader.cs index df32cda87..b288b6541 100644 --- a/src/Vorticity.Zstd/Internal/BackwardBitReader.cs +++ b/src/Vorticity.Zstd/Internal/BackwardBitReader.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Vorticity.Zstd.Internal; @@ -173,6 +174,73 @@ public BitStreamStatus Reload() return status; } + /// + /// Decodes symbols of a single-symbol Huffman table (entries of + /// : the code length in the low byte, the symbol in the high byte), + /// each looked up with the next bits, the stream reloaded before any + /// symbol once more than 64 - of its bits are consumed: + /// the reloads of , at the same symbols, its statuses ignored. + /// + /// + /// + /// The reader's state is held in locals for the loop: through the reader, every symbol would read + /// and write it in memory. A reload eight bytes or more from the start takes the loop's own path; + /// any other goes through . + /// + /// + /// The symbols form one chain, each code's length deciding where the next starts. So the chain + /// runs on the container shifted past the consumed bits, which a symbol shifts by its entry (whose + /// low six bits are the code's length): a shift, the lookup, a shift. The count of consumed bits, + /// which only the reloads read, follows on the side. Until 64 bits are consumed the shifted + /// container is the reader's container shifted by that count; past that, the stream has + /// overflowed and fails its end check whatever the symbols read. + /// + /// + public void DecodeHuffmanSymbols(ref ushort table, int tableLog, ref byte output, nint length) + { + ulong container = _container; + int consumed = _bitsConsumed; + int position = _position; + ulong bits = container << (consumed & 63); + bits = consumed >= 64 ? 0 : bits; + ref byte source = ref MemoryMarshal.GetReference(_source); + int shift = (64 - tableLog) & 63; + for (nint i = 0; i < length; i++) + { + if (consumed > 64 - HuffmanTable.MaxTableLog) + { + if ((consumed <= 64) & (position >= sizeof(ulong))) + { + position -= consumed >> 3; + consumed &= 7; + container = Unsafe.ReadUnaligned(ref Unsafe.Add(ref source, position)); + } + else + { + _container = container; + _bitsConsumed = consumed; + _position = position; + Reload(); + container = _container; + consumed = _bitsConsumed; + position = _position; + } + + bits = container << (consumed & 63); + bits = consumed >= 64 ? 0 : bits; + } + + uint entry = Unsafe.Add(ref table, (nint)(bits >> shift)); + Unsafe.Add(ref output, i) = (byte)(entry >> 8); + bits <<= (int)entry; + consumed += (int)(entry & 0xFF); + } + + _container = container; + _bitsConsumed = consumed; + _position = position; + } + /// libzstd's BIT_reloadDStreamFast: refuses to come within eight bytes of the start. public BitStreamStatus ReloadFast() { diff --git a/src/Vorticity.Zstd/Internal/BitWriter.cs b/src/Vorticity.Zstd/Internal/BitWriter.cs index 3c7e725af..b7b450a38 100644 --- a/src/Vorticity.Zstd/Internal/BitWriter.cs +++ b/src/Vorticity.Zstd/Internal/BitWriter.cs @@ -22,6 +22,7 @@ internal unsafe struct BitWriter private readonly byte* _end; /// libzstd's BIT_initCStream; is false for a buffer of eight bytes or less. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public BitWriter(byte* start, nuint capacity) { _start = start; @@ -70,6 +71,7 @@ public void Flush() /// libzstd's BIT_closeCStream: the end marker, then the last bytes. /// The size of the stream, or 0 when it did not fit. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public nuint Close() { AddBitsFast(1, 1); diff --git a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs new file mode 100644 index 000000000..ae2eb7442 --- /dev/null +++ b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.AttachedX64.cs @@ -0,0 +1,354 @@ +using System.Numerics; +using System.Runtime.CompilerServices; +using static Vorticity.Zstd.Internal.MatchFinder; + +namespace Vorticity.Zstd.Internal; + +/// The double-fast search with an attached dictionary, as x64 wants it. +internal static unsafe partial class DoubleFastMatchFinder +{ + /// + /// What the x64 search with an attached dictionary keeps out of registers, in a local whose address + /// is taken: read as the operands of its instructions, or once a match. The first fields are copies + /// of the search's own values, which it takes back after each match (see ). + /// + private struct AttachedCold + { + public byte* Base; + public uint* HashLong; + public uint* HashSmall; + public uint* DictHashLong; + public uint* DictHashSmall; + public int Shifts; + + public byte* Anchor; + public uint Offset1; + public uint Offset2; + public uint PrefixLowestIndex; + + /// The prefix's lowest index less one: as one subtraction. + public uint OverlapLimit; + public uint DictStartIndex; + public uint DictIndexDelta; + public uint Current; + public byte* Ilimit; + public byte* End; + public byte* PrefixLowest; + + /// The dictionary's base less the delta of its indices: a frame index's address there. + public byte* DictShifted; + public byte* DictBase; + public byte* DictStart; + public byte* DictEnd; + public byte* Lit; + public SequenceRecord* Sequence; + public uint* Counts; + } + + /// As , for . + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Expose(AttachedCold* cold) + { + } + + /// + /// on x64: the same search, the same sequences, with the + /// values its loop holds cut down to what x64 has registers for. + /// + /// + /// + /// libzstd's form, which Arm64 keeps, holds some twenty values across the loop, among them the + /// four tables, their four hash shifts, both segments' bounds, the repeat offsets and the cursors + /// of the store, live across the calls of the matches that cross the segments: x64, with 15 + /// registers and 8 kept across a call, spilled them all, the hashes and the entries read + /// included, a 440-byte frame whose stores and reloads made each position's probes twice as + /// long, the misses of the dictionary's tables less overlapped. + /// + /// + /// Here the search holds the cursor, the base and the four tables, and the four shifts in one + /// register, rotated to each. The rest is in , the anchor, the repeat + /// offset and the bounds read as operands. After a match the code reads only + /// , and the search's values are taken back from it before the next + /// position: none is live across a call. A dictionary entry is read with its tag's difference + /// from the hash's in its low byte: one value for the tag test and the index. + /// + /// + private static nuint CompressBlockAttachedX64(ref MatchState state, SequenceStore store, uint* rep, byte* source, nuint size) + where THash : IMatchHash + { + AttachedCold cold = default; + Expose(&cold); + + CompressionParameters parameters = state.Parameters; + MatchState* dictionary = state.Dictionary; + byte* @base = state.Base; + uint* hashLong = state.HashTable; + uint* hashSmall = state.ChainTable; + uint* dictHashLong = dictionary->HashTable; + uint* dictHashSmall = dictionary->ChainTable; + uint endIndex = (uint)(source - @base) + (uint)size; + uint prefixLowestIndex = state.LowestPrefixIndex(endIndex); + byte* dictBase = dictionary->Base; + + // The shifts of the frame's long hash, the dictionary's (tagged), the frame's short hash, the + // dictionary's, a byte each: a shift takes the low 6 bits of its count. + int shifts = (64 - parameters.HashLog) + | ((64 - (dictionary->Parameters.HashLog + TagBits)) << 8) + | ((64 - parameters.ChainLog) << 16) + | ((64 - (dictionary->Parameters.ChainLog + TagBits)) << 24); + + cold.Base = @base; + cold.HashLong = hashLong; + cold.HashSmall = hashSmall; + cold.DictHashLong = dictHashLong; + cold.DictHashSmall = dictHashSmall; + cold.Shifts = shifts; + cold.Offset1 = rep[0]; + cold.Offset2 = rep[1]; + cold.PrefixLowestIndex = prefixLowestIndex; + cold.OverlapLimit = prefixLowestIndex - 1; + cold.DictStartIndex = dictionary->DictLimit; + cold.DictIndexDelta = prefixLowestIndex - dictionary->End; + cold.End = source + size; + cold.Ilimit = cold.End - HashReadSize; + cold.PrefixLowest = @base + prefixLowestIndex; + cold.DictShifted = dictBase - (nuint)cold.DictIndexDelta; + cold.DictBase = dictBase; + cold.DictStart = dictBase + cold.DictStartIndex; + cold.DictEnd = dictBase + dictionary->End; + cold.Lit = store.Literals; + cold.Sequence = store.Sequences; + cold.Counts = store.Counts; + + byte* ip = source; + cold.Anchor = source; + uint dictAndPrefixLength = (uint)((ip - cold.PrefixLowest) + (cold.DictEnd - cold.DictStart)); + ip += dictAndPrefixLength == 0 ? 1 : 0; + + while (ip < cold.Ilimit) + { + nuint matchLength; + uint offset; + uint offBase; + byte* match; + byte* low; + + // The dictionary's entries first, the likeliest misses. + ulong bytes = Read64(ip); + ulong productLong = bytes * Hash8.Multiplier; + nuint dictHashAndTagLong = (nuint)(productLong >> (int)BitOperations.RotateRight((uint)shifts, 8)); + uint dictEntryLong = dictHashLong[dictHashAndTagLong >> TagBits] ^ ((uint)dictHashAndTagLong & TagMask); + nuint hashLongIndex = (nuint)(productLong >> shifts); + ulong productSmall = bytes * THash.Multiplier; + nuint dictHashAndTagSmall = (nuint)(productSmall >> (int)BitOperations.RotateRight((uint)shifts, 24)); + uint dictEntrySmall = dictHashSmall[dictHashAndTagSmall >> TagBits] ^ ((uint)dictHashAndTagSmall & TagMask); + nuint hashSmallIndex = (nuint)(productSmall >> (int)BitOperations.RotateRight((uint)shifts, 16)); + uint current = (uint)(ip - @base); + uint indexLong = hashLong[hashLongIndex]; + uint indexSmall = hashSmall[hashSmallIndex]; + hashLong[hashLongIndex] = hashSmall[hashSmallIndex] = current; + + // The first repeat offset at ip + 1. + uint repIndex = current + 1 - cold.Offset1; + if (cold.OverlapLimit - repIndex >= 3) + { + byte* repMatch = repIndex < cold.PrefixLowestIndex ? cold.DictShifted + repIndex : @base + repIndex; + if (Read32(repMatch) == Read32(ip + 1)) + { + cold.Current = current; + byte* repMatchEnd = repIndex < cold.PrefixLowestIndex ? cold.DictEnd : cold.End; + matchLength = CountAcross(ip + 1 + 4, repMatch + 4, cold.End, repMatchEnd, cold.PrefixLowest) + 4; + ip++; + offBase = RepeatCode1; + goto Store; + } + } + + if (indexLong >= cold.PrefixLowestIndex && Read64(@base + indexLong) == bytes) + { + // A long match in the prefix. + cold.Current = current; + match = @base + indexLong; + matchLength = Count(ip + 8, match + 8, cold.End) + 8; + offset = (uint)(ip - match); + low = cold.PrefixLowest; + goto ExtendBackward; + } + + if ((dictEntryLong & TagMask) == 0) + { + // A long match in the dictionary. + uint dictIndexLong = dictEntryLong >> TagBits; + match = cold.DictBase + dictIndexLong; + if (dictIndexLong > cold.DictStartIndex && Read64(match) == bytes) + { + cold.Current = current; + offset = current - dictIndexLong - cold.DictIndexDelta; + matchLength = CountAcross(ip + 8, match + 8, cold.End, cold.DictEnd, cold.PrefixLowest) + 8; + low = cold.DictStart; + goto ExtendBackward; + } + } + + if (indexSmall > cold.PrefixLowestIndex) + { + // A short match in the prefix. + match = @base + indexSmall; + if (Read32(match) == (uint)bytes) + { + goto SearchNextLong; + } + } + else if ((dictEntrySmall & TagMask) == 0) + { + // A short match in the dictionary. + uint dictIndexSmall = dictEntrySmall >> TagBits; + match = cold.DictBase + dictIndexSmall; + indexSmall = dictIndexSmall + cold.DictIndexDelta; + if (dictIndexSmall > cold.DictStartIndex && Read32(match) == (uint)bytes) + { + goto SearchNextLong; + } + } + + ip += ((nuint)(ip - cold.Anchor) >> SearchStrength) + 1; + continue; + + SearchNextLong: + cold.Current = current; + { + ulong nextBytes = Read64(ip + 1); + ulong nextProduct = nextBytes * Hash8.Multiplier; + nuint dictHashAndTagLong3 = (nuint)(nextProduct >> (int)BitOperations.RotateRight((uint)shifts, 8)); + uint dictEntryLong3 = dictHashLong[dictHashAndTagLong3 >> TagBits] ^ ((uint)dictHashAndTagLong3 & TagMask); + nuint hashLongIndex3 = (nuint)(nextProduct >> shifts); + uint indexLong3 = hashLong[hashLongIndex3]; + hashLong[hashLongIndex3] = current + 1; + + if (indexLong3 >= cold.PrefixLowestIndex && Read64(@base + indexLong3) == nextBytes) + { + // A long match at ip + 1 in the prefix. + match = @base + indexLong3; + matchLength = Count(ip + 9, match + 8, cold.End) + 8; + ip++; + offset = (uint)(ip - match); + low = cold.PrefixLowest; + goto ExtendBackward; + } + + if ((dictEntryLong3 & TagMask) == 0) + { + // A long match at ip + 1 in the dictionary. + uint dictIndexLong3 = dictEntryLong3 >> TagBits; + byte* dictMatchLong3 = cold.DictBase + dictIndexLong3; + if (dictIndexLong3 > cold.DictStartIndex && Read64(dictMatchLong3) == nextBytes) + { + offset = current + 1 - dictIndexLong3 - cold.DictIndexDelta; + matchLength = CountAcross(ip + 1 + 8, dictMatchLong3 + 8, cold.End, cold.DictEnd, cold.PrefixLowest) + 8; + ip++; + match = dictMatchLong3; + low = cold.DictStart; + goto ExtendBackward; + } + } + } + + // No long match at ip + 1: the short match found. + if (indexSmall < cold.PrefixLowestIndex) + { + offset = current - indexSmall; + matchLength = CountAcross(ip + 4, match + 4, cold.End, cold.DictEnd, cold.PrefixLowest) + 4; + low = cold.DictStart; + } + else + { + matchLength = Count(ip + 4, match + 4, cold.End) + 4; + offset = (uint)(ip - match); + low = cold.PrefixLowest; + } + + ExtendBackward: + { + byte* anchor = cold.Anchor; + while (ip > anchor && match > low && ip[-1] == match[-1]) + { + ip--; + match--; + matchLength++; + } + } + + cold.Offset2 = cold.Offset1; + cold.Offset1 = offset; + offBase = OffsetToOffBase(offset); + + Store: + { + byte* anchor = cold.Anchor; + byte* lit = cold.Lit; + SequenceRecord* sequence = cold.Sequence; + SequenceStore.Store(ref lit, ref sequence, cold.Counts, (nuint)(ip - anchor), anchor, cold.End, offBase, matchLength); + cold.Lit = lit; + cold.Sequence = sequence; + } + + ip += matchLength; + cold.Anchor = ip; + if (ip <= cold.Ilimit) + { + // Complementary insertion, after the limit test: the positions could pass the end - 8. + byte* b = cold.Base; + uint* tableLong = cold.HashLong; + uint* tableSmall = cold.HashSmall; + int shiftLong = cold.Shifts; + int shiftSmall = (int)BitOperations.RotateRight((uint)shiftLong, 16); + uint indexToInsert = cold.Current + 2; + ulong insertBytes = Read64(b + indexToInsert); + tableLong[(nuint)((insertBytes * Hash8.Multiplier) >> shiftLong)] = indexToInsert; + tableLong[(nuint)((Read64(ip - 2) * Hash8.Multiplier) >> shiftLong)] = (uint)(ip - 2 - b); + tableSmall[(nuint)((insertBytes * THash.Multiplier) >> shiftSmall)] = indexToInsert; + tableSmall[(nuint)((Read64(ip - 1) * THash.Multiplier) >> shiftSmall)] = (uint)(ip - 1 - b); + + // The second repeat offset, at once. + while (ip <= cold.Ilimit) + { + uint current2 = (uint)(ip - cold.Base); + uint offset2 = cold.Offset2; + uint repIndex2 = current2 - offset2; + bool inDictionary = repIndex2 < cold.PrefixLowestIndex; + byte* repMatch2 = (inDictionary ? cold.DictShifted : cold.Base) + repIndex2; + if (cold.OverlapLimit - repIndex2 < 3 || Read32(repMatch2) != Read32(ip)) + { + break; + } + + nuint repLength2 = CountAcross(ip + 4, repMatch2 + 4, cold.End, inDictionary ? cold.DictEnd : cold.End, cold.PrefixLowest) + 4; + cold.Offset2 = cold.Offset1; + cold.Offset1 = offset2; + SequenceRecord* sequence = cold.Sequence; + SequenceStore.StoreOnly(ref sequence, cold.Counts, 0, RepeatCode1, repLength2); + cold.Sequence = sequence; + ulong repBytes = Read64(ip); + cold.HashSmall[(nuint)((repBytes * THash.Multiplier) >> (int)BitOperations.RotateRight((uint)cold.Shifts, 16))] = current2; + cold.HashLong[(nuint)((repBytes * Hash8.Multiplier) >> cold.Shifts)] = current2; + ip += repLength2; + cold.Anchor = ip; + } + } + + // The search's values, taken back: none was live across the calls. + @base = cold.Base; + hashLong = cold.HashLong; + hashSmall = cold.HashSmall; + dictHashLong = cold.DictHashLong; + dictHashSmall = cold.DictHashSmall; + shifts = cold.Shifts; + } + + store.Literals = cold.Lit; + store.Sequences = cold.Sequence; + rep[0] = cold.Offset1; + rep[1] = cold.Offset2; + return (nuint)(cold.End - cold.Anchor); + } +} diff --git a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs index 2d74ee8d5..1b6816a3e 100644 --- a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs +++ b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.Dictionary.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -76,11 +77,16 @@ public static nuint CompressBlockAttached(ref MatchState state, SequenceStore st /// /// libzstd's ZSTD_compressBlock_doubleFast_dictMatchState_generic: the frame's tables first, /// the dictionary's (its own parameters, tagged indices) where they have nothing in the prefix; - /// the dictionary's indices sit just below the prefix's. + /// the dictionary's indices sit just below the prefix's. On x64, . /// private static nuint CompressBlockAttached(ref MatchState state, SequenceStore store, uint* rep, byte* source, nuint size) where THash : IMatchHash { + if (X86Base.IsSupported) + { + return CompressBlockAttachedX64(ref state, store, rep, source, size); + } + CompressionParameters parameters = state.Parameters; uint* hashLong = state.HashTable; int hashLogLong = parameters.HashLog; diff --git a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs index 0d65e7a5b..0744e1204 100644 --- a/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/DoubleFastMatchFinder.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -81,8 +82,10 @@ private struct Cold /// libzstd tests a candidate as read(safe) == read(ip) && safe == candidate, the /// safe address dummy bytes for a candidate below the window: the second test is the window test. /// Here a candidate below the window is read at the window's start instead, which is in the - /// source, and the two tests are one condition, combined with & into one chain of - /// compares: no select, no branch on the window. + /// source, and the two tests are one condition: on Arm64 combined with & into one chain + /// of compares, no select, no branch on the window; on x64, which has no conditional compare and + /// would combine them with setcc, two compares and branches that it fuses, the window's + /// taken only when the bytes match, as libzstd's. /// /// /// The values the search loop uses are locals; those it does not are in , read @@ -101,6 +104,7 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st uint endIndex = (uint)(istart - @base + (nint)size); nuint prefixLowestIndex = state.LowestPrefixIndex(endIndex); byte* ilimit = istart + size - HashReadSize; + byte* windowStart = @base + prefixLowestIndex; Cold cold = default; Expose(&cold); @@ -175,7 +179,9 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st // when the candidate is ip itself: compared with ip, it is not a test the JIT hoists and // keeps in a register. byte* repeat = ip - offset1; - if ((Read32(repeat + 1) == (uint)(ipBytes >> 8)) & (repeat != ip)) + if (X86Base.IsSupported + ? Read32(repeat + 1) == (uint)(ipBytes >> 8) && repeat != ip + : (Read32(repeat + 1) == (uint)(ipBytes >> 8)) & (repeat != ip)) { cold.Current = current; matchLength = Count(ip + 1 + 4, repeat + 1 + 4, cold.End) + 4; @@ -193,8 +199,11 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st hashLong1 = Hash8.Hash(ip1Bytes, hashLogLong); // A long match at ip. A candidate below the window is read at its start: see the remarks. - match = @base + ClampToWindow(indexLong0, prefixLowestIndex); - if ((Read64(match) == ipBytes) & (indexLong0 >= prefixLowestIndex)) + nint distanceLong0 = (nint)(indexLong0 - prefixLowestIndex); + match = X86Base.IsSupported ? InWindow(windowStart, distanceLong0) : @base + ClampToWindow(indexLong0, prefixLowestIndex); + if (X86Base.IsSupported + ? Read64(match) == ipBytes && distanceLong0 >= 0 + : (Read64(match) == ipBytes) & (indexLong0 >= prefixLowestIndex)) { cold.Current = current; matchLength = Count(ip + 8, match + 8, cold.End) + 8; @@ -205,8 +214,11 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st indexLong1 = hashLong[hashLong1]; // A short match at ip. - match = @base + ClampToWindow(indexShort0, prefixLowestIndex); - if ((Read32(match) == (uint)ipBytes) & (indexShort0 >= prefixLowestIndex)) + nint distanceShort0 = (nint)(indexShort0 - prefixLowestIndex); + match = X86Base.IsSupported ? InWindow(windowStart, distanceShort0) : @base + ClampToWindow(indexShort0, prefixLowestIndex); + if (X86Base.IsSupported + ? Read32(match) == (uint)ipBytes && distanceShort0 >= 0 + : (Read32(match) == (uint)ipBytes) & (indexShort0 >= prefixLowestIndex)) { cold.Current = current; goto SearchNextLong; diff --git a/src/Vorticity.Zstd/Internal/FastMatchFinder.cs b/src/Vorticity.Zstd/Internal/FastMatchFinder.cs index df0a016bb..f32b52ed4 100644 --- a/src/Vorticity.Zstd/Internal/FastMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/FastMatchFinder.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -161,7 +162,7 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st byte* repeat = ip2 - repOffset1; uint repValue = Read32(repeat); hashTable[hash0] = (uint)(ip0 - @base); - if ((Read32(ip2) == repValue) & (repeat != ip2)) + if (X86Base.IsSupported ? Read32(ip2) == repValue && repeat != ip2 : (Read32(ip2) == repValue) & (repeat != ip2)) { current0 = (uint)(ip0 - @base); @@ -295,9 +296,19 @@ private static nuint CompressBlock(ref MatchState state, SequenceStore st /// libzstd's ZSTD_match4Found_cmov: whether the candidate at , /// within the window, starts with the same four bytes. A candidate below the window is read at /// the window's start instead (see ), and the window test - /// joins the compare in one condition. + /// joins the compare in one condition: combined with & on Arm64, one chain of compares; + /// on x64, which would combine them with setcc, two compares and branches, which it fuses. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool MatchFound(byte* current, byte* @base, nuint matchIndex, nuint lowLimit) => - (Read32(current) == Read32(@base + ClampToWindow(matchIndex, lowLimit))) & (matchIndex >= lowLimit); + private static bool MatchFound(byte* current, byte* @base, nuint matchIndex, nuint lowLimit) + { + if (X86Base.IsSupported) + { + // The clamp as x64 takes it best: see MatchFinder.InWindow. + nint distance = (nint)(matchIndex - lowLimit); + return Read32(current) == Read32(InWindow(@base + lowLimit, distance)) && distance >= 0; + } + + return (Read32(current) == Read32(@base + ClampToWindow(matchIndex, lowLimit))) & (matchIndex >= lowLimit); + } } diff --git a/src/Vorticity.Zstd/Internal/Fse.cs b/src/Vorticity.Zstd/Internal/Fse.cs index bc320eb14..fedd66c77 100644 --- a/src/Vorticity.Zstd/Internal/Fse.cs +++ b/src/Vorticity.Zstd/Internal/Fse.cs @@ -158,6 +158,9 @@ public static int ReadNCount( ip += (bitCount + 7) >> 3; return ip; + // Inlined: as a call, it took the reader's position, count and bits by reference, which kept + // them in memory for the whole description. + [MethodImpl(MethodImplOptions.AggressiveInlining)] static void Advance(ReadOnlySpan header, int end, ref int ip, ref int bitCount, out uint bitStream) { if (ip <= end - 7 || ip + (bitCount >> 3) <= end - 4) diff --git a/src/Vorticity.Zstd/Internal/FseEncoder.cs b/src/Vorticity.Zstd/Internal/FseEncoder.cs index e1318cd2b..08812efc2 100644 --- a/src/Vorticity.Zstd/Internal/FseEncoder.cs +++ b/src/Vorticity.Zstd/Internal/FseEncoder.cs @@ -39,15 +39,6 @@ public FseCTable(int maxTableLog, int maxSymbolValue) public ushort* StateTable { get; } public FseSymbolTransform* Symbols { get; } - - /// libzstd's memcpy of a table: the parts the table uses. - public void CopyFrom(FseCTable other) - { - TableLog = other.TableLog; - MaxSymbolValue = other.MaxSymbolValue; - Unsafe.CopyBlockUnaligned(StateTable, other.StateTable, (uint)((1 << other.TableLog) * sizeof(ushort))); - Unsafe.CopyBlockUnaligned(Symbols, other.Symbols, (uint)((other.MaxSymbolValue + 1) * sizeof(FseSymbolTransform))); - } } /// An encoding state: libzstd's FSE_CState_t. diff --git a/src/Vorticity.Zstd/Internal/Histogram.cs b/src/Vorticity.Zstd/Internal/Histogram.cs index e44cec218..50a46a98a 100644 --- a/src/Vorticity.Zstd/Internal/Histogram.cs +++ b/src/Vorticity.Zstd/Internal/Histogram.cs @@ -12,6 +12,12 @@ internal static unsafe class Histogram /// , whose first + 1 counters it clears. /// On return is the largest byte present (0 for no byte). /// + /// + /// From 64 bytes, the bytes at even and odd places are counted apart, then added: a byte's count + /// waits on its last increment, which a frequent byte made a chain of, half as long so. The + /// largest count is taken four at a time as they are added: a maximum in a loop is a branch on + /// each count, which the counts make unpredictable. + /// /// The largest count. public static uint CountSimple(uint* count, ref uint maxSymbolValue, byte* source, nuint size) { @@ -24,9 +30,64 @@ public static uint CountSimple(uint* count, ref uint maxSymbolValue, byte* sourc } byte* end = source + size; - for (byte* p = source; p < end; p++) + byte* p = source; + if (size < 64) { - count[*p]++; + for (; p < end; p++) + { + count[*p]++; + } + } + else + { + // Rounded up to whole vectors, which the sum reads. + uint* odd = stackalloc uint[256]; + uint rounded = (max + 4) & ~3u; + Unsafe.InitBlockUnaligned(odd, 0, rounded * sizeof(uint)); + for (; p + 1 < end; p += 2) + { + count[p[0]]++; + odd[p[1]]++; + } + + if (p < end) + { + count[*p]++; + } + + if (Vector128.IsHardwareAccelerated) + { + Vector128 largestLanes = Vector128.Zero; + uint s = 0; + for (; s + 4 <= max + 1; s += 4) + { + Vector128 total = Vector128.Load(count + s) + Vector128.Load(odd + s); + total.Store(count + s); + largestLanes = Vector128.Max(largestLanes, total); + } + + uint largestCount = Math.Max( + Math.Max(largestLanes.GetElement(0), largestLanes.GetElement(1)), + Math.Max(largestLanes.GetElement(2), largestLanes.GetElement(3))); + for (; s <= max; s++) + { + count[s] += odd[s]; + largestCount = Math.Max(largestCount, count[s]); + } + + while (count[max] == 0) + { + max--; + } + + maxSymbolValue = max; + return largestCount; + } + + for (uint s = 0; s <= max; s++) + { + count[s] += odd[s]; + } } while (count[max] == 0) diff --git a/src/Vorticity.Zstd/Internal/Huffman.cs b/src/Vorticity.Zstd/Internal/Huffman.cs index dc010080d..80fce1bbb 100644 --- a/src/Vorticity.Zstd/Internal/Huffman.cs +++ b/src/Vorticity.Zstd/Internal/Huffman.cs @@ -457,8 +457,20 @@ private ref struct FourStreams /// /// The rounds of : a method of its own that calls nothing, so - /// that the twelve values of the four streams stay in registers, none of them living past it. + /// that the values of the four streams stay in registers, none of them living past it. /// + /// + /// + /// Each stream's next entry is read as soon as its container is shifted, as libzstd's assembly + /// does (): read where its symbol is written, after the three other streams' + /// work, the lookup started later, and the rounds took 3.0 cycles a symbol on x64 where they now + /// take 2.5, libzstd's. + /// + /// + /// The four outputs move together, a quarter apart: the second and the fourth are written at that + /// distance from the first and the third, a register less for the entries read ahead. + /// + /// [MethodImpl(MethodImplOptions.NoInlining)] private static void RunFourStreams(ref FourStreams s, ref ushort table, int shift, ref byte input, ref byte oend) { @@ -471,13 +483,13 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif ref byte ip2 = ref s.Ip2; ref byte ip3 = ref s.Ip3; ref byte op0 = ref s.Op0; - ref byte op1 = ref s.Op1; ref byte op2 = ref s.Op2; - ref byte op3 = ref s.Op3; + nint quarter = Unsafe.ByteOffset(ref s.Op0, ref s.Op1); + Debug.Assert(Unsafe.ByteOffset(ref s.Op2, ref s.Op3) == quarter && Unsafe.ByteOffset(ref s.Op1, ref s.Op2) == quarter); while (true) { - nint outputRounds = Unsafe.ByteOffset(ref op3, ref oend) / 5; + nint outputRounds = Unsafe.ByteOffset(ref Unsafe.Add(ref op2, quarter), ref oend) / 5; nint inputRounds = Unsafe.ByteOffset(ref input, ref ip0) / 7; nint rounds = Math.Min(outputRounds, inputRounds); if (rounds == 0 @@ -488,39 +500,45 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif break; } - ref byte olimit = ref Unsafe.Add(ref op3, rounds * 5); + ref byte olimit = ref Unsafe.Add(ref op0, rounds * 5); + nint entry0 = Unsafe.Add(ref table, (nint)(bits0 >> shift)); + nint entry1 = Unsafe.Add(ref table, (nint)(bits1 >> shift)); + nint entry2 = Unsafe.Add(ref table, (nint)(bits2 >> shift)); + nint entry3 = Unsafe.Add(ref table, (nint)(bits3 >> shift)); do { - Symbol(ref table, shift, ref bits0, ref op0, 0); - Symbol(ref table, shift, ref bits1, ref op1, 0); - Symbol(ref table, shift, ref bits2, ref op2, 0); - Symbol(ref table, shift, ref bits3, ref op3, 0); - Symbol(ref table, shift, ref bits0, ref op0, 1); - Symbol(ref table, shift, ref bits1, ref op1, 1); - Symbol(ref table, shift, ref bits2, ref op2, 1); - Symbol(ref table, shift, ref bits3, ref op3, 1); - Symbol(ref table, shift, ref bits0, ref op0, 2); - Symbol(ref table, shift, ref bits1, ref op1, 2); - Symbol(ref table, shift, ref bits2, ref op2, 2); - Symbol(ref table, shift, ref bits3, ref op3, 2); - Symbol(ref table, shift, ref bits0, ref op0, 3); - Symbol(ref table, shift, ref bits1, ref op1, 3); - Symbol(ref table, shift, ref bits2, ref op2, 3); - Symbol(ref table, shift, ref bits3, ref op3, 3); - Symbol(ref table, shift, ref bits0, ref op0, 4); - Symbol(ref table, shift, ref bits1, ref op1, 4); - Symbol(ref table, shift, ref bits2, ref op2, 4); - Symbol(ref table, shift, ref bits3, ref op3, 4); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 0); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 0); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 1); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter + 1); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 1); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter + 1); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 2); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter + 2); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 2); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter + 2); + Next(ref table, shift, ref bits0, ref entry0, ref op0, 3); + Next(ref table, shift, ref bits1, ref entry1, ref op0, quarter + 3); + Next(ref table, shift, ref bits2, ref entry2, ref op2, 3); + Next(ref table, shift, ref bits3, ref entry3, ref op2, quarter + 3); + Last(ref bits0, entry0, ref op0, 4); + Last(ref bits1, entry1, ref op0, quarter + 4); + Last(ref bits2, entry2, ref op2, 4); + Last(ref bits3, entry3, ref op2, quarter + 4); ip0 = ref ReloadFast(ref bits0, ref ip0); - op0 = ref Unsafe.Add(ref op0, 5); + entry0 = Unsafe.Add(ref table, (nint)(bits0 >> shift)); ip1 = ref ReloadFast(ref bits1, ref ip1); - op1 = ref Unsafe.Add(ref op1, 5); + entry1 = Unsafe.Add(ref table, (nint)(bits1 >> shift)); ip2 = ref ReloadFast(ref bits2, ref ip2); - op2 = ref Unsafe.Add(ref op2, 5); + entry2 = Unsafe.Add(ref table, (nint)(bits2 >> shift)); ip3 = ref ReloadFast(ref bits3, ref ip3); - op3 = ref Unsafe.Add(ref op3, 5); + entry3 = Unsafe.Add(ref table, (nint)(bits3 >> shift)); + op0 = ref Unsafe.Add(ref op0, 5); + op2 = ref Unsafe.Add(ref op2, 5); } - while (Unsafe.IsAddressLessThan(ref op3, ref olimit)); + while (Unsafe.IsAddressLessThan(ref op0, ref olimit)); } s.Bits0 = bits0; @@ -532,9 +550,9 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif s.Ip2 = ref ip2; s.Ip3 = ref ip3; s.Op0 = ref op0; - s.Op1 = ref op1; + s.Op1 = ref Unsafe.Add(ref op0, quarter); s.Op2 = ref op2; - s.Op3 = ref op3; + s.Op3 = ref Unsafe.Add(ref op2, quarter); } /// @@ -544,7 +562,8 @@ private static void RunFourStreams(ref FourStreams s, ref ushort table, int shif /// /// A round takes at most 5 x 11 bits from each stream, as the single-symbol rounds, and writes at /// most ten bytes to each: the rounds are capped by what the most filled quarter has left, and - /// recounted when they run out. + /// recounted when they run out. The entries are read ahead as in + /// (1.63 cycles a symbol before, 1.51 after; libzstd's assembly, 1.38). /// [MethodImpl(MethodImplOptions.NoInlining)] private static void RunFourStreamsDouble( @@ -577,32 +596,40 @@ private static void RunFourStreamsDouble( break; } + ulong entry0 = Unsafe.Add(ref table, (nint)(bits0 >> (64 - DoubleLog))); + ulong entry1 = Unsafe.Add(ref table, (nint)(bits1 >> (64 - DoubleLog))); + ulong entry2 = Unsafe.Add(ref table, (nint)(bits2 >> (64 - DoubleLog))); + ulong entry3 = Unsafe.Add(ref table, (nint)(bits3 >> (64 - DoubleLog))); do { - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); - op0 = ref Pair(ref table, ref bits0, ref op0); - op1 = ref Pair(ref table, ref bits1, ref op1); - op2 = ref Pair(ref table, ref bits2, ref op2); - op3 = ref Pair(ref table, ref bits3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairNext(ref table, ref bits0, ref entry0, ref op0); + op1 = ref PairNext(ref table, ref bits1, ref entry1, ref op1); + op2 = ref PairNext(ref table, ref bits2, ref entry2, ref op2); + op3 = ref PairNext(ref table, ref bits3, ref entry3, ref op3); + op0 = ref PairLast(ref bits0, entry0, ref op0); + op1 = ref PairLast(ref bits1, entry1, ref op1); + op2 = ref PairLast(ref bits2, entry2, ref op2); + op3 = ref PairLast(ref bits3, entry3, ref op3); ip0 = ref ReloadFast(ref bits0, ref ip0); + entry0 = Unsafe.Add(ref table, (nint)(bits0 >> (64 - DoubleLog))); ip1 = ref ReloadFast(ref bits1, ref ip1); + entry1 = Unsafe.Add(ref table, (nint)(bits1 >> (64 - DoubleLog))); ip2 = ref ReloadFast(ref bits2, ref ip2); + entry2 = Unsafe.Add(ref table, (nint)(bits2 >> (64 - DoubleLog))); ip3 = ref ReloadFast(ref bits3, ref ip3); + entry3 = Unsafe.Add(ref table, (nint)(bits3 >> (64 - DoubleLog))); } while (--rounds > 0); } @@ -621,11 +648,24 @@ private static void RunFourStreamsDouble( s.Op3 = ref op3; } - /// One lookup of the double-symbol table: both bytes written, the output moved by the count. + /// + /// One lookup of the double-symbol table, its entry read ahead as : both bytes + /// written, the output moved by the count, the next entry read. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref byte Pair(scoped ref uint table, scoped ref ulong bits, ref byte op) + private static ref byte PairNext(scoped ref uint table, scoped ref ulong bits, scoped ref ulong entry, ref byte op) + { + Unsafe.WriteUnaligned(ref op, (ushort)(entry >> 8)); + bits <<= (int)entry; + ref byte next = ref Unsafe.Add(ref op, (nint)(entry >> 30)); + entry = Unsafe.Add(ref table, (nint)(bits >> (64 - DoubleLog))); + return ref next; + } + + /// The last lookup of a round, as : the stream reloads first. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref byte PairLast(scoped ref ulong bits, ulong entry, ref byte op) { - ulong entry = Unsafe.Add(ref table, (nint)(bits >> (64 - DoubleLog))); Unsafe.WriteUnaligned(ref op, (ushort)(entry >> 8)); bits <<= (int)entry; return ref Unsafe.Add(ref op, (nint)(entry >> 30)); @@ -745,11 +785,20 @@ private static ulong InitFastStream(ref byte ip) return (Unsafe.ReadUnaligned(ref ip) | 1) << consumed; } + /// The symbol of an entry read ahead: the bits it takes skipped, its byte written, the next entry read. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Next(ref ushort table, int shift, ref ulong bits, ref nint entry, ref byte op, nint k) + { + bits <<= (int)entry; + Unsafe.Add(ref op, k) = (byte)(entry >> 8); + entry = Unsafe.Add(ref table, (nint)(bits >> shift)); + } + + /// The last symbol of a round: no entry read ahead, the stream reloads first. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Symbol(ref ushort table, int shift, ref ulong bits, ref byte op, int k) + private static void Last(ref ulong bits, nint entry, ref byte op, nint k) { - int entry = Unsafe.Add(ref table, (nint)(bits >> shift)); - bits <<= entry; + bits <<= (int)entry; Unsafe.Add(ref op, k) = (byte)(entry >> 8); } @@ -767,21 +816,9 @@ private static ref byte ReloadFast(scoped ref ulong bits, ref byte ip) } /// Decodes one symbol per entry until is full. - private void DecodeStream(ref BackwardBitReader bits, Span output) - { - ushort[] entries = Entries; - int tableLog = TableLog; - for (int i = 0; i < output.Length; i++) - { - // A reload leaves at least 57 bits while the stream has them; a code takes at most 12. - if (bits.BitsConsumed > 64 - MaxTableLog) - { - bits.Reload(); - } - - ushort entry = entries[bits.PeekBitsFast(tableLog)]; - output[i] = (byte)(entry >> 8); - bits.SkipBits(entry & 0xFF); - } - } + private void DecodeStream(ref BackwardBitReader bits, Span output) => + // A reload leaves at least 57 bits while the stream has them; a code takes at most 12. An + // entry index is below 2^TableLog: no bounds to check. + bits.DecodeHuffmanSymbols( + ref MemoryMarshal.GetArrayDataReference(Entries), TableLog, ref MemoryMarshal.GetReference(output), output.Length); } diff --git a/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs b/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs index 6225499a5..6e1fc7b97 100644 --- a/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs +++ b/src/Vorticity.Zstd/Internal/HuffmanEncoder.cs @@ -38,13 +38,6 @@ public HuffmanCTable() public int MaxSymbolValue; - public void CopyFrom(HuffmanCTable other) - { - TableLog = other.TableLog; - MaxSymbolValue = other.MaxSymbolValue; - Unsafe.CopyBlockUnaligned(Elements, other.Elements, HuffmanTable.MaxSymbols * sizeof(ulong)); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int NbBits(ulong element) => (int)(element & 0xFF); } @@ -58,7 +51,7 @@ internal sealed unsafe class HuffmanWorkspace /// public const int StreamCapacity = (((FrameFormat.MaxBlockSize / 4) * HuffmanTable.MaxTableLog) / 8) + 64; - /// libzstd's table->CTable: the new tree. + /// The trees the search of the optimal depth tries; the new tree is built into a block state's table. public readonly HuffmanCTable Table = new(); /// The encoding table of the tree's weights. @@ -123,30 +116,51 @@ private static uint BucketIndex(uint count) => /// libzstd's HUF_sort: the symbols by decreasing count, equal counts by increasing symbol /// except in the buckets the quicksort orders. /// + /// + /// The running totals and the symbols without count are in registers: in the table, each total + /// was a load of the entry just stored, and each symbol without count an increment of the same + /// entry, one store forwarded to the next load after another (clang keeps libzstd's totals in a + /// register; the JIT does not). + /// private static void Sort(Node* nodes, uint* count, uint maxSymbolValue, RankPosition* rankPosition) { uint maxSymbolValue1 = maxSymbolValue + 1; Unsafe.InitBlockUnaligned(rankPosition, 0, (uint)(sizeof(RankPosition) * RankPositionTableSize)); + uint zeros = 0; for (uint n = 0; n < maxSymbolValue1; n++) { - rankPosition[BucketIndex(count[n])].Base++; + uint c = count[n]; + if (c == 0) + { + zeros++; + } + else + { + rankPosition[BucketIndex(c)].Base++; + } } + rankPosition[0].Base = (ushort)zeros; + uint total = rankPosition[RankPositionTableSize - 1].Base; for (int n = RankPositionTableSize - 1; n > 0; n--) { - rankPosition[n - 1].Base += rankPosition[n].Base; - rankPosition[n - 1].Current = rankPosition[n - 1].Base; + total += rankPosition[n - 1].Base; + rankPosition[n - 1].Base = (ushort)total; + rankPosition[n - 1].Current = (ushort)total; } + // The symbols without count go to bucket 0's place, rankPosition[1], in order. + uint zeroPosition = rankPosition[1].Current; for (uint n = 0; n < maxSymbolValue1; n++) { uint c = count[n]; - uint r = BucketIndex(c) + 1; - uint position = rankPosition[r].Current++; + uint position = c == 0 ? zeroPosition++ : rankPosition[BucketIndex(c) + 1].Current++; nodes[position].Count = c; nodes[position].Symbol = (byte)n; } + rankPosition[1].Current = (ushort)zeroPosition; + for (int n = RankPositionDistinctCountCutoff; n < RankPositionTableSize - 1; n++) { int bucketSize = rankPosition[n].Current - rankPosition[n].Base; @@ -955,15 +969,15 @@ private static nuint CompressWithTable( /// Huffman-coded with the previous tree or a new one, whichever the heuristics prefer. /// /// - /// is nextHuf->CTable, a copy of the previous block's tree, - /// replaced by the new one when that one is used. says on entry whether - /// the previous tree may serve, and on return whether it did ( - /// otherwise). + /// is the previous block's tree. A new one is built into + /// , a table the previous state does not use, which the block's state + /// takes when the new tree is used. says on entry whether the previous + /// tree may serve, and on return whether it did ( otherwise). /// /// The size of the section after the literals header, 0 when not compressible, 1 for one repeated byte. public static nuint Compress( - byte* destination, nuint capacity, byte* source, nuint size, bool singleStream, HuffmanCTable previous, HuffmanWorkspace workspace, - ref HuffmanRepeat repeat, bool preferRepeat, bool optimalDepth, bool suspectUncompressible) + byte* destination, nuint capacity, byte* source, nuint size, bool singleStream, HuffmanCTable previous, HuffmanCTable fresh, + HuffmanWorkspace workspace, ref HuffmanRepeat repeat, bool preferRepeat, bool optimalDepth, bool suspectUncompressible) { const int SuspectIncompressibleSampleSize = 4096; const int SuspectIncompressibleSampleRatio = 10; @@ -1016,11 +1030,10 @@ public static nuint Compress( return CompressWithTable(destination, op, end, source, size, singleStream, previous, workspace); } - HuffmanCTable table = workspace.Table; uint huffLog = OptimalTableLog(LiteralsTableLog, size, maxSymbolValue, workspace, count, optimalDepth); - huffLog = BuildCTable(table, count, maxSymbolValue, huffLog); + huffLog = BuildCTable(fresh, count, maxSymbolValue, huffLog); - nuint headerSize = WriteCTable(op, capacity, table, maxSymbolValue, huffLog, workspace.Weights); + nuint headerSize = WriteCTable(op, capacity, fresh, maxSymbolValue, huffLog, workspace.Weights); if (headerSize == 0) { // libzstd's error, which its caller turns into raw literals. @@ -1031,7 +1044,7 @@ public static nuint Compress( { // The previous tree, unless the new one saves more than its description. nuint oldSize = EstimateCompressedSize(previous, count, maxSymbolValue); - nuint newSize = EstimateCompressedSize(table, count, maxSymbolValue); + nuint newSize = EstimateCompressedSize(fresh, count, maxSymbolValue); if (oldSize <= headerSize + newSize || headerSize + 12 >= size) { return CompressWithTable(destination, op, end, source, size, singleStream, previous, workspace); @@ -1045,7 +1058,6 @@ public static nuint Compress( op += headerSize; repeat = HuffmanRepeat.None; - previous.CopyFrom(table); - return CompressWithTable(destination, op, end, source, size, singleStream, table, workspace); + return CompressWithTable(destination, op, end, source, size, singleStream, fresh, workspace); } } diff --git a/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs b/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs index 92a77e869..aa910578c 100644 --- a/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/LazyMatchFinder.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -215,7 +216,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint repLength = ExtDictRepeatLength(ref state, ip, current, offset1, dictBase, dictEnd, iend); int gain2 = (int)(repLength * 3); int gain1 = (int)(((nint)matchLength * 3) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -228,7 +229,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 4); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -245,7 +246,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint repLength = ExtDictRepeatLength(ref state, ip, current, offset1, dictBase, dictEnd, iend); int gain2 = (int)(repLength * 4); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -258,7 +259,7 @@ private static nuint CompressBlockExtDict(ref MatchState state, nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 7); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -428,7 +429,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen } } } - else if ((offset1 > 0) & (Read32(ip + 1 - offset1) == Read32(ip + 1))) + else if (X86Base.IsSupported ? offset1 > 0 && Read32(ip + 1 - offset1) == Read32(ip + 1) : (offset1 > 0) & (Read32(ip + 1 - offset1) == Read32(ip + 1))) { matchLength = Count(ip + 1 + 4, ip + 1 + 4 - offset1, iend) + 4; if (TDepth.Depth == 0) @@ -470,19 +471,19 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint repLength = AttachedRepeatLength(ip, offset1, @base, prefixLowestIndex, dictBase, dictIndexDelta, dictEnd, iend); int gain2 = (int)(repLength * 3); int gain1 = (int)(((nint)matchLength * 3) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; start = ip; } } - else if ((offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) + else if (X86Base.IsSupported ? offset1 > 0 && Read32(ip) == Read32(ip - offset1) : (offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) { nuint repLength = Count(ip + 4, ip + 4 - offset1, iend) + 4; int gain2 = (int)(repLength * 3); int gain1 = (int)(((nint)matchLength * 3) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -495,7 +496,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 4); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -512,19 +513,19 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint repLength = AttachedRepeatLength(ip, offset1, @base, prefixLowestIndex, dictBase, dictIndexDelta, dictEnd, iend); int gain2 = (int)(repLength * 4); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; start = ip; } } - else if ((offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) + else if (X86Base.IsSupported ? offset1 > 0 && Read32(ip) == Read32(ip - offset1) : (offset1 > 0) & (Read32(ip) == Read32(ip - offset1))) { nuint repLength = Count(ip + 4, ip + 4 - offset1, iend) + 4; int gain2 = (int)(repLength * 4); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 1); - if ((repLength >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? repLength >= 4 && gain2 > gain1 : (repLength >= 4) & (gain2 > gain1)) { matchLength = repLength; offBase = RepeatCode1; @@ -537,7 +538,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen nuint length = TSearch.FindBestMatch(ref state, ip, iend, ref candidate); int gain2 = (int)(((nint)length * 4) - HighBit(candidate)); int gain1 = (int)(((nint)matchLength * 4) - HighBit(offBase) + 7); - if ((length >= 4) & (gain2 > gain1)) + if (X86Base.IsSupported ? length >= 4 && gain2 > gain1 : (length >= 4) & (gain2 > gain1)) { matchLength = length; offBase = candidate; @@ -570,7 +571,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen } else { - while (((start > anchor) & (start - offset > prefixLowest)) && start[-1] == (start - offset)[-1]) + while ((X86Base.IsSupported ? start > anchor && start - offset > prefixLowest : (start > anchor) & (start - offset > prefixLowest)) && start[-1] == (start - offset)[-1]) { start--; matchLength++; @@ -614,7 +615,7 @@ private static nuint CompressBlock(ref MatchState state, Sequen } else { - while (((ip <= ilimit) & (offset2 > 0)) && Read32(ip) == Read32(ip - offset2)) + while ((X86Base.IsSupported ? ip <= ilimit && offset2 > 0 : (ip <= ilimit) & (offset2 > 0)) && Read32(ip) == Read32(ip - offset2)) { matchLength = Count(ip + 4, ip + 4 - offset2, iend) + 4; (offset1, offset2) = (offset2, offset1); @@ -733,6 +734,10 @@ public static void FillHashCache(ref MatchState state, uint index, byte* limit) [MethodImpl(MethodImplOptions.NoInlining)] public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref nuint offBase) { + // The insertion first, then the search's values: computed before it, they were live across + // its loop, which x64 has not the registers for, and spilled, the loop's own with them. + nuint matchIndex = InsertAndFindFirstIndex(ref state, ip); + uint* chainTable = state.ChainTable; uint chainSize = 1u << state.Parameters.ChainLog; nuint chainMask = chainSize - 1; @@ -750,9 +755,8 @@ public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref byte* dictBase = state.DictionaryBase; byte* dictEnd = dictBase + dictLimit; - nuint matchIndex = InsertAndFindFirstIndex(ref state, ip); nuint bestLength = 4 - 1; - for (; (matchIndex >= lowLimit) & (attempts > 0); attempts--) + for (; X86Base.IsSupported ? matchIndex >= lowLimit && attempts > 0 : (matchIndex >= lowLimit) & (attempts > 0); attempts--) { if (TDictionary.Mode != ExtDictionary.Value || matchIndex >= dictLimit) { @@ -870,16 +874,25 @@ internal static uint InsertAndFindFirstIndex(ref MatchState state, byte* ip) byte* @base = state.Base; uint target = (uint)(ip - @base); uint index = state.NextToUpdate; - bool skipping = state.LazySkipping; - while (index < target) + if (state.LazySkipping) { - nuint hash = THash.Hash(Read64(@base + index), hashLog); - chainTable[index & chainMask] = hashTable[hash]; - hashTable[hash] = index; - index++; - if (skipping) + // Only the first position while skipping. + if (index < target) { - break; + nuint hash = THash.Hash(Read64(@base + index), hashLog); + chainTable[index & chainMask] = hashTable[hash]; + hashTable[hash] = index; + } + } + else + { + // The skipping test out of the loop, which then holds one value less: on x64, short of + // registers in the search it is inlined into, the tables and the mask came from memory. + for (; index < target; index++) + { + nuint hash = THash.Hash(Read64(@base + index), hashLog); + chainTable[index & chainMask] = hashTable[hash]; + hashTable[hash] = index; } } @@ -1308,9 +1321,9 @@ internal interface IRowLog /// /// /// libzstd prefetches the rows of the position eight ahead, whose hash its cache computes, and the -/// candidates before it compares them. .NET has no prefetch on arm64. The row of entries is brought -/// in by a store instead: of a 0 to its place 0, which no entry takes and nothing reads, in the line -/// the position writes once inserted. A store retires as a prefetch does, before its line arrives, +/// candidates before it compares them. So does x64 (see PrefetchRow); .NET has no prefetch on +/// Arm64. There, the row of entries is brought in by a store instead: of a 0 to its place 0, which no +/// entry takes and nothing reads, in the line the position writes once inserted. A store retires as a prefetch does, before its line arrives, /// where a load holds up the retirement of everything after it until its line is there. The row of /// tags has no such place (its first byte is the head, its others are tags, all read by the search, /// which a store still on its way would hold up): it is read, as are the candidates, summed into @@ -1372,7 +1385,7 @@ public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref MatchState* dictionary = state.Dictionary; dictionaryHash = (nuint)((Read64(ip) * state.RowHashMultiplier) >> (64 - (dictionary->RowHashLog + LazyMatchFinder.RowHashTagBits))); nuint dictionaryRow = RowOf(dictionaryHash); - read += dictionary->TagTable[dictionaryRow] + dictionary->HashTable[dictionaryRow]; + PrefetchRow(dictionary->TagTable, dictionary->HashTable, dictionaryRow, ref read, write: false); } // libzstd's ZSTD_row_update_internal: the positions before ip into their rows, then ip's @@ -1437,7 +1450,7 @@ public static nuint FindBestMatch(ref MatchState state, byte* ip, byte* end, ref candidates[count] = matchIndex; count++; - read += (TDictionary.Mode == ExtDictionary.Value && matchIndex < state.DictLimit ? state.DictionaryBase : @base)[matchIndex]; + PrefetchCandidate((TDictionary.Mode == ExtDictionary.Value && matchIndex < state.DictLimit ? state.DictionaryBase : @base) + matchIndex, ref read); if (--attempts == 0) { break; @@ -1539,7 +1552,7 @@ private static nuint SearchDictionary(ref MatchState state, byte* ip, byte* end, break; } - read += dictionaryBase[matchIndex]; + PrefetchCandidate(dictionaryBase + matchIndex, ref read); candidates[count++] = matchIndex; attempts--; } @@ -1584,8 +1597,7 @@ public static void FillHashCache(ref MatchState state, uint index, byte* limit) nuint hash = (nuint)(((Read64(@base + index) * multiplier) ^ salt) >> shift); cache[index & (LazyMatchFinder.RowHashCacheSize - 1)] = (uint)hash; nuint relativeRow = RowOf(hash); - read += state.TagTable[relativeRow]; - state.HashTable[relativeRow] = 0; + PrefetchRow(state.TagTable, state.HashTable, relativeRow, ref read, write: true); } state.Touched += read; @@ -1645,11 +1657,63 @@ private static nuint NextCachedHash( nuint next = (nuint)(((Read64(@base + index + LazyMatchFinder.RowHashCacheSize) * multiplier) ^ salt) >> shift); *slot = (uint)next; nuint nextRow = RowOf(next); - read += tagTable[nextRow]; - hashTable[nextRow] = 0; + PrefetchRow(tagTable, hashTable, nextRow, ref read, write: true); return hash; } + /// + /// Brings a row's entries and tags on their way. On x64, prefetches, as libzstd's + /// ZSTD_row_prefetch: they retire at once and wait on nothing. Without them (Arm64, see the + /// remarks), the row of tags is read, summed into , and the row of entries + /// written at its place 0 when it is the frame's own (), read otherwise. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void PrefetchRow(byte* tagTable, uint* hashTable, nuint relativeRow, ref uint read, bool write) + { + if (Sse.IsSupported) + { + Sse.Prefetch0(hashTable + relativeRow); + if (TRow.Log >= 5) + { + Sse.Prefetch0(hashTable + relativeRow + 16); + } + + Sse.Prefetch0(tagTable + relativeRow); + if (TRow.Log == 6) + { + Sse.Prefetch0(tagTable + relativeRow + 32); + } + + return; + } + + read += tagTable[relativeRow]; + if (write) + { + hashTable[relativeRow] = 0; + } + else + { + read += hashTable[relativeRow]; + } + } + + /// + /// Brings a candidate's first bytes on their way before it is compared: a prefetch on x64, as + /// libzstd's; a load summed into elsewhere. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void PrefetchCandidate(byte* candidate, ref uint read) + { + if (Sse.IsSupported) + { + Sse.Prefetch0(candidate); + return; + } + + read += *candidate; + } + /// /// The first entry of a hash's row: its bits above the tag, times the entries of a row, as one /// shift and one mask. diff --git a/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs b/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs index 6d38d2683..ad95193b9 100644 --- a/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs +++ b/src/Vorticity.Zstd/Internal/LongDistanceMatcher.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -470,7 +471,15 @@ private nuint GenerateBlock(byte* istart, nuint size, RawSequence* candidates[n].Split = split; candidates[n].Hash = hash; candidates[n].Checksum = (uint)(xxhash >> 32); - touched += table[(nuint)hash << bucketSizeLog]; + // The bucket on its way: a prefetch on x64, as libzstd's; a read kept in _touched elsewhere. + if (Sse.IsSupported) + { + Sse.Prefetch0(table + ((nuint)hash << bucketSizeLog)); + } + else + { + touched += table[(nuint)hash << bucketSizeLog]; + } } for (uint n = 0; n < splitCount; n++) diff --git a/src/Vorticity.Zstd/Internal/MatchFinder.cs b/src/Vorticity.Zstd/Internal/MatchFinder.cs index c7f9c9fce..9c4b0080a 100644 --- a/src/Vorticity.Zstd/Internal/MatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/MatchFinder.cs @@ -344,6 +344,15 @@ public static nuint ClampToWindow(nuint index, nuint low) return index - (nuint)(below & (below >> 63)); } + /// + /// x64's form of : a candidate's address from the + /// window's start and its signed distance past it, the start when the distance is negative. A + /// shift and a bit clear (andn), where the clamp takes eight instructions on x64, which has no + /// operand shifted in passing as Arm64's and; the distance's sign is then the window test. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte* InWindow(byte* windowStart, nint distance) => windowStart + (distance & ~(distance >> 63)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint Read32(byte* p) => Unsafe.ReadUnaligned(p); diff --git a/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs b/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs index dd39f0785..5b0582727 100644 --- a/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs +++ b/src/Vorticity.Zstd/Internal/OptimalMatchFinder.cs @@ -2,6 +2,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; using static Vorticity.Zstd.Internal.MatchFinder; namespace Vorticity.Zstd.Internal; @@ -1161,7 +1162,9 @@ private static uint GetAllMatches( // current > repIndex >= dictLimit, by an intentional overflow that discards 0 and -1. if (repOffset - 1 < current - dictLimit) { - if ((repIndex >= windowLow) & (ReadMinMatch(ip, minMatch) == ReadMinMatch(ip - repOffset, minMatch))) + if (X86Base.IsSupported + ? repIndex >= windowLow && ReadMinMatch(ip, minMatch) == ReadMinMatch(ip - repOffset, minMatch) + : (repIndex >= windowLow) & (ReadMinMatch(ip, minMatch) == ReadMinMatch(ip - repOffset, minMatch))) { repLength = Count(ip + minMatch, ip + minMatch - repOffset, end) + minMatch; } @@ -1195,7 +1198,7 @@ private static uint GetAllMatches( matches[count].OffBase = repCode - ll0 + 1; matches[count].Length = (uint)repLength; count++; - if ((repLength > sufficientLength) | (ip + repLength == end)) + if (X86Base.IsSupported ? repLength > sufficientLength || ip + repLength == end : (repLength > sufficientLength) | (ip + repLength == end)) { return count; } @@ -1206,7 +1209,9 @@ private static uint GetAllMatches( if ((TLength.MinLength == 3) && (bestLength < 3)) { uint matchIndex3 = InsertAndFindFirstIndexHash3(ref state, ref nextToUpdate3, ip); - if ((matchIndex3 >= matchLow) & (current - matchIndex3 < (1 << 18))) + if (X86Base.IsSupported + ? matchIndex3 >= matchLow && current - matchIndex3 < (1 << 18) + : (matchIndex3 >= matchLow) & (current - matchIndex3 < (1 << 18))) { nuint length = TDictionary.Mode != ExtDictionary.Value || matchIndex3 >= dictLimit ? Count(ip, @base + matchIndex3, end) @@ -1217,7 +1222,7 @@ private static uint GetAllMatches( matches[0].OffBase = OffsetToOffBase(current - matchIndex3); matches[0].Length = (uint)length; count = 1; - if ((length > sufficientLength) | (ip + length == end)) + if (X86Base.IsSupported ? length > sufficientLength || ip + length == end : (length > sufficientLength) | (ip + length == end)) { // The longest possible: ip is not inserted. state.NextToUpdate = current + 1; @@ -1292,7 +1297,7 @@ private static uint WalkTree( matches[count].OffBase = OffsetToOffBase((uint)(current - matchIndex)); matches[count].Length = (uint)matchLength; count++; - if ((matchLength > OptNum) | (ip + matchLength == end)) + if (X86Base.IsSupported ? matchLength > OptNum || ip + matchLength == end : (matchLength > OptNum) | (ip + matchLength == end)) { // Dropped, to keep the tree consistent; the attached dictionary is not searched. if (TDictionary.Mode == AttachedDictionary.Value) @@ -1391,7 +1396,7 @@ private static uint WalkDictionaryTree( matches[count].OffBase = OffsetToOffBase(current - matchIndex); matches[count].Length = (uint)matchLength; count++; - if ((matchLength > OptNum) | (ip + matchLength == end)) + if (X86Base.IsSupported ? matchLength > OptNum || ip + matchLength == end : (matchLength > OptNum) | (ip + matchLength == end)) { break; } diff --git a/src/Vorticity.Zstd/Internal/SequenceEncoder.cs b/src/Vorticity.Zstd/Internal/SequenceEncoder.cs index d1493ff8f..4ad304ae8 100644 --- a/src/Vorticity.Zstd/Internal/SequenceEncoder.cs +++ b/src/Vorticity.Zstd/Internal/SequenceEncoder.cs @@ -3,6 +3,8 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace Vorticity.Zstd.Internal; @@ -32,21 +34,41 @@ internal enum SymbolEncodingType /// libzstd's ZSTD_compressedBlockState_t: what a block hands the next, once it is emitted /// compressed: its entropy tables, their repeat modes, and the repeat offsets. /// +/// +/// The tables are references: a table that carries over from the previous state, a predefined one or +/// a dictionary's is that table, where libzstd copies it into the state (the previous tree into each +/// block's state, each table repeated or predefined: 2 to 5 KiB a block, the largest cost of a small +/// frame after its match finding). A new table is built into one the previous state does not use +/// (). +/// internal sealed unsafe class BlockState { private readonly uint[] _rep = GC.AllocateArray(SequenceEncoder.RepeatOffsetCount, pinned: true); + // The tables this state owns, which this state or the next builds its new tables into. + private readonly HuffmanCTable _huffman = new(); + private readonly FseCTable _literalLengths = new(SequenceEncoder.LiteralLengthFseLog, SequenceCodes.MaxLiteralLength); + private readonly FseCTable _offsets = new(SequenceEncoder.OffsetFseLog, SequenceCodes.MaxOffset); + private readonly FseCTable _matchLengths = new(SequenceEncoder.MatchLengthFseLog, SequenceCodes.MaxMatchLength); + public BlockState() { Rep = (uint*)Unsafe.AsPointer(ref _rep[0]); + Huffman = _huffman; + LiteralLengths = _literalLengths; + Offsets = _offsets; + MatchLengths = _matchLengths; Reset(); } - public readonly HuffmanCTable Huffman = new(); + /// The tree in use: a table this state or the previous one owns, or a dictionary's. + public HuffmanCTable Huffman; public HuffmanRepeat HuffmanRepeat; - public readonly FseCTable LiteralLengths = new(SequenceEncoder.LiteralLengthFseLog, SequenceCodes.MaxLiteralLength); - public readonly FseCTable Offsets = new(SequenceEncoder.OffsetFseLog, SequenceCodes.MaxOffset); - public readonly FseCTable MatchLengths = new(SequenceEncoder.MatchLengthFseLog, SequenceCodes.MaxMatchLength); + + /// The tables in use, as , or predefined ones. + public FseCTable LiteralLengths; + public FseCTable Offsets; + public FseCTable MatchLengths; public FseRepeat LiteralLengthRepeat; public FseRepeat OffsetRepeat; public FseRepeat MatchLengthRepeat; @@ -54,6 +76,27 @@ public BlockState() /// The three repeat offsets, pinned: the match finders update them in place. public uint* Rep { get; } + /// + /// The table this state builds a new tree into: its own, unless uses + /// it, then the previous state's own, which that state does not use then. + /// + /// + /// Only the previous state's tables must outlive a block, which may still be emitted raw; this + /// state's are replaced, and neither is ever a dictionary's or a predefined table. + /// + public HuffmanCTable FreshHuffman(BlockState previous) => previous.Huffman != _huffman ? _huffman : previous._huffman; + + /// As , for the literal lengths' table. + public FseCTable FreshLiteralLengths(BlockState previous) => + previous.LiteralLengths != _literalLengths ? _literalLengths : previous._literalLengths; + + /// As , for the offsets' table. + public FseCTable FreshOffsets(BlockState previous) => previous.Offsets != _offsets ? _offsets : previous._offsets; + + /// As , for the match lengths' table. + public FseCTable FreshMatchLengths(BlockState previous) => + previous.MatchLengths != _matchLengths ? _matchLengths : previous._matchLengths; + /// libzstd's ZSTD_reset_compressedBlockState: the state every frame starts from. public void Reset() { @@ -69,7 +112,7 @@ public void Reset() /// A copy of : its tables, their repeat modes, its repeat offsets. public void CopyFrom(BlockState other) { - Huffman.CopyFrom(other.Huffman); + Huffman = other.Huffman; HuffmanRepeat = other.HuffmanRepeat; CopySequenceTablesFrom(other); Rep[0] = other.Rep[0]; @@ -80,9 +123,9 @@ public void CopyFrom(BlockState other) /// libzstd's copy of prevEntropy->fse into nextEntropy->fse, when a block has no sequence. public void CopySequenceTablesFrom(BlockState other) { - LiteralLengths.CopyFrom(other.LiteralLengths); - Offsets.CopyFrom(other.Offsets); - MatchLengths.CopyFrom(other.MatchLengths); + LiteralLengths = other.LiteralLengths; + Offsets = other.Offsets; + MatchLengths = other.MatchLengths; LiteralLengthRepeat = other.LiteralLengthRepeat; OffsetRepeat = other.OffsetRepeat; MatchLengthRepeat = other.MatchLengthRepeat; @@ -155,15 +198,35 @@ private static FseCTable BuildDefault(ReadOnlySpan norm, int maxSymbolVal ]; /// The largest symbol counted, and the largest count: what a histogram of codes returns. - private static uint LargestCount(uint* count, ref uint max) + /// + /// The largest symbol is searched in a local: through the reference, each step down was a + /// decrement in memory and a load of it, one waiting on the other. The largest count four at a + /// time: a maximum in a loop is a branch on each count, which the counts make unpredictable, + /// where a vector maximum has none. + /// + private static uint LargestCount(uint* count, ref uint largestSymbol) { + uint max = largestSymbol; while (count[max] == 0) { max--; } + largestSymbol = max; uint largest = 0; - for (uint s = 0; s <= max; s++) + uint s = 0; + if (Vector128.IsHardwareAccelerated && max >= 3) + { + Vector128 best = Vector128.Load(count); + for (s = 4; s + 3 <= max; s += 4) + { + best = Vector128.Max(best, Vector128.Load(count + s)); + } + + largest = Math.Max(Math.Max(best.GetElement(0), best.GetElement(1)), Math.Max(best.GetElement(2), best.GetElement(3))); + } + + for (; s <= max; s++) { largest = Math.Max(largest, count[s]); } @@ -320,26 +383,28 @@ public static SymbolEncodingType SelectEncodingType( /// libzstd's ZSTD_buildCTable: the table of a code in its mode, and its description when /// it has one. A new table leaves out the last sequence's symbol once, which the state that /// starts the bitstream encodes for free; libzstd takes it off its histogram, which it counts - /// again for its estimates, where it is put back here. + /// again for its estimates, where it is put back here. A table repeated or predefined is that + /// table, a new one is built into : is set to it. /// /// The size of the description. public static nuint BuildCTable( - byte* destination, FseCTable next, uint fseLog, SymbolEncodingType type, uint* count, uint max, uint firstCode, uint lastCode, - nuint sequenceCount, FseCTable defaultTable, FseCTable previous) + byte* destination, ref FseCTable next, FseCTable fresh, uint fseLog, SymbolEncodingType type, uint* count, uint max, uint firstCode, + uint lastCode, nuint sequenceCount, FseCTable defaultTable, FseCTable previous) { switch (type) { case SymbolEncodingType.Rle: - FseEncoder.BuildCTableRle(next, (byte)max); + FseEncoder.BuildCTableRle(fresh, (byte)max); + next = fresh; *destination = (byte)firstCode; return 1; case SymbolEncodingType.Repeat: - next.CopyFrom(previous); + next = previous; return 0; case SymbolEncodingType.Basic: - next.CopyFrom(defaultTable); + next = defaultTable; return 0; default: @@ -353,7 +418,8 @@ public static nuint BuildCTable( FseEncoder.NormalizeCount(normalized, tableLog, count, total, max, UseLowProbCount(total)); count[lastCode] += taken; nuint size = FseEncoder.WriteNCount(destination, normalized, max, tableLog); - FseEncoder.BuildCTable(next, normalized, max, tableLog); + FseEncoder.BuildCTable(fresh, normalized, max, tableLog); + next = fresh; return size; } } @@ -393,7 +459,7 @@ public static Statistics BuildStatistics(in SequenceSection store, BlockState pr ref next.LiteralLengthRepeat, llCount, max, mostFrequent, sequenceCount, LiteralLengthFseLog, previous.LiteralLengths, SequenceCodes.LiteralLengthDefaultNorm, LiteralLengthDefaultNormLog, isDefaultAllowed: true, strategy); nuint countSize = BuildCTable( - op, next.LiteralLengths, LiteralLengthFseLog, stats.LiteralLengths, llCount, max, first & 0x7F, last & 0x7F, sequenceCount, + op, ref next.LiteralLengths, next.FreshLiteralLengths(previous), LiteralLengthFseLog, stats.LiteralLengths, llCount, max, first & 0x7F, last & 0x7F, sequenceCount, DefaultLiteralLengths, previous.LiteralLengths); if (stats.LiteralLengths == SymbolEncodingType.Compressed) { @@ -414,7 +480,7 @@ public static Statistics BuildStatistics(in SequenceSection store, BlockState pr ref next.OffsetRepeat, ofCount, max, mostFrequent, sequenceCount, OffsetFseLog, previous.Offsets, SequenceCodes.OffsetDefaultNorm, OffsetDefaultNormLog, defaultAllowed, strategy); nuint countSize = BuildCTable( - op, next.Offsets, OffsetFseLog, stats.Offsets, ofCount, max, ((first >> 8) & 0x7F) - SequenceStore.OffsetCodes, ((last >> 8) & 0x7F) - SequenceStore.OffsetCodes, sequenceCount, + op, ref next.Offsets, next.FreshOffsets(previous), OffsetFseLog, stats.Offsets, ofCount, max, ((first >> 8) & 0x7F) - SequenceStore.OffsetCodes, ((last >> 8) & 0x7F) - SequenceStore.OffsetCodes, sequenceCount, DefaultOffsets, previous.Offsets); if (stats.Offsets == SymbolEncodingType.Compressed) { @@ -434,7 +500,7 @@ public static Statistics BuildStatistics(in SequenceSection store, BlockState pr ref next.MatchLengthRepeat, mlCount, max, mostFrequent, sequenceCount, MatchLengthFseLog, previous.MatchLengths, SequenceCodes.MatchLengthDefaultNorm, MatchLengthDefaultNormLog, isDefaultAllowed: true, strategy); nuint countSize = BuildCTable( - op, next.MatchLengths, MatchLengthFseLog, stats.MatchLengths, mlCount, max, ((first >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, ((last >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, + op, ref next.MatchLengths, next.FreshMatchLengths(previous), MatchLengthFseLog, stats.MatchLengths, mlCount, max, ((first >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, ((last >> 16) & 0x7F) - SequenceStore.MatchLengthCodes, sequenceCount, DefaultMatchLengths, previous.MatchLengths); if (stats.MatchLengths == SymbolEncodingType.Compressed) @@ -492,6 +558,11 @@ public static nuint EncodeSequences( return 0; } + if (X86Base.IsSupported) + { + return EncodeSequencesInPlace(destination, capacity, literalLengthTable, offsetTable, matchLengthTable, sequences, sequenceCount); + } + // The three tables in two arrays: the transforms by code at the codes' places, the states one // table after the other, each transform's next-state offset moved by its table's place. The // loop then holds two table pointers instead of six. @@ -605,6 +676,96 @@ static void Gather(FseCTable table, FseSymbolTransform* transforms, ushort* stat } } + /// + /// as x64 runs it: the three tables read in place, as libzstd reads + /// them, and the three states encoded one after the other, in the stream's order, each one's bits + /// into the container as soon as they are known. + /// + /// + /// Gathering the tables costs a copy of their 2.5 KiB of states and a pass over their transforms + /// a block, which a small block (a record of a few hundred bytes, some fifty sequences) pays in + /// full; the gathered loop saves registers, which x64 has too few of to keep the three states' + /// parts anyway (the JIT spilled them), and which its six table pointers here only take in + /// memory, read without a store. The bits are those of the gathered loop: every state's low bits + /// in the stream's order, offset, match length, literal length, then the extra bits. + /// + private static nuint EncodeSequencesInPlace( + byte* destination, nuint capacity, FseCTable literalLengthTable, FseCTable offsetTable, FseCTable matchLengthTable, + SequenceRecord* sequences, nuint sequenceCount) + { + // Each table's transforms moved down by its codes' place, which the records index them by. + FseSymbolTransform* llTransforms = literalLengthTable.Symbols; + FseSymbolTransform* ofTransforms = offsetTable.Symbols - SequenceStore.OffsetCodes; + FseSymbolTransform* mlTransforms = matchLengthTable.Symbols - SequenceStore.MatchLengthCodes; + ushort* llStates = literalLengthTable.StateTable; + ushort* ofStates = offsetTable.StateTable; + ushort* mlStates = matchLengthTable.StateTable; + + byte* ptr = destination; + byte* end = destination + capacity - sizeof(ulong); + ulong container; + nint bitPosition; + + // ---- the last sequence: its states start the stream, then its extra bits + SequenceRecord* last = sequences + sequenceCount - 1; + nint lastCodes = (nint)last->Codes; + nint mlState = InitialState(mlStates, mlTransforms, (lastCodes >> 16) & 0x7F); + nint ofState = InitialState(ofStates, ofTransforms, (lastCodes >> 8) & 0x7F); + nint llState = InitialState(llStates, llTransforms, lastCodes & 0x7F); + container = last->Extras; + bitPosition = lastCodes >> 24; + Flush(ref container, ref bitPosition, ref ptr, end); + + for (SequenceRecord* sequence = last - 1; sequence >= sequences; sequence--) + { + nint codes = (nint)sequence->Codes; + EncodeState(ref ofState, ofTransforms[(codes >> 8) & 0x7F], ofStates, ref container, ref bitPosition); + EncodeState(ref mlState, mlTransforms[(codes >> 16) & 0x7F], mlStates, ref container, ref bitPosition); + EncodeState(ref llState, llTransforms[codes & 0x7F], llStates, ref container, ref bitPosition); + nint extraBits = codes >> 24; + if (extraBits >= 64 - 7 - (LiteralLengthFseLog + MatchLengthFseLog + OffsetFseLog)) + { + Flush(ref container, ref bitPosition, ref ptr, end); + } + + ulong extras = sequence->Extras; + if (extraBits <= 56) + { + container |= extras << (int)bitPosition; + bitPosition += extraBits; + } + else + { + // The lengths' bits, then a flush, then the offset's. + nint ofBits = (nint)BitOperations.Log2(sequence->OffBase); + nint lengthBits = extraBits - ofBits; + container |= LowBits(extras, lengthBits) << (int)bitPosition; + bitPosition += lengthBits; + Flush(ref container, ref bitPosition, ref ptr, end); + container |= (extras >> (int)lengthBits) << (int)bitPosition; + bitPosition += ofBits; + } + + Flush(ref container, ref bitPosition, ref ptr, end); + } + + // ---- the final states, which the decoder starts from, then the end marker + AddBits(ref container, ref bitPosition, (ulong)mlState, matchLengthTable.TableLog); + Flush(ref container, ref bitPosition, ref ptr, end); + AddBits(ref container, ref bitPosition, (ulong)ofState, offsetTable.TableLog); + Flush(ref container, ref bitPosition, ref ptr, end); + AddBits(ref container, ref bitPosition, (ulong)llState, literalLengthTable.TableLog); + Flush(ref container, ref bitPosition, ref ptr, end); + AddBits(ref container, ref bitPosition, 1, 1); + Flush(ref container, ref bitPosition, ref ptr, end); + if (ptr >= end) + { + return 0; + } + + return (nuint)(ptr - destination) + (bitPosition > 0 ? 1u : 0u); + } + /// libzstd's FSE_initCState2: see . [MethodImpl(MethodImplOptions.AggressiveInlining)] private static nint InitialState(ushort* states, FseSymbolTransform* symbols, nint symbol) @@ -615,6 +776,22 @@ private static nint InitialState(ushort* states, FseSymbolTransform* symbols, ni return states[(nint)(value >> (int)nbBitsOut) + transform.DeltaFindState]; } + /// + /// libzstd's FSE_encodeSymbol on a state of the gathered tables: its low bits into the + /// container, then its successor. As in the loop that gathers the three: the high part of the + /// state indexes the successor, the state less the high part shifted back is what it outputs. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EncodeState(ref nint state, FseSymbolTransform transform, ushort* states, ref ulong container, ref nint bitPosition) + { + nint nb = (state + (nint)transform.DeltaNbBits) >> 16; + nint high = state >> (int)nb; + ulong low = (ulong)(state - (high << (int)(nb + 64))); + container |= low << (int)bitPosition; + bitPosition += nb; + state = states[high + transform.DeltaFindState]; + } + /// The low bits (0 to 63) of : a shift and a bit clear. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static ulong LowBits(ulong value, nint count) => value & ~(ulong.MaxValue << (int)count); diff --git a/src/Vorticity.Zstd/Internal/SequenceStore.cs b/src/Vorticity.Zstd/Internal/SequenceStore.cs index a1492b15c..ae68d10a4 100644 --- a/src/Vorticity.Zstd/Internal/SequenceStore.cs +++ b/src/Vorticity.Zstd/Internal/SequenceStore.cs @@ -4,13 +4,15 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace Vorticity.Zstd.Internal; /// /// One sequence of a block, as its encoding wants it: libzstd's SeqDef and what /// ZSTD_seqToCodes derives from it, computed when the match finder stores the sequence (see -/// ). +/// ), or on x64 +/// once the block's sequences are found (). /// [StructLayout(LayoutKind.Sequential)] internal struct SequenceRecord @@ -35,8 +37,10 @@ internal struct SequenceRecord /// /// /// -/// The codes are computed as each sequence is stored, rather than in passes of their own as libzstd -/// does: the match finders wait on their loads and mispredictions, and have the issue slots to spare. +/// On Arm64 the codes are computed as each sequence is stored, rather than in passes of their own as +/// libzstd does: the match finders wait on their loads and mispredictions, and have the issue slots to +/// spare. On x64 they do not have the registers: the codes are computed in one pass once the block's +/// sequences are found (see ). /// Computed from the full lengths, a length over 16 bits gets the largest code and its low 16 bits /// as extra bits, which is what libzstd's long-length fix-up gives. /// @@ -139,23 +143,87 @@ public static void Store( StoreOnly(ref sequence, counts, litLength, offBase, matchLength); } + /// + /// Whether the match finders store a sequence as its two lengths and its offset alone, as libzstd + /// does, its codes computed once the block is found (): on x64. Computed + /// as each sequence is stored, the codes take some forty instructions and a dozen values the + /// match finders have no registers left for on x64, which spill them; in a pass of their own, + /// they take a loop that holds them all. Arm64, with its 31 registers, computes them as it stores. + /// + public static bool DefersCodes => X86Base.IsSupported; + + /// + /// libzstd's ZSTD_storeSeqOnly: with its codes (), or, where + /// , the lengths in the place of the extra bits and the offset in its + /// own, to complete them. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint litLength, uint offBase, nuint matchLength) + { + if (DefersCodes) + { + sequence->Extras = (ulong)litLength | ((ulong)matchLength << 32); + sequence->OffBase = offBase; + sequence++; + return; + } + + StoreCoded(ref sequence, counts, litLength, offBase, matchLength); + } + + /// + /// The codes of the sequences stored since the store was reset, where : + /// each record completed from its lengths and offset, its codes counted. Called once the block's + /// sequences are found, before anything reads them. + /// + public void ComputeCodes() + { + if (!DefersCodes) + { + return; + } + + uint* counts = Counts; + SequenceRecord* record = SequencesStart; + SequenceRecord* end = Sequences; + while (record < end) + { + ulong lengths = record->Extras; + StoreCoded(ref record, counts, (nuint)(uint)lengths, record->OffBase, (nuint)(lengths >> 32)); + } + } + /// /// libzstd's ZSTD_storeSeqOnly, and its codes: those of the two lengths from tables below 64 /// and 128, from their logarithms above (libzstd's ZSTD_LLcode, ZSTD_MLcode), the /// offset's from its logarithm; the extra bits gathered in the stream's order; the codes counted. /// + /// + /// On x64 an entry is read as its two bytes, the code and the bits (an entry split took four + /// instructions), the extra bits are kept with bzhi (a mask took three), and the offset's + /// logarithm, of a value never 0, without the guard against 0. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint litLength, uint offBase, nuint matchLength) + public static void StoreCoded(ref SequenceRecord* sequence, uint* counts, nuint litLength, uint offBase, nuint matchLength) { - Debug.Assert(matchLength >= 3 && litLength <= FrameFormat.MaxBlockSize); + Debug.Assert(matchLength >= 3 && litLength <= FrameFormat.MaxBlockSize && offBase != 0); nuint matchLengthBase = matchLength - 3; nuint llCode; nuint llBits; if (litLength < 64) { - nuint entry = ((ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(LiteralLengthCodeTable)))[litLength]; - llCode = entry & 0x3F; - llBits = entry >> 8; + ushort* table = (ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(LiteralLengthCodeTable)); + if (X86Base.IsSupported) + { + llCode = ((byte*)(table + litLength))[0]; + llBits = ((byte*)(table + litLength))[1]; + } + else + { + nuint entry = table[litLength]; + llCode = entry & 0x3F; + llBits = entry >> 8; + } } else { @@ -167,9 +235,18 @@ public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint l nuint mlBits; if (matchLengthBase < 128) { - nuint entry = ((ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(MatchLengthCodeTable)))[matchLengthBase]; - mlCode = entry & 0x3F; - mlBits = entry >> 8; + ushort* table = (ushort*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(MatchLengthCodeTable)); + if (X86Base.IsSupported) + { + mlCode = ((byte*)(table + matchLengthBase))[0]; + mlBits = ((byte*)(table + matchLengthBase))[1]; + } + else + { + nuint entry = table[matchLengthBase]; + mlCode = entry & 0x3F; + mlBits = entry >> 8; + } } else { @@ -177,9 +254,11 @@ public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint l mlCode = mlBits + 36; } - nuint ofCode = (nuint)(uint)BitOperations.Log2(offBase); - ulong extras = ((ulong)litLength & ~(ulong.MaxValue << (int)llBits)) - | (((ulong)matchLengthBase & ~(ulong.MaxValue << (int)mlBits)) << (int)llBits) + nuint ofCode = X86Base.IsSupported + ? (nuint)(uint)(31 ^ BitOperations.LeadingZeroCount(offBase)) + : (nuint)(uint)BitOperations.Log2(offBase); + ulong extras = LowBits(litLength, llBits) + | (LowBits(matchLengthBase, mlBits) << (int)llBits) | ((ulong)(offBase ^ (1u << (int)ofCode)) << (int)(llBits + mlBits)); // The codes at their places in the common arrays, which index the counts as they are. @@ -194,6 +273,11 @@ public static void StoreOnly(ref SequenceRecord* sequence, uint* counts, nuint l sequence++; } + /// The low bits of , below 64. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong LowBits(nuint value, nuint count) => + Bmi2.X64.IsSupported ? Bmi2.X64.ZeroHighBits(value, count) : value & ~(ulong.MaxValue << (int)count); + /// The whole store, as a section the entropy coding takes. public SequenceSection Whole => new(LiteralsStart, LiteralCount, SequencesStart, SequenceCount, Counts); @@ -226,7 +310,7 @@ public static void ReplaceOffBase(SequenceRecord* record, uint offBase) nuint matchLength = MatchLengthOf(record); uint* scratch = stackalloc uint[AllCodes]; SequenceRecord* sequence = record; - StoreOnly(ref sequence, scratch, literalLength, offBase, matchLength); + StoreCoded(ref sequence, scratch, literalLength, offBase, matchLength); } /// The codes of records counted into , cleared first. diff --git a/src/Vorticity.Zstd/README.md b/src/Vorticity.Zstd/README.md index 95aa8f559..b36586fa6 100644 --- a/src/Vorticity.Zstd/README.md +++ b/src/Vorticity.Zstd/README.md @@ -227,6 +227,17 @@ dotnet publish bench/Vorticity.Zstd.Perf -c Release -o artifacts/perf artifacts/perf/Vorticity.Zstd.Perf --corpus silesia,github,github-dict --results corpus.tsv --markdown corpus.md ``` +On Windows x64, from Git Bash, the reference is built with a GNU-compatible clang such as +[llvm-mingw](https://github.com/mstorsjo/llvm-mingw): MSVC builds neither zstd's assembly Huffman +decoder nor its BMI2 dispatch, which libzstd has wherever a GNU-compatible compiler builds it. +`--core N` keeps the timed thread on one logical processor: + +```sh +CC=/bin/clang tools/native-ref/build.sh && bench/zstd-corpus.sh +dotnet publish bench/Vorticity.Zstd.Perf -c Release -o artifacts/perf +artifacts/perf/Vorticity.Zstd.Perf --core 2 --corpus silesia,github,github-dict --results corpus.tsv --markdown corpus.md +``` + ## Testing Correctness is checked against libzstd itself: @@ -256,8 +267,10 @@ dotnet test The SDK version is pinned in `global.json`. `testdata.sh` downloads the zstd 1.5.7 sources once, checks their SHA-256, builds zstd's `decodecorpus` with the system C compiler and writes the test data; it -runs on macOS and Linux. The native reference that `tools/native-ref/build.sh` builds is needed only -for the multithreaded-compression tests and for the benchmarks; that script targets macOS. +runs on macOS and Linux, and on Windows from Git Bash with a GNU-compatible clang as `CC`. The +native reference that `tools/native-ref/build.sh` builds is needed only for the +multithreaded-compression tests and for the benchmarks: dylibs on macOS, DLLs on Windows (with the +same `CC`), shared objects elsewhere. ## Repository layout diff --git a/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs b/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs index 61b581a53..92ed2745f 100644 --- a/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs +++ b/src/Vorticity.Zstd/ZstdCompressor.Benchmarks.cs @@ -142,7 +142,7 @@ internal long EntropyOnly(List blocks, int level, bool literals, bo total += (long)CompressLiterals(destination, capacity, _store.LiteralsStart, _store.LiteralCount, disabled, suspect); if (block.Compressed) { - _previous.Huffman.CopyFrom(_next.Huffman); + _previous.Huffman = _next.Huffman; _previous.HuffmanRepeat = _next.HuffmanRepeat; } } diff --git a/src/Vorticity.Zstd/ZstdCompressor.Block.cs b/src/Vorticity.Zstd/ZstdCompressor.Block.cs index 5299f9287..60d1febf3 100644 --- a/src/Vorticity.Zstd/ZstdCompressor.Block.cs +++ b/src/Vorticity.Zstd/ZstdCompressor.Block.cs @@ -392,6 +392,7 @@ private void FindSequences(byte* source, int size) // attached dictionary. nuint dictionaryLiterals = FindSequencesWithDictionary(source, size, rep); _store.StoreLastLiterals(source + size - (nint)dictionaryLiterals, dictionaryLiterals); + _store.ComputeCodes(); SkipJobSequences(size); return; } @@ -405,6 +406,7 @@ private void FindSequences(byte* source, int size) }; _store.StoreLastLiterals(source + size - (nint)lastLiterals, lastLiterals); + _store.ComputeCodes(); SkipJobSequences(size); } @@ -508,8 +510,8 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, bool singleStream = size < 256; SymbolEncodingType type = SymbolEncodingType.Compressed; - // The previous tree, assumed reused. - _next.Huffman.CopyFrom(_previous.Huffman); + // The previous tree, assumed reused: the state takes a new one only when it is used. + _next.Huffman = _previous.Huffman; _next.HuffmanRepeat = _previous.HuffmanRepeat; if (disabled || size < MinLiteralsToCompress(strategy, _previous.HuffmanRepeat)) { @@ -525,9 +527,10 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, singleStream = true; } + HuffmanCTable fresh = _next.FreshHuffman(_previous); nuint compressed = HuffmanEncoder.Compress( - destination + headerSize, capacity - headerSize, source, size, singleStream, _next.Huffman, _huffmanWorkspace, - ref repeat, preferRepeat, optimalDepth, suspectUncompressible); + destination + headerSize, capacity - headerSize, source, size, singleStream, _previous.Huffman, fresh, + _huffmanWorkspace, ref repeat, preferRepeat, optimalDepth, suspectUncompressible); if (repeat != HuffmanRepeat.None) { type = SymbolEncodingType.Repeat; @@ -536,8 +539,6 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, nuint minGain = MinGain(size, strategy); if (compressed == 0 || compressed >= size - minGain) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return WriteRawLiterals(destination, source, size); } @@ -546,14 +547,13 @@ private nuint CompressLiterals(byte* destination, nuint capacity, byte* source, // One symbol, or, for fewer than 8 literals, possibly one byte of stream: checked. if (size >= 8 || new ReadOnlySpan(source + 1, (int)size - 1).IndexOfAnyExcept(source[0]) < 0) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return WriteRleLiterals(destination, source, size); } } if (type == SymbolEncodingType.Compressed) { + _next.Huffman = fresh; _next.HuffmanRepeat = HuffmanRepeat.Check; } diff --git a/src/Vorticity.Zstd/ZstdCompressor.Split.cs b/src/Vorticity.Zstd/ZstdCompressor.Split.cs index 3796e5501..4b38dfc56 100644 --- a/src/Vorticity.Zstd/ZstdCompressor.Split.cs +++ b/src/Vorticity.Zstd/ZstdCompressor.Split.cs @@ -388,7 +388,7 @@ private nuint EstimateSize(in SequenceSection section) private SymbolEncodingType BuildLiteralStatistics(byte* literals, nuint size, out nuint descriptionSize) { descriptionSize = 0; - _next.Huffman.CopyFrom(_previous.Huffman); + _next.Huffman = _previous.Huffman; _next.HuffmanRepeat = _previous.HuffmanRepeat; nuint minimumSize = _previous.HuffmanRepeat == HuffmanRepeat.Valid ? 6u : 63u; @@ -415,16 +415,17 @@ private SymbolEncodingType BuildLiteralStatistics(byte* literals, nuint size, ou repeat = HuffmanRepeat.None; } - new Span(_next.Huffman.Elements, HuffmanTable.MaxSymbols).Clear(); - _next.Huffman.TableLog = 0; - _next.Huffman.MaxSymbolValue = 0; + HuffmanCTable fresh = _next.FreshHuffman(_previous); + new Span(fresh.Elements, HuffmanTable.MaxSymbols).Clear(); + fresh.TableLog = 0; + fresh.MaxSymbolValue = 0; uint huffLog = HuffmanEncoder.OptimalTableLog( HuffmanEncoder.LiteralsTableLog, size, maxSymbol, _huffmanWorkspace, count, _parameters.Strategy >= Strategy.BinaryTreeUltra); - huffLog = HuffmanEncoder.BuildCTable(_next.Huffman, count, maxSymbol, huffLog); + huffLog = HuffmanEncoder.BuildCTable(fresh, count, maxSymbol, huffLog); - nuint newSize = HuffmanEncoder.EstimateCompressedSize(_next.Huffman, count, maxSymbol); + nuint newSize = HuffmanEncoder.EstimateCompressedSize(fresh, count, maxSymbol); byte* description = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(_splitScratch)); - nuint headerSize = HuffmanEncoder.WriteCTable(description, MaxHuffmanHeaderSize, _next.Huffman, maxSymbol, huffLog, _huffmanWorkspace.Weights); + nuint headerSize = HuffmanEncoder.WriteCTable(description, MaxHuffmanHeaderSize, fresh, maxSymbol, huffLog, _huffmanWorkspace.Weights); if (headerSize == 0) { descriptionSize = nuint.MaxValue; @@ -436,19 +437,16 @@ private SymbolEncodingType BuildLiteralStatistics(byte* literals, nuint size, ou nuint oldSize = HuffmanEncoder.EstimateCompressedSize(_previous.Huffman, count, maxSymbol); if (oldSize < size && (oldSize <= headerSize + newSize || headerSize + 12 >= size)) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return SymbolEncodingType.Repeat; } } if (newSize + headerSize >= size) { - _next.Huffman.CopyFrom(_previous.Huffman); - _next.HuffmanRepeat = _previous.HuffmanRepeat; return SymbolEncodingType.Basic; } + _next.Huffman = fresh; _next.HuffmanRepeat = HuffmanRepeat.Check; descriptionSize = headerSize; return SymbolEncodingType.Compressed; diff --git a/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs b/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs index bcd6fa8b3..6ab226c1e 100644 --- a/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs +++ b/src/Vorticity.Zstd/ZstdDecompressor.Execution.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using Vorticity.Zstd.Internal; namespace Vorticity.Zstd; @@ -190,9 +191,16 @@ private int RunSequences( state.HistoryLength = history.Length; ref byte frameStart = ref MemoryMarshal.GetReference(destination); nint fastRoom = Math.Min((nint)blockStart + blockSizeMax, destination.Length - WildCopyOverlength); - ExecuteSequencesFast( - ref state, ref frameStart, ref Unsafe.Add(ref frameStart, Math.Max(fastRoom, Unsafe.ByteOffset(ref frameStart, ref state.Dst))), - ref Unsafe.Add(ref MemoryMarshal.GetReference(literals), literalCount)); + ref byte fastLimit = ref Unsafe.Add(ref frameStart, Math.Max(fastRoom, Unsafe.ByteOffset(ref frameStart, ref state.Dst))); + ref byte litEnd = ref Unsafe.Add(ref MemoryMarshal.GetReference(literals), literalCount); + if (X86Base.IsSupported) + { + ExecuteSequencesFastX64(ref state, ref frameStart, ref fastLimit, ref litEnd); + } + else + { + ExecuteSequencesFast(ref state, ref frameStart, ref fastLimit, ref litEnd); + } } return ExecuteSequencesCareful(ref state, literals, literalCount, destination, blockStart, blockSizeMax, history); @@ -1010,7 +1018,8 @@ private static void OverlapCopy(ref byte dst, ref byte match, nuint offset, nint Debug.Assert(offset is > 0 and < 16); Vector128 source = Unsafe.ReadUnaligned>(ref match); Vector128 indices = Unsafe.ReadUnaligned>(ref Unsafe.Add(ref MemoryMarshal.GetReference(PatternIndices), (nint)offset * 16)); - Vector128 pattern = Vector128.Shuffle(source, indices); + // The indices are all below 16: the native shuffle (tbl, pshufb) needs no masking of the others. + Vector128 pattern = Vector128.ShuffleNative(source, indices); nint step = PatternStep[(int)offset]; nint i = 0; do diff --git a/src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs b/src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs new file mode 100644 index 000000000..a43e5f66d --- /dev/null +++ b/src/Vorticity.Zstd/ZstdDecompressor.ExecutionX64.cs @@ -0,0 +1,365 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using Vorticity.Zstd.Internal; + +namespace Vorticity.Zstd; + +/// The fused sequence loop, as x64 runs it. +/// +/// +/// keeps its whole state in registers, which Arm64's 31 hold: +/// x64 has 15, and the JIT, short of them, spills the states it has just computed and reloads them +/// at the top of the next iteration, putting a store and a load on the chain every sequence waits +/// on. This loop decodes and executes the same sequences, in the same order, with the same results +/// and the same handoff to the careful path, from fewer live values: +/// +/// +/// The bitstream is one moving reference, as in , and its bound +/// is proven a batch of sequences at a time: a sequence reads at most 64 bits on the common paths, +/// so a batch of (position / 8) sequences never reloads below the start, and the sequences need no +/// test of their own. A sequence that reloads twice ends its batch. +/// What a sequence only reads or rarely writes stays in memory: the limits, the sequences left +/// outside the batch, the second and third repeat offsets. x64 compares with a memory operand at the +/// cost of a register compare. +/// Each test is a compare and a branch of its own, which x64 fuses into one operation: the +/// conditions the Arm64 loop combines with & become, without conditional compares, +/// chains of setcc. +/// +/// +public sealed partial class ZstdDecompressor +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ExecuteSequencesFastX64(ref SequenceState s, ref byte frameStart, ref byte fastLimit, ref byte litEnd) + { + ref byte stream = ref Unsafe.Add(ref s.Bits, s.Ptr); + nint bc = s.Consumed; + ulong container = s.Container; + ref SeqSymbol tables = ref s.Tables; + nint llState = s.LiteralLengthState; + nint mlState = s.MatchLengthState; + nint ofState = s.OffsetState; + nuint rep0 = s.Rep0; + ref byte dst = ref s.Dst; + ref byte lit = ref s.Lit; + + // The second and third repeat offsets stay in s, and so do the sequences before the last one + // (which the careful path takes) that are not in the batch: read and written there, in memory. + s.NbSeq--; + nint batch; + ulong llEntry; + ulong mlEntry; + ulong ofEntry; + nint litLength; + nint matchLength; + nuint offset; + + Batch: + batch = Math.Min(s.NbSeq, Unsafe.ByteOffset(ref s.Bits, ref stream) >> 3); + if (batch <= 0) + { + goto Edge; + } + + s.NbSeq -= batch; + + Loop: + llEntry = SeqEntry.Load(ref tables, llState); + mlEntry = SeqEntry.Load(ref tables, mlState); + ofEntry = SeqEntry.Load(ref tables, ofState); + if (((llEntry | mlEntry) & 0x1F00) != 0) + { + goto Extra; + } + + if ((ofEntry & 0x1E) == 0) + { + goto CommonRepeat; + } + + // ---- the common sequence with a new offset: lengths without extra bits, the states first + { + nint bcBefore = bc; + ulong containerBefore = container; + nint consumed = CommonNextStates(llEntry, mlEntry, ofEntry, bc, container, ref llState, ref mlState, ref ofState); + stream = ref Unsafe.Subtract(ref stream, (consumed >> 3) & 0x1F); + bc = consumed & 7; + container = Unsafe.ReadUnaligned(ref stream); + offset = SeqEntry.OffsetBase(ofEntry) + ReadBitsFast(containerBefore, bcBefore, (nint)ofEntry); + s.Rep2 = s.Rep1; + s.Rep1 = rep0; + rep0 = offset; + goto CommonCopy; + } + + CommonRepeat: + { + nint bcBefore = bc; + ulong containerBefore = container; + nint consumed = CommonNextStates(llEntry, mlEntry, ofEntry, bc, container, ref llState, ref mlState, ref ofState); + stream = ref Unsafe.Subtract(ref stream, (consumed >> 3) & 0x1F); + bc = consumed & 7; + container = Unsafe.ReadUnaligned(ref stream); + nuint raw = SeqEntry.OffsetBase(ofEntry) + ReadBits(containerBefore, bcBefore, SeqEntry.OffsetExtraBits(ofEntry)); + offset = ResolveOffsetX64(raw, SeqEntry.LengthBase(llEntry) == 0, ref rep0, ref s); + } + + CommonCopy: + { + matchLength = (nint)SeqEntry.LengthBase(mlEntry); + litLength = (nint)SeqEntry.LengthBase(llEntry); + ref byte matchStart = ref Unsafe.Add(ref dst, litLength); + + // A literal length without extra bits is its code, 0 to 15: one 16-byte copy. + if (Unsafe.IsAddressGreaterThan(ref Unsafe.Add(ref matchStart, matchLength), ref fastLimit)) + { + goto Execute; + } + + if (Unsafe.IsAddressGreaterThan(ref Unsafe.Add(ref lit, litLength), ref litEnd)) + { + goto Execute; + } + + if (offset > (nuint)Unsafe.ByteOffset(ref frameStart, ref matchStart)) + { + goto Execute; + } + + if ((offset < 16) | (matchLength > 16)) + { + goto Execute; + } + + Copy16(ref dst, ref lit); + Copy16(ref matchStart, ref Unsafe.Subtract(ref matchStart, offset)); + dst = ref Unsafe.Add(ref matchStart, matchLength); + lit = ref Unsafe.Add(ref lit, litLength); + if (--batch != 0) + { + goto Loop; + } + + goto Batch; + } + + // ---- lengths with extra bits + Extra: + { + nint ofBits = SeqEntry.OffsetExtraBits(ofEntry); + nint mlBits = SeqEntry.LengthExtraBits(mlEntry); + nint llBits = SeqEntry.LengthExtraBits(llEntry); + if (ofBits + mlBits + llBits >= 57 - (SequenceCodes.LiteralLengthMaxLog + SequenceCodes.MatchLengthMaxLog + SequenceCodes.OffsetMaxLog)) + { + // A reload inside the sequence, then another after it: up to 16 bytes back, which + // the batch did not count. It ends with this sequence. + s.NbSeq += batch - 1; + batch = 1; + goto General; + } + + // Every position from the entries, the states first, as the common path; one reload. + nint bcBefore = bc; + ulong containerBefore = container; + nint consumed = ExtraNextStates(llEntry, mlEntry, ofEntry, bc, container, ref llState, ref mlState, ref ofState, out nint mlAt); + stream = ref Unsafe.Subtract(ref stream, (consumed >> 3) & 0x1F); + bc = consumed & 7; + container = Unsafe.ReadUnaligned(ref stream); + nuint raw = SeqEntry.OffsetBase(ofEntry) + ReadBits(containerBefore, bcBefore, (nint)ofEntry); + matchLength = (nint)SeqEntry.LengthBase(mlEntry) + (nint)ReadBits(containerBefore, mlAt, (nint)(mlEntry >> 8)); + litLength = (nint)SeqEntry.LengthBase(llEntry) + (nint)ReadBits(containerBefore, mlAt + (nint)(mlEntry >> 8), (nint)(llEntry >> 8)); + if ((ofEntry & 0x1E) != 0) + { + offset = raw; + s.Rep2 = s.Rep1; + s.Rep1 = rep0; + rep0 = raw; + goto Execute; + } + + offset = ResolveOffsetX64(raw, SeqEntry.LengthBase(llEntry) == 0, ref rep0, ref s); + goto Execute; + } + + // ---- the stream within eight bytes of its start: one sequence at a time, with clamped reloads + Edge: + if (s.NbSeq <= 0) + { + batch = 0; + goto Stop; + } + + s.NbSeq--; + batch = 1; + llEntry = SeqEntry.Load(ref tables, llState); + mlEntry = SeqEntry.Load(ref tables, mlState); + ofEntry = SeqEntry.Load(ref tables, ofState); + + // ---- libzstd's ZSTD_decodeSequence in full + General: + { + nint ofBits = SeqEntry.OffsetExtraBits(ofEntry); + nint mlBits = SeqEntry.LengthExtraBits(mlEntry); + nint llBits = SeqEntry.LengthExtraBits(llEntry); + nint ptr = Unsafe.ByteOffset(ref s.Bits, ref stream); + nuint raw = SeqEntry.OffsetBase(ofEntry) + ReadBits(container, bc, ofBits); + bc += ofBits; + if (ofBits > 1) + { + offset = raw; + s.Rep2 = s.Rep1; + s.Rep1 = rep0; + rep0 = raw; + } + else + { + offset = ResolveOffsetX64(raw, SeqEntry.LengthBase(llEntry) == 0, ref rep0, ref s); + } + + matchLength = (nint)SeqEntry.LengthBase(mlEntry) + (nint)ReadBits(container, bc, mlBits); + bc += mlBits; + if (ofBits + mlBits + llBits >= 57 - (SequenceCodes.LiteralLengthMaxLog + SequenceCodes.MatchLengthMaxLog + SequenceCodes.OffsetMaxLog)) + { + ReloadClamped(ref s.Bits, ref ptr, ref bc, ref container); + } + + litLength = (nint)SeqEntry.LengthBase(llEntry) + (nint)ReadBits(container, bc, llBits); + bc += llBits; + NextStates(llEntry, mlEntry, ofEntry, container, ref bc, ref llState, ref mlState, ref ofState); + ReloadClamped(ref s.Bits, ref ptr, ref bc, ref container); + stream = ref Unsafe.Add(ref s.Bits, ptr); + } + + // ---- any sequence, decoded: libzstd's ZSTD_execSequence + Execute: + { + ref byte litAfter = ref Unsafe.Add(ref lit, litLength); + ref byte matchStart = ref Unsafe.Add(ref dst, litLength); + ref byte matchEnd = ref Unsafe.Add(ref matchStart, matchLength); + if (Unsafe.IsAddressGreaterThan(ref matchEnd, ref fastLimit)) + { + goto Pending; + } + + if (Unsafe.IsAddressGreaterThan(ref litAfter, ref litEnd)) + { + goto Pending; + } + + nuint prefix = (nuint)Unsafe.ByteOffset(ref frameStart, ref matchStart); + if (offset > prefix) + { + // A match that starts in the dictionary and ends there: see ExecuteSequencesFast. + nuint back = offset - prefix; + if (back > (nuint)s.HistoryLength) + { + goto Pending; + } + + if ((nint)back < matchLength) + { + goto Pending; + } + + WildCopyWide(ref dst, ref lit, litLength); + WildCopyWide(ref matchStart, ref Unsafe.Subtract(ref s.HistoryEnd, back), matchLength); + } + else + { + // The literals from their own buffer, then the match 32 bytes at a time when it is 32 + // or more back, else 16 at a time, else as a pattern: a copy of up to 32 bytes is one + // store, its loop not taken. + ref byte match = ref Unsafe.Subtract(ref matchStart, offset); + WildCopyWide(ref dst, ref lit, litLength); + if (offset >= 32) + { + WildCopyWide(ref matchStart, ref match, matchLength); + } + else if (offset >= 16) + { + WildCopy16(ref matchStart, ref match, matchLength); + } + else + { + OverlapCopy(ref matchStart, ref match, offset, matchLength); + } + } + + dst = ref matchEnd; + lit = ref litAfter; + if (--batch != 0) + { + goto Loop; + } + + goto Batch; + } + + Pending: + s.Pending = true; + s.LitLength = litLength; + s.MatchLength = matchLength; + s.Offset = offset; + + Stop: + s.Ptr = Unsafe.ByteOffset(ref s.Bits, ref stream); + s.Consumed = bc; + s.Container = container; + s.LiteralLengthState = llState; + s.MatchLengthState = mlState; + s.OffsetState = ofState; + s.Rep0 = rep0; + s.Dst = ref dst; + s.Lit = ref lit; + s.NbSeq += batch + 1; + } + + /// + /// Copies bytes 32 at a time, one block at least, between places 32 or + /// more apart: writing, and reading, up to 31 past the end, as , which it + /// is where 256-bit vectors are not accelerated. With AVX2, one load and one store a block. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void WildCopyWide(ref byte dst, ref byte src, nint length) + { + if (!Vector256.IsHardwareAccelerated) + { + WildCopy32(ref dst, ref src, length); + return; + } + + nint i = 0; + do + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dst, i), Unsafe.ReadUnaligned>(ref Unsafe.Add(ref src, i))); + i += 32; + } + while (i < length); + } + + /// + /// with the second and third repeat offsets in , + /// where the x64 loop keeps them. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static nuint ResolveOffsetX64(nuint raw, bool noLiteral, ref nuint rep0, ref SequenceState s) + { + nuint index = raw + (noLiteral ? 1u : 0u); + if (index == 0) + { + return rep0; + } + + nuint rep1 = s.Rep1; + nuint offset = index == 1 ? rep1 : index == 2 ? s.Rep2 : rep0 - 1; + offset = offset == 0 ? nuint.MaxValue : offset; + if (index != 1) + { + s.Rep2 = rep1; + } + + s.Rep1 = rep0; + rep0 = offset; + return offset; + } +} diff --git a/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs b/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs index 405682a07..0b56288c7 100644 --- a/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs +++ b/src/Vorticity.Zstd/ZstdDecompressor.Pairs.cs @@ -3,6 +3,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; using Vorticity.Zstd.Internal; namespace Vorticity.Zstd; @@ -55,8 +56,12 @@ public sealed partial class ZstdDecompressor /// The ulongs a step of the pair loop writes: A's record, then B's, each lengths then offset. private const int RecordStride = 4; - /// Whether blocks are decoded in pairs; tests turn it off to compare with one at a time. - internal bool PairsBlocks { get; set; } = true; + /// + /// Whether blocks are decoded in pairs; tests set it to compare with one at a time. Not on x64, + /// whose 15 registers cannot hold two chains: the JIT spills them, and the fused loop (see + /// ) decodes the same blocks one at a time 1.2 to 1.3 times as fast. + /// + internal bool PairsBlocks { get; set; } = !X86Base.IsSupported; /// The sequences decoded side by side so far, for the tests. internal long PairedSequences { get; private set; } diff --git a/tests/Vorticity.Tests/Scan/PathAllocationTests.cs b/tests/Vorticity.Tests/Scan/PathAllocationTests.cs index 93b7f86f2..dbcd6155a 100644 --- a/tests/Vorticity.Tests/Scan/PathAllocationTests.cs +++ b/tests/Vorticity.Tests/Scan/PathAllocationTests.cs @@ -45,6 +45,7 @@ using Vorticity.Columns; using Vorticity.Expressions; using Vorticity.File; +using Vorticity.IO; using Vorticity.Scanning; using Xunit; @@ -132,13 +133,43 @@ public sealed class PathAllocationTests private const int Runs = 10; /// - /// What the runtime's own file handle weighs on Windows beyond the platform the ceilings were - /// measured on: every axis opens one file, and every axis, the footer-only one included, reads - /// 24 bytes more there. The handle is the one allocation of an open that differs by platform: - /// Windows' SafeFileHandle carries its file type, its cached length and its thread-pool - /// binding, 80 bytes. The library allocates the same bytes on both. + /// What an open of allocates on Windows beyond the platforms the ceilings + /// were measured on, which open every file from a handle. /// - private static readonly long HandleAllowance = OperatingSystem.IsWindows() ? 24 : 0; + /// + /// + /// A file opened from a handle: 24 bytes. Windows' SafeFileHandle carries its file type, + /// its cached length and its thread-pool binding, 80 bytes; the library allocates the same bytes + /// around it. + /// + /// + /// A file the session keeps mapped, which Windows 11 24H2 reopens by its name, with no handle + /// (): at most 40 bytes. Without a handle the tail is read as + /// slices of the mapping rather than into pooled buffers: a tail read in one go, as the main + /// file's, is one owner where the other open had none, 16 bytes over a handle's open there; a + /// footer past the first read, as the map file's, costs less by name, the handle's open then + /// allocating the larger buffer. Every axis that scans the main file reopens it so once its + /// warm-ups have mapped it; the footer-only one does when an earlier test left it mapped; a file + /// the open reads whole is never mapped. So the allowance is asked after the warm-ups, of the + /// state the measured runs found. + /// + /// + private static long WindowsAllowance(string file) + { + if (!OperatingSystem.IsWindows()) + { + return 0; + } + + if (VortexSession.Default.Mappings is { } mappings && FileInode.TryGetByName(file, out FileInode identity, out long length) + && mappings.TryTake(identity, length) is { } kept) + { + kept.Release(); + return 40; + } + + return 24; + } /// Samples taken at each end of the retention check, of which the minimum counts. private const int Samples = 3; @@ -350,8 +381,9 @@ public void EveryReadPathAllocatesWithinItsCeiling() List over = []; foreach ((string axis, string file, long measured, Func> path) in Axes) { - long ceiling = measured + HandleAllowance; - long floor = Floor(path, Corpus.Path(file)); + string resolved = Corpus.Path(file); + long floor = Floor(path, resolved); + long ceiling = measured + WindowsAllowance(resolved); long headroom = ceiling - floor; report.Append(" ") .Append(axis.PadRight(32)) diff --git a/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs b/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs index 121f38c3c..159995c4a 100644 --- a/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs +++ b/tests/Vorticity.Zstd.Tests/Compression/EncoderUnitTests.cs @@ -75,6 +75,7 @@ public void Huffman_streams_decode_back(int seed) var random = new Random(seed); var workspace = new HuffmanWorkspace(); var previous = new HuffmanCTable(); + var fresh = new HuffmanCTable(); var table = new HuffmanTable(); byte[] output = new byte[1 << 17]; for (int round = 0; round < 100; round++) @@ -103,7 +104,7 @@ public void Huffman_streams_decode_back(int seed) fixed (byte* dst = destination) { compressed = HuffmanEncoder.Compress( - dst, (nuint)destination.Length, src, (nuint)size, singleStream, previous, workspace, ref repeat, + dst, (nuint)destination.Length, src, (nuint)size, singleStream, previous, fresh, workspace, ref repeat, preferRepeat: false, optimalDepth: random.Next(2) == 0, suspectUncompressible: false); } diff --git a/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs b/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs index daf1b081b..ebe9f635c 100644 --- a/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs +++ b/tests/Vorticity.Zstd.Tests/Fuzz/PairEquivalenceTests.cs @@ -37,8 +37,8 @@ public void Paired_blocks_decode_and_fail_as_blocks_one_at_a_time(int seed) if (!decoders.TryGetValue(dictionary ?? Array.Empty(), out var pair)) { pair = dictionary is null - ? (new ZstdDecompressor(), new ZstdDecompressor { PairsBlocks = false }) - : (new ZstdDecompressor(dictionary), new ZstdDecompressor(dictionary) { PairsBlocks = false }); + ? (new ZstdDecompressor { PairsBlocks = true }, new ZstdDecompressor { PairsBlocks = false }) + : (new ZstdDecompressor(dictionary) { PairsBlocks = true }, new ZstdDecompressor(dictionary) { PairsBlocks = false }); decoders.Add(dictionary ?? Array.Empty(), pair); } diff --git a/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs b/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs index cc535095d..8bf038838 100644 --- a/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs +++ b/tests/Vorticity.Zstd.Tests/Support/LibzstdMt.cs @@ -42,10 +42,10 @@ private LibzstdMt(nint library) _errorName = (delegate* unmanaged)NativeLibrary.GetExport(library, "ZSTD_getErrorName"); } - /// The library, loaded once; null when tools/native-ref/out/libzstd_mt.dylib is missing. + /// The library, loaded once; null when tools/native-ref/out/libzstd_mt (build.sh mt) is missing. public static LibzstdMt? Instance { get; } = Load(); - public static string LibraryPath => Path.Combine(RepositoryRoot(), "tools", "native-ref", "out", "libzstd_mt.dylib"); + public static string LibraryPath => Path.Combine(RepositoryRoot(), "tools", "native-ref", "out", "libzstd_mt" + (OperatingSystem.IsWindows() ? ".dll" : OperatingSystem.IsMacOS() ? ".dylib" : ".so")); private static LibzstdMt? Load() => File.Exists(LibraryPath) && NativeLibrary.TryLoad(LibraryPath, out nint library) ? new LibzstdMt(library) : null; diff --git a/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs b/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs index 6e605e621..a14b43d20 100644 --- a/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs +++ b/tests/Vorticity.Zstd.Tests/Support/LibzstdOracle.cs @@ -41,7 +41,7 @@ private LibzstdOracle(nint library) private static LibzstdOracle? Load() { - string path = Path.Combine(TestData.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref.dylib"); + string path = Path.Combine(TestData.RepositoryRoot, "tools", "native-ref", "out", "libzstd_ref" + (OperatingSystem.IsWindows() ? ".dll" : OperatingSystem.IsMacOS() ? ".dylib" : ".so")); return File.Exists(path) && NativeLibrary.TryLoad(path, out nint library) ? new LibzstdOracle(library) : null; } diff --git a/tests/scalar-pass.sh b/tests/scalar-pass.sh index 56e3258c5..03facba20 100755 --- a/tests/scalar-pass.sh +++ b/tests/scalar-pass.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Runs what CI runs with intrinsics disabled: the classes of Vorticity.Tests that tests/scalar-pass.txt -# names, which reach every scalar fallback the whole suite does, and the conformance corpus, through -# the scalar decoders. Arguments go to each dotnet test, --no-build say. +# names, which reach every scalar fallback the whole suite does, Vorticity.Zstd's suite whole, through +# its portable paths rather than x64's, and the conformance corpus, through the scalar decoders. +# Arguments go to each dotnet test, --no-build say. # # bash tests/scalar-pass.sh set -euo pipefail @@ -15,5 +16,6 @@ done < <(grep -v -E '^[[:space:]]*(#|$)' "$root/tests/scalar-pass.txt") export DOTNET_EnableHWIntrinsic=0 status=0 dotnet test --project "$root/tests/Vorticity.Tests" -c Release "${filters[@]}" "$@" || status=1 +dotnet test --project "$root/tests/Vorticity.Zstd.Tests" -c Release "$@" || status=1 dotnet test --project "$root/tests/Vorticity.Conformance" -c Release "$@" || status=1 exit $status diff --git a/tools/native-ref/build.sh b/tools/native-ref/build.sh index 28906a356..74983c823 100755 --- a/tools/native-ref/build.sh +++ b/tools/native-ref/build.sh @@ -1,16 +1,20 @@ #!/usr/bin/env bash # Builds the native reference side of the comparisons, into tools/native-ref/out/: # -# libzstd_ref.dylib zstd 1.5.7 compiled with today's compiler at -O3: the libzstd a well-built +# libzstd_ref. zstd 1.5.7 compiled with today's compiler at -O3: the libzstd a well-built # C program would ship, and the speed this project aims for. -# zd the timing harness (zd.c): compresses the reference frame with the first -# library, writes it out, and times ZSTD_decompressDCtx in each library given. -# libzstd_mt.dylib the same library with ZSTD_MULTITHREAD: zstdmt, what libzstd writes with +# zd the timing harness (zd.c, macOS only): compresses the reference frame with the +# first library, writes it out, and times ZSTD_decompressDCtx in each library given. +# libzstd_mt. the same library with ZSTD_MULTITHREAD: zstdmt, what libzstd writes with # ZSTD_c_nbWorkers >= 1, the oracle of the parallel compressor. `build.sh mt` # builds it alone. # -# The sources are the zstd 1.5.7 release tarball (zstd-source.sh). macOS: the libraries are dylibs, -# and zd times with mach_absolute_time. The test data comes from testdata.sh, which runs anywhere. +# The sources are the zstd 1.5.7 release tarball (zstd-source.sh). The libraries are dylibs on macOS, +# where zd times with mach_absolute_time; DLLs on Windows, from Git Bash with a GNU-compatible clang +# (CC=/bin/clang: MSVC builds neither the assembly nor the BMI2 dispatch libzstd ships +# with); shared objects elsewhere. On x86-64 the library takes zstd's assembly Huffman decoder, as +# libzstd built by a GNU-compatible compiler does there. The test data comes from testdata.sh, which +# runs anywhere. set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -20,19 +24,39 @@ lib="$src/lib" cc="${CC:-cc}" echo "compiler: $($cc --version | head -1)" -# libzstd_mt.dylib: compression with worker threads (zstdmt), decompression for the round trips. +pic=() +pdb= +threads=(-pthread) +case "$(uname -s)" in + Darwin) shared=(-dynamiclib); ext=dylib ;; + # Windows threads for zstdmt (lib/common/threading.c), not winpthreads, which would be one more DLL. + # A PDB beside each DLL, for profilers: debug information leaves the code as it is. + MINGW* | MSYS* | CYGWIN*) shared=(-shared); ext=dll; threads=(); pdb=1 ;; + *) shared=(-shared); ext=so; pic=(-fPIC) ;; +esac + +asm=() +if [[ "$(uname -m)" == x86_64 ]]; then + asm=("$lib/decompress/huf_decompress_amd64.S") +fi + +sources=( + "$lib"/common/*.c "$lib"/compress/*.c + "$lib/decompress/zstd_ddict.c" "$lib/decompress/zstd_decompress.c" + "$lib/decompress/zstd_decompress_block.c" "$lib/decompress/huf_decompress.c" + ${asm[@]+"${asm[@]}"} +) + +# libzstd_mt: compression with worker threads (zstdmt), decompression for the round trips. build_mt() { local objs="$out/obj-mt" rm -rf "$objs" && mkdir -p "$objs" ( cd "$objs" - $cc -O3 -DZSTD_MULTITHREAD -pthread -c -I"$lib" -I"$lib/common" \ - "$lib"/common/*.c "$lib"/compress/*.c \ - "$lib/decompress/zstd_ddict.c" "$lib/decompress/zstd_decompress.c" \ - "$lib/decompress/zstd_decompress_block.c" "$lib/decompress/huf_decompress.c" - $cc -dynamiclib -pthread -o "$out/libzstd_mt.dylib" ./*.o + $cc -O3 ${pic[@]+"${pic[@]}"} -DZSTD_MULTITHREAD ${threads[@]+"${threads[@]}"} -c -I"$lib" -I"$lib/common" "${sources[@]}" + $cc "${shared[@]}" ${threads[@]+"${threads[@]}"} -o "$out/libzstd_mt.$ext" ./*.o ) - echo "built: $out/libzstd_mt.dylib" + echo "built: $out/libzstd_mt.$ext" } if [[ "${1:-}" == mt ]]; then @@ -40,20 +64,19 @@ if [[ "${1:-}" == mt ]]; then exit 0 fi -# libzstd_ref.dylib: the reference library, with the flags of the measurement it reproduces. +# libzstd_ref: the reference library, with the flags of the measurement it reproduces. objs="$out/obj-ref" rm -rf "$objs" && mkdir -p "$objs" ( cd "$objs" - $cc -O3 -c -I"$lib" -I"$lib/common" \ - "$lib"/common/*.c "$lib"/compress/*.c \ - "$lib/decompress/zstd_ddict.c" "$lib/decompress/zstd_decompress.c" \ - "$lib/decompress/zstd_decompress_block.c" "$lib/decompress/huf_decompress.c" - $cc -dynamiclib -o "$out/libzstd_ref.dylib" ./*.o + $cc -O3 ${pic[@]+"${pic[@]}"} ${pdb:+-g -gcodeview} -c -I"$lib" -I"$lib/common" "${sources[@]}" + $cc "${shared[@]}" ${pdb:+-Wl,--pdb=$out/libzstd_ref.pdb} -o "$out/libzstd_ref.$ext" ./*.o ) -# zd: the timing harness. -$cc -O2 -o "$out/zd" "$here/zd.c" +# zd: the timing harness, on macOS. +if [[ "$ext" == dylib ]]; then + $cc -O2 -o "$out/zd" "$here/zd.c" +fi build_mt -echo "built: $out/libzstd_ref.dylib $out/zd" +echo "built: $out/libzstd_ref.$ext" diff --git a/tools/native-ref/testdata.sh b/tools/native-ref/testdata.sh index e9ecdd04c..7d1b14794 100755 --- a/tools/native-ref/testdata.sh +++ b/tools/native-ref/testdata.sh @@ -10,7 +10,8 @@ # kept, with a manifest of each original's size and SHA-256: the originals would weigh # tens of megabytes. The seeds below make them the same bytes on every machine. # -# Needs curl, tar and a C compiler (CC, or cc); runs on macOS and Linux. CI runs it before the tests, +# Needs curl, tar and a C compiler (CC, or cc); runs on macOS and Linux, and on Windows from Git Bash +# with a GNU-compatible clang as CC (llvm-mingw). CI runs it before the tests, # which fail there without the data, and skip elsewhere. set -euo pipefail