From b54aa2e3da2293fa1c6a05eaa7bdeda92db37abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 14:00:21 +0200 Subject: [PATCH 01/56] bench: the runner's cycles per round, and its file kept mapped on demand Each round line now carries the process's processor time, instructions and cycles over the round, on every thread, read with proc_pid_rusage as a user (macOS; zero elsewhere): a round whose time moves while its cycles do not was slowed by the machine, not by the code. The counters are read outside the timed span, without allocating. VORTICITY_RUNNER_WARM=1 keeps the session's default of 64 mappings instead of mapping the file anew at each round: the file already in memory, the base a comparison against a table loaded beforehand takes. On 2026-10-09 a count and sum by 10^3 keys at fourteen lanes read 14.5 ms with a new mapping a round and 7.1 kept: its 33 000 page faults cost five times more in the kernel at fourteen lanes than at one. --- bench/Vorticity.Benchmarks.Runner/Program.cs | 15 +++-- .../ProcessCost.cs | 64 +++++++++++++++++++ 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/bench/Vorticity.Benchmarks.Runner/Program.cs b/bench/Vorticity.Benchmarks.Runner/Program.cs index c2f3b895a..a18c8ba4b 100644 --- a/bench/Vorticity.Benchmarks.Runner/Program.cs +++ b/bench/Vorticity.Benchmarks.Runner/Program.cs @@ -71,11 +71,14 @@ private static async Task Main(string[] args) // A round maps its file anew, as the reference maps it at each call: the default session // would hand round two the mapping round one left, pages already mapped. The bench host // sets the same session in `Scenarios.OpenFilesCold`. A group by takes its lanes from the - // session, as the queries bench gives them. + // session, as the queries bench gives them. VORTICITY_RUNNER_WARM=1 keeps the session's + // default instead, 64 files mapped once and taken over by the next open: the file already in + // memory, the base a comparison against a table loaded beforehand takes (bench/duckdb.sh). QueryMemoryBudget? budget = grouped && GroupScenarios.BudgetBytes is long bytes ? new QueryMemoryBudget(bytes) : null; + bool warm = Environment.GetEnvironmentVariable("VORTICITY_RUNNER_WARM") == "1"; ScenarioSet.Session = VortexSession.Create(options => { - options.MappedFileCacheCount = 0; + options.MappedFileCacheCount = warm ? 64 : 0; if (grouped) { options.MaxDegreeOfParallelism = threads; @@ -88,15 +91,19 @@ private static async Task Main(string[] args) long[] delivered = new long[repeat]; long[] workMicros = new long[repeat]; long[] allocated = new long[repeat]; + ProcessCounters[] spent = new ProcessCounters[repeat]; long before = AllocatedSoFar(); for (int round = 0; round < repeat; round++) { // The action is timed from inside the process, once it is up: what the report compares // is the work, and the process start is a property of the build that the parent times - // apart. + // apart. The processor's counters are read outside the timed span, on every thread of the + // process: what the round cost the machine beside how long it took. + ProcessCounters counted = ProcessCost.ReadCounters(); long started = Stopwatch.GetTimestamp(); delivered[round] = await scenario(args[2]).ConfigureAwait(false); workMicros[round] = (long)(Stopwatch.GetElapsedTime(started).TotalMilliseconds * 1000); + spent[round] = ProcessCost.ReadCounters().Since(counted); long after = AllocatedSoFar(); allocated[round] = after - before; before = after; @@ -108,7 +115,7 @@ private static async Task Main(string[] args) { Console.WriteLine(string.Create( CultureInfo.InvariantCulture, - $"round={round} rows={delivered[round]} work_us={workMicros[round]} allocated_bytes={allocated[round]}")); + $"round={round} rows={delivered[round]} work_us={workMicros[round]} allocated_bytes={allocated[round]} cpu_us={spent[round].CpuNanoseconds / 1000} instructions={spent[round].Instructions} cycles={spent[round].Cycles}")); } } diff --git a/bench/Vorticity.Benchmarks.Scenarios/ProcessCost.cs b/bench/Vorticity.Benchmarks.Scenarios/ProcessCost.cs index f714aa3db..042010a36 100644 --- a/bench/Vorticity.Benchmarks.Scenarios/ProcessCost.cs +++ b/bench/Vorticity.Benchmarks.Scenarios/ProcessCost.cs @@ -38,6 +38,70 @@ public static unsafe (long CpuMs, long RssBytes) Read() RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? maxrss : maxrss * 1024); } + /// + /// What the process has spent so far, to be read before and after a round: its processor time in + /// nanoseconds, user and system, and on macOS the instructions it retired and the cycles it ran, + /// the counters /usr/bin/time -l prints, which a user reads without privileges. Zero where + /// the platform has no such counters. A round whose time moves while its cycles do not was slowed by + /// the machine, not by its code. + /// + /// + /// proc_pid_rusage with RUSAGE_INFO_V4: the times at bytes 16 and 24 in Mach absolute + /// units, the instructions at 248 and the cycles at 256 (sys/resource.h, checked against + /// offsetof on 2026-10-09). No allocation: a round reads it twice. + /// + public static unsafe ProcessCounters ReadCounters() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + (long cpuMs, _) = Read(); + return new ProcessCounters(cpuMs * 1_000_000, 0, 0); + } + + byte* info = stackalloc byte[RusageInfoV4Bytes]; + if (proc_pid_rusage(Environment.ProcessId, RusageInfoV4, info) != 0) + { + return default; + } + + ulong ticks = *(ulong*)(info + 16) + *(ulong*)(info + 24); + return new ProcessCounters(TicksToNanoseconds(ticks), (long)*(ulong*)(info + 248), (long)*(ulong*)(info + 256)); + } + + private const int RusageInfoV4 = 4; + + /// The bytes of struct rusage_info_v4, 296 on macOS 26, with room to spare. + private const int RusageInfoV4Bytes = 512; + + private static readonly (uint Numer, uint Denom) Timebase = ReadTimebase(); + + private static long TicksToNanoseconds(ulong ticks) => (long)(ticks * Timebase.Numer / Timebase.Denom); + + private static unsafe (uint, uint) ReadTimebase() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return (1, 1); + } + + uint* timebase = stackalloc uint[2]; + return mach_timebase_info(timebase) == 0 && timebase[1] != 0 ? (timebase[0], timebase[1]) : (1, 1); + } + [DllImport("libc", EntryPoint = "getrusage")] private static extern unsafe int getrusage(int who, byte* usage); + + [DllImport("libc", EntryPoint = "proc_pid_rusage")] + private static extern unsafe int proc_pid_rusage(int pid, int flavor, byte* buffer); + + [DllImport("libc", EntryPoint = "mach_timebase_info")] + private static extern unsafe int mach_timebase_info(uint* timebase); +} + +/// A process's processor time in nanoseconds, and the instructions and cycles it ran, so far. +public readonly record struct ProcessCounters(long CpuNanoseconds, long Instructions, long Cycles) +{ + /// What was spent between and this reading. + public ProcessCounters Since(ProcessCounters before) => + new ProcessCounters(CpuNanoseconds - before.CpuNanoseconds, Instructions - before.Instructions, Cycles - before.Cycles); } From ce4605d1b11dbeb4555bd21482e500fe8e27577a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 14:00:21 +0200 Subject: [PATCH 02/56] bench: pairs until the ratio holds, canaries and controls in runners.sh alt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Mac always runs other processes beside the bench, and a few fixed pairs could not tell a change from them. alt now runs pairs whose order alternates (ABBA) until the 95 % interval of the median ratio, a bootstrap of the pairs, is within ±3 % (three pairs at least, eight at most); it frames each scenario with two canaries of the before runner (a scan of one lane and a group by on fourteen), against the session's first, and runs a scenario again once when a canary is 5 % off or swap grew; it holds a scenario back while another process takes more than half a core, and notes those above a fifth; it runs a run again when more than one round in five passes 1.5 times its median. The table prints the interval, the pairs, the ratio of the fastest rounds (a gap of 10 % from the median marks the scenario noisy) and the ratio of the rounds' cycles; runs.tsv keeps every run, canary and control with its time of day. --warm keeps each side's file mapped. Nothing needs sudo. Two identical runners, q4 over 10^7 rows: x0.997 [0.990-1.001] at one lane and x1.006 at fourteen, three pairs each. duckdb.sh --only keeps the rows it names, by the runner's scenario without "group-", so that a change looks at DuckDB on its own rows in a minute. --- bench/README.md | 5 +- bench/duckdb.sh | 23 +++- bench/runners.sh | 299 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 261 insertions(+), 66 deletions(-) diff --git a/bench/README.md b/bench/README.md index 4bd07282f..76eec48cd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -58,8 +58,9 @@ finds. | every branch a scenario takes | `bench/profile.sh trace ` | 30 s | Processor Trace, for Instruments to open. An M4 or later, and Processor Trace allowed in System Settings → Privacy & Security → Developer Tools | | who ran, and on what, when a scenario has lanes | `bench/profile.sh cycles --threads all` | 20 s | the cycles above, on as many lanes as the report's all-core rows, plus `threads.md` from `bench/profile/threads.py`: each thread's share, how many threads ran in each millisecond, and the hottest functions of the caller's thread and of the pool's workers. The profiler records fewer of each thread's cycles as more of them run: take shares from it, and the process's processor time from the runner's `cpu_ms` | | where a group by spends its cycles | `bench/profile.sh cycles group-total-k6+core ~/.cache/vorticity/queries/spread-random-4000000.vortex 0 --threads 14` | 30 s | the cycles above for a group by of the high-cardinality bench (`group--`, the plan's switches after `+`: `GroupScenarios.cs` in the runner), over the spread files `vortex-queries --matrix small` writes; the runner prints the engine's counts, the core's with it. The bench compiles as it goes, and Instruments names no frame of a method compiled at run time | -| a change to the engine, against the last change to the same part | `bench/runners.sh keep ` once, then `bench/runners.sh alt HEAD --threads 1,14 group--[@file] …` | 30 s for three scenarios at one lane | the **Native AOT runner** of a commit published once and kept under `~/.cache/vorticity/runners/`, and two kept runners run in turns, a process each, three pairs by default, the median of the rounds after the first: the ratio of the two binaries, which a loaded machine moves less than it moves a number of milliseconds taken another day. Without a commit, `keep` publishes the working tree's tracked changes. The table prints the machine's load beside each scenario: replay a degree-14 ratio taken under a load above the cores before believing it | -| our group bys against DuckDB's | `bench/duckdb.sh [--set hc\|db\|all] [--rows N] [--threads 1,14]` | 4 min for both sets, at 1 and 14 threads, its files written | the same files, the bench's own at edition core2025.10, read by the Native AOT runner, by DuckDB's vortex extension and from a DuckDB table in memory, at the same threads: the median of the last 7 of 10 runs, each answer held to the others. `hc` is the high-cardinality table (keys of 10³ to 10⁷ values), `db` db-benchmark's group by, the queries our native aggregates cover. Needs `duckdb` and its vortex extension; informs, gates nothing | +| a change to the engine, against the last change to the same part | `bench/runners.sh keep ` once, then `bench/runners.sh alt HEAD --threads 1,14 [--warm] group--[@file] …` | 1 min for three scenarios at one lane | the **Native AOT runner** of a commit published once and kept under `~/.cache/vorticity/runners/`, and two kept runners run in turns, a process each, in pairs whose order alternates: three pairs, then more while the 95 % interval of the median ratio is wider than ±3 % (eight at most), the median of the rounds after the first: the ratio of the two binaries, which a loaded machine moves less than it moves a number of milliseconds taken another day. Two canary runs before and after each scenario, against the session's first, catch what other processes did meanwhile; a scenario framed by a canary 5 % off, or during which swap grew, runs again once. The table prints the interval, the pairs, the ratio of the fastest rounds (a gap of 10 % from the median marks the scenario noisy), the ratio of the rounds' cycles, the load and the processes that were busy. `--warm` keeps each side's file mapped (`VORTICITY_RUNNER_WARM=1`). Without a commit, `keep` publishes the working tree's tracked changes. Every run is kept with its time of day in `bench/.ab/runners-/runs.tsv` | +| our group bys against DuckDB's | `bench/duckdb.sh [--set hc\|db\|all] [--only ] [--rows N] [--threads 1,14]` | 4 min for both sets, at 1 and 14 threads, its files written; a minute for the rows `--only` names | the same files, the bench's own at edition core2025.10, read by the Native AOT runner, by DuckDB's vortex extension and from a DuckDB table in memory, at the same threads: the median of the last 7 of 10 runs, each answer held to the others. `hc` is the high-cardinality table (keys of 10³ to 10⁷ values), `db` db-benchmark's group by, the queries our native aggregates cover; `--only db-q4,total-k7` keeps the rows named by the runner's scenario without `group-`. Needs `duckdb` and its vortex extension; informs, gates nothing | +| the runner, its file kept mapped, and what a round cost | `VORTICITY_RUNNER_WARM=1 Vorticity.Benchmarks.Runner --scenario … --repeat N` | — | by default each round maps its file anew, as the reference maps it at each call; `VORTICITY_RUNNER_WARM=1` keeps the session's default of 64 mappings, the file already in memory, which is what a comparison against a table loaded beforehand needs. Each round line carries `cpu_us`, `instructions` and `cycles`, the process's on every thread, read with `proc_pid_rusage` as a user (macOS; zero elsewhere): a time that moves while the cycles do not was moved by the machine | | a ratchet | `dotnet test Vorticity.slnx -c Release` | ~1 min | the suite plus the ten allocation, count, budget and work ratchets | | a change, and only the tests it can affect | `dotnet run -c Release --project tools/testimpact -- run` (after `-- map` once) | seconds | the test classes that ran a method the working tree changed against `HEAD` (`--base REF` for another), from a map of what each class runs: every class in a process of its own with inlining off, the JIT's list of compiled methods recorded (35 s). A constant, a build file or the source generator runs everything; a new class runs; a comment or a `using` runs nothing | diff --git a/bench/duckdb.sh b/bench/duckdb.sh index 6981629d4..6c25564ac 100755 --- a/bench/duckdb.sh +++ b/bench/duckdb.sh @@ -2,12 +2,16 @@ # Our group bys against DuckDB's, on the same files: DuckDB through its vortex extension and on a native # table already in memory, our side the Native AOT runner, at the same threads. # -# bench/duckdb.sh [--set hc|db|all] [--rows N] [--db-rows N] [--threads N[,M…]] [--runner |] -# [--rounds N] [--rest S] +# bench/duckdb.sh [--set hc|db|all] [--only ] [--rows N] [--db-rows N] [--threads N[,M…]] +# [--runner |] [--rounds N] [--rest S] # # bench/duckdb.sh PR #43's table: 40M rows, 1 and 14 threads # bench/duckdb.sh --set db --db-rows 10000000 db-benchmark's group by at 10^7 rows # bench/duckdb.sh --rows 4000000 --threads 1 a quick look +# bench/duckdb.sh --set all --only db-q4,total-k7 --threads 14 +# the rows a change touches, and no others: the +# runner's scenario names without "group-" (total-k3, +# range-k6, strided-k7, db-q10…) # # THE SAME WORK ON BOTH SIDES. Each query is a group by whose result is aggregated once more: DuckDB's outer # query sums every column the group by aggregates, and the runner sums the same columns as it reads the @@ -34,6 +38,7 @@ set -uo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" duckdb="${DUCKDB:-$(command -v duckdb || echo /opt/homebrew/bin/duckdb)}" set_name="hc" +only="" rows=40000000 db_rows=10000000 threads="1,14" @@ -42,13 +47,14 @@ rounds=10 rest=0 usage() { - echo "usage: bench/duckdb.sh [--set hc|db|all] [--rows N] [--db-rows N] [--threads N[,M…]] [--runner |] [--rounds N] [--rest S]" >&2 + echo "usage: bench/duckdb.sh [--set hc|db|all] [--only ] [--rows N] [--db-rows N] [--threads N[,M…]] [--runner |] [--rounds N] [--rest S]" >&2 exit 2 } while [ $# -gt 0 ]; do case "$1" in --set) set_name="$2"; shift 2 ;; + --only) only=",$2,"; shift 2 ;; --rows) rows="$2"; shift 2 ;; --db-rows) db_rows="$2"; shift 2 ;; --threads) threads="$2"; shift 2 ;; @@ -105,6 +111,17 @@ if [ "$set_name" = "db" ] || [ "$set_name" = "all" ]; then queries+=("db q10: sum v3, count by id1:id6|group-db-q10|$db|SELECT sum(v3) + sum(c) FROM (SELECT Id1, Id2, Id3, Id4, Id5, Id6, sum(V3) AS v3, count(*) AS c FROM x GROUP BY Id1, Id2, Id3, Id4, Id5, Id6)") fi +# --only keeps the rows it names, by their scenario without "group-". +if [ -n "$only" ]; then + kept_queries=() + for entry in "${queries[@]}"; do + IFS='|' read -r _ scenario _ _ <<< "$entry" + case "$only" in *",${scenario#group-},"*) kept_queries+=("$entry") ;; esac + done + + queries=(${kept_queries[@]+"${kept_queries[@]}"}) +fi + [ ${#queries[@]} -gt 0 ] || usage IFS=',' read -r -a degrees <<< "$threads" diff --git a/bench/runners.sh b/bench/runners.sh index 6f38ebb1c..4aa4e6061 100755 --- a/bench/runners.sh +++ b/bench/runners.sh @@ -4,7 +4,8 @@ # # bench/runners.sh keep [] [--force] # bench/runners.sh list -# bench/runners.sh alt [--alternations N] [--repeat N] [--rest S] [--threads N[,M…]] [@] … +# bench/runners.sh alt [--alternations N] [--most N] [--target P] [--repeat N] [--rest S] +# [--threads N[,M…]] [--warm] [--no-canary] [@] … # # bench/runners.sh keep main # bench/runners.sh alt main HEAD --threads 1,14 group-pages-url group-users-day@visits-20000000.vortex @@ -21,18 +22,36 @@ # the same scenarios; when they do not build against it, the commit's own are, and the runner says so # in its commit.txt. A runner kept already is kept as it is, unless --force. # -# alt runs each scenario at each degree in the two runners, in turns: pairs (3 by -# default), the order of a pair alternating, each run a process of its own that runs the scenario -# times (8 by default). A run's time is the median of its rounds after the first, which pays -# the page faults and the pools' first rents; a side's, the median of its runs. A commit not kept yet is -# kept first; a directory is taken as a runner kept by hand. A scenario's file is under -# ~/.cache/vorticity/queries unless it holds a slash; without one, each family has its file, the one -# bench/Vorticity.Benchmarks.Runner/GroupScenarios.cs names (vortex-queries --matrix small and full write -# them). The table prints the machine's load before and after each scenario: on a machine whose load -# passes its cores, a degree-14 ratio is to be replayed before it is believed. With --rest S, S -# seconds of rest before every run: fourteen lanes heat a laptop, and on 2026-10-08 a group by of 10^7 -# keys run back to back read 133 ms in its first process and 165 to 245 in the next, where five -# seconds of rest before each kept every process within 5 % of the others. +# alt runs each scenario at each degree in the two runners, in turns, each run a process of its own that +# runs the scenario times (8 by default). A run's time is the median of its rounds after the +# first, which pays the page faults and the pools' first rents. A pair is a run of each side, the order +# alternating from one pair to the next (ABBA), which cancels a drift of the machine; its ratio is +# after over before. A side's time is the median of its runs, the scenario's ratio the median of its +# pairs' ratios. A commit not kept yet is kept first; a directory is taken as a runner kept by hand. A +# scenario's file is under ~/.cache/vorticity/queries unless it holds a slash; without one, each family +# has its file, the one bench/Vorticity.Benchmarks.Runner/GroupScenarios.cs names (vortex-queries +# --matrix small and full write them). +# +# THE NOISE IS OTHER PROCESSES: a Mac always runs some beside the bench. So: +# - pairs until the answer holds: pairs first (3), then one more while the 95 % interval +# of the median ratio (a bootstrap of the pairs) is wider than ± percent (3), at most +# (8). The pairs and the interval are printed: a clear answer stops at three; +# - canaries: before and after each scenario, two fixed runs of the before runner, the file kept mapped +# (a scan of one lane, group-stridedfloor-k7, and a count and sum of 10^3 keys on 14 lanes). Their +# first values of the session are the standard; a scenario framed by a canary more than 5 % off is +# run again once, and marked if it still is; +# - controls, user-level, never sudo: another process above 50 % of a core holds a scenario back (30 s +# at most), one above 20 % is noted; swap that grows during a scenario has it run again once; the +# free memory, the power source and the load are noted; +# - a run whose rounds pass 1.5 times its median more than one time in five is run again once; +# - the cross-check: the ratio of the two sides' fastest runs beside the median of the pairs' ratios, a +# gap of more than 10 % marking the scenario noisy; +# - the cycles: the runner prints the processor's cycles of each round (macOS), and the table their +# ratio beside the time's. A time that moves while the cycles do not was moved by the machine. +# --warm runs both sides with their file kept mapped (VORTICITY_RUNNER_WARM=1): the engine without the +# page faults of a new mapping, which other processes make dearer at fourteen lanes. --rest S rests S +# seconds before every run (1 is the most this machine needs). Every run, canary and control is kept, +# with its time of day, in runs.tsv under bench/.ab/runners-/. set -uo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -41,7 +60,7 @@ corpus="${VORTICITY_QUERIES_CORPUS:-$HOME/.cache/vorticity/queries}" usage() { echo "usage: bench/runners.sh keep [] [--force]" >&2 echo " bench/runners.sh list" >&2 - echo " bench/runners.sh alt [--alternations N] [--repeat N] [--rest S] [--threads N[,M…]] [@] …" >&2 + echo " bench/runners.sh alt [--alternations N] [--most N] [--target P] [--repeat N] [--rest S] [--threads N[,M…]] [--warm] [--no-canary] [@] …" >&2 exit 2 } [ $# -ge 1 ] || usage @@ -173,22 +192,30 @@ case "$1" in before="$1" after="$2" shift 2 alternations=3 + most=8 + target=3 repeat=8 rest=0 degrees="1" + canary=1 + warm=0 scenarios=() while [ $# -gt 0 ]; do case "$1" in --alternations) alternations="$2"; shift 2 ;; + --most) most="$2"; shift 2 ;; + --target) target="$2"; shift 2 ;; --repeat) repeat="$2"; shift 2 ;; --rest) rest="$2"; shift 2 ;; --threads) degrees="$2"; shift 2 ;; + --warm) warm=1; shift ;; + --no-canary) canary=0; shift ;; --*) usage ;; *) scenarios+=("$1"); shift ;; esac done - [ ${#scenarios[@]} -gt 0 ] && [ "$repeat" -ge 2 ] || usage + [ ${#scenarios[@]} -gt 0 ] && [ "$repeat" -ge 2 ] && [ "$alternations" -ge 1 ] && [ "$most" -ge "$alternations" ] || usage runner_of "$before" || exit 1 before_bin="$kept/Vorticity.Benchmarks.Runner" before_name="$(basename "$kept")" @@ -199,28 +226,128 @@ case "$1" in out="$root/bench/.ab/runners-$(date +%Y%m%d-%H%M%S)" mkdir -p "$out" tsv="$out/runs.tsv" - echo "before $before_name, after $after_name: $alternations pairs of $repeat rounds; runs kept in ${out#"$root"/}" + mode="cold" + [ "$warm" -eq 1 ] && mode="warm" + echo "before $before_name, after $after_name: $alternations to $most pairs of $repeat rounds, ±$target % at 95 %, $mode; runs kept in ${out#"$root"/}" - # One run: the median of its rounds after the first, in microseconds, and the result it gave. - run() { - local side="$1" bin="$2" name="$3" file="$4" threads="$5" log="$6" label="$7" + now() { date +%H:%M:%S; } + + # The awk functions the decisions and the table share: a median, and the 95 % interval of a + # median by a bootstrap of 2 000 draws, seeded so that the same ratios give the same interval. + stats=' + function sortn(a, n, gap, i, j, t) { + for (gap = int(n / 2); gap > 0; gap = int(gap / 2)) + for (i = gap; i < n; i++) { t = a[i]; for (j = i; j >= gap && a[j - gap] + 0 > t + 0; j -= gap) a[j] = a[j - gap]; a[j] = t } + } + function median(a, n, c, i) { + for (i = 0; i < n; i++) c[i] = a[i] + sortn(c, n) + return n % 2 ? c[int(n / 2)] : (c[n / 2 - 1] + c[n / 2]) / 2 + } + function interval(r, n, b, i, s, m) { + srand(7) + for (b = 0; b < 2000; b++) { for (i = 0; i < n; i++) s[i] = r[int(rand() * n)]; m[b] = median(s, n) } + sortn(m, 2000) + low = m[49]; high = m[1950] + }' + + # One run of a side: "\t\t\t\t\t" + # of its rounds after the first. + measure() { + local bin="$1" name="$2" file="$3" threads="$4" log="$5" keep="$6" rounds="${7:-$repeat}" [ "$rest" -gt 0 ] && sleep "$rest" - "$bin" --scenario "$name" "$file" 0 --repeat "$repeat" --threads "$threads" > "$log" 2>&1 || { - echo "the $side runner failed on $name at $threads:" >&2; tail -5 "$log" >&2; return 1; } - awk -v side="$side" -v name="$label" -v threads="$threads" ' + VORTICITY_RUNNER_WARM="$keep" "$bin" --scenario "$name" "$file" 0 --repeat "$rounds" --threads "$threads" > "$log" 2>&1 || { + echo "the runner failed on $name at $threads:" >&2; tail -5 "$log" >&2; return 1; } + awk "$stats"' /^round=/ { + delete v split($0, f, " ") for (i in f) { split(f[i], kv, "="); v[kv[1]] = kv[2] } - if (v["round"] > 0) times[n++] = v["work_us"] + if (v["round"] > 0) { times[n + 0] = v["work_us"]; cycles[n + 0] = ("cycles" in v) ? v["cycles"] : 0; n++ } rows = v["rows"] } END { - for (i = 0; i < n; i++) for (j = i + 1; j < n; j++) if (times[j] + 0 < times[i] + 0) { t = times[i]; times[i] = times[j]; times[j] = t } - median = n % 2 ? times[int(n / 2)] : (times[n / 2 - 1] + times[n / 2]) / 2 - printf "%s\t%s\t%s\t%.0f\t%s\n", name, threads, side, median, rows + m = median(times, n); c = median(cycles, n); fastest = times[0]; past = 0 + for (i = 0; i < n; i++) { if (times[i] + 0 < fastest + 0) fastest = times[i]; if (times[i] > 1.5 * m) past++ } + printf "%.0f\t%s\t%d\t%d\t%.0f\t%s\n", m, rows, n, past, c, fastest }' "$log" } + # A side's run in a pair, recorded; run again once when more than one round in five passes 1.5 + # times its median. Prints the run's median. + run() { + local side="$1" bin="$2" name="$3" file="$4" threads="$5" log="$6" label="$7" pair="$8" result again=0 rounds past + result="$(measure "$bin" "$name" "$file" "$threads" "$log" "$warm")" || return 1 + IFS=$'\t' read -r _ _ rounds past _ _ <<< "$result" + if [ $((past * 5)) -gt "$rounds" ]; then + again=1 + result="$(measure "$bin" "$name" "$file" "$threads" "$log" "$warm")" || return 1 + fi + + printf 'run\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$(now)" "$label" "$threads" "$attempt" "$side" "$pair" "$again" "$result" >> "$tsv" + echo "${result%%$'\t'*}" + } + + # The canaries, fixed runs of the before runner with their files kept mapped: a scan of one lane + # and a count and sum of 10^3 keys on 14 lanes. Their first values are the session's standard; + # fails when either is more than 5 % off it. + # Each takes ten rounds of 7 to 11 ms: shorter, it would move more than the 5 % it judges. + canary_scan="$corpus/spread-strided-20000000.vortex" + [ -e "$canary_scan" ] || canary_scan="$corpus/spread-strided-4000000.vortex" + canary_lanes="$corpus/spread-random-40000000-core2025.10.vortex" + [ -e "$canary_lanes" ] || canary_lanes="$corpus/spread-random-20000000.vortex" + [ -e "$canary_lanes" ] || canary_lanes="$corpus/spread-random-4000000.vortex" + standard_scan="" + standard_lanes="" + canaries() { + local where="$1" label="$2" threads="$3" scan lanes + scan="$(measure "$before_bin" group-stridedfloor-k7 "$canary_scan" 1 "$out/canary.log" 1 10)" || return 2 + lanes="$(measure "$before_bin" group-total-k3 "$canary_lanes" 14 "$out/canary.log" 1 10)" || return 2 + scan="${scan%%$'\t'*}" + lanes="${lanes%%$'\t'*}" + [ -n "$standard_scan" ] || { standard_scan="$scan"; standard_lanes="$lanes"; } + printf 'canary\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$(now)" "$label" "$threads" "$attempt" "$where" "$scan" "$lanes" "$standard_scan/$standard_lanes" >> "$tsv" + awk -v a="$scan" -v b="$lanes" -v sa="$standard_scan" -v sb="$standard_lanes" \ + 'BEGIN { da = a / sa - 1; db = b / sb - 1; if (da < 0) da = -da; if (db < 0) db = -db; exit (da > 0.05 || db > 0.05) ? 1 : 0 }' + } + + # The controls, read as a user: the swap in use (MB), the busiest process but the runners + # ("\t"), the free memory, the power source. + swap_used() { sysctl -n vm.swapusage 2> /dev/null | sed -E 's/.*used = ([0-9.]+)M.*/\1/'; } + free_memory() { memory_pressure -Q 2> /dev/null | awk -F ': ' '/free percentage/ { print $2 }'; } + power() { if pmset -g batt 2> /dev/null | head -1 | grep -q "AC Power"; then echo ac; else echo battery; fi; } + busiest() { + ps -Ao pcpu=,comm= | awk ' + { cpu = $1 + 0; name = $0; sub(/^ *[0-9.]+ +/, "", name); sub(/.*\//, "", name) + if (name ~ /Vorticity\.Benchmarks\.Runner/ || name == "ps" || name == "awk") next + if (cpu > top) top = cpu + if (cpu > 20) busy = busy sprintf("%s %.0f%%; ", name, cpu) } + END { printf "%.0f\t%s\n", top, busy }' + } + + # Holds a scenario back while another process takes more than half a core, 30 s at most; prints the seconds waited. + quiet() { + local waited=0 top + while :; do + IFS=$'\t' read -r top _ <<< "$(busiest)" + if [ "${top:-0}" -le 50 ] || [ "$waited" -ge 30 ]; then + break + fi + + sleep 2 + waited=$((waited + 2)) + done + + echo "$waited" + } + + control() { + local where="$1" label="$2" threads="$3" waited="$4" top busy + IFS=$'\t' read -r top busy <<< "$(busiest)" + printf 'control\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$(now)" "$label" "$threads" "$attempt" "$where" \ + "$waited" "$(swap_used)" "$(free_memory)" "$(power)" "$(load)" "$busy" >> "$tsv" + } + for spec in "${scenarios[@]}"; do name="${spec%%@*}" file="${spec#*@}" @@ -229,53 +356,103 @@ case "$1" in [ -e "$file" ] || { echo "no file $file for $name: vortex-queries --matrix small or full, or --fixture, writes it" >&2; exit 2; } label="$name@$(basename "$file")" for threads in ${degrees//,/ }; do - started="$(load)" - for ((i = 1; i <= alternations; i++)); do - log="$out/$name-$(basename "$file")-$threads-$i" - if ((i % 2 == 1)); then - run before "$before_bin" "$name" "$file" "$threads" "$log-before.log" "$label" >> "$tsv" || exit 1 - run after "$after_bin" "$name" "$file" "$threads" "$log-after.log" "$label" >> "$tsv" || exit 1 - else - run after "$after_bin" "$name" "$file" "$threads" "$log-after.log" "$label" >> "$tsv" || exit 1 - run before "$before_bin" "$name" "$file" "$threads" "$log-before.log" "$label" >> "$tsv" || exit 1 + attempt=0 + while :; do + attempt=$((attempt + 1)) + control before "$label" "$threads" "$(quiet)" + swap_before="$(swap_used)" + off=0 + if [ "$canary" -eq 1 ]; then + canaries before "$label" "$threads" || off=1 fi - done - printf '%s\t%s\tload\t%s\t%s\n' "$label" "$threads" "$started" "$(load)" >> "$tsv" + # Pairs, ABBA, until the interval of the median ratio is within the target. + ratios=() + pair=0 + while :; do + pair=$((pair + 1)) + log="$out/$name-$(basename "$file")-$threads-$attempt-$pair" + if ((pair % 2 == 1)); then + b="$(run before "$before_bin" "$name" "$file" "$threads" "$log-before.log" "$label" "$pair")" || exit 1 + a="$(run after "$after_bin" "$name" "$file" "$threads" "$log-after.log" "$label" "$pair")" || exit 1 + else + a="$(run after "$after_bin" "$name" "$file" "$threads" "$log-after.log" "$label" "$pair")" || exit 1 + b="$(run before "$before_bin" "$name" "$file" "$threads" "$log-before.log" "$label" "$pair")" || exit 1 + fi + + ratios+=("$(awk -v a="$a" -v b="$b" 'BEGIN { printf "%.5f", a / b }')") + [ "$pair" -ge "$alternations" ] || continue + [ "$pair" -ge "$most" ] && break + awk -v target="$target" "$stats"' BEGIN { + n = ARGC - 1; for (i = 0; i < n; i++) r[i] = ARGV[i + 1] + 0 + interval(r, n); m = median(r, n) + exit (100 * (high - low) / 2 / m <= target) ? 0 : 1 }' "${ratios[@]}" && break + done + + if [ "$canary" -eq 1 ]; then + canaries after "$label" "$threads" || off=1 + fi + + control after "$label" "$threads" 0 + grew=0 + awk -v a="${swap_before:-0}" -v b="$(swap_used)" 'BEGIN { exit (b > a + 1) ? 0 : 1 }' && grew=1 + if { [ "$off" -eq 1 ] || [ "$grew" -eq 1 ]; } && [ "$attempt" -eq 1 ]; then + printf 'replay\t%s\t%s\t%s\t%s\tcanary off %s, swap grew %s\n' "$(now)" "$label" "$threads" "$attempt" "$off" "$grew" >> "$tsv" + continue + fi + + printf 'scenario\t%s\t%s\t%s\t%s\t%s\t%s\n' "$(now)" "$label" "$threads" "$attempt" "$off" "$grew" >> "$tsv" + break + done done done - # A side's time is the median of its runs; the pairs' ratios give the spread. The results must - # agree, run after run and side against side: else the two did not do the same work. + # The table, from the last attempt of each scenario. A side's time is the median of its runs; the + # ratio, the median of the pairs' ratios, with its 95 % interval; the fastest, the ratio of the two + # sides' fastest rounds, which a gap of more than 10 % from the median marks noisy; the cycles, the + # ratio of the medians of the rounds' cycles. The results must agree, run after run and side + # against side: else the two did not do the same work. echo - awk -F '\t' ' - function median(list, count, i, j, t) { - for (i = 0; i < count; i++) for (j = i + 1; j < count; j++) if (list[j] + 0 < list[i] + 0) { t = list[i]; list[i] = list[j]; list[j] = t } - return count % 2 ? list[int(count / 2)] : (list[count / 2 - 1] + list[count / 2]) / 2 - } - { - key = $1 " (" $2 ")" + awk -F '\t' "$stats"' + $1 == "scenario" { key = $3 " (" $4 ")"; last[key] = $5; off[key] = $6; grew[key] = $7; next } + $1 == "control" && $6 == "before" { key = $3 " (" $4 ")"; waited[key, $5] = $7; loadb[key, $5] = $11; busy[key, $5] = $12; next } + $1 == "control" && $6 == "after" { key = $3 " (" $4 ")"; loada[key, $5] = $11; next } + $1 == "run" { + key = $3 " (" $4 ")" if (!(key in seen)) { seen[key] = 1; order[++n] = key } - if ($3 == "load") { loads[key] = $4 " -> " $5; next } - k = key SUBSEP $3 + k = key SUBSEP $5 SUBSEP $6 at = count[k]++ - times[k, at] = $4 - if (key in result && result[key] != $5) differs[key] = 1 - result[key] = $5 + time[k, at] = $9; pairof[k, at] = $7; cyc[k, at] = $13; fast[k, at] = $14 + if ($8 == 1) again[key, $5]++ + if ((key SUBSEP $5) in result && result[key, $5] != $10) differs[key] = 1 + result[key, $5] = $10 } END { - printf "%-64s %10s %10s %7s %15s %13s\n", "scenario@file (degree)", "before ms", "after ms", "ratio", "pairs min-max", "load" + printf "%-56s %9s %9s %7s %15s %5s %8s %8s %s\n", "scenario@file (degree)", "before ms", "after ms", "ratio", "95 %", "pairs", "fastest", "cycles", "notes" for (i = 1; i <= n; i++) { - key = order[i]; b = key SUBSEP "before"; a = key SUBSEP "after" - nb = count[b]; na = count[a] - for (j = 0; j < nb; j++) lb[j] = times[b, j] - for (j = 0; j < na; j++) la[j] = times[a, j] - lo = 1e9; hi = 0 - for (j = 0; j < nb && j < na; j++) { r = times[a, j] / times[b, j]; lo = r < lo ? r : lo; hi = r > hi ? r : hi } - mb = median(lb, nb); ma = median(la, na) - printf "%-64s %10.2f %10.2f %7.3f %7.3f-%-7.3f %13s%s\n", key, mb / 1000, ma / 1000, ma / mb, lo, hi, loads[key], (key in differs) ? " RESULT DIFFERS" : "" + key = order[i]; t = last[key]; b = key SUBSEP t SUBSEP "before"; a = key SUBSEP t SUBSEP "after" + nb = count[b]; na = count[a]; np = 0; fb = 1e18; fa = 1e18 + for (j = 0; j < nb; j++) { lb[j] = time[b, j]; cb[j] = cyc[b, j]; if (fast[b, j] + 0 < fb) fb = fast[b, j] + 0; tb[pairof[b, j]] = time[b, j] } + for (j = 0; j < na; j++) { la[j] = time[a, j]; ca[j] = cyc[a, j]; if (fast[a, j] + 0 < fa) fa = fast[a, j] + 0; ta[pairof[a, j]] = time[a, j] } + for (p = 1; p <= nb && p <= na; p++) r[np++] = ta[p] / tb[p] + ratio = median(r, np); interval(r, np) + mb = median(lb, nb); ma = median(la, na); mcb = median(cb, nb); mca = median(ca, na) + fastest = fa / fb; cycles = mcb > 0 ? sprintf("%8.3f", mca / mcb) : sprintf("%8s", "-") + notes = "" + gap = fastest / ratio - 1; if (gap < 0) gap = -gap + if (gap > 0.10) notes = notes "noisy; " + if (100 * (high - low) / 2 / ratio > target) notes = notes "wide; " + if (off[key]) notes = notes "canary off; " + if (grew[key]) notes = notes "swap grew; " + if (t > 1) notes = notes "replayed; " + if (waited[key, t] > 0) notes = notes "waited " waited[key, t] " s; " + if (again[key, t] > 0) notes = notes again[key, t] " runs again; " + if (key in differs) notes = notes "RESULT DIFFERS; " + if (busy[key, t] != "") notes = notes "busy: " busy[key, t] + printf "%-56s %9.2f %9.2f %7.3f %7.3f-%-7.3f %5d %8.3f %s load %s -> %s %s\n", key, mb / 1000, ma / 1000, ratio, low, high, np, fastest, cycles, loadb[key, t], loada[key, t], notes + delete tb; delete ta } - }' "$tsv" + }' target="$target" "$tsv" ;; *) usage ;; From f589ed44753c5a4ac36366956512edabdddb1f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 14:09:45 +0200 Subject: [PATCH 03/56] bench: --micro words, a short text key's groups found alone, in memory vortex-queries --micro words hands ShortTextKeys.Assign batches of the canonical views a decoder makes of texts of 5, 8 and 12 bytes, a hundred to a hundred thousand of them drawn at random, as --micro keys does for integers: what a change to the words does, in seconds, with no file, decode or aggregate. Run it under DOTNET_TieredCompilation=0. --- bench/Vorticity.Benchmarks.Queries/Program.cs | 6 + .../Vorticity.Benchmarks.Queries/WordMicro.cs | 115 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 bench/Vorticity.Benchmarks.Queries/WordMicro.cs diff --git a/bench/Vorticity.Benchmarks.Queries/Program.cs b/bench/Vorticity.Benchmarks.Queries/Program.cs index 5291a08e0..ed3f4c689 100644 --- a/bench/Vorticity.Benchmarks.Queries/Program.cs +++ b/bench/Vorticity.Benchmarks.Queries/Program.cs @@ -53,6 +53,12 @@ return KeyMicro.Run(rounds); } +// --micro words: a short text key's groups found alone, in memory (WordMicro.cs). +if (Text(args, "--micro") == "words") +{ + return WordMicro.Run(rounds); +} + // Each file is written the first time a query asks for it, and kept. Dictionary>> fixtures = new Dictionary>>(StringComparer.Ordinal) { diff --git a/bench/Vorticity.Benchmarks.Queries/WordMicro.cs b/bench/Vorticity.Benchmarks.Queries/WordMicro.cs new file mode 100644 index 000000000..89f974f28 --- /dev/null +++ b/bench/Vorticity.Benchmarks.Queries/WordMicro.cs @@ -0,0 +1,115 @@ +using System; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using Vorticity.Aggregating; +using Vorticity.Arrays; +using Vorticity.Buffers; +using Vorticity.Expressions; +using Vorticity.Types; + +namespace Vorticity.Benchmarks.Queries; + +/// +/// A short text key's groups found alone, in memory (--micro words): texts of 5, 8 and 12 bytes, a +/// hundred to a hundred thousand of them drawn at random, handed batch after batch to +/// .Assign in the canonical views a decoder makes of them, every view holding +/// its value whole. No file, no decode, no aggregate: what a change to the words does, in seconds. +/// +/// +/// The shapes are db-benchmark's: id1 and id2, a hundred texts of 5 bytes; id3, texts +/// of 12 bytes, a hundred thousand of them at 10⁷ rows. Run it with DOTNET_TieredCompilation=0, so +/// that every pass runs the code at its last tier. +/// +internal static class WordMicro +{ + /// The rows of a batch: a chunk of the bench's db-benchmark file. + private const int BatchRows = 65_536; + + private const int ViewSize = 16; + + internal static int Run(int rounds) + { + Console.WriteLine($"{"words",-28} {"rows",11} {"groups",10} {"best ms",10} {"ns/row",8}"); + Measure("5 bytes, 1e2 (id1)", 10_000_000, 5, 100, rounds); + Measure("8 bytes, 1e4", 10_000_000, 8, 10_000, rounds); + Measure("12 bytes, 1e2", 10_000_000, 12, 100, rounds); + Measure("12 bytes, 1e5 (id3)", 10_000_000, 12, 100_000, rounds); + return 0; + } + + /// The best of passes over rows of texts of bytes. + private static void Measure(string name, int rows, int length, int values, int rounds) + { + byte[] views = Views(rows, length, values); + double best = double.MaxValue; + int groups = 0; + for (int round = 0; round < rounds; round++) + { + (double ms, int count) = Pass(views, rows); + best = Math.Min(best, ms); + groups = count; + } + + Console.WriteLine(string.Create( + CultureInfo.InvariantCulture, + $"{name,-28} {rows,11:N0} {groups,10:N0} {best,10:F1} {best * 1e6 / rows,8:F2}")); + } + + /// + /// Every row's view: its length, then its text, id and a number of − 2 + /// digits drawn among , zero past it, as a decoder writes an inline view. + /// + private static byte[] Views(int rows, int length, int values) + { + byte[] views = new byte[rows * ViewSize]; + string format = "D" + (length - 2).ToString(CultureInfo.InvariantCulture); + for (int row = 0; row < rows; row++) + { + int value = (int)(Mix((ulong)row) % (ulong)values); + Span view = views.AsSpan(row * ViewSize, ViewSize); + BinaryPrimitives.WriteUInt32LittleEndian(view, (uint)length); + Encoding.ASCII.GetBytes("id" + value.ToString(format, CultureInfo.InvariantCulture), view[4..]); + } + + return views; + } + + /// One pass: every batch's views a canonical node of an arena, assigned to groups. + private static (double Ms, int Groups) Pass(byte[] views, int rows) + { + GC.Collect(); + RetainingArena arena = new RetainingArena(); + DType dtype = new DTypeArena().Utf8(Nullability.NonNullable); + ColumnShape shape = new ColumnShape(new ColumnSym(Expr.Field("k"), VortexType.Utf8, null, null, -1, [])); + ShortTextKeys groups = new ShortTextKeys(shape, sorted: false); + ulong[] selection = []; + int[] rowGroups = new int[BatchRows]; + GroupRanges ranges = new GroupRanges(); + long start = Stopwatch.GetTimestamp(); + for (int at = 0; at < rows; at += BatchRows) + { + int count = Math.Min(BatchRows, rows - at); + arena.ResetKeepingBlocks(); + VortexBuffer buffer = arena.Allocate(count * ViewSize, ViewSize, out Span bytes); + views.AsSpan(at * ViewSize, count * ViewSize).CopyTo(bytes); + int node = arena.AddVarBinView(dtype, count, Validity.NonNullable, buffer, []); + ranges.Clear(); + groups.Assign(arena, [node], count, selection, rowGroups, ranges); + } + + double ms = Stopwatch.GetElapsedTime(start).TotalMilliseconds; + arena.Reset(); + return (ms, groups.Count); + } + + private static ulong Mix(ulong x) + { + x += 0x9E3779B97F4A7C15UL; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9UL; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBUL; + return x ^ (x >> 31); + } +} From 80b1fad844521f2b0526685f7d5de8fc42b12561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 14:09:45 +0200 Subject: [PATCH 04/56] perf(query): find a short text's group with no branch and no second hash Four costs of the first pass over a block of short texts, seen in the disassembly of KeyTable.FindAtHome and in the profile of db-benchmark's q1 (a hundred texts of 5 bytes), where the key took 81 % of the cycles against DuckDB's table: - the seed rapidhash mixes was prepared at every row, a test and four instructions of a constant the native compiler left in the loop: it is prepared once a batch (TextWord.HomePrepared), in KeyTable and WideKeyTable; - TextWord.Equals compared the low words then the high ones with &&, compiled to a branch, mispredicted at every key not in its home slot: both words are compared at once; - a key whose home its neighbour took sent each of its rows to the search, which hashed it again: one row in five, a sixth of q1's cycles. A line of KeyTable holds two slots of a 16-byte key, and the first pass now compares both, the group taken by masks; - FillWords copied each view to a word with two branches and the validity's test a row: a block with no null now copies them with no branch, the bytes past a value zeroed by a mask its length picks (BytesBlock.TryWords), the longest length judged once. --micro words under DOTNET_TieredCompilation=0, best of three: a hundred texts of 5 bytes 8.06 -> 4.70 ns a row (x0.58), a hundred of 12 bytes 7.21 -> 5.14 (x0.71), ten thousand of 8 bytes 6.64 -> 5.89 (x0.89), a hundred thousand of 12 bytes 7.81 -> 7.27 (x0.93). --- src/Vorticity/Aggregation/ColumnAccess.cs | 47 ++++++++++++++++++++++ src/Vorticity/Aggregation/KeyTable.cs | 41 +++++++++++++++---- src/Vorticity/Aggregation/ShortTextKeys.cs | 28 +++++++++++-- src/Vorticity/Aggregation/WideKeyTable.cs | 17 ++++++-- 4 files changed, 118 insertions(+), 15 deletions(-) diff --git a/src/Vorticity/Aggregation/ColumnAccess.cs b/src/Vorticity/Aggregation/ColumnAccess.cs index 7c5021f79..7c4e74a67 100644 --- a/src/Vorticity/Aggregation/ColumnAccess.cs +++ b/src/Vorticity/Aggregation/ColumnAccess.cs @@ -444,6 +444,53 @@ internal bool TryWord(int row, out TextWord word) return true; } + /// + /// The words of rows on, as many as holds, every row + /// valid, as makes each: the view's sixteen bytes, those past the value zeroed by a + /// mask its length picks. False when a value is longer than 12 bytes, which the words cannot hold. + /// + /// + /// No branch on a row: the longest length is kept and judged once. Row by row through + /// , two branches a row and the validity's test took a sixth of the cycles of a sum + /// by a hundred short texts (2026-10-09). + /// + internal bool TryWords(int start, Span words) + { + ReadOnlySpan views = _views.Slice(start * ViewSize, words.Length * ViewSize); + ref byte view = ref MemoryMarshal.GetReference(views); + ref TextWord word = ref MemoryMarshal.GetReference(words); + ref ulong lows = ref MemoryMarshal.GetReference(LowMasks); + ref ulong highs = ref MemoryMarshal.GetReference(HighMasks); + uint longest = 0; + for (int i = 0; i < words.Length; i++) + { + ref byte at = ref Unsafe.Add(ref view, i * ViewSize); + ulong low = Unsafe.ReadUnaligned(ref at); + ulong high = Unsafe.ReadUnaligned(ref Unsafe.Add(ref at, 8)); + uint size = (uint)low; + longest = Math.Max(longest, size); + nint masked = (nint)Math.Min(size, (uint)MaxInline); + Unsafe.Add(ref word, i) = new TextWord(low & Unsafe.Add(ref lows, masked), high & Unsafe.Add(ref highs, masked)); + } + + return longest <= MaxInline; + } + + // By length, from 0 to 12: the bits of a word's low half, then of its high half, that its length and its + // bytes take, 32 then 8 a byte. + private static ReadOnlySpan LowMasks => + [ + 0x0000_0000_FFFF_FFFFUL, 0x0000_00FF_FFFF_FFFFUL, 0x0000_FFFF_FFFF_FFFFUL, 0x00FF_FFFF_FFFF_FFFFUL, + ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, + ulong.MaxValue, ulong.MaxValue, + ]; + + private static ReadOnlySpan HighMasks => + [ + 0, 0, 0, 0, 0, 0xFFUL, 0xFFFFUL, 0xFF_FFFFUL, 0xFFFF_FFFFUL, 0xFF_FFFF_FFFFUL, 0xFFFF_FFFF_FFFFUL, + 0xFF_FFFF_FFFF_FFFFUL, ulong.MaxValue, + ]; + /// Data buffer of the block, which row names. private ReadOnlySpan Data(int row, uint buffer) { diff --git a/src/Vorticity/Aggregation/KeyTable.cs b/src/Vorticity/Aggregation/KeyTable.cs index a046a631b..f303a6dbd 100644 --- a/src/Vorticity/Aggregation/KeyTable.cs +++ b/src/Vorticity/Aggregation/KeyTable.cs @@ -198,6 +198,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp // The table's fields in locals: a store to the homes could alias them for all the JIT knows. ulong seed = _seed; + ulong prepared = Prepared(seed); ulong multiplier = _multiplier; uint length = (uint)slots.Length; int shift = _shift; @@ -211,11 +212,31 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp { // No read ahead: each home probed as it is computed, none stored to be read back; with no // bit left out, a loop without the shift, a cycle a row. - if (shift == 0) + if (shift == 0 && Width == 2) { + // A line of two slots, a key of 16 bytes: the key its home's neighbour took is found here + // too, both slots compared and the group taken by masks, rather than in the search. Its + // home alone sent a hundred short texts' rows to the search one time in five, re-hashed + // there, a sixth of the cycles of a sum by them (2026-10-09). The last line of a table of a + // prime number of slots holds one: its home is then compared twice. + uint last = length - 1; for (int i = 0; i < keys.Length; i++) { - ref Slot home = ref Unsafe.Add(ref first, (nint)HomeOf(keys[i], seed, multiplier, length, 0)); + uint at = HomeOf(keys[i], seed, prepared, multiplier, length, 0); + ref Slot home = ref Unsafe.Add(ref first, (nint)at); + ref Slot next = ref Unsafe.Add(ref first, (nint)Math.Min(at ^ 1, last)); + int here = Unsafe.BitCast(home.Key.Equals(keys[i])); + int there = Unsafe.BitCast(next.Key.Equals(keys[i])); + int group = ((home.Group & -here) | (next.Group & -there)) - 1; + groups[i] = group; + any |= group; + } + } + else if (shift == 0) + { + for (int i = 0; i < keys.Length; i++) + { + ref Slot home = ref Unsafe.Add(ref first, (nint)HomeOf(keys[i], seed, prepared, multiplier, length, 0)); int match = Unsafe.BitCast(home.Key.Equals(keys[i])); int group = (home.Group & -match) - 1; groups[i] = group; @@ -226,7 +247,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp { for (int i = 0; i < keys.Length; i++) { - ref Slot home = ref Unsafe.Add(ref first, (nint)HomeOf(keys[i], seed, multiplier, length, shift)); + ref Slot home = ref Unsafe.Add(ref first, (nint)HomeOf(keys[i], seed, prepared, multiplier, length, shift)); int match = Unsafe.BitCast(home.Key.Equals(keys[i])); int group = (home.Group & -match) - 1; groups[i] = group; @@ -241,7 +262,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp homes = homes[..keys.Length]; for (int i = 0; i < keys.Length; i++) { - homes[i] = HomeOf(keys[i], seed, multiplier, length, shift); + homes[i] = HomeOf(keys[i], seed, prepared, multiplier, length, shift); } int sink = 0; @@ -409,17 +430,21 @@ private readonly int Shift(ReadOnlySpan slots, ReadOnlySpan overflo /// The slot starts from: its folded bits, mixed under the seed once there is one, modulo the slots. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly uint Home(TValue key) => HomeOf(key, _seed, _multiplier, (uint)_length, _shift); + private readonly uint Home(TValue key) => HomeOf(key, _seed, Prepared(_seed), _multiplier, (uint)_length, _shift); + + /// The seed a short text's word is homed by, prepared once (); the seed itself for any other key. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Prepared(ulong seed) => typeof(TValue) == typeof(TextWord) ? TextWord.Prepared(seed) : seed; - /// As , the table's seed, multiplier, length and shift given. + /// As , the table's seed, its preparation (), multiplier, length and shift given. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint HomeOf(TValue key, ulong seed, ulong multiplier, uint length, int shift) + private static uint HomeOf(TValue key, ulong seed, ulong prepared, ulong multiplier, uint length, int shift) { (ulong low, ulong high) = KeyWords.Of(key); uint hash; if (typeof(TValue) == typeof(TextWord)) { - hash = TextWord.Home(low, high, seed); + hash = TextWord.HomePrepared(low, high, prepared); } else if (seed != 0) { diff --git a/src/Vorticity/Aggregation/ShortTextKeys.cs b/src/Vorticity/Aggregation/ShortTextKeys.cs index 1ad458985..407965217 100644 --- a/src/Vorticity/Aggregation/ShortTextKeys.cs +++ b/src/Vorticity/Aggregation/ShortTextKeys.cs @@ -26,7 +26,13 @@ internal TextWord(ulong low, ulong high) High = high; } - public bool Equals(TextWord other) => Low == other.Low && High == other.High; + /// + /// Both words at once, with no branch: the first passes of the key tables compare a row's word with a + /// slot's and take the group or none by a mask, and a && compiled to a branch on the low + /// word, mispredicted at every key that is not in its home slot (2026-10-09, the disassembly of + /// ). + /// + public bool Equals(TextWord other) => ((Low ^ other.Low) | (High ^ other.High)) == 0; public override bool Equals(object? obj) => obj is TextWord other && Equals(other); @@ -57,13 +63,24 @@ internal TextWord(ulong low, ulong high) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static uint Home(ulong low, ulong high, ulong seed) + internal static uint Home(ulong low, ulong high, ulong seed) => HomePrepared(low, high, Prepared(seed)); + + /// + /// of a seed already prepared by : a table's first pass prepares + /// it once a batch, where preparing it at every row cost a test and four instructions of the constant a + /// row, in the loop the native compiler left it in (2026-10-09). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint HomePrepared(ulong low, ulong high, ulong prepared) { - ulong prepared = seed == 0 ? Unseeded : seed ^ Mix(seed ^ Secret2, Secret1) ^ 16; ulong upper = Math.BigMul(low ^ Secret1, high ^ prepared, out ulong lower); return (uint)(Mix(lower ^ Secret7, upper ^ Secret1 ^ 16) >> 32); } + /// The seed as rapidhash prepares it, the length of 16 folded in: seed ^ mix(seed ^ secret[2], secret[1]) ^ 16. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static ulong Prepared(ulong seed) => seed == 0 ? Unseeded : seed ^ Mix(seed ^ Secret2, Secret1) ^ 16; + // rapidhash's secrets (V3) its path for 16 bytes reads. private const ulong Secret1 = 0x8bb84b93962eacc9UL; private const ulong Secret2 = 0x4b33a62ed433d4a3UL; @@ -233,6 +250,11 @@ internal override bool Assign(CanonicalArena arena, ReadOnlySpan nodes, int /// The words of rows to into the block's scratch from its start; false at the first value too long for one. private bool FillWords(BytesBlock canonical, ReadOnlySpan validity, int start, int end) { + if (validity.IsEmpty) + { + return canonical.TryWords(start, _block.AsSpan(0, end - start)); + } + TextWord[] block = _block; for (int row = start; row < end; row++) { diff --git a/src/Vorticity/Aggregation/WideKeyTable.cs b/src/Vorticity/Aggregation/WideKeyTable.cs index e1e93337d..01b6a53a7 100644 --- a/src/Vorticity/Aggregation/WideKeyTable.cs +++ b/src/Vorticity/Aggregation/WideKeyTable.cs @@ -210,6 +210,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp groups = groups[..keys.Length]; ulong seed = _seed; + ulong prepared = Prepared(seed); ulong multiplier = _multiplier; uint length = (uint)slots.Length; ref Slot first = ref MemoryMarshal.GetReference(slots); @@ -223,7 +224,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp // 0.86 of one loop's time, measured on 2026-10-08. for (int i = 0; i < keys.Length; i++) { - uint hash = HashOf(keys[i], seed); + uint hash = HashOf(keys[i], seed, prepared); ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.FastMod(hash, length, multiplier)); groups[i] = (home.Group & -Unsafe.BitCast(home.Hash == hash)) - 1; } @@ -244,7 +245,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp hashes = hashes[..keys.Length]; for (int i = 0; i < keys.Length; i++) { - hashes[i] = HashOf(keys[i], seed); + hashes[i] = HashOf(keys[i], seed, prepared); } int sink = 0; @@ -442,12 +443,20 @@ internal void Reseed(ulong seed, ReadOnlySpan keys) /// rapidhash (). /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint HashOf(TValue key, ulong seed) + private static uint HashOf(TValue key, ulong seed) => HashOf(key, seed, Prepared(seed)); + + /// The seed a short text's word is homed by, prepared once a batch (); the seed itself for any other key. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Prepared(ulong seed) => typeof(TValue) == typeof(TextWord) ? TextWord.Prepared(seed) : seed; + + /// As , the seed already prepared. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint HashOf(TValue key, ulong seed, ulong prepared) { (ulong low, ulong high) = KeyWords.Of(key); if (typeof(TValue) == typeof(TextWord)) { - return TextWord.Home(low, high, seed); + return TextWord.HomePrepared(low, high, prepared); } if (seed != 0) From 2a0aa88ac0f3894d12a9330028ef5636857dc035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 14:24:23 +0200 Subject: [PATCH 05/56] perf(query): a short text's slot by one multiplication, its small tables a quarter full Two more costs of the first pass over short texts: - the slot of a word was its rapidhash modulo the prime length by the runtime's fast modulo, two multiplications; it is now the high half of the hash times the length (Lemire's fast range), one, in KeyTable and WideKeyTable. Integers keep the modulo, which lays keys in a row in slots in a row; - six tenths full, a hundred texts laid a tenth of their keys in the chains past a full line, each of their rows sent to the search: a short text's table of 512 slots or fewer (128 keys) now fills to a quarter. Past it, the table grows by the same primes and lands on the length six tenths would have given it. Up to 8 192 slots, a stream of sorted names grew its table once more after its first read, where its reads allocate nothing (AllocationContractTests); and a table cleared to keep some groups, a stream's or a cache's, fills to six tenths from then on. --micro words under DOTNET_TieredCompilation=0, against the previous commit: a hundred texts of 5 bytes 4.70 -> 3.89 ns a row (x0.83), a hundred of 12 bytes 5.14 -> 3.89-3.96 (x0.76), ten thousand of 8 bytes 5.89 -> 5.23-5.29 (x0.89), a hundred thousand of 12 bytes 7.27 -> 6.38-6.42 (x0.88). A view a vector in TryWords, its mask made of the tables' two halves, ran 5 % slower than two words a view, and was not kept; nor was a first pass of WideKeyTable that compared the slot after the home too (ten thousand texts x1.17, a hundred thousand x1.06). The work counters of "count by a name, each once then all again" come down: 70 arrays of 11 901 724 bytes instead of 72 of 11 905 852. --- src/Vorticity/Aggregation/ColumnAccess.cs | 3 +- src/Vorticity/Aggregation/KeyTable.cs | 62 +++++++++++++++++-- src/Vorticity/Aggregation/WideKeyTable.cs | 12 ++-- .../Aggregation/WorkCounterTests.cs | 2 +- 4 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/Vorticity/Aggregation/ColumnAccess.cs b/src/Vorticity/Aggregation/ColumnAccess.cs index 7c4e74a67..c6788b7ff 100644 --- a/src/Vorticity/Aggregation/ColumnAccess.cs +++ b/src/Vorticity/Aggregation/ColumnAccess.cs @@ -452,7 +452,8 @@ internal bool TryWord(int row, out TextWord word) /// /// No branch on a row: the longest length is kept and judged once. Row by row through /// , two branches a row and the validity's test took a sixth of the cycles of a sum - /// by a hundred short texts (2026-10-09). + /// by a hundred short texts (2026-10-09). A view a vector, its mask made of the two halves read from + /// the tables, ran 5 % slower than these two words a view (`--micro words`, the same day). /// internal bool TryWords(int start, Span words) { diff --git a/src/Vorticity/Aggregation/KeyTable.cs b/src/Vorticity/Aggregation/KeyTable.cs index f303a6dbd..ffdd2cd36 100644 --- a/src/Vorticity/Aggregation/KeyTable.cs +++ b/src/Vorticity/Aggregation/KeyTable.cs @@ -63,6 +63,9 @@ internal struct KeyTable private int _shift; private int _count; private int _growAt; + + // Whether the table was cleared to keep some groups, a stream's or a cache's: it then fills to six tenths (Clear). + private bool _carried; private readonly ArrayShelf? _shelf; public KeyTable() @@ -87,8 +90,9 @@ internal KeyTable(ArrayShelf shelf) internal readonly int Count => _count; /// - /// The bytes the table would take more were new keys to come: past six tenths - /// full it doubles, its store and its chains twice what they hold. Its chains may make it grow sooner. + /// The bytes the table would take more were new keys to come: past its fill (six + /// tenths, a quarter for a short text's small table, ) it doubles, its store and its + /// chains twice what they hold. Its chains may make it grow sooner. /// internal readonly long GrowthFor(int more) { @@ -294,12 +298,40 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp internal void Reserve(int keys) { long slots = ((10L * keys) / 6) + 1; + if (!_carried && Roomy(slots)) + { + slots = (4L * keys) + 1; + } + if (slots > _length) { Resize(PrimeAtLeast((int)Math.Min(int.MaxValue / 2, slots))); } } + /// + /// The slots a short text's table holds a quarter of at most, where any other holds six tenths: a + /// hundred and twenty-eight keys. Past it, a table grows by the same primes as before, a quarter full + /// sooner, and lands on the length six tenths would have given it: up to 8 192 slots, a stream of sorted + /// names grew its table once more after its first read, where its reads allocate nothing + /// (AllocationContractTests, 2026-10-09). + /// + private const int RoomySlots = 512; + + /// + /// Whether a table of slots fills to a quarter only: a short text's, while it is + /// small. Six tenths full, a hundred texts lay a tenth of their keys in the chains, past a full line, + /// each of their rows sent to the search (2026-10-09); a quarter full, few. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Roomy(long slots) => typeof(TValue) == typeof(TextWord) && slots <= RoomySlots; + + /// + /// The keys a table of slots holds before it doubles: a quarter of a small + /// short text's table, six tenths of any other, and of every table cleared to keep groups (). + /// + private readonly int FillOf(int length) => !_carried && Roomy(length) ? length / 4 : (int)(6L * length / 10); + /// /// Forgets every key, keeping the slots and the seed, with room for keys: a table /// about to take as many keys as it held, beside those kept, doubles now, empty, rather than in the @@ -311,10 +343,20 @@ internal void Clear(int room) Array.Clear(_chains); _overflowed = 0; _count = 0; + + // A table cleared to keep some groups, a stream's from batch to batch or a cache's, fills to six tenths + // from then on: a quarter full, a stream of sorted names grew its table once more after its first + // read, where its reads allocate nothing (AllocationContractTests, 2026-10-09). + if (!_carried) + { + _carried = true; + _growAt = FillOf(_length); + } + if (_length > 0 && room > _growAt) { int length = _length; - while ((int)(6L * length / 10) < room) + while (FillOf(length) < room) { length = PrimeAtLeast(GroupKeys.Doubled(length)); } @@ -460,9 +502,19 @@ private static uint HomeOf(TValue key, ulong seed, ulong prepared, ulong multipl hash = (uint)(folded ^ (folded >> 32)); } - return FastMod(hash, length, multiplier); + return SlotOf(hash, length, multiplier); } + /// + /// The slot of among : a short text's word, mixed by + /// rapidhash, by the high half of the hash times the length (Lemire's fast range), one multiplication; + /// any other key by the hash modulo the prime length (), two, which keeps integers + /// in a row in slots in a row. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint SlotOf(uint hash, uint length, ulong multiplier) => + typeof(TValue) == typeof(TextWord) ? (uint)(((ulong)hash * length) >> 32) : FastMod(hash, length, multiplier); + /// Places every key again in slots, a prime, under the seed the table has. private void Resize(int length) { @@ -484,7 +536,7 @@ private void Resize(int length) _overflow = kept ? overflow : NewArray(overflowed); _overflowed = 0; _multiplier = (ulong.MaxValue / (uint)length) + 1; - _growAt = (int)(6L * length / 10); + _growAt = FillOf(length); // The low bits no key has set, left out of the homes; under a seed, every bit goes to the mix. _shift = Shift(old, overflow.AsSpan(0, overflowed)); diff --git a/src/Vorticity/Aggregation/WideKeyTable.cs b/src/Vorticity/Aggregation/WideKeyTable.cs index 01b6a53a7..5dae5e0b1 100644 --- a/src/Vorticity/Aggregation/WideKeyTable.cs +++ b/src/Vorticity/Aggregation/WideKeyTable.cs @@ -167,7 +167,7 @@ internal int GetOrAdd(TValue key, int next, ReadOnlySpan keys) if (slots.Length != 0) { uint hash = HashOf(key, _seed); - ref Slot slot = ref slots[(int)KeyTable.FastMod(hash, (uint)slots.Length, _multiplier)]; + ref Slot slot = ref slots[(int)KeyTable.SlotOf(hash, (uint)slots.Length, _multiplier)]; if (slot.Group != 0) { if (slot.Hash == hash && keys[slot.Group - 1].Equals(key)) @@ -225,7 +225,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp for (int i = 0; i < keys.Length; i++) { uint hash = HashOf(keys[i], seed, prepared); - ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.FastMod(hash, length, multiplier)); + ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.SlotOf(hash, length, multiplier)); groups[i] = (home.Group & -Unsafe.BitCast(home.Hash == hash)) - 1; } @@ -252,7 +252,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp int row = 0; for (int touched = keys.Length - ahead; row < touched; row++) { - sink ^= Unsafe.Add(ref first, (nint)KeyTable.FastMod(hashes[row + ahead], length, multiplier)).Group; + sink ^= Unsafe.Add(ref first, (nint)KeyTable.SlotOf(hashes[row + ahead], length, multiplier)).Group; int group = Found(ref first, ref firstKey, hashes[row], keys[row], length, multiplier); groups[row] = group; any |= group; @@ -276,7 +276,7 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int Found(ref Slot first, ref TValue firstKey, uint hash, TValue key, uint length, ulong multiplier) { - ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.FastMod(hash, length, multiplier)); + ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.SlotOf(hash, length, multiplier)); int candidate = (home.Group & -Unsafe.BitCast(home.Hash == hash)) - 1; int same = Unsafe.BitCast(Unsafe.Add(ref firstKey, (nint)(uint)(candidate & ~(candidate >> 31))).Equals(key)); return ((candidate + 1) & -same) - 1; @@ -350,7 +350,7 @@ private int Search(TValue key, int next, ReadOnlySpan keys) Span slots = Slots; uint hash = HashOf(key, _seed); - uint at = KeyTable.FastMod(hash, (uint)slots.Length, _multiplier); + uint at = KeyTable.SlotOf(hash, (uint)slots.Length, _multiplier); uint first = at & ~(uint)(Width - 1); uint end = Math.Min(first + (uint)Width, (uint)slots.Length); uint probe = at; @@ -516,7 +516,7 @@ private void Resize(int length) private void Place(uint hash, int group) { Span slots = Slots; - uint at = KeyTable.FastMod(hash, (uint)slots.Length, _multiplier); + uint at = KeyTable.SlotOf(hash, (uint)slots.Length, _multiplier); uint first = at & ~(uint)(Width - 1); uint end = Math.Min(first + (uint)Width, (uint)slots.Length); uint probe = at; diff --git a/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs b/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs index 284ef6560..1ce68570c 100644 --- a/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs +++ b/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs @@ -39,7 +39,7 @@ public sealed partial class WorkCounterTests "distinct values of 10^6: result 129508936719, rows 0, lanes 0, lane groups 0, arrays 0 of 0 B, copied 0 B, blocks 0/0/0, merge parts 0 entries, peak 0 B", "count by a text: result 200000, rows 200000, lanes 1, lane groups 49081, arrays 50 of 7730472 B, copied 1833792 B, blocks 0/12/1, merge 0 parts 0 entries, peak 6029368 B", "count by a text and an int: result 200000, rows 200000, lanes 1, lane groups 196058, arrays 94 of 14492304 B, copied 2357952 B, blocks 0/0/0, merge 0 parts 0 entries, peak 12126112 B", - "count by a name, each once then all again: result 200000, rows 200000, lanes 1, lane groups 100000, arrays 72 of 11905852 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 7947288 B", + "count by a name, each once then all again: result 200000, rows 200000, lanes 1, lane groups 100000, arrays 70 of 11901724 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 7947288 B", "least and greatest text by 10^3 values: result 56000, rows 200000, lanes 1, lane groups 1000, arrays 21 of 171968 B, copied 0 B, blocks 0/0/13, merge 0 parts 0 entries, peak 1310720 B", "distinct users by day: result 299552, rows 300000, lanes 1, lane groups 1, arrays 370 of 2940924 B, copied 0 B, blocks 4/0/0, merge 0 parts 0 entries, peak 2621440 B", ]; From 11998932e5092dc1ce0b18a8e3345cc658f58fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 15:02:28 +0200 Subject: [PATCH 06/56] bench: a canary off replays a scenario only when its pairs did not hold runners.sh alt ran a scenario again whenever a canary that framed it was more than 5 % off its standard. Pairs share whatever the machine does while they run: a scenario whose pairs held the target already says the perturbation did not reach the ratio. Under a video call that kept every canary off, replaying them all doubled a session. A canary off now asks for a replay only when the pairs stopped at the most without holding the target; swap that grew still asks for one. --- bench/runners.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/bench/runners.sh b/bench/runners.sh index 4aa4e6061..76e708ed9 100755 --- a/bench/runners.sh +++ b/bench/runners.sh @@ -38,8 +38,10 @@ # (8). The pairs and the interval are printed: a clear answer stops at three; # - canaries: before and after each scenario, two fixed runs of the before runner, the file kept mapped # (a scan of one lane, group-stridedfloor-k7, and a count and sum of 10^3 keys on 14 lanes). Their -# first values of the session are the standard; a scenario framed by a canary more than 5 % off is -# run again once, and marked if it still is; +# first values of the session are the standard; a scenario framed by a canary more than 5 % off whose +# pairs did not hold the target is run again once, and marked if a canary is still off. Pairs that +# held it shared whatever the machine did: a whole scenario replayed for them, under a video call that +# kept every canary off, doubled a session (2026-10-09); # - controls, user-level, never sudo: another process above 50 % of a core holds a scenario back (30 s # at most), one above 20 % is noted; swap that grows during a scenario has it run again once; the # free memory, the power source and the load are noted; @@ -369,6 +371,7 @@ case "$1" in # Pairs, ABBA, until the interval of the median ratio is within the target. ratios=() pair=0 + held=0 while :; do pair=$((pair + 1)) log="$out/$name-$(basename "$file")-$threads-$attempt-$pair" @@ -382,11 +385,15 @@ case "$1" in ratios+=("$(awk -v a="$a" -v b="$b" 'BEGIN { printf "%.5f", a / b }')") [ "$pair" -ge "$alternations" ] || continue - [ "$pair" -ge "$most" ] && break - awk -v target="$target" "$stats"' BEGIN { + if awk -v target="$target" "$stats"' BEGIN { n = ARGC - 1; for (i = 0; i < n; i++) r[i] = ARGV[i + 1] + 0 interval(r, n); m = median(r, n) - exit (100 * (high - low) / 2 / m <= target) ? 0 : 1 }' "${ratios[@]}" && break + exit (100 * (high - low) / 2 / m <= target) ? 0 : 1 }' "${ratios[@]}"; then + held=1 + break + fi + + [ "$pair" -ge "$most" ] && break done if [ "$canary" -eq 1 ]; then @@ -396,7 +403,9 @@ case "$1" in control after "$label" "$threads" 0 grew=0 awk -v a="${swap_before:-0}" -v b="$(swap_used)" 'BEGIN { exit (b > a + 1) ? 0 : 1 }' && grew=1 - if { [ "$off" -eq 1 ] || [ "$grew" -eq 1 ]; } && [ "$attempt" -eq 1 ]; then + # A canary off asks for the scenario again only when its pairs did not hold the target + # anyway: the pairs share what the machine did, and a tight interval already says so. + if { { [ "$off" -eq 1 ] && [ "$held" -eq 0 ]; } || [ "$grew" -eq 1 ]; } && [ "$attempt" -eq 1 ]; then printf 'replay\t%s\t%s\t%s\t%s\tcanary off %s, swap grew %s\n' "$(now)" "$label" "$threads" "$attempt" "$off" "$grew" >> "$tsv" continue fi From e37132feb9d1f81b1e9eda29c36aaa3c4e9a833b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 15:03:53 +0200 Subject: [PATCH 07/56] perf(query): a span of one page numbers its keys from the page itself FixedKeys numbers an integer key by its value when the statistics bound it, through pages of 4 096 numbers. Each row read its page from the array of pages, then the page's start, then its number: a chain of three loads a row, the first two the same for every row of a span of one page, which is every key of a few thousand values (k3, db-benchmark q4 and q5). DirectTwoPasses now takes the page and its start once a chunk when the span has one page, and reads the number from them. --micro keys under DOTNET_TieredCompilation=0, best of five, twice, twenty million keys numbered by their bounds: a hundred values 16.4-16.6 -> 11.8-12.6 ms (x0.73), a thousand 16.5 -> 11.4-11.5 ms (x0.70); a million, many pages, 29.4-29.7 -> 29.8-30.4 ms, unchanged. The micro gains the two cases. --- .../Vorticity.Benchmarks.Queries/KeyMicro.cs | 2 ++ src/Vorticity/Aggregation/GroupKeys.cs | 36 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/bench/Vorticity.Benchmarks.Queries/KeyMicro.cs b/bench/Vorticity.Benchmarks.Queries/KeyMicro.cs index 003c4279f..c50143d75 100644 --- a/bench/Vorticity.Benchmarks.Queries/KeyMicro.cs +++ b/bench/Vorticity.Benchmarks.Queries/KeyMicro.cs @@ -23,6 +23,8 @@ internal static class KeyMicro internal static int Run(int rounds) { Console.WriteLine($"{"keys",-28} {"rows",11} {"groups",10} {"bounds ms",10} {"hashed ms",10} {"ratio",7}"); + Measure("random 1e2", 20_000_000, row => (int)((Mix(Mix((ulong)row)) >> 32) % 100), rounds); + Measure("random 1e3", 20_000_000, row => (int)((Mix(Mix((ulong)row)) >> 32) % 1_000), rounds); Measure("random 1e6", 20_000_000, row => (int)((Mix(Mix((ulong)row)) >> 32) % 1_000_000), rounds); Measure("random 1e7", 20_000_000, row => (int)(Mix(Mix(Mix((ulong)row))) % 10_000_000), rounds); Measure("tenfold", 20_000_000, row => (int)((Mix(Mix(Mix((ulong)row))) >> 32) % 2_000_000), rounds); diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index d323d207e..e905ae5e0 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -1174,16 +1174,34 @@ private void DirectTwoPasses(ReadOnlySpan canonical, ReadOnlySpan } int missed = 0; - for (int i = 0; i < values.Length; i++) + if (pages.Length == 1) + { + // A span of one page, a few thousand values: its numbers read from the page itself, taken once + // a chunk, where the page and its start read again at every row made a chain of three loads. + ref int numbers = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(pages[0]), starts[0]); + for (int i = 0; i < values.Length; i++) + { + ulong number = (ulong)(Integer(values[i]) - min); + bool inside = number < span; + int group = Unsafe.Add(ref numbers, (nint)(inside ? number : 0)); + group = inside ? group : -1; + groups[i] = group; + missed |= group; + } + } + else { - ulong number = (ulong)(Integer(values[i]) - min); - bool inside = number < span; - ulong at = inside ? number : 0; - int page = (int)(at >> PageBits); - int group = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(pages[page]), starts[page] + ((int)at & PageMask)); - group = inside ? group : -1; - groups[i] = group; - missed |= group; + for (int i = 0; i < values.Length; i++) + { + ulong number = (ulong)(Integer(values[i]) - min); + bool inside = number < span; + ulong at = inside ? number : 0; + int page = (int)(at >> PageBits); + int group = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(pages[page]), starts[page] + ((int)at & PageMask)); + group = inside ? group : -1; + groups[i] = group; + missed |= group; + } } // Every row found its group, none null: the usual chunk once the values are known. From ddc081d53737337e0f20f4ad56cf6e2dd0723142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 15:38:23 +0200 Subject: [PATCH 08/56] perf(query): a small span of integers numbered whole, a row's group its value less the least An integer key the statistics bound to a span is numbered by its value through pages from the value's number to its group, a group given as each value first comes: every row of k3 (a thousand values) read its group from the page, a dependent load, and a third of its cycles went there. A lane's key now numbers its span whole when the span's records fit the private cache (FixedKeys.WholeBytes, 1 MiB with the keys and the numbers): each value of the span is a group from the first row, its number, met or not. The pages tell the values met from the others, a value met for the first time taking its number where it took the next group; once every value is met, the first pass of a chunk is the value less the least, in vectors for keys of four bytes, a value past the bounds left for the lookup as before (FixedKeys.Numbers). The groups no row met are dropped, keys and records kept in order, before anyone else reads the lane's groups: at the end of its lane, or when it turns to the core or writes its table to the scratch (AggregationPartition.DropUnmet). A spill test that left the drop out of EvictAsync failed. Never with a top, the core, a key that streams, a distinct query's or the core's partitions; and only when every slot keeps its state in the records: a keep moves a distinct count's pairs to arrays beside the old, and its peak at one lane went from 29 to 47 MB by a thousand values three in ten unmet. runners.sh alt against the runner of 8d4a53cc, warm, on a busy Mac (a video call): group-total-k3 on 20M rows x0.559 at one lane (29.6 -> 16.8 ms) and x0.628 at fourteen (5.0 -> 3.1 ms); group-db-q4 x0.872 and x0.920, its exact float mean left to D3; group-total-k4 x0.733; tenfold x0.992 and k5 x1.026 (0.96-1.11, cycles x0.988), unchanged past the span. The work counters of "count and sum by 10^3 values" come down: 3 arrays of 36 824 bytes instead of 8 of 40 888. --- .../Aggregation/AggregationEngine.cs | 67 +++++++ src/Vorticity/Aggregation/GroupKeys.cs | 187 +++++++++++++++++- .../Aggregation/DirectGroupKeysTests.cs | 84 +++++++- .../Aggregation/WorkCounterTests.cs | 2 +- 4 files changed, 329 insertions(+), 11 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 31dac0794..0f23e0c09 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -381,6 +381,13 @@ internal AggregationPlan(ReadOnlySpan results, ColumnShape[] keys) /// internal bool MergeByValue { get; set; } = true; + /// + /// Whether a lane's key numbered by value over a span whose groups fit the private cache numbers it + /// whole (): false to number values as they first come, the switch + /// the tests compare them with. + /// + internal bool NumberWhole { get; set; } = true; + /// The result a symbol stands for. /// The symbol is a column, not an aggregate or a key. internal static ResultNode Result(Sym symbol) => Result(symbol, []); @@ -968,6 +975,7 @@ internal bool CanEvict /// internal async ValueTask EvictAsync(CancellationToken cancellationToken) { + DropUnmet(); if (Keys is not { Count: > 0 } keys) { return; @@ -1645,6 +1653,7 @@ internal async ValueTask TurnAsync(CancellationToken cancellationToken) // A lane whose table holds less than the core would cost it keeps its table, but for one already // past its budget when the core spills, which a lane's table cannot; and but when a lane turned // on what its rows showed, where every lane turns while its table is small. + DropUnmet(); CorePressure pressure = Pressure!; // The core cannot hold the query's groups: what the lane's rows show changes nothing, and its @@ -2127,6 +2136,49 @@ internal void Settle(ZoneSettling settling, RowRange rows) /// The first groups of an order on the key the query takes, which the partition keeps alone as it goes; null to keep every group. internal KeyTop? Top { get; init; } + /// + /// Whether a key numbered by value may number its span whole (): a + /// lane of a group by's pass, whose groups no one reads before its rows are folded but through + /// ; never with a top, the core or a key that streams. + /// + internal bool NumbersWhole { get; init; } + + // Whether the partition judged at its first batch whether its key numbers its span whole. + private bool _wholeJudged; + + /// + /// The bytes of a group's record, or -1 when a slot keeps its states apart. The groups no row met are + /// dropped by a keep, which moves the records within their array, but a set of values a group to + /// arrays beside the old: a distinct count by a thousand values numbered whole, three in ten never + /// met, held 47 MB at its peak at one lane rather than 29. + /// + private long RecordBytes() + { + for (int i = 0; i < Slots.Length; i++) + { + if (_inputs[i] != Settled && Slots[i].StateBytes == 0) + { + return -1; + } + } + + return (long)(Records?.Layout.Stride ?? 0) * sizeof(ulong); + } + + /// + /// The groups of a span numbered whole that no row met, dropped, keys and states: before anyone else + /// reads the partition's groups, at the end of its lane, or when it turns to the core or writes its + /// table to the scratch. Its key numbers values as they first come from then on. + /// + internal void DropUnmet() + { + if (Keys?.Met() is { } met) + { + Keep(met); + GroupsAtEnd = met.Length; + } + } + /// The groups the partition held at most since its top last counted them. internal int PeakGroups { get; set; } @@ -2342,6 +2394,17 @@ private void Fold(RecordBatch batch) return; } + // A key numbered by value over a span whose records fit the private cache numbers it whole from + // the first row: a row's group is its value less the least once every value is met. + if (NumbersWhole && !_wholeJudged) + { + _wholeJudged = true; + if (Top is null && _core is null && _componentKeys is null && RecordBytes() is long bytes and >= 0) + { + Keys.NumberWhole(bytes); + } + } + Scratch.Grow(ref _rowGroups, rows); _ranges.Clear(); int before = Keys.Count; @@ -2968,6 +3031,7 @@ internal static async ValueTask RunAsync( AggregationPartition only = new AggregationPartition(plan, settled, columns, inputs, sorted, source: source, facts: facts, memory: memory) { Top = top, + NumbersWhole = plan.NumberWhole, Core = core?.Lane(), Pressure = pressure, Spill = spill, @@ -2987,6 +3051,7 @@ internal static async ValueTask RunAsync( partitions = [only]; plan.Watch?.Invoke(partitions); await RunPartitionAsync(source, pass, metrics, only, cancellationToken).ConfigureAwait(false); + only.DropUnmet(); } else { @@ -2998,6 +3063,7 @@ internal static async ValueTask RunAsync( partitions[p] = new AggregationPartition(plan, settled, columns, inputs, sorted, source: source, facts: facts, memory: memory) { Top = top is { FirstMet: true } ? null : top, + NumbersWhole = plan.NumberWhole, Core = core?.Lane(), Pressure = pressure, TurnOnNew = pressure is not null && plan.CoreOnNew && lanes >= (plan.CoreLanes ?? GroupCore.DefaultLanes) @@ -4334,6 +4400,7 @@ private static async Task RunQueueAsync( await RunPartitionAsync(source, lane with { Rows = ranges[at] }, metrics, partition, token).ConfigureAwait(false); } + partition.DropUnmet(); if (partition.Retiring) { await partition.RetireAsync(token).ConfigureAwait(false); diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index e905ae5e0..62e38f026 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -149,6 +149,19 @@ internal static int DoubledUpTo(int length, int bound) /// internal virtual (long Least, ulong Span)? ValueSpan => null; + /// + /// For an index that numbers its groups by value, before a row is assigned: each value of the span a + /// group from the start, its number, when the span's groups at of + /// states each fit the private cache (); false otherwise. + /// + internal virtual bool NumberWhole(long groupBytes) => false; + + /// + /// The groups an index that numbered its span whole keeps once the rows are folded, the values no + /// row met left out; null when it keeps them all. It numbers values as they first come from then on. + /// + internal virtual int[]? Met() => null; + /// /// For an index that numbers its groups by value over a span of values or /// more: the bins of , 4 096 bits over the span, that the selected values of a @@ -615,6 +628,10 @@ internal sealed class FixedKeys : GroupKeys // Whether the last chunk of rows met mostly values for the first time: the next goes by the lookup alone (DirectTwoPasses). private bool _directNew; + // Numbering the span whole (NumberWhole): the values of the span no row met yet, the group of each + // value its number; -1 when the keys number values as they first come. + private int _unmet = -1; + // Every key a group of its own, no table looked up (Appending); the shelf a sub-table's arrays come from (ForTable). private readonly bool _appending; private readonly ArrayShelf? _shelf; @@ -1174,7 +1191,11 @@ private void DirectTwoPasses(ReadOnlySpan canonical, ReadOnlySpan } int missed = 0; - if (pages.Length == 1) + if (_unmet == 0) + { + missed = Numbers(values, groups, min, span); + } + else if (pages.Length == 1) { // A span of one page, a few thousand values: its numbers read from the page itself, taken once // a chunk, where the page and its start read again at every row made a chain of three loads. @@ -1249,7 +1270,10 @@ private void DirectTwoPasses(ReadOnlySpan canonical, ReadOnlySpan } } - /// The group of a value within the bounds, by its number: added when it is new, its page allocated at the first value it holds. + /// + /// The group of a value within the bounds, by its number: added when it is new, its page allocated at + /// the first value it holds; numbering the span whole, the number itself, the value met. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private int Numbered(TValue value, ulong number) { @@ -1263,12 +1287,153 @@ private int Numbered(TValue value, ulong number) ref int group = ref slab[_pageStarts![at] + ((int)number & PageMask)]; if (group < 0) { - group = Add(value); + group = _unmet >= 0 ? Meet(number) : Add(value); } return group; } + /// A value of the span met for the first time while the keys number it whole: its group is its number. + private int Meet(ulong number) + { + _unmet--; + return (int)number; + } + + /// + /// The bytes a span numbered whole may take at most, its groups' states, keys and numbers: 1 MiB, the + /// private second level of cache of a current core, which 's windows + /// take as the line past which records leave it. + /// + internal const long WholeBytes = 1 << 20; + + /// + /// Numbers the span whole before a row is assigned, when its groups, at + /// of states each, fit : each value of the span + /// is a group from the start, its number, whether a row holds it or not, a null's group and the + /// values past the bounds after them. The pages tell the values met from the others; once every one + /// is met, a row's group is its number, read with no page (). The groups no row + /// met are dropped once the rows are folded (). + /// + /// Whether the keys number the span whole. + internal override bool NumberWhole(long groupBytes) + { + if (_pages is null || _sorted || _appending || Count != 0 + || (long)_span * (groupBytes + Unsafe.SizeOf() + sizeof(int)) > WholeBytes) + { + return false; + } + + int span = (int)_span; + if (_keys.Length < span) + { + Grow(span); + } + + for (int number = 0; number < span; number++) + { + _keys[number] = FromInteger(_directMin + number); + } + + Count = span; + _unmet = span; + return true; + } + + /// + /// The groups a span numbered whole keeps once its rows are folded: the values a row met, in their + /// order, then a null's group and the values past the bounds; null when every value was met, or the + /// span is not numbered whole. The keys number values as they first come from then on. + /// + internal override int[]? Met() + { + int unmet = _unmet; + _unmet = -1; + if (unmet <= 0) + { + return null; + } + + int[] met = new int[Count - unmet]; + int kept = 0; + int span = (int)_span; + for (int at = 0; at < _pages!.Length; at++) + { + int[] slab = _pages[at]; + if (slab == Unmet) + { + continue; + } + + int start = _pageStarts![at]; + int first = at << PageBits; + int numbers = Math.Min(1 << PageBits, span - first); + for (int n = 0; n < numbers; n++) + { + if (slab[start + n] >= 0) + { + met[kept++] = first + n; + } + } + } + + for (int group = span; group < Count; group++) + { + met[kept++] = group; + } + + return met; + } + + /// The first pass of a chunk once every value of a span numbered whole is met: a row's group is its number, -1 past the bounds. + /// The groups or-ed together: negative when a row is left for the lookup. + private static int Numbers(ReadOnlySpan values, Span groups, long min, ulong span) + { + int left = 0; + int i = 0; + ref int group = ref MemoryMarshal.GetReference(groups); + if (Vector.IsHardwareAccelerated && Unsafe.SizeOf() == sizeof(int) && values.Length >= Vector.Count) + { + // Four bytes, an int or a uint: the number less the least wraps as the value does, and the span + // never does, so that one comparison of the difference as unsigned tells a value within it. + ref int value = ref Unsafe.As(ref MemoryMarshal.GetReference(values)); + Vector least = new Vector((int)min); + Vector width = new Vector((uint)span); + Vector lefts = Vector.Zero; + for (; i <= values.Length - Vector.Count; i += Vector.Count) + { + Vector number = Vector.LoadUnsafe(ref value, (nuint)i) - least; + Vector numbered = number | Vector.AsVectorInt32(Vector.GreaterThanOrEqual(Vector.AsVectorUInt32(number), width)); + numbered.StoreUnsafe(ref group, (nuint)i); + lefts |= numbered; + } + + left = Vector.LessThanAny(lefts, Vector.Zero) ? -1 : 0; + } + + for (; i < values.Length; i++) + { + ulong number = (ulong)(Integer(values[i]) - min); + int numbered = number < span ? (int)number : -1; + Unsafe.Add(ref group, i) = numbered; + left |= numbered; + } + + return left; + } + + /// The value of as a key, the inverse of . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TValue FromInteger(long integer) => + typeof(TValue) == typeof(sbyte) ? Unsafe.BitCast((sbyte)integer) + : typeof(TValue) == typeof(short) ? Unsafe.BitCast((short)integer) + : typeof(TValue) == typeof(int) ? Unsafe.BitCast((int)integer) + : typeof(TValue) == typeof(long) ? Unsafe.BitCast(integer) + : typeof(TValue) == typeof(byte) ? Unsafe.BitCast((byte)integer) + : typeof(TValue) == typeof(ushort) ? Unsafe.BitCast((ushort)integer) + : typeof(TValue) == typeof(uint) ? Unsafe.BitCast((uint)integer) + : Unsafe.BitCast((ulong)integer); + /// The most pages a slab holds: 256 KiB. private const int MostSlabPages = 16; @@ -1476,6 +1641,7 @@ internal override void Release() _keys = []; Count = 0; _null = -1; + _unmet = -1; } internal override int NullNumber => _null; @@ -1506,13 +1672,20 @@ internal override int CompareKeys(int a, int b, int component) private void Keep(ReadOnlySpan groups, int room) { // The table of groups forgets every group's number, then learns the kept ones' new ones; a - // value it holds is in no index. + // value it holds is in no index. A span numbered whole is numbered as values come from then on. + _unmet = -1; for (int g = 0; g < Count; g++) { - if (g != _null && DirectSlot(_keys[g], out ulong number)) + if (g == _null || !DirectSlot(_keys[g], out ulong number)) { - int page = (int)(number >> PageBits); - _pages![page][_pageStarts![page] + ((int)number & PageMask)] = -1; + continue; + } + + // A value of a span numbered whole that no row met may lie in a page never allocated. + int page = (int)(number >> PageBits); + if (_pages![page] != Unmet) + { + _pages[page][_pageStarts![page] + ((int)number & PageMask)] = -1; } } diff --git a/tests/Vorticity.Tests/Aggregation/DirectGroupKeysTests.cs b/tests/Vorticity.Tests/Aggregation/DirectGroupKeysTests.cs index 4ae03a7f8..4fd049efd 100644 --- a/tests/Vorticity.Tests/Aggregation/DirectGroupKeysTests.cs +++ b/tests/Vorticity.Tests/Aggregation/DirectGroupKeysTests.cs @@ -41,6 +41,76 @@ public void ATableOfGroupsFollowsTheGroupsAKeepRenumbers() Assert.Equal(6, keys.Count); } + // A span numbered whole: each value's group is its number from the start, a value past the bounds a + // group past the span; the values no row met are dropped once the rows are folded, the others kept in + // their order, and the keys number values as they first come from then on. + [Fact] + public void ASpanNumberedWholeDropsTheValuesNoRowMet() + { + GroupKeys keys = new FixedKeys(Shape(VortexType.Int32), sorted: false, new KeyBounds(-5, 5)); + Assert.True(keys.NumberWhole(groupBytes: 16)); + AssertGroupedAs(keys, [3, -5, 3, 0, 5, -5], [8, 0, 8, 5, 10, 0]); + AssertGroupedAs(keys, [9, 0], [11, 5]); + Assert.Equal(12, keys.Count); + + int[] met = keys.Met()!; + Assert.Equal([0, 5, 8, 10, 11], met); + keys.Keep(met); + AssertGroupedAs(keys, [5, 1, -5, 9], [3, 5, 0, 4]); + Assert.Equal(6, keys.Count); + Assert.Null(keys.Met()); + } + + // Every value of a span numbered whole met, a row's group is its number with no page read, a value past + // the bounds left for the lookup, in vectors and then row by row; nothing is dropped. A span whose + // groups would not fit the private cache is numbered as values come. + [Fact] + public void ASpanMetWholeNumbersItsRowsByTheirValue() + { + GroupKeys keys = new FixedKeys(Shape(VortexType.Int32), sorted: false, new KeyBounds(100, 163)); + Assert.True(keys.NumberWhole(groupBytes: 16)); + AssertGroupedAs(keys, [.. Enumerable.Range(100, 64)], [.. Enumerable.Range(0, 64)]); + int[] values = [.. Enumerable.Range(0, 37).Select(i => i == 20 ? 99 : i == 33 ? 500 : 100 + (i * 7 % 64))]; + AssertGroupedAs(keys, values, [.. values.Select(value => value == 99 ? 64 : value == 500 ? 65 : value - 100)]); + Assert.Null(keys.Met()); + + GroupKeys wide = new FixedKeys(Shape(VortexType.Int32), sorted: false, new KeyBounds(0, (1 << 16) - 1)); + Assert.True(((FixedKeys)wide).ByValue); + Assert.False(wide.NumberWhole(groupBytes: 16)); + } + + // A span numbered whole whose rows meet one value in three, a null in thirteen among them, under a filter + // and not: the groups .NET makes at every degree, as numbering values as they come makes them. + [Theory] + [InlineData(1, true)] + [InlineData(4, true)] + [InlineData(4, false)] + public async Task ASpanNumberedWholeGroupsAsTheRowsDo(int degree, bool whole) + { + Reading[] rows = Readings(); + string path = await WriteAsync(rows); + try + { + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = degree); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + foreach (bool filtered in (bool[])[false, true]) + { + Scan scan = filtered ? file.Scan().Where(r => r.Value > 3) : file.Scan(); + Vorticity.Aggregation grouped = scan.GroupBy(r => r.Sparse).Select(g => (g.Key, g.Count(), g.Sum(r => r.Value))); + grouped.Plan.NumberWhole = whole; + List totals = await ListAsync(grouped.As()); + Assert.Equal( + rows.Where(r => !filtered || r.Value > 3).GroupBy(r => r.Sparse).Select(g => new SparseTotal(g.Key, g.Count(), g.Sum(r => r.Value))) + .OrderBy(t => t.Sparse ?? int.MinValue), + totals.OrderBy(t => t.Sparse ?? int.MinValue)); + } + } + finally + { + System.IO.File.Delete(path); + } + } + [Theory] [InlineData(1)] [InlineData(4)] @@ -99,14 +169,19 @@ private static void AssertGroupedAs(GroupKeys keys, int[] values, int[] groups) private static ColumnShape Shape(VortexType type) => new ColumnShape(new ColumnSym(Expr.Field("k"), type, null, null, -1, [])); - /// Keys scattered over a few hundred values: negative ones, nullable ones, unsigned ones. + /// Keys scattered over a few hundred values: negative ones, nullable ones, unsigned ones, one value in three of their span. private static Reading[] Readings() { Reading[] rows = new Reading[Rows]; for (int row = 0; row < Rows; row++) { int scattered = (int)((uint)(row * 2_654_435_761u) >> 23); - rows[row] = new Reading(scattered - 256, scattered % 9 == 0 ? null : (short)(scattered % 40 - 20), (uint)(scattered % 300) + 4_000_000_000u, row % 7); + rows[row] = new Reading( + scattered - 256, + scattered % 9 == 0 ? null : (short)(scattered % 40 - 20), + (uint)(scattered % 300) + 4_000_000_000u, + row % 7, + row % 13 == 0 ? null : (scattered % 200 * 3) - 300); } return rows; @@ -139,7 +214,10 @@ private static async Task WriteAsync(Reading[] rows) } [VortexRecord] - public partial record struct Reading(int Small, short? Maybe, uint Unsigned, long Value); + public partial record struct Reading(int Small, short? Maybe, uint Unsigned, long Value, int? Sparse); + + [VortexRecord] + public partial record struct SparseTotal(int? Sparse, long Count, long Sum); [VortexRecord] public partial record struct Total(int Small, long Count, long Sum); diff --git a/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs b/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs index 1ce68570c..3a2dd9397 100644 --- a/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs +++ b/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs @@ -29,7 +29,7 @@ public sealed partial class WorkCounterTests /// The counts at one lane, a line a query, as writes them. private static readonly string[] Expected = [ - "count and sum by 10^3 values: result 1501672601, rows 300000, lanes 1, lane groups 1000, arrays 8 of 40888 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 1310720 B", + "count and sum by 10^3 values: result 1501672601, rows 300000, lanes 1, lane groups 1000, arrays 3 of 36824 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 1310720 B", "count and sum by 10^5 values: result 1501672601, rows 300000, lanes 1, lane groups 95013, arrays 19 of 3653560 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 4177976 B", "count and sum by 10^6 values: result 1501672601, rows 300000, lanes 1, lane groups 259343, arrays 35 of 12566512 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 12566640 B", "count and sum by an ordered key: result 1501672601, rows 300000, lanes 1, lane groups 1, arrays 51 of 3720508 B, copied 0 B, blocks 4/0/0, merge 0 parts 0 entries, peak 3606448 B", From 5c4cd24ba77d7c27b6b18501168c332bb44f3924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 15:43:21 +0200 Subject: [PATCH 09/56] perf(query): an integer mean's add inlined, no call a row The profile of db-benchmark's q4 at one lane, its key numbered whole, showed NarrowSignedSum.Add apart from the loop that called it: 19 % of the cycles in a function of two statements, a call a row and a value. The native compiler, with no profile to guide it, leaves out of line an add whose conversion (long.CreateTruncating) checks its type in its IL; a total of one statement it inlines. The adds of the four sums that carry a count, the integer means' states, are now marked AggressiveInlining. runners.sh alt against the runner of the previous commit, warm, on a busy Mac: group-db-q4 at one lane x0.897 (46.4 -> 41.6 ms), cycles x0.920. --- src/Vorticity/Aggregation/ValueOps.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Vorticity/Aggregation/ValueOps.cs b/src/Vorticity/Aggregation/ValueOps.cs index 1143bb51b..a1293ebdc 100644 --- a/src/Vorticity/Aggregation/ValueOps.cs +++ b/src/Vorticity/Aggregation/ValueOps.cs @@ -74,6 +74,7 @@ internal struct ExtremeState { public SumState Seed() => default; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add(ref SumState state, TValue value) { state.Sum += Int128.CreateTruncating(value); @@ -123,6 +124,9 @@ public void AddWords(ref SumState state, ReadOnlySpan block, Rea { public SumState Seed() => default; + // Inlined by hand: two statements and the conversion's checks of its type, the native compiler left + // the add a call a row, 19 % of the cycles of db-benchmark's q4 at one lane (2026-10-09). + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add(ref SumState state, TValue value) { state.Sum += long.CreateTruncating(value); @@ -168,6 +172,7 @@ public void AddWords(ref SumState state, ReadOnlySpan block, ReadO { public SumState Seed() => default; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add(ref SumState state, TValue value) { state.Sum += ulong.CreateTruncating(value); @@ -213,6 +218,7 @@ public void AddWords(ref SumState state, ReadOnlySpan block, Read { public SumState Seed() => default; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add(ref SumState state, TValue value) { state.Sum += UInt128.CreateTruncating(value); From 113d7130a249a5d04c865d92cf1a3d349abd8240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 15:50:37 +0200 Subject: [PATCH 10/56] perf(query): two aggregates of one kind over two columns fold in one pass db-benchmark's q4 takes the means of V1 and V2, two integer columns, and q5 their sums: two slots of the same type, their states in the same record, each folding the window in a pass of its own, its group read and its record reached again. A slot now has a twin, the next slot of its type, both unfiltered, their states in the partition's records (AggregationPartition.TwinOf), and FixedSlot.StepRowsPaired folds both in one pass: a row's group read once, its record reached once, each value read from its column. A batch with nulls on either side, a column coded or constant, or a selection folds them apart as before. PairedSlotsTests checks sums and means of integers against LINQ, nulls in half the batches, a coded column and a filtered twin, at one and four lanes; it fails when the twin's add is left out. runners.sh alt against the runner of the previous commit, warm, on a busy Mac (load 14): group-db-q4 at one lane x0.895 (42.1 -> 37.7 ms), group-db-q5 x0.923 (79.4 -> 73.0 ms). The plan gave the pass up below a tenth on q4: it holds it, just. --- .../Aggregation/AggregationEngine.cs | 57 +++++++++- src/Vorticity/Aggregation/FixedSlot.cs | 49 +++++++++ src/Vorticity/Aggregation/Slots.cs | 8 ++ .../Aggregation/PairedSlotsTests.cs | 102 ++++++++++++++++++ 4 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 tests/Vorticity.Tests/Aggregation/PairedSlotsTests.cs diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 0f23e0c09..964e24200 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -2464,12 +2464,25 @@ private void Fold(RecordBatch batch) Slots[first].StepRows(Input(number, arena, first, rows, selection).Window(start, end), rowGroups); } + // A slot and its twin, the same aggregate over another column, fold in one pass when they can. + ulong paired = 0; for (int i = 0; i < Slots.Length; i++) { - if (i != first && Folds(i) && !(carried && (i == count || i == carrier))) + if (i == first || !Folds(i) || (carried && (i == count || i == carrier)) || (i < 64 && ((paired >> i) & 1) != 0)) { - Slots[i].StepRows(Input(number, arena, i, rows, selection).Window(start, end), rowGroups); + continue; } + + BatchInput input = Input(number, arena, i, rows, selection).Window(start, end); + int twin = TwinOf(i); + if (twin >= 0 && twin != first && Folds(twin) && !(carried && (twin == count || twin == carrier)) + && Slots[i].StepRowsPaired(input, rowGroups, Slots[twin], Input(number, arena, twin, rows, selection).Window(start, end))) + { + paired |= 1UL << twin; + continue; + } + + Slots[i].StepRows(input, rowGroups); } } @@ -2757,6 +2770,46 @@ private BatchInput Input(long number, CanonicalArena arena, int slot, int rows, /// Whether the aggregate has rows of the batch to fold: none when its filter keeps none. private bool Folds(int slot) => _filterOf[slot] < 0 || !_masks!.KeepsNone(_filterOf[slot]); + // Each slot's twin (TwinOf), made at the first batch. + private int[]? _twins; + + /// + /// The slot that folds in 's pass (), or + /// -1: the next one of the same type, the same aggregate over another column, both unfiltered, their + /// states in the same records, among the first 64 slots; each slot the twin of one at most. + /// + private int TwinOf(int slot) + { + if (_twins is null) + { + int[] twins = new int[Slots.Length]; + twins.AsSpan().Fill(-1); + ulong taken = 0; + for (int i = 0; i < Math.Min(Slots.Length, 64); i++) + { + if (((taken >> i) & 1) != 0 || _inputs[i] == Settled || _filterOf[i] >= 0 || Slots[i].Bound is not { } records) + { + continue; + } + + for (int j = i + 1; j < Math.Min(Slots.Length, 64); j++) + { + if (((taken >> j) & 1) == 0 && _inputs[j] != Settled && _filterOf[j] < 0 && Slots[j].GetType() == Slots[i].GetType() + && ReferenceEquals(Slots[j].Bound, records)) + { + twins[i] = j; + taken |= (1UL << i) | (1UL << j); + break; + } + } + } + + _twins = twins; + } + + return _twins[slot]; + } + /// /// The slot of a count of rows in the records, unfiltered, and the fixed slot that can carry it in its /// own pass: unfiltered too, its state in the same record, and when records diff --git a/src/Vorticity/Aggregation/FixedSlot.cs b/src/Vorticity/Aggregation/FixedSlot.cs index 63ee4386e..5ea2d5e30 100644 --- a/src/Vorticity/Aggregation/FixedSlot.cs +++ b/src/Vorticity/Aggregation/FixedSlot.cs @@ -293,6 +293,55 @@ private bool StepCounted(in BatchInput input, ReadOnlySpan groups, return true; } + internal override bool StepRowsPaired(in BatchInput input, ReadOnlySpan groups, AggregateSlot twin, in BatchInput twinInput) + { + if (twin is not FixedSlot other || RuntimeHelpers.IsReferenceOrContainsReferences() + || Bound is not { } records || !ReferenceEquals(records, other.Bound) || !input.Selection.IsEmpty || !twinInput.Selection.IsEmpty + || FixedReader.EncodingOf(input.Arena, input.Node, _kind) is ColumnEncoding.Constant or ColumnEncoding.Dictionary + || FixedReader.EncodingOf(twinInput.Arena, twinInput.Node, other._kind) is ColumnEncoding.Constant or ColumnEncoding.Dictionary) + { + return false; + } + + ReadOnlySpan values = _values.Of(input.Arena, input.Batch, input.Node, _kind, out ReadOnlySpan valid); + ReadOnlySpan twinValues = other._values.Of(twinInput.Arena, twinInput.Batch, twinInput.Node, other._kind, out ReadOnlySpan twinValid); + if (!_rows.And(input, input.Selection, valid).IsEmpty || !other._rows.And(twinInput, twinInput.Selection, twinValid).IsEmpty) + { + return false; + } + + // Both states in one record: a row's group read once, its record reached once, each value read + // from its own column. + TOp op = _op; + TOp twinOp = other._op; + StateView states = States; + nint state = states.Offset; + nint twinState = other.States.Offset; + ref int groupOf = ref MemoryMarshal.GetReference(groups); + ref TValue valueOf = ref MemoryMarshal.GetReference(values); + ref TValue twinOf = ref MemoryMarshal.GetReference(twinValues); + if (input.Settled) + { + for (int row = input.Start; row < input.End; row++) + { + ref byte record = ref states.Record(Unsafe.Add(ref groupOf, row)); + op.Add(ref Unsafe.As(ref Unsafe.AddByteOffset(ref record, state)), Unsafe.Add(ref valueOf, row)); + twinOp.Add(ref Unsafe.As(ref Unsafe.AddByteOffset(ref record, twinState)), Unsafe.Add(ref twinOf, row)); + } + } + else + { + for (int row = input.Start; row < input.End; row++) + { + ref byte record = ref states.Record(Unsafe.Add(ref groupOf, row)); + op.AddSelected(ref Unsafe.As(ref Unsafe.AddByteOffset(ref record, state)), Unsafe.Add(ref valueOf, row)); + twinOp.AddSelected(ref Unsafe.As(ref Unsafe.AddByteOffset(ref record, twinState)), Unsafe.Add(ref twinOf, row)); + } + } + + return true; + } + /// /// A run-end or a constant column folds a range as a weighted value per run it overlaps, where /// its rows would expand the column first: ranges however short. A dictionary or a canonical diff --git a/src/Vorticity/Aggregation/Slots.cs b/src/Vorticity/Aggregation/Slots.cs index 576507848..8a804a382 100644 --- a/src/Vorticity/Aggregation/Slots.cs +++ b/src/Vorticity/Aggregation/Slots.cs @@ -91,6 +91,14 @@ internal abstract class AggregateSlot /// Whether the slot can count rows in its own pass (). internal virtual bool CarriesCount => false; + /// + /// As , 's rows folded in the same pass, a slot of the + /// same aggregate over another column, its states in the same records: a row's group read and its + /// record reached once for both, when every row of the window is selected and none is null on either + /// side. False when it cannot, nothing folded. + /// + internal virtual bool StepRowsPaired(in BatchInput input, ReadOnlySpan groups, AggregateSlot twin, in BatchInput twinInput) => false; + /// /// Whether folds ranges of a few rows of for /// less than a group per row: in one call, its column's form read once, and in less work per diff --git a/tests/Vorticity.Tests/Aggregation/PairedSlotsTests.cs b/tests/Vorticity.Tests/Aggregation/PairedSlotsTests.cs new file mode 100644 index 000000000..2e4b85b36 --- /dev/null +++ b/tests/Vorticity.Tests/Aggregation/PairedSlotsTests.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Vorticity.Tests.Aggregation; + +/// +/// Two aggregates of one kind over two columns fold in one pass (AggregateSlot.StepRowsPaired), and +/// apart when a batch has nulls on either side, a column comes as a dictionary, or one is filtered: sums +/// and means of integers against LINQ over the same rows, at every degree. +/// +public sealed partial class PairedSlotsTests +{ + private const int Rows = 60_000; + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task TwoAggregatesOfOneKindFoldAsTheRowsDo(int degree) + { + Reading[] rows = Readings(); + string path = await WriteAsync(rows); + try + { + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = degree); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + List totals = []; + Scan grouped = file.Scan().GroupBy(r => r.Key) + .Select(g => ( + g.Key, + g.Count(), + g.Sum(r => r.A), + g.Sum(r => r.B), + g.Where(r => r.A > 500).Sum(r => r.B), + g.Average(r => r.Maybe), + g.Average(r => r.A), + g.Average(r => r.Coded), + g.Average(r => r.B))) + .As(); + await foreach (Totals total in grouped.ToRecordsAsync(Ct)) + { + totals.Add(total); + } + + Assert.Equal( + rows.GroupBy(r => r.Key).Select(g => new Totals( + g.Key, + g.Count(), + g.Sum(r => (long)r.A), + g.Sum(r => (long)r.B), + g.Where(r => r.A > 500).Sum(r => (long)r.B), + g.Average(r => r.Maybe), + g.Average(r => r.A), + g.Average(r => r.Coded), + g.Average(r => r.B))).OrderBy(t => t.Key), + totals.OrderBy(t => t.Key)); + } + finally + { + System.IO.File.Delete(path); + } + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// A key of 128 values; nulls in the second half of the rows alone; a column of five values, which the writer codes. + private static Reading[] Readings() + { + Reading[] rows = new Reading[Rows]; + for (int row = 0; row < Rows; row++) + { + int key = (int)((uint)(row * 2_654_435_761u) >> 25); + rows[row] = new Reading(key, row * 7 % 1_000, (row * 13 % 1_000) - 500, row >= Rows / 2 && row % 11 == 0 ? null : row * 3 % 700, row % 5 * 10); + } + + return rows; + } + + private static async Task WriteAsync(Reading[] rows) + { + string directory = Path.Combine(AppContext.BaseDirectory, "paired-slots"); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, $"readings-{Environment.ProcessId}-{Guid.NewGuid():N}.vortex"); + await using (VortexFileWriter writer = VortexSession.Default.CreateWriter(path, new VortexWriteOptions { RowBlockSize = 8_192, ChunkTargetBytes = 1 << 16 })) + { + await writer.WriteAsync(rows, Ct); + await writer.CompleteAsync(Ct); + } + + return path; + } + + [VortexRecord] + public partial record struct Reading(int Key, int A, int B, int? Maybe, int Coded); + + [VortexRecord] + public partial record struct Totals(int Key, long Count, long SumA, long SumB, long BigB, double? MeanMaybe, double? MeanA, double? MeanCoded, double? MeanB); +} From 0e06b92fd27d7416c3c521ccb4b5bd65a33850b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 16:07:45 +0200 Subject: [PATCH 11/56] test: a grouped float sum's bits against its values one at a time; --micro floatsum The two tools the grouped exact float sum is changed by, before it is. GroupedFloatSumTests holds a group by's float sums and means to the bits an IndexedSum gives each group's values one at a time, in the order of the rows, at one, four and fourteen lanes, the rows cut into small blocks: groups of values of one range beside groups of tiny, subnormal and huge values, of zeros of both signs, of values that grow along the rows, of NaN and infinities; and a key that opens a group at every row. vortex-queries --micro floatsum folds a column of doubles, batch after batch, into the sums of their groups by the slot a sum or a mean of doubles takes, the groups given, in memory, and prints a checksum of every group's bits: q4's hundred groups, q3's hundred thousand, a hundred groups of ranges of their own, and q10's group a row. Under DOTNET_TieredCompilation=0 today: 2.6, 4.0, 2.6 and 5.0 ns a row. --- .../FloatSumMicro.cs | 105 ++++++++++++ bench/Vorticity.Benchmarks.Queries/Program.cs | 6 + .../Aggregation/GroupedFloatSumTests.cs | 153 ++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs create mode 100644 tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs diff --git a/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs b/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs new file mode 100644 index 000000000..a26939827 --- /dev/null +++ b/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs @@ -0,0 +1,105 @@ +using System; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using Vorticity.Aggregating; +using Vorticity.Arrays; +using Vorticity.Buffers; +using Vorticity.Types; + +namespace Vorticity.Benchmarks.Queries; + +/// +/// A grouped exact float sum alone, in memory (--micro floatsum): a column of doubles folded batch +/// after batch into the of each row's group, by the slot a sum or a mean of +/// doubles takes, the rows' groups given. No file, no decode, no key: what a change to the sum does, in +/// seconds. The checksum, the bits of every group's sum folded together, tells two builds' sums apart. +/// +/// +/// The shapes are db-benchmark's: q4's mean of v3 by a hundred groups, q3's by a hundred thousand, +/// q10's sum where every row opens its group; and a hundred groups whose values lie in ranges of their +/// own. Run it with DOTNET_TieredCompilation=0, so that every pass runs the code at its last tier. +/// +internal static class FloatSumMicro +{ + /// The rows of a batch: a chunk of the bench's db-benchmark file. + private const int BatchRows = 65_536; + + internal static int Run(int rounds) + { + Console.WriteLine($"{"float sum",-32} {"rows",11} {"groups",10} {"best ms",10} {"ns/row",8} {"checksum",18}"); + Measure("1e2 groups, one range (q4)", 10_000_000, 100, mixed: false, rounds); + Measure("1e5 groups, one range (q3)", 10_000_000, 100_000, mixed: false, rounds); + Measure("1e2 groups, ranges of their own", 10_000_000, 100, mixed: true, rounds); + Measure("a group a row (q10)", 2_000_000, 0, mixed: false, rounds); + return 0; + } + + /// The best of passes; 0 for a group a row, each row its own in order. + private static void Measure(string name, int rows, int groups, bool mixed, int rounds) + { + double[] values = new double[rows]; + int[] rowGroups = new int[rows]; + for (int row = 0; row < rows; row++) + { + ulong mix = Mix((ulong)row); + int group = groups == 0 ? row : (int)((mix >> 40) % (ulong)groups); + rowGroups[row] = group; + double unit = (mix >> 11) * (1.0 / (1UL << 53)); + values[row] = mixed ? Math.ScaleB(unit + 0.5, (group % 16 * 9) - 70) * ((mix & 1) == 0 ? 1 : -1) : unit * 100; + } + + double best = double.MaxValue; + long checksum = 0; + for (int round = 0; round < rounds; round++) + { + (double ms, long sum) = Pass(values, rowGroups, groups == 0 ? rows : groups); + best = Math.Min(best, ms); + checksum = sum; + } + + Console.WriteLine(string.Create( + CultureInfo.InvariantCulture, + $"{name,-32} {rows,11:N0} {(groups == 0 ? rows : groups),10:N0} {best,10:F1} {best * 1e6 / rows,8:F2} {checksum,18:X16}")); + } + + /// One pass: every batch's values a canonical node of an arena, folded into their groups' sums. + private static (double Ms, long Checksum) Pass(double[] values, int[] rowGroups, int groups) + { + GC.Collect(); + RetainingArena arena = new RetainingArena(); + DType dtype = new DTypeArena().Primitive(PType.F64, Nullability.NonNullable); + FixedSlot, double> slot = + new FixedSlot, double>(StorageKind.Primitive, static s => s.Value); + slot.EnsureGroups(groups); + long start = Stopwatch.GetTimestamp(); + long batch = 0; + for (int at = 0; at < values.Length; at += BatchRows) + { + int count = Math.Min(BatchRows, values.Length - at); + arena.ResetKeepingBlocks(); + VortexBuffer buffer = arena.Allocate(count * sizeof(double), sizeof(double), out Span bytes); + values.AsSpan(at, count).CopyTo(MemoryMarshal.Cast(bytes)); + int node = arena.AddPrimitive(dtype, count, Validity.NonNullable, PType.F64, buffer); + slot.StepRows(new BatchInput(++batch, arena, node, count, []), rowGroups.AsSpan(at, count)); + } + + double ms = Stopwatch.GetElapsedTime(start).TotalMilliseconds; + long checksum = 0; + for (int group = 0; group < groups; group++) + { + checksum = (checksum * 31) ^ BitConverter.DoubleToInt64Bits(slot.Result(group)); + } + + arena.Reset(); + return (ms, checksum); + } + + private static ulong Mix(ulong x) + { + x += 0x9E3779B97F4A7C15UL; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9UL; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBUL; + return x ^ (x >> 31); + } +} diff --git a/bench/Vorticity.Benchmarks.Queries/Program.cs b/bench/Vorticity.Benchmarks.Queries/Program.cs index ed3f4c689..7afc20f44 100644 --- a/bench/Vorticity.Benchmarks.Queries/Program.cs +++ b/bench/Vorticity.Benchmarks.Queries/Program.cs @@ -59,6 +59,12 @@ return WordMicro.Run(rounds); } +// --micro floatsum: a grouped exact float sum alone, in memory (FloatSumMicro.cs). +if (Text(args, "--micro") == "floatsum") +{ + return FloatSumMicro.Run(rounds); +} + // Each file is written the first time a query asks for it, and kept. Dictionary>> fixtures = new Dictionary>>(StringComparer.Ordinal) { diff --git a/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs b/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs new file mode 100644 index 000000000..59914c700 --- /dev/null +++ b/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs @@ -0,0 +1,153 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Vorticity.Aggregating; +using Xunit; + +namespace Vorticity.Tests.Aggregation; + +/// +/// The float sums and means of a group by are the bits each group's values give an +/// one at a time, in the order of the rows, whatever the degree and the cut of +/// the rows into batches: groups of values of one range, which share a top, beside groups of tiny, +/// subnormal and huge values, of zeros, of values that grow, of NaN and infinities; and a key that opens +/// a group at every row. +/// +public sealed partial class GroupedFloatSumTests +{ + private const int Rows = 300_000; + + private const int Groups = 1_000; + + [Theory] + [InlineData(1)] + [InlineData(4)] + [InlineData(14)] + public async Task AGroupsFloatSumsAreTheBitsOfItsValuesOneAtATime(int degree) + { + (Reading[] rows, string path) = await Fixture.Async; + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = degree); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + + List sums = []; + Scan grouped = file.Scan().GroupBy(r => r.Key) + .Select(g => (g.Key, g.Sum(r => r.Mixed), g.Sum(r => r.Uniform), g.Average(r => r.Uniform), g.Average(r => r.Mixed))) + .As(); + await foreach (KeySums sum in grouped.ToRecordsAsync(Ct)) + { + sums.Add(sum); + } + + Dictionary expected = []; + foreach (Reading row in rows) + { + (IndexedSum mixed, IndexedSum uniform) = expected.GetValueOrDefault(row.Key); + mixed.Add(row.Mixed); + uniform.Add(row.Uniform); + expected[row.Key] = (mixed, uniform); + } + + Assert.Equal(expected.Count, sums.Count); + Assert.Equal( + expected.OrderBy(e => e.Key).Select(e => (e.Key, Bits(e.Value.Mixed.Value), Bits(e.Value.Uniform.Value), Bits(Mean(e.Value.Uniform)), Bits(Mean(e.Value.Mixed)))), + sums.OrderBy(s => s.Key).Select(s => (s.Key, Bits(s.Mixed), Bits(s.Uniform), Bits(s.UniformMean), Bits(s.MixedMean)))); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task AKeyThatOpensAGroupAtEveryRowSumsAsItsValuesDo(int degree) + { + (Reading[] rows, string path) = await Fixture.Async; + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = degree); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + + List sums = []; + await foreach (IdSum sum in file.Scan().GroupBy(r => r.Id).Select(g => (g.Key, g.Sum(r => r.Mixed))).As().ToRecordsAsync(Ct)) + { + sums.Add(sum); + } + + Assert.Equal( + rows.Select(r => (r.Id, Bits(Single(r.Mixed)))), + sums.OrderBy(s => s.Id).Select(s => (s.Id, Bits(s.Mixed)))); + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static long Bits(double value) => BitConverter.DoubleToInt64Bits(value); + + private static long Bits(double? value) => value is double some ? BitConverter.DoubleToInt64Bits(some) : long.MinValue; + + private static double? Mean(IndexedSum sum) => sum.Count == 0 ? null : sum.Value / sum.Count; + + private static double Single(double value) + { + IndexedSum sum = default; + sum.Add(value); + return sum.Value; + } + + /// + /// A value of the row's group's kind: of one range shared by most groups; tiny, subnormal or huge; + /// zeros of both signs; growing from small to large along the rows; or of any range, a NaN and an + /// infinity among them now and then. + /// + private static double MixedOf(int key, int row, Random random) + { + double unit = random.NextDouble(); + double sign = random.Next(4) == 0 ? -1 : 1; + return (key % 10) switch + { + 0 or 1 or 2 or 3 => sign * unit * 100, + 4 => sign * unit * 1e-200, + 5 => sign * unit * 1e-310, + 6 => sign * unit * 1e300, + 7 => random.Next(2) == 0 ? 0.0 : -0.0, + 8 => sign * Math.ScaleB(unit + 0.5, (int)((long)row * 120 / Rows) - 60), + _ => row % 97 == 0 ? double.NaN : row % 1_013 == 0 ? (sign > 0 ? double.PositiveInfinity : double.NegativeInfinity) + : sign * Math.ScaleB(unit + 0.5, random.Next(-80, 81)), + }; + } + + private static class Fixture + { + private static readonly Lazy> s_written = new Lazy>(WriteAsync); + + internal static Task<(Reading[] Rows, string Path)> Async => s_written.Value; + + private static async Task<(Reading[], string)> WriteAsync() + { + Random random = new Random(23); + Reading[] rows = new Reading[Rows]; + for (int row = 0; row < Rows; row++) + { + int key = (int)((uint)(row * 2_654_435_761u) % Groups); + rows[row] = new Reading(row, key, MixedOf(key, row, random), random.NextDouble() * 1_000); + } + + string directory = Path.Combine(AppContext.BaseDirectory, "grouped-float-sum"); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, $"readings-{Environment.ProcessId}.vortex"); + await using (VortexFileWriter writer = VortexSession.Default.CreateWriter(path, new VortexWriteOptions { RowBlockSize = 4_096, ChunkTargetBytes = 1 << 15 })) + { + await writer.WriteAsync(rows, CancellationToken.None); + await writer.CompleteAsync(CancellationToken.None); + } + + return (rows, path); + } + } + + [VortexRecord] + public partial record struct Reading(int Id, int Key, double Mixed, double Uniform); + + [VortexRecord] + public partial record struct KeySums(int Key, double Mixed, double Uniform, double? UniformMean, double? MixedMean); + + [VortexRecord] + public partial record struct IdSum(int Id, double Mixed); +} From 1f422d4691be23d8af64228ab4ca9284eebe9d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 16:25:50 +0200 Subject: [PATCH 12/56] perf(query): a grouped float sum split with one top's extractors held A row of a grouped exact float sum went through IndexedSum.Add: the group's top read from its state, the limit and the three extractors read by it from their tables, then the value split by seven additions into the bins. The split waited on the group's state, and a row whose group a row still in flight held waited for that chain whole: 2.6 ns a row at a hundred groups (--micro floatsum), 60 % of db-benchmark's q4. IndexedSum.Fold now splits a window's rows at the top of its first row's group, or of its first value when the group holds none yet, the top's limit and extractors held in registers. A row whose group sits at that top, its count under the endurance and its value under the limit, the two read at once (the group's top and count less the top's, unsigned), is split with no wait on the group: only its count and its three bins are. A group that holds no value yet takes its first at the top the value calls for, with no bin to move, as AddSlow does. Any other row, a NaN, an infinity or a value past the limit among them, goes through Add. A window where more than a quarter of the rows did sends the next eight through Add straight (FixedSlot._ownTops): a hundred groups of ranges of their own ran x1.13 with the branch between the two paths taken at random, and x1.03 with the windows left to Add. The top a magnitude calls for is now read off its exponent's bits, or for a subnormal off its highest bit, where Math.ILogB was a call (IndexedSum.Needed, which a test holds to ILogB's for every binade). The parts are those Add gives: GroupedFloatSumTests now adds a column whose groups sit at tops of their own, a third of the rows at a time, and sums it by a key that opens a group a row; with the first value of a group taken at the window's top rather than its own, it fails at every degree. --micro floatsum under DOTNET_TieredCompilation=0, checksums unchanged: a hundred groups 2.61 -> 1.46 ns a row, a hundred thousand 4.01 -> 2.21, a group a row 5.02 -> 2.90, a hundred groups of ranges of their own 2.62 -> 2.75. runners.sh alt against the runner of 8be9f89e, warm, at one lane: group-db-q4 x0.668 (36.5 -> 24.4 ms), group-db-q3 x0.855 (127.1 -> 109.0 ms), group-mean-k6 x0.881 (88.6 -> 77.9 ms). --- src/Vorticity/Aggregation/FixedSlot.cs | 22 ++++ src/Vorticity/Aggregation/IndexedSum.cs | 103 +++++++++++++++++- .../Aggregation/GroupedFloatSumTests.cs | 47 +++++--- .../Aggregation/IndexedSumTests.cs | 28 +++++ 4 files changed, 184 insertions(+), 16 deletions(-) diff --git a/src/Vorticity/Aggregation/FixedSlot.cs b/src/Vorticity/Aggregation/FixedSlot.cs index 5ea2d5e30..e79c7d5d3 100644 --- a/src/Vorticity/Aggregation/FixedSlot.cs +++ b/src/Vorticity/Aggregation/FixedSlot.cs @@ -27,6 +27,14 @@ internal sealed class FixedSlot : RecordSlotThe windows an exact float sum whose groups sat at tops of their own folds a row at a time before it tries the window's top again. + private const int OwnTopWindows = 8; + internal FixedSlot(StorageKind kind, Func finish) : this(kind, finish, default(TOp)) { @@ -205,6 +213,20 @@ internal override void StepRows(in BatchInput input, ReadOnlySpan groups) ReadOnlySpan mask = _rows.And(input, input.Selection, valid); if (mask.IsEmpty) { + // An exact float sum: the extractors of the window's top held in registers, unless the + // last window's groups sat at tops of their own (IndexedSum.Fold). + if (typeof(TState) == typeof(IndexedSum) && typeof(TValue) == typeof(double)) + { + if (_ownTops == 0) + { + int added = IndexedSum.Fold(Unsafe.As, StateView>(ref states), groups, MemoryMarshal.Cast(values), input.Start, input.End); + _ownTops = added > (input.End - input.Start) / 4 ? OwnTopWindows : 0; + return; + } + + _ownTops--; + } + // Every row and no null: a loop with nothing but the fold, the cursor's test of its mask // out of it; with no branch on the values while the groups have seen few rows. if (input.Settled) diff --git a/src/Vorticity/Aggregation/IndexedSum.cs b/src/Vorticity/Aggregation/IndexedSum.cs index 33201a62b..04308aac0 100644 --- a/src/Vorticity/Aggregation/IndexedSum.cs +++ b/src/Vorticity/Aggregation/IndexedSum.cs @@ -347,16 +347,20 @@ private static double[] Limits() /// /// The top a magnitude calls for: the lowest whose limit, 2^(27 · top − 1048), the whole - /// binade of the magnitude stays below. + /// binade of the magnitude stays below. The binade read off the exponent's bits, or for a subnormal + /// off its highest bit, as gives it, which is a call. /// - private static int Needed(double magnitude) + internal static int Needed(double magnitude) { if (magnitude == 0) { return MinTop; } - int bound = Math.ILogB(magnitude) + 1 - (Lowest + Width - 1); + ulong bits = (ulong)BitConverter.DoubleToInt64Bits(magnitude); + int biased = (int)(bits >> 52); + int binade = biased != 0 ? biased - 1023 : 63 - BitOperations.LeadingZeroCount(bits) + Lowest; + int bound = binade + 1 - (Lowest + Width - 1); return Math.Max(MinTop, (bound + Width - 1) / Width); } @@ -536,6 +540,99 @@ private void DepositSpan(ReadOnlySpan values, int top) Deposit(values[i], top); } } + + /// + /// Folds the rows [, ) of into the + /// sums of their groups, : the window's largest magnitude calls for a top, whose + /// extractors stay in registers. A row whose group sits at that top is split with them, its group read + /// for its bins alone; a group that holds no value yet takes its first at the top the value calls for; + /// any other row is folded as folds it. A window with a NaN or an infinity, or that + /// calls for a top split scaled, is folded a row at a time. The parts are those Add gives. + /// + /// + /// Add reads the group's top, then its extractors by it, then splits the value: a chain from the group's + /// state through seven additions to its bins, which a row whose group a row still in flight holds + /// waits for whole. A hundred groups took ≈ 12 cycles a value (vortex-queries --micro floatsum). Here + /// the split does not wait on the group: only the three bins and the count do. + /// + /// The rows folded as Add folds them: groups at another top than the window's. + internal static int Fold(StateView states, ReadOnlySpan groups, ReadOnlySpan values, int start, int end) + { + int top = TopOf(states, groups, values, start, end); + if (top is < MinTop or >= HugeTop) + { + for (int row = start; row < end; row++) + { + states[groups[row]].Add(values[row]); + } + + return end - start; + } + + // The group's top and count read at once: their difference from the top's, unsigned, is the count + // when the group sits at the top, past the endurance otherwise. A value at or past the top's limit, + // a NaN or an infinity among them, is folded as Add folds it. + long topBits = (long)top << TopShift; + double limit = Limit[top]; + double s0 = Sigma[top]; + double s1 = Sigma[top - 1]; + double s2 = Sigma[top - 2]; + long b0 = SigmaBits[top]; + long b1 = SigmaBits[top - 1]; + long b2 = SigmaBits[top - 2]; + nint offset = states.Offset; + ref int groupOf = ref MemoryMarshal.GetReference(groups); + ref double valueOf = ref MemoryMarshal.GetReference(values); + int added = 0; + for (int row = start; row < end; row++) + { + double value = Unsafe.Add(ref valueOf, row); + double magnitude = Math.Abs(value); + ref IndexedSum sum = ref Unsafe.As(ref Unsafe.AddByteOffset(ref states.Record(Unsafe.Add(ref groupOf, row)), offset)); + long meta = sum.Meta; + if (((ulong)((meta & (TopMask | CountMask)) - topBits) < Endurance) & (magnitude < limit)) + { + sum.Meta = meta + 1; + double y = s0 + value; + sum.M0 += BitConverter.DoubleToInt64Bits(y) - b0; + double r = value - (y - s0); + y = s1 + r; + sum.M1 += BitConverter.DoubleToInt64Bits(y) - b1; + r -= y - s1; + sum.M2 += BitConverter.DoubleToInt64Bits(s2 + r) - b2; + } + else if (meta == 0 && magnitude < double.PositiveInfinity && Needed(magnitude) is int own && own < HugeTop) + { + // A group's first value, which no NaN came before: counted, its top the value's, as AddSlow + // does with no bin to move. + sum.Meta = ((long)own << TopShift) + 1; + sum.Deposit(value, own); + } + else + { + sum.Add(value); + added++; + } + } + + return added; + } + + /// + /// The top a window is split at: that of its first row's group, or of the first row's value when its + /// group holds none yet; 0 when that value is a NaN or an infinity. + /// + private static int TopOf(StateView states, ReadOnlySpan groups, ReadOnlySpan values, int start, int end) + { + if (start >= end) + { + return 0; + } + + int top = states[groups[start]].Top; + double magnitude = Math.Abs(values[start]); + return top != 0 ? top : magnitude < double.PositiveInfinity ? Needed(magnitude) : 0; + } } /// A sum of floating-point values in an : the same bits in any order and under any cut. diff --git a/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs b/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs index 59914c700..25260a22b 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupedFloatSumTests.cs @@ -34,26 +34,28 @@ public async Task AGroupsFloatSumsAreTheBitsOfItsValuesOneAtATime(int degree) List sums = []; Scan grouped = file.Scan().GroupBy(r => r.Key) - .Select(g => (g.Key, g.Sum(r => r.Mixed), g.Sum(r => r.Uniform), g.Average(r => r.Uniform), g.Average(r => r.Mixed))) + .Select(g => (g.Key, g.Sum(r => r.Mixed), g.Sum(r => r.Uniform), g.Average(r => r.Uniform), g.Average(r => r.Mixed), g.Sum(r => r.Ranged))) .As(); await foreach (KeySums sum in grouped.ToRecordsAsync(Ct)) { sums.Add(sum); } - Dictionary expected = []; + Dictionary expected = []; foreach (Reading row in rows) { - (IndexedSum mixed, IndexedSum uniform) = expected.GetValueOrDefault(row.Key); + (IndexedSum mixed, IndexedSum uniform, IndexedSum ranged) = expected.GetValueOrDefault(row.Key); mixed.Add(row.Mixed); uniform.Add(row.Uniform); - expected[row.Key] = (mixed, uniform); + ranged.Add(row.Ranged); + expected[row.Key] = (mixed, uniform, ranged); } Assert.Equal(expected.Count, sums.Count); Assert.Equal( - expected.OrderBy(e => e.Key).Select(e => (e.Key, Bits(e.Value.Mixed.Value), Bits(e.Value.Uniform.Value), Bits(Mean(e.Value.Uniform)), Bits(Mean(e.Value.Mixed)))), - sums.OrderBy(s => s.Key).Select(s => (s.Key, Bits(s.Mixed), Bits(s.Uniform), Bits(s.UniformMean), Bits(s.MixedMean)))); + expected.OrderBy(e => e.Key).Select(e => + (e.Key, Bits(e.Value.Mixed.Value), Bits(e.Value.Uniform.Value), Bits(Mean(e.Value.Uniform)), Bits(Mean(e.Value.Mixed)), Bits(e.Value.Ranged.Value))), + sums.OrderBy(s => s.Key).Select(s => (s.Key, Bits(s.Mixed), Bits(s.Uniform), Bits(s.UniformMean), Bits(s.MixedMean), Bits(s.Ranged)))); } [Theory] @@ -66,14 +68,14 @@ public async Task AKeyThatOpensAGroupAtEveryRowSumsAsItsValuesDo(int degree) await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); List sums = []; - await foreach (IdSum sum in file.Scan().GroupBy(r => r.Id).Select(g => (g.Key, g.Sum(r => r.Mixed))).As().ToRecordsAsync(Ct)) + await foreach (IdSum sum in file.Scan().GroupBy(r => r.Id).Select(g => (g.Key, g.Sum(r => r.Mixed), g.Sum(r => r.Ranged))).As().ToRecordsAsync(Ct)) { sums.Add(sum); } Assert.Equal( - rows.Select(r => (r.Id, Bits(Single(r.Mixed)))), - sums.OrderBy(s => s.Id).Select(s => (s.Id, Bits(s.Mixed)))); + rows.Select(r => (r.Id, Bits(Single(r.Mixed)), Bits(Single(r.Ranged)))), + sums.OrderBy(s => s.Id).Select(s => (s.Id, Bits(s.Mixed), Bits(s.Ranged)))); } private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -113,6 +115,25 @@ private static double MixedOf(int key, int row, Random random) }; } + /// + /// A finite value whose binade the row's group picks among a few, a top apart or less: the groups at + /// tops of their own, most windows of a third of the rows within one top, the next third from one + /// range of groups to many, the last third every group at one top but a few, with no NaN, no infinity + /// and nothing split scaled. + /// + private static double RangedOf(int key, int row, Random random) + { + double value = (random.NextDouble() + 0.5) * (random.Next(4) == 0 ? -1 : 1); + int binade = (row / (Rows / 3)) switch + { + 0 => key % 2 == 0 ? 3 : -40, + 1 => (key % 7 * 15) - 45, + _ => key % 50 == 0 ? -60 : 10, + }; + + return Math.ScaleB(value, binade); + } + private static class Fixture { private static readonly Lazy> s_written = new Lazy>(WriteAsync); @@ -126,7 +147,7 @@ private static class Fixture for (int row = 0; row < Rows; row++) { int key = (int)((uint)(row * 2_654_435_761u) % Groups); - rows[row] = new Reading(row, key, MixedOf(key, row, random), random.NextDouble() * 1_000); + rows[row] = new Reading(row, key, MixedOf(key, row, random), random.NextDouble() * 1_000, RangedOf(key, row, random)); } string directory = Path.Combine(AppContext.BaseDirectory, "grouped-float-sum"); @@ -143,11 +164,11 @@ private static class Fixture } [VortexRecord] - public partial record struct Reading(int Id, int Key, double Mixed, double Uniform); + public partial record struct Reading(int Id, int Key, double Mixed, double Uniform, double Ranged); [VortexRecord] - public partial record struct KeySums(int Key, double Mixed, double Uniform, double? UniformMean, double? MixedMean); + public partial record struct KeySums(int Key, double Mixed, double Uniform, double? UniformMean, double? MixedMean, double Ranged); [VortexRecord] - public partial record struct IdSum(int Id, double Mixed); + public partial record struct IdSum(int Id, double Mixed, double Ranged); } diff --git a/tests/Vorticity.Tests/Aggregation/IndexedSumTests.cs b/tests/Vorticity.Tests/Aggregation/IndexedSumTests.cs index 921b21e3f..5193fe837 100644 --- a/tests/Vorticity.Tests/Aggregation/IndexedSumTests.cs +++ b/tests/Vorticity.Tests/Aggregation/IndexedSumTests.cs @@ -13,6 +13,34 @@ namespace Vorticity.Tests.Aggregation; /// public sealed class IndexedSumTests { + // The top a magnitude calls for, its binade read off its bits, against Math.ILogB's: every binade of + // the normal doubles at both ends of its mantissas, every bit a subnormal may hold highest, and zero. + [Fact] + public void TheTopReadOffTheBitsIsTheOneILogBGives() + { + static int ByILogB(double magnitude) => + magnitude == 0 ? 2 : Math.Max(2, (Math.ILogB(magnitude) + 1 - (-1074 + 27 - 1) + 27 - 1) / 27); + + Assert.Equal(ByILogB(0), IndexedSum.Needed(0)); + for (long biased = 1; biased < 2047; biased++) + { + foreach (long mantissa in (long[])[0, 1, 1L << 51, (1L << 52) - 1]) + { + double magnitude = BitConverter.Int64BitsToDouble((biased << 52) | mantissa); + Assert.Equal(ByILogB(magnitude), IndexedSum.Needed(magnitude)); + } + } + + for (int bit = 0; bit < 52; bit++) + { + foreach (long below in (long[])[0, (1L << bit) - 1]) + { + double subnormal = BitConverter.Int64BitsToDouble((1L << bit) | below); + Assert.Equal(ByILogB(subnormal), IndexedSum.Needed(subnormal)); + } + } + } + // The rounding in two words, which a sum's value takes, against the rounding of an Int128: the same // bits for totals of every length, both signs, ties to even, and exponents down past the subnormals. [Fact] From 4a2161e1c7425f8d0b90cddbdc1d2e7d89504a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 16:34:03 +0200 Subject: [PATCH 13/56] perf(query): an exact float sum folds its own pass rather than carry the count A count rode on a fixed slot's pass when both folded a window into one record, the float sum's among them: its rows then went through IndexedSum.Add a value at a time, never through the split held to one top. group-mean-k6, a count and a float's mean by 10^6 keys, carried its count so through its first batches, 6.6 % of its cycles. A slot of an exact float sum no longer carries it: the count folds first, alone, as it does past the cache, and the sum's own pass splits its values at one top. runners.sh alt against the runner of the previous commit, warm, at one lane: group-mean-k6 x0.973 (80.0 -> 77.8 ms); group-db-q10 x1.006 (0.97-1.03) and group-db-q4 x0.994, unchanged. --micro floatsum now prints what reading every group's sum out costs, as the result's batches read them (FixedSlot.Results): 10.7 ns a group at a hundred thousand groups, 4.3 at a group a row. Building the double from its bits rather than through Math.ScaleB did not change it. --- .../FloatSumMicro.cs | 31 ++++++++++++++----- src/Vorticity/Aggregation/FixedSlot.cs | 4 ++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs b/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs index a26939827..9316d2bee 100644 --- a/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs +++ b/bench/Vorticity.Benchmarks.Queries/FloatSumMicro.cs @@ -27,7 +27,7 @@ internal static class FloatSumMicro internal static int Run(int rounds) { - Console.WriteLine($"{"float sum",-32} {"rows",11} {"groups",10} {"best ms",10} {"ns/row",8} {"checksum",18}"); + Console.WriteLine($"{"float sum",-32} {"rows",11} {"groups",10} {"best ms",10} {"ns/row",8} {"ns/group out",13} {"checksum",18}"); Measure("1e2 groups, one range (q4)", 10_000_000, 100, mixed: false, rounds); Measure("1e5 groups, one range (q3)", 10_000_000, 100_000, mixed: false, rounds); Measure("1e2 groups, ranges of their own", 10_000_000, 100, mixed: true, rounds); @@ -50,21 +50,24 @@ private static void Measure(string name, int rows, int groups, bool mixed, int r } double best = double.MaxValue; + double bestOut = double.MaxValue; long checksum = 0; + int count = groups == 0 ? rows : groups; for (int round = 0; round < rounds; round++) { - (double ms, long sum) = Pass(values, rowGroups, groups == 0 ? rows : groups); + (double ms, double outMs, long sum) = Pass(values, rowGroups, count); best = Math.Min(best, ms); + bestOut = Math.Min(bestOut, outMs); checksum = sum; } Console.WriteLine(string.Create( CultureInfo.InvariantCulture, - $"{name,-32} {rows,11:N0} {(groups == 0 ? rows : groups),10:N0} {best,10:F1} {best * 1e6 / rows,8:F2} {checksum,18:X16}")); + $"{name,-32} {rows,11:N0} {count,10:N0} {best,10:F1} {best * 1e6 / rows,8:F2} {bestOut * 1e6 / count,13:F2} {checksum,18:X16}")); } - /// One pass: every batch's values a canonical node of an arena, folded into their groups' sums. - private static (double Ms, long Checksum) Pass(double[] values, int[] rowGroups, int groups) + /// One pass: every batch's values a canonical node of an arena, folded into their groups' sums; then every group's sum read out. + private static (double Ms, double OutMs, long Checksum) Pass(double[] values, int[] rowGroups, int groups) { GC.Collect(); RetainingArena arena = new RetainingArena(); @@ -85,14 +88,26 @@ private static (double Ms, long Checksum) Pass(double[] values, int[] rowGroups, } double ms = Stopwatch.GetElapsedTime(start).TotalMilliseconds; - long checksum = 0; + + // Every group's sum read out at once, as the result's batches read them. + int[] order = new int[groups]; for (int group = 0; group < groups; group++) { - checksum = (checksum * 31) ^ BitConverter.DoubleToInt64Bits(slot.Result(group)); + order[group] = group; + } + + double[] sums = new double[groups]; + long reading = Stopwatch.GetTimestamp(); + slot.Results(order, sums); + double outMs = Stopwatch.GetElapsedTime(reading).TotalMilliseconds; + long checksum = 0; + foreach (double sum in sums) + { + checksum = (checksum * 31) ^ BitConverter.DoubleToInt64Bits(sum); } arena.Reset(); - return (ms, checksum); + return (ms, outMs, checksum); } private static ulong Mix(ulong x) diff --git a/src/Vorticity/Aggregation/FixedSlot.cs b/src/Vorticity/Aggregation/FixedSlot.cs index e79c7d5d3..b3b7f5f76 100644 --- a/src/Vorticity/Aggregation/FixedSlot.cs +++ b/src/Vorticity/Aggregation/FixedSlot.cs @@ -258,7 +258,9 @@ internal override void StepRows(in BatchInput input, ReadOnlySpan groups) } } - internal override bool CarriesCount => true; + // An exact float sum folds its own pass, its split held to one top (IndexedSum.Fold), rather than carry + // a count through IndexedSum.Add, a value at a time. + internal override bool CarriesCount => typeof(TState) != typeof(IndexedSum); internal override bool StepRowsCounted(in BatchInput input, ReadOnlySpan groups, AggregateSlot count) => count switch { From ab4ef6e6fdda9f885bb18d93225d47561073a2b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 17:33:15 +0200 Subject: [PATCH 14/56] perf(query): the core pages a key numbered by value, a part's pages side by side At fourteen lanes the core holds a key of ten million values (k7) in sub-tables of hashes: a key hashed into its part, its sub-table, its slot, the tables grown and split, half the cycles for a key whose value is its number. A key the statistics bound to a span no wider than the rows, whose pages its values fill, is now paged (GroupCore.Paged): the span cut into pages of 256 values, a page in the part the hash of its number names (LaneCore.PartOf, the same for every lane), a part's pages side by side in one sub-table of FixedKeys numbered whole (GroupKeys.ForPages): an entry's group is its page's place among the part's, then its place in the page, read off its value with no hash, no probe, no growth and no split (FixedKeys.PagedGroupsOf). A bit a value tells those an entry met: a table of a group a value, 156 KB a part where its bits take 5, cost a miss to memory an entry as the parts went from core to core, 30 % of the cycles. The values no entry met are dropped before anything reads a part's groups: at the end, before a part is read as one, and before its groups go to the scratch (SubTable.DropUnmet, SubTable.CopyEntries), the keep that follows reading the bits in order (FixedKeys.KeepNumbered); a lookup into a part once dropped throws. Never for the core the governor turns to, which holds as little past its groups as it can, nor under a budget that would not hold the span's pages twice over, nor with an emitter. Pages of their own, a sub-table each, ran x1.21: the cut by page cost what the hashes did, and a part held from none to six pages. GroupCoreTests checks every state of a record at one, two and fourteen lanes in pages too; SpillTests.APagedCoreSpillsTheValuesItMetAlone pages two million values, one in two met, under a budget its pending entries pass, and reads them back exact. runners.sh alt against the runner of 500e7c6e, warm, at fourteen lanes on a busy Mac: group-total-k7 x0.665 (84.4 -> 56.9 ms, cycles x0.658); k6 x0.978 and range-k6 x0.987, which do not take the core; tenfold x1.009, unique x0.965, drift x1.032 and zipf x1.093, their intervals wide and their cycles x0.994 to x1.032. --- .../Aggregation/AggregationEngine.cs | 7 + src/Vorticity/Aggregation/GroupCore.Spill.cs | 2 + src/Vorticity/Aggregation/GroupCore.cs | 202 ++++++++++-- src/Vorticity/Aggregation/GroupKeys.cs | 297 ++++++++++++++++-- src/Vorticity/Aggregation/RawKeys.cs | 2 +- .../Aggregation/GroupCoreTests.cs | 27 +- .../Vorticity.Tests/Aggregation/SpillTests.cs | 64 ++++ 7 files changed, 544 insertions(+), 57 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 964e24200..eb7ef7293 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -388,6 +388,13 @@ internal AggregationPlan(ReadOnlySpan results, ColumnShape[] keys) /// internal bool NumberWhole { get; set; } = true; + /// + /// Whether the core pages a key numbered by value over a span no wider than the rows + /// (): false for sub-tables of hashes, the switch the tests and the bench + /// compare them with. + /// + internal bool CorePages { get; set; } = true; + /// The result a symbol stands for. /// The symbol is a column, not an aggregate or a key. internal static ResultNode Result(Sym symbol) => Result(symbol, []); diff --git a/src/Vorticity/Aggregation/GroupCore.Spill.cs b/src/Vorticity/Aggregation/GroupCore.Spill.cs index 1cb3dc032..b1d0b8c37 100644 --- a/src/Vorticity/Aggregation/GroupCore.Spill.cs +++ b/src/Vorticity/Aggregation/GroupCore.Spill.cs @@ -133,6 +133,8 @@ private async ValueTask EvictAsync(CorePart part, CancellationToken cancellation { foreach (SubTable table in taken) { + // A paged part's values no entry met are no groups to write (SubTable.CopyEntries). + table.DropUnmet(); int from = 0; while (from < table.Keys.Count) { diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 1119d1f3c..2dbc77323 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; @@ -164,7 +165,7 @@ internal sealed partial class GroupCore private GroupCore( AggregationPlan plan, AggregateSlot?[] settled, ColumnShape[] columns, int[] inputs, ScanSource? source, KeyFacts? facts, GroupKeys kind, RecordLayout? layout, int lanes, - QueryMemory? memory, bool lean) + QueryMemory? memory, bool lean, (long Least, ulong Span)? values = null) { _plan = plan; _settled = settled; @@ -223,8 +224,89 @@ private GroupCore( LaneBytes = ((long)PartCount * _batchWords * sizeof(ulong)) + (Capacity * groupBytes); for (int p = 0; p < PartCount; p++) { - _parts[p] = new CorePart(); + _parts[p] = new CorePart { Index = p }; + } + + // Paged: the span cut into pages of 2^PageBits values, each page in the part its hash names, a + // part's pages side by side in one sub-table numbered whole. Never for the core the governor + // turns to, which holds as little past its groups as it can, nor under a budget that would not + // hold the span's records twice over: a part's sub-table is made whole at its first entry. + long perValue = ((layout?.Stride ?? 0) * sizeof(ulong)) + keyBytes + sizeof(int); + if (values is { } span && !lean && span.Span <= int.MaxValue && (memory is null || memory.Ceiling / 2 >= perValue * (long)span.Span)) + { + _pageLeast = span.Least; + int pages = (int)((span.Span + (1UL << PageBits) - 1) >> PageBits); + _pageSlots = new int[pages]; + Span counts = stackalloc int[PartCount]; + counts.Clear(); + for (int page = 0; page < pages; page++) + { + _pageSlots[page] = counts[PageHome(page)]++; + } + + _partPages = new long[PartCount][]; + for (int p = 0; p < PartCount; p++) + { + _partPages[p] = new long[counts[p]]; + } + + for (int page = 0; page < pages; page++) + { + _partPages[PageHome(page)][_pageSlots[page]] = page; + } + } + } + + /// + /// The bits of a page's values in a core that pages its key: 256 values, a few hundred pages a part + /// at ten million keys, whose shares of the span differ by a few percent. + /// + internal const int PageBits = 8; + + // A key the core pages (Paged): the least value of its span, each page's place among its part's + // pages, and each part's pages in that order; null for a core of hashes. + private readonly long _pageLeast; + private readonly int[]? _pageSlots; + private readonly long[][]? _partPages; + + /// The part a page of the span is in: the top bits of the hash of its number. + internal static int PageHome(long page) => (int)(MergeHash.Of((ulong)page, MergeHash.Seed) >> PartShift); + + /// + /// Whether the core pages its key: a key numbered by value over a span no wider than the rows. A + /// part is then the pages the hashes of their numbers name, side by side in one sub-table numbered + /// whole: an entry's group is read off its value, no hash, no probe, no growth, no split; the values + /// no entry met are dropped before anything reads the groups. + /// + internal bool Paged => _pageSlots is not null; + + /// The least value of the span a paged core's pages cut. + internal long PageLeast => _pageLeast; + + /// + /// A part's first sub-table: its pages side by side, numbered whole, in a paged core; else one of + /// hashes made at once for the groups entries may hold, which would + /// otherwise double some ten times, rehashing, through the part's first bursts. + /// + internal SubTable FirstTable(CorePart part, int entries) + { + if (Paged) + { + return NewPagedTable(part.Index); } + + SubTable first = NewTable(0); + first.Reserve((int)Math.Min(entries, _tableGroups)); + return first; + } + + /// The sub-table of a part of a paged core: the part's pages side by side, numbered whole. + private SubTable NewPagedTable(int part) + { + AggregateSlot[] slots = AggregationPartition.NewSlots(_plan, _settled, _source, out GroupRecords? records, _shelf); + GroupKeys keys = Kind.ForPages(_pageLeast, PageBits, _pageSlots!, _partPages![part], _shelf); + keys.NumberWhole((long)(records?.Layout.Stride ?? 0) * sizeof(ulong)); + return new SubTable(keys, slots, records, MostDepth); } /// An empty index of the keys' kind, called for what it does on entries. @@ -316,7 +398,11 @@ internal CorePressure? Pressure return null; } - return new GroupCore(plan, settled, columns, inputs, source, CacheFacts(facts), kind.ForPart(), records?.Layout, lanes, memory, lean); + // A key numbered by value over a span no wider than the rows, whose pages its values fill: the core + // pages it. A sparser one keeps sub-tables of hashes, which hold the groups met alone. + (long Least, ulong Span)? values = plan.CorePages && plan.Emitter is null && facts?.Rows is long rows and > 0 + && plan.CreateKeys(sorted: false, facts).ValueSpan is { } span && span.Span <= (ulong)rows ? span : null; + return new GroupCore(plan, settled, columns, inputs, source, CacheFacts(facts), kind.ForPart(), records?.Layout, lanes, memory, lean, values); } /// @@ -759,6 +845,7 @@ internal void Merge(AggregateSlot[] from, ReadOnlySpan groups, SubTable int foreach (SubTable table in part.Tables) { + table.DropUnmet(); keys.Add(table.Keys); slots.Add(table.Slots); bytes += table.Footprint; @@ -1014,6 +1101,7 @@ internal async ValueTask BuildAsync(CorePart part, PartBuilder build List slots = []; foreach (SubTable table in part.Tables) { + table.DropUnmet(); keys.Add(table.Keys); slots.Add(table.Slots); } @@ -1314,18 +1402,41 @@ internal PartBatch(ulong[] slab, int start, int capacity) internal bool Alone { get; init; } } -/// A part's groups whose hashes share the bits of its places in the part's directory: their keys and their records, final. +/// +/// A part's groups whose hashes share the bits of its places in the part's directory, or, in a core that +/// pages its key, the values of the part's pages: their keys and their records, final. +/// internal sealed class SubTable(GroupKeys keys, AggregateSlot[] slots, GroupRecords? records, int depth) { internal GroupKeys Keys { get; } = keys; internal AggregateSlot[] Slots { get; } = slots; - /// The bits of the hash past the part's its groups share. + /// The bits of the hash past the part's its groups share; for a paged part's, which never splits. internal int Depth { get; } = depth; internal long Footprint => Keys.Footprint + AggregateSlot.FootprintOf(Slots); + /// + /// A paged part's values no entry met, dropped, keys and states, before anything reads its groups: + /// the part delivered, read as one, or written to the scratch. Its keys number values as they come + /// from then on. + /// + internal void DropUnmet() + { + if (Keys.Met() is not { } met) + { + return; + } + + Keys.Keep(met); + records?.Keep(met); + foreach (AggregateSlot slot in Slots) + { + slot.Keep(met); + } + } + /// Makes room for groups at once: keys and records. internal void Reserve(int groups) { @@ -1340,10 +1451,16 @@ internal void Release() records?.Release(); } - /// The groups from on copied into entries of , as many as holds. + /// + /// The groups from on copied into entries of , as many as + /// holds: a paged part's values no entry met dropped first, which no run holds. + /// /// The group to copy next. - internal int CopyEntries(EntryShape shape, int from, Span entries, out int written) => - Keys.CopyEntries(records is null ? default : records.Made, shape, from, entries, out written); + internal int CopyEntries(EntryShape shape, int from, Span entries, out int written) + { + DropUnmet(); + return Keys.CopyEntries(records is null ? default : records.Made, shape, from, entries, out written); + } } /// A part's groups written to the query's scratch as it was spilled: their entries, one after the other, from . @@ -1400,6 +1517,9 @@ internal sealed class CorePart /// The groups of the sub-tables, as the last application left them. internal int Groups; + /// The part's place among the core's, which its pages' hashes name in a paged core. + internal int Index { get; init; } + /// /// The part's groups written to the scratch, one run each time it was spilled; null for a /// part never spilled. Written by the eviction alone, which one task runs at a time. @@ -1665,11 +1785,29 @@ internal LaneCore(GroupCore core) _core = core; Shape = core.Shape; _entryWords = Shape.Words; + _pageLeast = core.PageLeast; + _pageBits = core.Paged ? GroupCore.PageBits : -1; } + // The core's pages, when it pages its key: the least value of the span and the bits of a page; -1 + // bits for a core of hashes. + private readonly long _pageLeast; + private readonly int _pageBits; + /// The core the lane's side belongs to. internal GroupCore Core => _core; + /// + /// The part an entry of goes to: the top bits of the key's hash under + /// , or in a core that pages its key, of its page's. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal int PartOf(TKey key) + where TKey : unmanaged => + _pageBits < 0 + ? (int)(EntryKeys.Hash(key, MergeHash.Seed) >> GroupCore.PartShift) + : (int)(MergeHash.Of((ulong)((EntryKeys.Integer(key) - _pageLeast) >> _pageBits), MergeHash.Seed) >> GroupCore.PartShift); + internal EntryShape Shape { get; } /// @@ -1825,7 +1963,7 @@ internal void Empty(AggregationPartition table) { Span counts = stackalloc int[GroupCore.PartCount]; counts.Clear(); - table.Keys!.CountParts(counts); + table.Keys!.CountParts(counts, this); for (int p = 0; p < counts.Length; p++) { if (counts[p] > 0) @@ -1973,12 +2111,11 @@ internal int Apply(CorePart part, PartBatch? stack, bool burst) if (entries > 0) { // A part's first sub-table made at once for the groups its first entries may hold: it - // would otherwise double some ten times, rehashing, through its first bursts. + // would otherwise double some ten times, rehashing, through its first bursts. A paged part's + // is its pages, whole. if (part.Tables.Count == 0) { - SubTable first = _core.NewTable(0); - first.Reserve((int)Math.Min(entries, _core.TableGroups)); - part.Tables.Add(first); + part.Tables.Add(_core.FirstTable(part, entries)); } // A split appends its upper half: the sub-tables cut here are the first ones. Only the end @@ -1990,8 +2127,7 @@ internal int Apply(CorePart part, PartBatch? stack, bool burst) _core.Presplit(part, entries, this); } - int tables = part.Tables.Count; - Cut(part, count, entries); + int tables = Cut(part, count, entries); _emitting = _core.Emitter is not null && !part.Silent; for (int t = 0; t < tables; t++) { @@ -2006,10 +2142,11 @@ internal int Apply(CorePart part, PartBatch? stack, bool burst) _core.Emitter!.Flush(this); } + // A page counts the values its entries met, not those it numbers ahead of them. int groups = 0; foreach (SubTable table in part.Tables) { - groups += table.Keys.Count; + groups += table.Keys.MetGroups; } Volatile.Write(ref part.Groups, groups); @@ -2051,11 +2188,12 @@ internal int Apply(CorePart part, PartBatch? stack, bool burst) /// the batch and the entry at each place, from each sub-table's start. A part of one sub-table takes /// its entries as they come. /// - private void Cut(CorePart part, int count, int entries) + /// The part's sub-tables. + private int Cut(CorePart part, int count, int entries) { - int tables = part.Tables.Count; Scratch.Grow(ref _entryAt, entries); Scratch.Grow(ref _batchAt, entries); + int tables = part.Tables.Count; Scratch.Grow(ref _starts, tables + 1); Span starts = _starts.AsSpan(0, tables + 1); starts.Clear(); @@ -2073,7 +2211,7 @@ private void Cut(CorePart part, int count, int entries) } starts[1] = entries; - return; + return tables; } int shift = GroupCore.PartShift - part.Depth; @@ -2117,6 +2255,8 @@ private void Cut(CorePart part, int count, int entries) at += length; } + + return tables; } /// @@ -2269,7 +2409,6 @@ internal static void Scatter(ReadOnlySpan keys, int skip, ReadOnlySp EntryShape shape = lane.Shape; int stride = shape.RecordWords; nint keyOffset = shape.KeyOffset; - ulong seed = MergeHash.Seed; if (records.Length < keys.Length * stride) { throw new InvalidOperationException("A cache's records are fewer than its groups."); @@ -2284,7 +2423,7 @@ internal static void Scatter(ReadOnlySpan keys, int skip, ReadOnlySp } TKey key = keys[g]; - ref ulong entry = ref lane.Entry((int)(Hash(key, seed) >> GroupCore.PartShift)); + ref ulong entry = ref lane.Entry(lane.PartOf(key)); ref ulong record = ref Unsafe.Add(ref first, (nint)g * stride); for (int w = 0; w < stride; w++) { @@ -2331,20 +2470,33 @@ internal static int Copy(ReadOnlySpan keys, int skip, ReadOnlySpanCounts each key but group 's by its part, as would place it. - internal static void CountParts(ReadOnlySpan keys, int skip, Span counts) + /// Counts each key but group 's by its part, as would place it in 's core. + internal static void CountParts(ReadOnlySpan keys, int skip, Span counts, LaneCore lane) where TKey : unmanaged { - ulong seed = MergeHash.Seed; for (int g = 0; g < keys.Length; g++) { if (g != skip) { - counts[(int)(Hash(keys[g], seed) >> GroupCore.PartShift)]++; + counts[lane.PartOf(keys[g])]++; } } } + /// An integer key as a long, as numbers it by value; an unsigned one past the longs as a negative. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long Integer(TKey key) + where TKey : unmanaged => + typeof(TKey) == typeof(sbyte) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(short) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(int) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(long) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(byte) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(ushort) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(uint) ? Unsafe.BitCast(key) + : typeof(TKey) == typeof(ulong) ? (long)Unsafe.BitCast(key) + : throw new NotSupportedException("Only integer keys are numbered by value."); + /// The bits of each entry's key's hash from up, under . internal static void TablesOf(PartBatch batch, EntryShape shape, int shift, int mask, Span tables) where TKey : unmanaged diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index 62e38f026..4bd65dced 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -181,6 +181,18 @@ internal static int DoubledUpTo(int length, int bound) /// An empty index a sub-table of the core holds its groups in, its arrays taken from and given back to . internal virtual GroupKeys ForTable(ArrayShelf shelf) => ForPart(); + /// + /// An empty index of a part of a core that pages its key, its arrays from : the + /// part's pages side by side, in their order, each of 2^ + /// values from plus its number shifted by them, each + /// page's place among its part's. + /// + internal virtual GroupKeys ForPages(long least, int bits, int[] slots, long[] pages, ArrayShelf shelf) => + throw new NotSupportedException("Only keys numbered by value are paged."); + + /// The groups some row met: , less the values of a span numbered whole that none did yet. + internal virtual int MetGroups => Count; + /// Gives the index's arrays back to its shelf, if it has one: a sub-table split, whose groups another holds. The index is empty after. internal virtual void Release() { @@ -217,13 +229,14 @@ internal virtual void Reserve(int groups) /// /// Copies every group but the null one into an entry of the lane's batch of the part its key falls - /// in, the top bits of its hash under as cuts: the - /// group's record, read from , then its key. + /// in (), the top bits of its hash under as + /// cuts, or of its page's in a core that pages it: the group's record, read from + /// , then its key. /// internal virtual void Scatter(ReadOnlySpan records, LaneCore lane) => throw NotEntries(); - /// Counts every group but the null one by the part copies it to. - internal virtual void CountParts(Span counts) => throw NotEntries(); + /// Counts every group but the null one by the part copies it to in 's core. + internal virtual void CountParts(Span counts, LaneCore lane) => throw NotEntries(); /// /// The groups from on, but the null one, copied into entries of @@ -632,6 +645,18 @@ internal sealed class FixedKeys : GroupKeys // value its number; -1 when the keys number values as they first come. private int _unmet = -1; + // A part of a core that pages its key (ForPages): its pages side by side, a value's number its page's + // place among them then its place in the page. The bits of a page, each page's place among its + // part's, the core's, and the part's pages in that order; null for a span in one piece. + private readonly int _mapBits; + private readonly int[]? _mapSlots; + private readonly long[]? _mapPages; + + // A paged core's part numbered whole: a bit a number, set once an entry met its value, where a group's + // number is its group and needs no table; null before NumberWhole, and once the part's unmet values are + // dropped, when its keys are final. + private ulong[]? _metBits; + // Every key a group of its own, no table looked up (Appending); the shelf a sub-table's arrays come from (ForTable). private readonly bool _appending; private readonly ArrayShelf? _shelf; @@ -671,6 +696,55 @@ internal FixedKeys( } } + /// + /// A part of a core that pages its key (): the part's pages side by + /// side, numbered by value, from , 2^ values a page, + /// each page's place among its part's, the part's. + /// + private FixedKeys(ColumnShape shape, long least, int bits, int[] slots, long[] pages, ArrayShelf shelf) + : this(shape, sorted: false, shelf: shelf) + { + ulong span = (ulong)pages.Length << bits; + _pages = new int[(int)((span + PageMask) >> PageBits)][]; + _pages.AsSpan().Fill(Unmet); + _pageStarts = new int[_pages.Length]; + _directMin = least; + _span = span; + _mapBits = bits; + _mapSlots = slots; + _mapPages = pages; + } + + /// + /// The number of in a part of a core that pages its key: its page's place + /// among the part's, then its place in the page; false for a value of another part's page or past the + /// span, which exact statistics never give. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool Mapped(long integer, out ulong number) + { + long offset = integer - _directMin; + long page = offset >> _mapBits; + number = 0; + if ((ulong)page >= (ulong)_mapSlots!.Length) + { + return false; + } + + int slot = _mapSlots[page]; + if ((uint)slot >= (uint)_mapPages!.Length || _mapPages[slot] != page) + { + return false; + } + + number = ((ulong)slot << _mapBits) | (ulong)(offset & ((1L << _mapBits) - 1)); + return true; + } + + /// The value numbered in the span, in one piece or in a paged core's part's pages. + private long ValueOf(ulong number) => + _mapPages is null ? _directMin + (long)number : _directMin + (_mapPages[(int)(number >> _mapBits)] << _mapBits) + (long)(number & ((1UL << _mapBits) - 1)); + /// /// The page of the table of groups every number reads until its own is allocated: no value met, -1 /// throughout, and never written. A first pass reads it as any other, with no branch on the keys. @@ -1017,7 +1091,8 @@ internal bool AssignValues(ReadOnlySpan canonical, ReadOnlySpan v return true; } - if (_pages is not null) + // A paged core's part is merged into, never assigned rows: its span is not in one piece. + if (_pages is not null && _mapSlots is null) { if (_probeAhead >= 0 && selection.IsEmpty) { @@ -1318,8 +1393,9 @@ private int Meet(ulong number) /// Whether the keys number the span whole. internal override bool NumberWhole(long groupBytes) { + // A paged core's part holds its pages whatever their bytes: the span's density chose them. if (_pages is null || _sorted || _appending || Count != 0 - || (long)_span * (groupBytes + Unsafe.SizeOf() + sizeof(int)) > WholeBytes) + || (_mapSlots is null && (long)_span * (groupBytes + Unsafe.SizeOf() + sizeof(int)) > WholeBytes)) { return false; } @@ -1332,7 +1408,13 @@ internal override bool NumberWhole(long groupBytes) for (int number = 0; number < span; number++) { - _keys[number] = FromInteger(_directMin + number); + _keys[number] = FromInteger(ValueOf((ulong)number)); + } + + if (_mapSlots is not null) + { + int words = (span + 63) >> 6; + _metBits = _shelf?.Take(words, zeroed: true) ?? new ulong[words]; } Count = span; @@ -1354,25 +1436,42 @@ internal override bool NumberWhole(long groupBytes) return null; } + // The groups are still the span's numbers in order, which the keep that follows reads once (KeepNumbered). + _numbered = true; + int[] met = new int[Count - unmet]; int kept = 0; int span = (int)_span; - for (int at = 0; at < _pages!.Length; at++) + if (_metBits is { } bits) { - int[] slab = _pages[at]; - if (slab == Unmet) + // A paged core's part: its bits, a word at a time. + for (int w = 0; w < bits.Length; w++) { - continue; + for (ulong word = bits[w]; word != 0; word &= word - 1) + { + met[kept++] = (w << 6) + BitOperations.TrailingZeroCount(word); + } } - - int start = _pageStarts![at]; - int first = at << PageBits; - int numbers = Math.Min(1 << PageBits, span - first); - for (int n = 0; n < numbers; n++) + } + else + { + for (int at = 0; at < _pages!.Length; at++) { - if (slab[start + n] >= 0) + int[] slab = _pages[at]; + if (slab == Unmet) { - met[kept++] = first + n; + continue; + } + + int start = _pageStarts![at]; + int first = at << PageBits; + int numbers = Math.Min(1 << PageBits, span - first); + for (int n = 0; n < numbers; n++) + { + if (slab[start + n] >= 0) + { + met[kept++] = first + n; + } } } } @@ -1524,6 +1623,11 @@ private bool DirectSlot(TValue value, out ulong number) return false; } + if (_mapSlots is not null) + { + return Mapped(Integer(value), out number); + } + number = (ulong)(Integer(value) - _directMin); return number < _span; } @@ -1539,12 +1643,12 @@ private bool DirectSlot(TValue value, out ulong number) internal override long Footprint => _index.Footprint + _wide.Footprint + ((long)(_keys.Length + _values.Length) * Unsafe.SizeOf()) + (_slabsHeld * sizeof(int)) + ((long)(_pages?.Length ?? 0) * (IntPtr.Size + sizeof(int))) - + ((long)(_homes.Length + _left.Length) * sizeof(int)); + + ((long)(_homes.Length + _left.Length) * sizeof(int)) + ((long)(_metBits?.Length ?? 0) * sizeof(ulong)); /// A part of a merge is merged into, never assigned rows: no table of groups. internal override GroupKeys ForPart() => new FixedKeys(_shape, _sorted); - internal override (long Least, ulong Span)? ValueSpan => _pages is null ? null : (_directMin, _span); + internal override (long Least, ulong Span)? ValueSpan => _pages is null || _mapSlots is not null ? null : (_directMin, _span); /// The bits of a number below its part's in a merge by value: the span cut into 2^ runs of a power of two each. private int ValuePartShift(int partBits) => Math.Max(0, 64 - System.Numerics.BitOperations.LeadingZeroCount(_span - 1) - partBits); @@ -1642,6 +1746,11 @@ internal override void Release() Count = 0; _null = -1; _unmet = -1; + if (_metBits is not null) + { + _shelf?.Give(_metBits); + _metBits = null; + } } internal override int NullNumber => _null; @@ -1671,6 +1780,12 @@ internal override int CompareKeys(int a, int b, int component) private void Keep(ReadOnlySpan groups, int room) { + if (_numbered) + { + KeepNumbered(groups, room); + return; + } + // The table of groups forgets every group's number, then learns the kept ones' new ones; a // value it holds is in no index. A span numbered whole is numbered as values come from then on. _unmet = -1; @@ -1728,6 +1843,83 @@ private void Keep(ReadOnlySpan groups, int room) Renumbered(); } + // Whether the groups are the numbers of a span numbered whole, in order, the groups past it after them, + // and Met just listed those some row met: the keep that follows reads the table of groups once. + private bool _numbered; + + /// + /// The keep that follows : the groups kept, the numbers met in order then the groups + /// past the span. A met number's entry in the table of groups is its rank among them, read in order, + /// with no lookup; the keys move down, and those past the span are found again in the index. + /// + private void KeepNumbered(ReadOnlySpan groups, int room) + { + _numbered = false; + int span = (int)_span; + int rank = 0; + if (_metBits is not null) + { + // A paged core's part: no table of groups to follow the keep, its keys final from now on. + while (rank < groups.Length && groups[rank] < span) + { + rank++; + } + + _shelf?.Give(_metBits); + _metBits = null; + } + + for (int at = 0; _mapSlots is null && at < _pages!.Length; at++) + { + int[] slab = _pages[at]; + if (slab == Unmet) + { + continue; + } + + int first = at << PageBits; + Span numbers = slab.AsSpan(_pageStarts![at], Math.Min(1 << PageBits, span - first)); + for (int n = 0; n < numbers.Length; n++) + { + if (numbers[n] >= 0) + { + numbers[n] = rank++; + } + } + } + + for (int i = 0; i < groups.Length; i++) + { + _keys[i] = _keys[groups[i]]; + } + + if (Compact) + { + _wide.Clear(room); + } + else + { + _index.Clear(room); + } + + int nullGroup = -1; + for (int i = rank; i < groups.Length; i++) + { + if (groups[i] == _null) + { + nullGroup = i; + } + else + { + GetOrAdd(_keys[i], i); + } + } + + _null = nullGroup; + Count = groups.Length; + Renumbered(); + } + internal override void MergeInto(GroupKeys target, ReadOnlySpan groups, Span map) { FixedKeys into = (FixedKeys)target; @@ -1800,7 +1992,12 @@ internal override long GrowthFor(int more) internal override void Scatter(ReadOnlySpan records, LaneCore lane) => EntryKeys.Scatter(_keys.AsSpan(0, Count), _null, records, lane); - internal override void CountParts(Span counts) => EntryKeys.CountParts(_keys.AsSpan(0, Count), _null, counts); + internal override void CountParts(Span counts, LaneCore lane) => EntryKeys.CountParts(_keys.AsSpan(0, Count), _null, counts, lane); + + internal override GroupKeys ForPages(long least, int bits, int[] slots, long[] pages, ArrayShelf shelf) => + new FixedKeys(_shape, least, bits, slots, pages, shelf); + + internal override int MetGroups => _unmet >= 0 ? Count - _unmet : Count; internal override int CopyEntries(ReadOnlySpan records, EntryShape shape, int from, Span entries, out int written) => EntryKeys.Copy(_keys.AsSpan(0, Count), _null, records, shape, from, entries, out written); @@ -1818,6 +2015,12 @@ internal override void TablesOf(PartBatch batch, EntryShape shape, int shift, in internal override void GroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan entries, Span groups, Span scratch) { int count = entries.Length; + if (_unmet >= 0 && _mapSlots is { } slots) + { + PagedGroupsOf(batch, shape, entries, groups, slots); + return; + } + if (_pages is not null || _appending) { for (int i = 0; i < count; i++) @@ -1852,6 +2055,53 @@ internal override void GroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan< } } + /// + /// for a paged core's part, numbered whole: an entry's group is its number, its + /// page's place among the part's then its place in the page, read off its value with no call, its bit + /// set; one past the span looked up. + /// + private void PagedGroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan entries, Span groups, int[] slots) + { + long least = _directMin; + int bits = _mapBits; + long mask = (1L << bits) - 1; + long[] pages = _mapPages!; + ulong[] met = _metBits!; + for (int i = 0; i < entries.Length; i++) + { + TValue key = EntryKeys.KeyAt(batch, shape, entries[i]); + long offset = Integer(key) - least; + long page = offset >> bits; + int slot = (ulong)page < (ulong)slots.Length ? slots[page] : -1; + if ((uint)slot < (uint)pages.Length && pages[slot] == page) + { + int number = (slot << bits) | (int)(offset & mask); + ref ulong word = ref met[number >> 6]; + _unmet -= (int)((~word >> number) & 1); + word |= 1UL << number; + groups[i] = number; + } + else + { + groups[i] = Lookup(key); + } + } + } + + /// The group of a number in a paged core's part numbered whole: the number itself, its bit set, counted met the first time. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int MeetBit(ulong number) + { + ref ulong word = ref _metBits![(int)(number >> 6)]; + _unmet -= (int)((~word >> (int)number) & 1); + word |= 1UL << (int)number; + return (int)number; + } + + /// A value looked up in a paged core's part once its unmet values are dropped, which nothing does: its keys are final. + private static InvalidOperationException FinalPart() => + new InvalidOperationException("A part of a paged core is looked up once its unmet values are dropped: its groups are final."); + internal override int CompareKeys(GroupKeys other, int a, int b, int component) { FixedKeys right = (FixedKeys)other; @@ -1974,6 +2224,11 @@ internal int Lookup(TValue value) if (DirectSlot(value, out ulong number)) { + if (_mapSlots is not null) + { + return _metBits is not null ? MeetBit(number) : throw FinalPart(); + } + return Numbered(value, number); } diff --git a/src/Vorticity/Aggregation/RawKeys.cs b/src/Vorticity/Aggregation/RawKeys.cs index d93523cda..13e593581 100644 --- a/src/Vorticity/Aggregation/RawKeys.cs +++ b/src/Vorticity/Aggregation/RawKeys.cs @@ -330,7 +330,7 @@ internal override void ReadKeys(ref SpillReader reader, Span groups) internal override void Scatter(ReadOnlySpan records, LaneCore lane) => EntryKeys.Scatter(_keys.AsSpan(0, Count), -1, records, lane); - internal override void CountParts(Span counts) => EntryKeys.CountParts(_keys.AsSpan(0, Count), -1, counts); + internal override void CountParts(Span counts, LaneCore lane) => EntryKeys.CountParts(_keys.AsSpan(0, Count), -1, counts, lane); internal override int CopyEntries(ReadOnlySpan records, EntryShape shape, int from, Span entries, out int written) => EntryKeys.Copy(_keys.AsSpan(0, Count), -1, records, shape, from, entries, out written); diff --git a/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs b/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs index 4175166fa..0eefd94af 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs @@ -20,14 +20,18 @@ public sealed partial class GroupCoreTests private const int Rows = 60_000; [Theory] - [InlineData(1, 0.0)] - [InlineData(1, 1.0)] - [InlineData(2, 1.0)] - [InlineData(14, 0.0)] - [InlineData(14, 1.0)] - public async Task EveryStateInARecordComesOutTheSame(int degree, double bypass) + [InlineData(1, 0.0, false)] + [InlineData(1, 1.0, false)] + [InlineData(2, 1.0, false)] + [InlineData(14, 0.0, false)] + [InlineData(14, 1.0, false)] + [InlineData(1, 0.0, true)] + [InlineData(2, 1.0, true)] + [InlineData(14, 1.0, true)] + public async Task EveryStateInARecordComesOutTheSame(int degree, double bypass, bool pages) { - // Batches enough for a lane to fill its cache, measure it, then bypass it. + // Batches enough for a lane to fill its cache, measure it, then bypass it. Nine thousand keys + // over four times as many rows, which the core pages, or holds in sub-tables of hashes. Row[] rows = Rows_(keys: 9_000, count: 4 * Rows); string path = await WriteAsync(rows); try @@ -39,15 +43,17 @@ public async Task EveryStateInARecordComesOutTheSame(int degree, double bypass) { Tiny(plan); plan.CoreBypass = bypass; + plan.CorePages = pages; }); Dictionary held = ByKey(await ListAsync(core.As())); Assert.Equal(reference, held); - // The caches copied, the parts burst and split; bypassed, or never. At fourteen lanes a lane - // may take a single batch, before its cache has filled, as the queue hands them out. + // The caches copied, the parts burst, the sub-tables of hashes split, the pages never; + // bypassed, or never. At fourteen lanes a lane may take a single batch, before its cache has + // filled, as the queue hands them out. CoreRun run = core.Plan.LastRun!.Core!; - Assert.True(run.Flushes > 0 && run.Bursts > 0 && run.Splits > 0, run.ToString()); + Assert.True(run.Flushes > 0 && run.Bursts > 0 && (pages ? run.Splits == 0 : run.Splits > 0), run.ToString()); Assert.True(bypass > 0 ? run.BypassedRows > 0 || degree > 2 : run.BypassedRows == 0, run.ToString()); // Delivered part by part, every group once, a part let go before the next; and whole, as @@ -59,6 +65,7 @@ public async Task EveryStateInARecordComesOutTheSame(int degree, double bypass) Tiny(plan); plan.CoreBypass = bypass; plan.CoreParted = false; + plan.CorePages = pages; }); Assert.Equal(reference, ByKey(await ListAsync(whole.As()))); diff --git a/tests/Vorticity.Tests/Aggregation/SpillTests.cs b/tests/Vorticity.Tests/Aggregation/SpillTests.cs index 4fbaea90d..cf4340d26 100644 --- a/tests/Vorticity.Tests/Aggregation/SpillTests.cs +++ b/tests/Vorticity.Tests/Aggregation/SpillTests.cs @@ -68,6 +68,70 @@ public async Task AGroupByItsBudgetCannotHoldSpillsAndEndsExact(int degree, int } } + [Fact] + public async Task APagedCoreSpillsTheValuesItMetAlone() + { + // Two million values, one in two met, over eight million rows, at fourteen lanes: the core pages + // the key under a budget three times its pages, which its pending entries pass, and its parts go + // to the scratch, then come back into their pages, the values no entry met dropped. + const int span = 2_000_000; + Row[] rows = new Row[8_000_000]; + for (int row = 0; row < rows.Length; row++) + { + rows[row] = new Row((int)(((ulong)row * 0x9E37_79B9_7F4A_7C15UL >> 24) % (span / 2)) * 2, row % 100); + } + + // Both ends of the span met, so that the statistics bound the key to all of it. + rows[0] = new Row(0, 0); + rows[1] = new Row(span - 1, 0); + Dictionary expected = []; + foreach (Row row in rows) + { + expected[row.Key] = expected.GetValueOrDefault(row.Key) + row.Value; + } + + string path = Path.Combine(AppContext.BaseDirectory, "spill", $"paged-{Environment.ProcessId}-{Guid.NewGuid():N}.vortex"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await using (VortexFileWriter writer = VortexSession.Default.CreateWriter(path)) + { + await writer.WriteAsync(rows, Ct); + await writer.CompleteAsync(Ct); + } + + string scratch = Directory.CreateTempSubdirectory("vorticity-spill-").FullName; + try + { + // A value's page takes its record, its key and a number: 16 bytes. + QueryMemoryBudget budget = new QueryMemoryBudget(3L * span * 16); + await using VortexSession session = VortexSession.Create(options => + { + options.MaxDegreeOfParallelism = 14; + options.MemoryBudget = budget; + options.ScratchDirectory = scratch; + }); + + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + Vorticity.Aggregation sums = file.Scan().GroupBy(r => r.Key).Select(g => (g.Key, g.Sum(x => x.Value))); + sums.Plan.Core = true; + Dictionary read = []; + await foreach (KeySum sum in sums.As().ToRecordsAsync(Ct)) + { + read.Add(sum.Key, sum.Sum); + } + + Assert.Equal(expected, read); + CoreRun run = sums.Plan.LastRun!.Core!; + Assert.True(run.Splits == 0 && run.SpilledParts > 0, $"{run}, {run.SpilledParts} parts spilled"); + Assert.Equal(0, budget.ReservedBytes); + Assert.Empty(Directory.EnumerateFileSystemEntries(scratch)); + } + finally + { + Directory.Delete(scratch, recursive: true); + System.IO.File.Delete(path); + } + } + [Fact] public async Task TwoQueriesSpillAtOnceAndBothEndExact() { From 79068b4d08411702101ac230031f78d0771448a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 17:40:07 +0200 Subject: [PATCH 15/56] perf(query): a lane bypassing its cache takes a batch's keys whole A lane whose cache finds too few of its keys folds its rows into a partition of a group a row, then copies them into the core's batches. Its keys, appending, still looked each row up: a comparison with the row before, a call to Lookup, an add to the keys, a sixth of k7's cycles at fourteen lanes, where 82 % of the rows bypass the caches. A batch whose rows are all selected and none null is now taken whole: its values copied after the keys, row i's group the i-th new one. A row whose key is the row before's makes a group of its own too, which the core merges. runners.sh alt against the runner of cfc27311, warm, at fourteen lanes: group-total-k7 x0.898 (55.7 -> 50.5 ms), group-strided-k7 x0.978. --- src/Vorticity/Aggregation/GroupKeys.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index 4bd65dced..d5a70854e 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -1112,6 +1112,28 @@ internal bool AssignValues(ReadOnlySpan canonical, ReadOnlySpan v return false; } + // Appending, every row selected and none null: the batch's values taken whole, a group a row in + // their order, where a lookup a row cost a call and an add, a sixth of k7's cycles at fourteen + // lanes as they bypassed their caches. A row whose key is the row before's makes a group of its + // own too, which the core merges. + if (_appending && selection.IsEmpty && validity.IsEmpty) + { + int first = Count; + if (_keys.Length < first + rows) + { + Grow(Math.Max(first + rows, DoubledUpTo(_keys.Length, _doublesUpTo))); + } + + canonical[..rows].CopyTo(_keys.AsSpan(first)); + for (int i = 0; i < rows; i++) + { + rowGroups[i] = first + i; + } + + Count = first + rows; + return false; + } + RowCursor selected = new RowCursor(selection, 0, rows); bool hasLast = false; TValue last = default; From c97daa5b05cc7027745df9306aafffbf936941df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 17:50:23 +0200 Subject: [PATCH 16/56] perf(query): a page's part by one multiplication A paged core sends an entry to the part of its page, which every lane computes for every row it copies into its batches: the page's number mixed, two multiplications and three shifts. It is now the page's number offset by the session's seed, times the golden ratio's odd constant, its top byte: pages in order spread over every part, and no layout of keys always lands on one. runners.sh alt against the runner of 8eb56c55, warm, at fourteen lanes: group-total-k7 x0.949 (0.87-1.00), cycles x0.927. --- src/Vorticity/Aggregation/GroupCore.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 2dbc77323..44bf65215 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -269,8 +269,14 @@ private GroupCore( private readonly int[]? _pageSlots; private readonly long[][]? _partPages; - /// The part a page of the span is in: the top bits of the hash of its number. - internal static int PageHome(long page) => (int)(MergeHash.Of((ulong)page, MergeHash.Seed) >> PartShift); + /// + /// The part a page of the span is in: the top bits of its number, offset by , + /// times the golden ratio's odd constant, which spreads pages in order over every part. One + /// multiplication where the two of a mix cost k7's lanes 7 % of their cycles at fourteen lanes, its + /// rows copied into their parts' batches. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int PageHome(long page) => (int)((((ulong)page + MergeHash.Seed) * 0x9E37_79B9_7F4A_7C15UL) >> PartShift); /// /// Whether the core pages its key: a key numbered by value over a span no wider than the rows. A @@ -1806,7 +1812,7 @@ internal int PartOf(TKey key) where TKey : unmanaged => _pageBits < 0 ? (int)(EntryKeys.Hash(key, MergeHash.Seed) >> GroupCore.PartShift) - : (int)(MergeHash.Of((ulong)((EntryKeys.Integer(key) - _pageLeast) >> _pageBits), MergeHash.Seed) >> GroupCore.PartShift); + : GroupCore.PageHome((EntryKeys.Integer(key) - _pageLeast) >> _pageBits); internal EntryShape Shape { get; } From 0c397f65bcb512e1e6cd29a26ec6509106252218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:19:37 +0200 Subject: [PATCH 17/56] perf(query): the core's parts wait for the end while a burst reduces nothing A lane that deposits past a part's threshold applies the part's stack, a burst: its entries made groups in sub-tables that grew, then split at the end, their groups moved again. Ten million keys over forty million rows (s7) burst twice a part, three entries in four making a group, and split 1 792 times. Past the query's first applications, a few parts' floors, the core now lets its parts wait for the end while an entry applied made a group three times in four or more, and the budget could still grant half its ceiling (GroupCore.Defers): the end applies each part once, split ahead to its depth. A million keys, each forty times, reduce four in five: they burst as before. runners.sh alt against the runner of 0957c943, warm, at fourteen lanes on 40M rows: group-strided-k7 x0.745 (249.8 -> 186.0 ms), its peak 1.58 -> 1.16 GB; group-total-k7, paged, x0.944, its peak 953 -> 785 MB; group-strided-k6 x1.012, its peak unchanged. group-total-unique x0.986, group-pages-uuid x0.991, group-pairs-k100k x1.020 (0.95-1.06, twelve pairs). --- src/Vorticity/Aggregation/GroupCore.cs | 27 ++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 44bf65215..fd79cf75a 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -627,8 +627,31 @@ private bool Waits(long pending) => /// The most a part's application of entries may take: a group an entry, twice over for a doubling, and a split's two halves. private long Growth(long pending) => (2 * pending * _groupBytes) + (2 * _tableBytes); - /// The entries pending past which a part is applied: its floor, or α times its groups. - private long Threshold(CorePart part) => Math.Max(Floor, (long)Alpha * Volatile.Read(ref part.Groups)); + /// + /// The entries pending past which a part is applied: its floor, or α times its groups; none while a + /// burst reduces nothing (). + /// + private long Threshold(CorePart part) => Defers ? long.MaxValue : Math.Max(Floor, (long)Alpha * Volatile.Read(ref part.Groups)); + + /// The entries the query applies before it judges whether its bursts reduce: a few parts' floors. + private const long JudgedEntries = 4L * DefaultFloor; + + /// + /// Whether the parts wait for the end rather than burst: past the query's first applications, an entry + /// applied made a group nine times in ten or more, and the budget could still grant half its ceiling. + /// Such a burst only moves entries into sub-tables that split at the end, their groups moved again: + /// ten million keys over forty million rows at fourteen lanes took 205 ms with no burst against 273, + /// their peak 1.30 GB against 1.57 (2026-10-09). A million keys, each forty times, reduce: they burst. + /// + private bool Defers + { + get + { + long applied = Interlocked.Read(ref _applied); + return applied >= JudgedEntries && Interlocked.Read(ref _made) * 4 >= applied * 3 + && (_memory is null || _memory.CanGrow(_memory.Ceiling / 2)); + } + } /// Applies the part's stack while the lock is free and the count past the threshold. private void Burst(CorePart part, CoreApplier applier) From 081e804e8b451739c1c8343488ff212646119876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 18:41:19 +0200 Subject: [PATCH 18/56] perf(query): a float sum merged into an empty one takes it whole A merge of lanes or parts folds each group's IndexedSum into the target's: the count added, the tops compared, the bins shifted and added. Into an empty sum, no value and no infinity, that fold gives the other sum exactly, which nearly every group of db-benchmark's q10, a group nearly a row, met: 9 % of its cycles at fourteen lanes. Merge now copies it. runners.sh alt against the runner of a71cd499, warm, at fourteen lanes: group-db-q10 x0.958 (281.3 -> 269.6 ms). --- src/Vorticity/Aggregation/IndexedSum.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Vorticity/Aggregation/IndexedSum.cs b/src/Vorticity/Aggregation/IndexedSum.cs index 04308aac0..684e5fa75 100644 --- a/src/Vorticity/Aggregation/IndexedSum.cs +++ b/src/Vorticity/Aggregation/IndexedSum.cs @@ -262,6 +262,15 @@ internal void AddSpan(ReadOnlySpan values) /// Folds another sum: the higher top for both, the bins added. internal void Merge(in IndexedSum other) { + // An empty sum takes the other whole: its count, its top, its infinities and its bins are what the + // fold would give, with no shift. Nearly every group of db-benchmark's q10 merges into an empty + // one, 9 % of its cycles at fourteen lanes. + if (Meta == 0) + { + this = other; + return; + } + IndexedSum theirs = other; Counted(theirs.Count); int mine = Top; From 0e84181340cb8e6c993f3a43ad086b73e768af07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 19:43:08 +0200 Subject: [PATCH 19/56] fix(query): a lane on a core that spills takes past its budget in a batch A lane on a core the plan chose from the start grows its cache within a batch from a shelf that refused anything past the budget: when the core's pages and pending entries held the whole budget first, the lane failed the query rather than let the core spill (APagedCoreSpillsTheValuesItMetAlone, twice under the whole suite's load). Its shelf now takes past the budget, as a lane that may turn to the core does: its next deposit finds no room, and the core spills its largest part before the lane's next batch. The core's spill loop also stops at the query's cancellation: a query that failed in a burst left the part it emptied counted pending, which the loop took again and again, nothing given back, its lane never done (a test run hung on it). --- src/Vorticity/Aggregation/AggregationEngine.cs | 12 +++++++++++- src/Vorticity/Aggregation/GroupCore.Spill.cs | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index eb7ef7293..5000d1094 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -813,7 +813,17 @@ internal AggregationPartition( internal LaneCore? Core { get => _core; - init => _core = value; + init + { + // A lane on a core that spills takes past its budget in the middle of a batch, its cache + // growing: its next deposit finds no room, and the core spills its largest part before the + // lane's next batch. Refused, the lane failed the query when it met the budget first. + _core = value; + if (_arrays is not null && value is not null && value.Core.Spills) + { + _arrays.Overdraws = true; + } + } } /// diff --git a/src/Vorticity/Aggregation/GroupCore.Spill.cs b/src/Vorticity/Aggregation/GroupCore.Spill.cs index b1d0b8c37..30a9af9b0 100644 --- a/src/Vorticity/Aggregation/GroupCore.Spill.cs +++ b/src/Vorticity/Aggregation/GroupCore.Spill.cs @@ -59,6 +59,9 @@ internal async ValueTask SpillAsync(CancellationToken cancellationToken) QueryMemory memory = _memory!; do { + // A query that failed leaves what it held, and a part a failed burst emptied counted still + // pending: the loop would take it again and again, nothing given back, the lane never done. + cancellationToken.ThrowIfCancellationRequested(); if (Largest() is not { } part) { break; From dd5e7e65199ccbc23271143025a5eeee57d2cabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 19:46:58 +0200 Subject: [PATCH 20/56] perf(query): a key of five to eight columns as the tuple of its values db-benchmark's q10 groups by three short texts and three integers, a group nearly a row. PackedKeys numbered each column in an index of its own, then looked the tuple of the numbers up: seven lookups a row, 64 % of its cycles at one thread; and the numbers differ from one lane to the next, so that the core could not take the key, and fourteen lanes each built a table of 714 000 groups, then merged them by copying. TupleKeys holds such a key, of text and integer columns, as the tuple of its values in 64 bytes (KeyTuple): a text of 12 bytes or less as its word, a longer one as a mark and its number among the run's long texts, an integer as its bytes, the nulls in a byte of flags. One table of tuples, a FixedKeys of them, hashed once a row; and a key of a known width, the same in every lane, which the core takes. The tuples of a block are made 1 024 rows at a time, 64 KB, where a block whole held 4 MB a lane that a lane on the core reserved twice past its budget. The long texts are numbered once for the run, by whichever lane meets one first, in a layout the plan keeps per run (keyed by the run's QueryMemory), and counted under its memory beside the groups (QueryMemory.HoldBeside), kept with the result until it is delivered. Under pressure the governor now turns such keys to the core, which takes past the budget what a stack asks more, as for every key it takes (CorePressureTests): GroupSpillTests checks the peak of the lanes' tables alone, keeps its six-column test on the numbers (Tuples = false) and adds one on the tuples. TupleKeysTests: texts short, long, empty and null, integers of every width, filters, one to fourteen lanes, the core, the order of the components, one numbering a run. The runner's +numbers switch keeps the numbers. --- .../GroupScenarios.cs | 6 +- .../Aggregation/AggregationEngine.cs | 39 +- src/Vorticity/Aggregation/ArrayShelf.cs | 3 + src/Vorticity/Aggregation/FixedSlot.cs | 5 + src/Vorticity/Aggregation/GroupCore.cs | 15 +- src/Vorticity/Aggregation/GroupKeys.cs | 78 ++- src/Vorticity/Aggregation/QueryMemory.cs | 32 +- src/Vorticity/Aggregation/TupleKeys.cs | 655 ++++++++++++++++++ .../Aggregation/GroupSpillTests.cs | 60 +- .../Aggregation/TupleKeysTests.cs | 302 ++++++++ 10 files changed, 1153 insertions(+), 42 deletions(-) create mode 100644 src/Vorticity/Aggregation/TupleKeys.cs create mode 100644 tests/Vorticity.Tests/Aggregation/TupleKeysTests.cs diff --git a/bench/Vorticity.Benchmarks.Runner/GroupScenarios.cs b/bench/Vorticity.Benchmarks.Runner/GroupScenarios.cs index 12717011f..f83f6cdd1 100644 --- a/bench/Vorticity.Benchmarks.Runner/GroupScenarios.cs +++ b/bench/Vorticity.Benchmarks.Runner/GroupScenarios.cs @@ -38,7 +38,8 @@ namespace Vorticity.Bench.Runner; /// streams at every degree), blocking (no stream), core (the core at every degree), /// whole (the core's groups delivered whole), tables (the lanes' tables, the core taken /// neither from the first batch nor from the first rows), nofan (the lanes all start at the head -/// of the queue), noretire (lanes under pressure write their tables rather than retire into +/// of the queue), numbers (a key of five to eight columns as the numbers of its columns rather +/// than its tuples), noretire (lanes under pressure write their tables rather than retire into /// one), budget=N (the session's memory budget, in MiB, under which the runner prints its peak), /// capacity=N, alpha=N, floor=N, table=N, batch=N, window=N, probe=N, bypass=N /// (the core's ε, in percent). The degree is the runner's --threads. @@ -196,6 +197,7 @@ private static Action Switch(string option) plan.CoreOnNew = false; }, "nofan" => static plan => plan.Fan = false, + "numbers" => static plan => plan.Tuples = false, "capacity" => plan => plan.CoreCapacity = Valued(value, name), "alpha" => plan => plan.CoreAlpha = Valued(value, name), "floor" => plan => plan.CoreFloor = Valued(value, name), @@ -206,7 +208,7 @@ private static Action Switch(string option) "bypass" => plan => plan.CoreBypass = Valued(value, name) / 100.0, "noretire" => static plan => plan.LanesRetire = false, "budget" => Budget(Valued(value, name)), - _ => throw new ArgumentException($"No switch named '{name}': zones, blocking, core, whole, tables, nofan, noretire, budget=N (MiB), capacity=N, alpha=N, floor=N, table=N, batch=N, window=N, probe=N, bypass=N (percent).", nameof(option)), + _ => throw new ArgumentException($"No switch named '{name}': zones, blocking, core, whole, tables, nofan, numbers, noretire, budget=N (MiB), capacity=N, alpha=N, floor=N, table=N, batch=N, window=N, probe=N, bypass=N (percent).", nameof(option)), }; } diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 5000d1094..c3f8d35bf 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -447,21 +447,56 @@ private static (IAggregateNode Row, int[] Columns)[] Rows(IChosenColumn[] chosen /// /// The index of the groups of one partition: a key of one column by its own index, of two to - /// eight by their indexes' numbers packed into a word, of more by their values encoded into bytes. + /// eight by their indexes' numbers packed into a word, or of five to eight texts and integers by + /// the tuple of their values, of more by their values encoded into bytes. /// /// Whether the statistics say the key of one column is sorted. /// What the statistics say of each column, which a composite's parts and a bounded integer read. /// The lane's shelf under its query's memory, which the key's tables grow from; null for tables nothing counts. - internal GroupKeys CreateKeys(bool sorted, KeyFacts? facts = null, ArrayShelf? shelf = null) => Keys.Length switch + /// + /// The run's memory, the shelf's when not given: a key held as its tuples numbers its long texts once + /// for the run, under it. Null for keys that only tell what they are. + /// + internal GroupKeys CreateKeys(bool sorted, KeyFacts? facts = null, ArrayShelf? shelf = null, QueryMemory? memory = null) => Keys.Length switch { 1 => Single(Keys[0], sorted, sorted ? null : facts?.Bounds[0], ProbeAhead, facts?.Rows ?? -1, shelf), 2 or 3 or 4 when Raw(facts) is { } layout => layout.Bits <= 64 ? new RawKeys(layout, shelf: shelf) : new RawKeys(layout, shelf: shelf), 2 => new PackedKeys(Keys, facts, shelf: shelf), 3 or 4 => new PackedKeys(Keys, facts, shelf: shelf), + <= 8 when Tuples && TupleLayout(memory ?? shelf?.Memory) is { } layout => new TupleKeys(layout, ProbeAhead, shelf), <= 8 => new PackedKeys(Keys, facts, shelf: shelf), _ => new CompositeKeys(Keys, shelf), }; + /// + /// Whether a key of five to eight text and integer columns is held as the tuple of its values + /// (), which the core takes, rather than as the numbers its columns have in + /// indexes of their own: false for those numbers, the switch the tests and the bench compare them with. + /// + internal bool Tuples { get; set; } = true; + + // Whether a tuple holds the key (null before it is asked), and the layout of each run's tuples, keyed + // by the run's memory: its long texts numbered once for the run, the same in every lane, and dropped + // with it. + private bool? _tuplesFit; + private readonly System.Runtime.CompilerServices.ConditionalWeakTable _tupleLayouts = new(); + + /// + /// The layout of the key's tuples for the run counts, the same for every lane of + /// it; one of its own without a run, which only tells what the keys are. Null when a tuple cannot hold + /// the key. + /// + private TupleLayout? TupleLayout(QueryMemory? run) + { + _tuplesFit ??= Aggregating.TupleLayout.Of(Keys, memory: null) is not null; + if (_tuplesFit != true) + { + return null; + } + + return run is null ? Aggregating.TupleLayout.Of(Keys, memory: null) : _tupleLayouts.GetValue(run, memory => Aggregating.TupleLayout.Of(Keys, memory)!); + } + /// /// The layout of the key as the tuple of its values in one word, unless the statistics say a /// column is sorted, which hands its rows by range, or bound the diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index ee63fab15..a71568538 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -78,6 +78,9 @@ internal ArrayShelf(QueryMemory memory) /// Whether the shelf is a lane's: under a query's memory, with no pile and no process's shelf behind it. private bool Lane => _memory is not null && _parent is null; + /// The query's memory the shelf counts its arrays under: its run's; null for a shelf that counts nothing. + internal QueryMemory? Memory => _memory; + private ArrayShelf(ArrayShelf? parent, long budget) { _parent = parent; diff --git a/src/Vorticity/Aggregation/FixedSlot.cs b/src/Vorticity/Aggregation/FixedSlot.cs index b3b7f5f76..dc8003021 100644 --- a/src/Vorticity/Aggregation/FixedSlot.cs +++ b/src/Vorticity/Aggregation/FixedSlot.cs @@ -1424,6 +1424,11 @@ internal static (ulong Low, ulong High) Of(TValue value) return (Half.IsNaN(h) ? (ushort)0x7E00 : h == Half.Zero ? (ushort)0 : BitConverter.HalfToUInt16Bits(h), 0); } + if (typeof(TValue) == typeof(KeyTuple)) + { + return Unsafe.BitCast(value).Words(); + } + if (Unsafe.SizeOf() == 32) { // A decimal of 256 bits: the high words folded into the low ones through a multiply, diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index fd79cf75a..b9925bf40 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -399,7 +399,7 @@ internal CorePressure? Pressure AggregationPlan plan, AggregateSlot?[] settled, ColumnShape[] columns, int[] inputs, ScanSource source, KeyFacts? facts, bool sorted, KeyTop? top, int lanes, QueryMemory? memory, bool lean) { - if (!plan.Grouped || sorted || top is not null || !Holds(plan, settled, source, facts, out GroupKeys? kind, out GroupRecords? records)) + if (!plan.Grouped || sorted || top is not null || !Holds(plan, settled, source, facts, memory, out GroupKeys? kind, out GroupRecords? records)) { return null; } @@ -413,17 +413,18 @@ internal CorePressure? Pressure /// /// Whether the core can hold the plan's groups: a key that travels in entries, of fixed width, and - /// every state in the records. The others, a text key, a composite holding one, a text's extremes, a - /// distinct count, only the lanes' tables hold, which spill under pressure. + /// every state in the records. The others, a text key, a composite of two to four columns holding one, + /// a text's extremes, a distinct count, only the lanes' tables hold, which spill under pressure. /// internal static bool Holds(AggregationPlan plan, AggregateSlot?[] settled, ScanSource source, KeyFacts? facts) => - plan.Grouped && Holds(plan, settled, source, facts, out _, out _); + plan.Grouped && Holds(plan, settled, source, facts, memory: null, out _, out _); + /// Whether the core can hold the plan's groups, and the keys it holds them by: the run's under , which its lanes share. private static bool Holds( - AggregationPlan plan, AggregateSlot?[] settled, ScanSource source, KeyFacts? facts, [NotNullWhen(true)] out GroupKeys? kind, out GroupRecords? records) + AggregationPlan plan, AggregateSlot?[] settled, ScanSource source, KeyFacts? facts, QueryMemory? memory, [NotNullWhen(true)] out GroupKeys? kind, out GroupRecords? records) { records = null; - kind = plan.CreateKeys(sorted: false, CacheFacts(facts)); + kind = plan.CreateKeys(sorted: false, CacheFacts(facts), memory: memory); if (kind.EntryBytes == 0) { return false; @@ -638,7 +639,7 @@ private bool Waits(long pending) => /// /// Whether the parts wait for the end rather than burst: past the query's first applications, an entry - /// applied made a group nine times in ten or more, and the budget could still grant half its ceiling. + /// applied made a group three times in four or more, and the budget could still grant half its ceiling. /// Such a burst only moves entries into sub-tables that split at the end, their groups moved again: /// ten million keys over forty million rows at fourteen lanes took 205 ms with no burst against 273, /// their peak 1.30 GB against 1.57 (2026-10-09). A million keys, each forty times, reduce: they burst. diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index d5a70854e..fd87aaddd 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -1112,25 +1112,9 @@ internal bool AssignValues(ReadOnlySpan canonical, ReadOnlySpan v return false; } - // Appending, every row selected and none null: the batch's values taken whole, a group a row in - // their order, where a lookup a row cost a call and an add, a sixth of k7's cycles at fourteen - // lanes as they bypassed their caches. A row whose key is the row before's makes a group of its - // own too, which the core merges. if (_appending && selection.IsEmpty && validity.IsEmpty) { - int first = Count; - if (_keys.Length < first + rows) - { - Grow(Math.Max(first + rows, DoubledUpTo(_keys.Length, _doublesUpTo))); - } - - canonical[..rows].CopyTo(_keys.AsSpan(first)); - for (int i = 0; i < rows; i++) - { - rowGroups[i] = first + i; - } - - Count = first + rows; + AppendAll(canonical[..rows], rowGroups.AsSpan(0, rows)); return false; } @@ -1160,6 +1144,66 @@ internal bool AssignValues(ReadOnlySpan canonical, ReadOnlySpan v return false; } + /// + /// Appending, every row selected and none null: taken whole, a group a value + /// in their order, theirs, where a lookup a row cost a call and an add, a sixth + /// of k7's cycles at fourteen lanes as they bypassed their caches. A value that is the one before makes + /// a group of its own too, which the core merges. + /// + internal void AppendAll(ReadOnlySpan values, Span groups) + { + int first = Count; + if (_keys.Length < first + values.Length) + { + Grow(Math.Max(first + values.Length, DoubledUpTo(_keys.Length, _doublesUpTo))); + } + + values.CopyTo(_keys.AsSpan(first)); + for (int i = 0; i < values.Length; i++) + { + groups[i] = first + i; + } + + Count = first + values.Length; + } + + /// + /// The groups of the rows of a block from that + /// selects, theirs, none null: a block made a chunk at a time + /// (), each found as finds a block whole. + /// + internal void AssignChunk(ReadOnlySpan values, int start, ReadOnlySpan selection, int[] rowGroups) + { + if (selection.IsEmpty && _appending) + { + AppendAll(values, rowGroups.AsSpan(start, values.Length)); + return; + } + + if (selection.IsEmpty && _probeAhead >= 0) + { + TwoPasses(values, default, start, rowGroups); + return; + } + + RowCursor selected = new RowCursor(selection, start, start + values.Length); + bool hasLast = false; + TValue last = default; + int lastGroup = -1; + while (selected.Next(out int row)) + { + TValue value = values[row - start]; + if (!hasLast || !value.Equals(last)) + { + last = value; + lastGroup = Lookup(value); + hasLast = true; + } + + rowGroups[row] = lastGroup; + } + } + /// /// The group of each row from of a block, theirs, /// every row selected, in two passes: the group of each key that sits in its home slot, with no branch diff --git a/src/Vorticity/Aggregation/QueryMemory.cs b/src/Vorticity/Aggregation/QueryMemory.cs index c3629b3f2..91536d649 100644 --- a/src/Vorticity/Aggregation/QueryMemory.cs +++ b/src/Vorticity/Aggregation/QueryMemory.cs @@ -79,6 +79,7 @@ internal sealed class QueryMemory : IDisposable private long _held; private long _peak; private long _measured; + private long _beside; private int _left; /// A query's memory under , one more query to share its ceiling with until it is given back. @@ -200,16 +201,39 @@ internal void Discard(long bytes) } } - /// Keeps of what the query holds and measures, the result it delivers, and gives back the rest. + /// + /// Reserves and measures the query holds beside its groups for its whole run, + /// which its lanes share: the long texts of a key's tuples (). Past the + /// ceiling when the budget refuses them, as a table that grows within a batch: the lanes' next readings + /// spill or turn to the core. Kept with the result until it is delivered. + /// + internal void HoldBeside(long bytes) + { + if (!TryGrow(bytes)) + { + Force(bytes); + } + + Measure(bytes); + Interlocked.Add(ref _beside, bytes); + } + + /// + /// Keeps of what the query holds and measures, the result it delivers, and + /// what it holds beside its groups (), and gives back the rest. + /// internal void Keep(long bytes) { + // What it holds beside its groups stays, whether the bytes count it (what it holds) or not (a + // result's own). long held = Held; - if (held > bytes) + long kept = Math.Max(bytes, Math.Min(held, bytes + Volatile.Read(ref _beside))); + if (held > kept) { - Shrink(held - bytes); + Shrink(held - kept); } - Measure(bytes - Measured); + Measure(kept - Measured); } /// Gives back everything the query holds. diff --git a/src/Vorticity/Aggregation/TupleKeys.cs b/src/Vorticity/Aggregation/TupleKeys.cs new file mode 100644 index 000000000..f125e9f3e --- /dev/null +++ b/src/Vorticity/Aggregation/TupleKeys.cs @@ -0,0 +1,655 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Vorticity.Arrays; +using Vorticity.Types; +using Vorticity.Writing; + +namespace Vorticity.Aggregating; + +/// +/// The values of a key of several columns side by side in 64 bytes, eight words: each text a word, each +/// integer its bytes, a byte of flags for the nulls (). Its words compared at +/// once; its order is the words', which only a table needs. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct KeyTuple : IEquatable, IComparable +{ + internal ulong W0; + internal ulong W1; + internal ulong W2; + internal ulong W3; + internal ulong W4; + internal ulong W5; + internal ulong W6; + internal ulong W7; + + public readonly bool Equals(KeyTuple other) => + ((W0 ^ other.W0) | (W1 ^ other.W1) | (W2 ^ other.W2) | (W3 ^ other.W3) | (W4 ^ other.W4) | (W5 ^ other.W5) | (W6 ^ other.W6) | (W7 ^ other.W7)) == 0; + + public readonly int CompareTo(KeyTuple other) + { + int c = W0.CompareTo(other.W0); + c = c != 0 ? c : W1.CompareTo(other.W1); + c = c != 0 ? c : W2.CompareTo(other.W2); + c = c != 0 ? c : W3.CompareTo(other.W3); + c = c != 0 ? c : W4.CompareTo(other.W4); + c = c != 0 ? c : W5.CompareTo(other.W5); + c = c != 0 ? c : W6.CompareTo(other.W6); + return c != 0 ? c : W7.CompareTo(other.W7); + } + + public override readonly bool Equals(object? obj) => obj is KeyTuple other && Equals(other); + + public override readonly int GetHashCode() => (int)KeyWords.Of(this).Low; + + /// + /// The tuple's eight words folded into two by four products of 128 bits, the halves of each xored, as + /// rapidhash mixes: a table that hashes the two words weakly, unseeded, still spreads tuples that differ + /// in any word. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (ulong Low, ulong High) Words() => + (Fold(W0 ^ 0x2D35_8DCC_AA6C_78A5UL, W1 ^ 0x8BB8_4B93_962E_ACC9UL) ^ Fold(W2 ^ 0x4B33_A62E_D433_D4A3UL, W3 ^ 0x4D5A_2DA5_1DE1_AA47UL), + Fold(W4 ^ 0xA076_1D64_78BD_642FUL, W5 ^ 0xE703_7ED1_A0B4_28DBUL) ^ Fold(W6 ^ 0x9E37_79B9_7F4A_7C15UL, W7 ^ 0xC2B2_AE3D_27D4_EB4FUL)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Fold(ulong a, ulong b) + { + ulong high = Math.BigMul(a, b, out ulong low); + return low ^ high; + } +} + +/// +/// Where each column of a key lies in a , a run's: a text a word of 16 bytes, its +/// length then its bytes as a canonical view holds a value of 12 bytes or less (); +/// an integer its bytes, as wide as its type; the nulls a bit each in the last byte. A text longer than a +/// word is a mark, a length no word has, and its number among the run's long texts, the same in every +/// lane, numbered once by whichever lane meets it first: a tuple is the same tuple in every lane, which +/// the core takes as a key of a known width. The long texts are held once for the run, under its memory. +/// +internal sealed class TupleLayout +{ + /// The bytes of a tuple. + internal const int Bytes = 64; + + /// The byte of the null flags, a bit a column. + internal const int FlagsAt = Bytes - 1; + + /// The bytes of a text's word. + internal const int WordBytes = 16; + + /// The length a long text's mark holds in place of a word's: none has it. + private const uint LongMark = uint.MaxValue; + + /// What a long text holds past its bytes: its array's header, its node in the dictionary and its place in the list. + private const int LongOverhead = 96; + + private readonly QueryMemory? _memory; + private readonly ConcurrentDictionary _longIds; + private readonly ConcurrentDictionary.AlternateLookup> _longLookup; + private readonly Lock _gate = new Lock(); + private byte[][] _longs = new byte[16][]; + private int _longCount; + + private TupleLayout(ColumnShape[] shapes, int[] offsets, QueryMemory? memory) + { + Shapes = shapes; + Offsets = offsets; + _memory = memory; + _longIds = new ConcurrentDictionary(LongComparer.Instance); + _longLookup = _longIds.GetAlternateLookup>(); + } + + internal ColumnShape[] Shapes { get; } + + /// The byte each column starts at. + internal int[] Offsets { get; } + + /// + /// The layout of a key of for a run under , or null + /// when a tuple cannot hold it: more than eight columns, a column neither a text nor an integer, or more + /// than 63 bytes. Texts first, each on a word, then the integers from the widest down. + /// + internal static TupleLayout? Of(ColumnShape[] shapes, QueryMemory? memory) + { + if (shapes.Length is < 2 or > 8) + { + return null; + } + + int[] widths = new int[shapes.Length]; + for (int p = 0; p < shapes.Length; p++) + { + widths[p] = WidthOf(shapes[p]); + if (widths[p] == 0) + { + return null; + } + } + + int[] offsets = new int[shapes.Length]; + int at = 0; + foreach (int width in (int[])[WordBytes, 8, 4, 2, 1]) + { + for (int p = 0; p < shapes.Length; p++) + { + if (widths[p] == width) + { + offsets[p] = at; + at += width; + } + } + } + + return at <= FlagsAt ? new TupleLayout(shapes, offsets, memory) : null; + } + + /// The bytes column takes in a tuple: a word for a text, its width for an integer; 0 for any other. + private static int WidthOf(ColumnShape shape) => shape.Kind switch + { + StorageKind.Bytes => WordBytes, + StorageKind.Primitive => shape.PType switch + { + PType.I8 or PType.U8 => 1, + PType.I16 or PType.U16 => 2, + PType.I32 or PType.U32 => 4, + PType.I64 or PType.U64 => 8, + _ => 0, + }, + _ => 0, + }; + + /// Whether column is a text. + internal bool Text(int component) => Shapes[component].Kind == StorageKind.Bytes; + + /// The word of a text longer than a word holds: its mark and its number, numbered the first time. + internal TextWord LongWord(ReadOnlySpan value) + { + if (!_longLookup.TryGetValue(value, out int id)) + { + id = AddLong(value.ToArray()); + } + + return new TextWord(LongMark | ((ulong)(uint)id << 32), 0); + } + + private int AddLong(byte[] value) + { + lock (_gate) + { + if (_longIds.TryGetValue(value, out int known)) + { + return known; + } + + // Counted before it is kept: a text the run's memory refuses fails the query with nothing kept. + _memory?.HoldBeside(value.Length + LongOverhead); + int id = _longCount; + if (id == _longs.Length) + { + Array.Resize(ref _longs, id * 2); + } + + _longs[id] = value; + _longCount = id + 1; + _longIds[value] = id; + return id; + } + } + + /// The long texts numbered so far. + internal int LongTexts => Volatile.Read(ref _longCount); + + /// The bytes of the text a word holds, through of 16 bytes for a short one. + internal ReadOnlySpan TextOf(TextWord word, Span buffer) + { + if ((uint)word.Low != LongMark) + { + return ShortTextKeys.BytesOf(word, buffer); + } + + lock (_gate) + { + return _longs[(int)(word.Low >> 32)]; + } + } + + /// Byte arrays equal by their bytes, found by a span of them. + private sealed class LongComparer : IEqualityComparer, IAlternateEqualityComparer, byte[]> + { + internal static readonly LongComparer Instance = new LongComparer(); + + public bool Equals(byte[]? x, byte[]? y) => x.AsSpan().SequenceEqual(y); + + public int GetHashCode(byte[] obj) => (int)MergeHash.Of(obj, MergeHash.Seed); + + public bool Equals(ReadOnlySpan alternate, byte[] other) => alternate.SequenceEqual(other); + + public int GetHashCode(ReadOnlySpan alternate) => (int)MergeHash.Of(alternate, MergeHash.Seed); + + public byte[] Create(ReadOnlySpan alternate) => alternate.ToArray(); + } +} + +/// +/// A key of two to eight text and integer columns as the tuple of its values (): one +/// table of tuples, where numbers each column in an index of its own, then +/// the tuple of the numbers; and a key of a known width, the same in every lane, which the core takes. Its +/// tables, entries, merges and spills are those of of tuples; its columns +/// come out of the tuples, each in its type. +/// +/// +/// db-benchmark's q10, six columns of which three short texts and a group nearly a row, held a table of +/// numbers a lane at fourteen lanes, ≈ 714 000 groups each, then merged them, copying every one: half its +/// time (2026-10-09). Its tuples go to the core, which holds each group once. +/// +internal sealed class TupleKeys : GroupKeys +{ + /// + /// The rows whose tuples are made and found at a time: 64 KB of tuples a lane, in the first level of + /// cache of current cores, where a block of 65 536 rows at once held 4 MB of them, which a lane on the + /// core reserved twice past its budget. + /// + private const int Chunk = 1 << 10; + + private readonly TupleLayout _layout; + private readonly FixedKeys _tuples; + private KeyTuple[] _block = []; + private TextWord[] _words = []; + + internal TupleKeys(TupleLayout layout, FixedKeys tuples) + { + _layout = layout; + _tuples = tuples; + Count = tuples.Count; + } + + /// The keys of a plan's columns as tuples, their arrays from . + internal TupleKeys(TupleLayout layout, int probeAhead, ArrayShelf? shelf) + : this(layout, new FixedKeys(TupleShape(layout), sorted: false, probeAhead: probeAhead, shelf: shelf)) + { + } + + /// The shape a tuple's table reads: the first column's, which it never reads by. + private static ColumnShape TupleShape(TupleLayout layout) => layout.Shapes[0]; + + /// Where the tuples' columns lie, and the run's long texts. + internal TupleLayout Layout => _layout; + + private TupleKeys Wrap(GroupKeys tuples) => new TupleKeys(_layout, (FixedKeys)tuples); + + internal override bool Assign(CanonicalArena arena, ReadOnlySpan nodes, int rows, ReadOnlySpan selection, int[] rowGroups, GroupRanges ranges) + { + Scratch.Grow(ref _block, Math.Min(rows, Chunk)); + for (int start = 0; start < rows; start += Chunk) + { + int end = Math.Min(rows, start + Chunk); + _tuples.AssignChunk(Fill(arena, nodes, start, end), start, selection, rowGroups); + } + + Count = _tuples.Count; + return false; + } + + /// The tuples of rows to , from the scratch's start. + private Span Fill(CanonicalArena arena, ReadOnlySpan nodes, int start, int end) + { + Span block = _block.AsSpan(0, end - start); + block.Clear(); + ref byte first = ref Unsafe.As(ref MemoryMarshal.GetReference(block)); + for (int p = 0; p < _layout.Shapes.Length; p++) + { + int offset = _layout.Offsets[p]; + if (_layout.Text(p)) + { + Texts(arena, nodes[p], start, end, ref first, offset, p); + continue; + } + + switch (_layout.Shapes[p].PType) + { + case PType.I8: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + case PType.U8: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + case PType.I16: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + case PType.U16: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + case PType.I32: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + case PType.U32: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + case PType.I64: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + default: + Integers(arena, nodes[p], start, end, ref first, offset, p); + break; + } + } + + return block; + } + + /// + /// A text column's words of rows to into the tuples at + /// : rows all short and none null at once, any others a row at a time. + /// + private void Texts(CanonicalArena arena, int node, int start, int end, ref byte first, int offset, int component) + { + BytesBlock values = BytesBlock.Canonical(arena, node, out ReadOnlySpan validity); + int rows = end - start; + Scratch.Grow(ref _words, rows); + Span words = _words.AsSpan(0, rows); + if (validity.IsEmpty && values.TryWords(start, words)) + { + for (int i = 0; i < rows; i++) + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref first, ((nint)i * TupleLayout.Bytes) + offset), words[i]); + } + + return; + } + + byte flag = (byte)(1 << component); + for (int i = 0; i < rows; i++) + { + int row = start + i; + ref byte tuple = ref Unsafe.Add(ref first, (nint)i * TupleLayout.Bytes); + if (!StorageValues.IsValid(validity, row)) + { + Unsafe.Add(ref tuple, TupleLayout.FlagsAt) |= flag; + continue; + } + + TextWord word = values.TryWord(row, out TextWord shortWord) ? shortWord : _layout.LongWord(values[row]); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref tuple, offset), word); + } + } + + /// An integer column's values of rows to into the tuples at , its nulls flagged. + private static void Integers(CanonicalArena arena, int node, int start, int end, ref byte first, int offset, int component) + where T : unmanaged + { + // A primitive column is read where it lies: no scratch. + T[] none = []; + ReadOnlySpan values = FixedReader.Values(arena, node, StorageKind.Primitive, ref none, out ReadOnlySpan validity)[start..end]; + if (validity.IsEmpty) + { + for (int i = 0; i < values.Length; i++) + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref first, ((nint)i * TupleLayout.Bytes) + offset), values[i]); + } + + return; + } + + byte flag = (byte)(1 << component); + for (int i = 0; i < values.Length; i++) + { + ref byte tuple = ref Unsafe.Add(ref first, (nint)i * TupleLayout.Bytes); + if (StorageValues.IsValid(validity, start + i)) + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref tuple, offset), values[i]); + } + else + { + Unsafe.Add(ref tuple, TupleLayout.FlagsAt) |= flag; + } + } + } + + /// Whether column of is null. + private static bool IsNull(KeyTuple tuple, int component) => + ((Unsafe.Add(ref Unsafe.As(ref tuple), TupleLayout.FlagsAt) >> component) & 1) != 0; + + /// The value of a column of a tuple at byte , as its bytes. + private static T Read(KeyTuple tuple, int offset) + where T : unmanaged => + Unsafe.ReadUnaligned(ref Unsafe.Add(ref Unsafe.As(ref tuple), offset)); + + /// The bytes of text column of a tuple, through of 16 bytes. + private ReadOnlySpan TextOf(KeyTuple tuple, int component, Span buffer) => + _layout.TextOf(Read(tuple, _layout.Offsets[component]), buffer); + + /// Column of two tuples, by its type: a text by its bytes, an integer by its value, a null last. + private int Compare(KeyTuple left, KeyTuple right, int component) + { + bool leftNull = IsNull(left, component); + bool rightNull = IsNull(right, component); + if (leftNull || rightNull) + { + return leftNull == rightNull ? 0 : leftNull ? 1 : -1; + } + + int offset = _layout.Offsets[component]; + if (_layout.Text(component)) + { + Span a = stackalloc byte[16]; + Span b = stackalloc byte[16]; + return TextOf(left, component, a).SequenceCompareTo(TextOf(right, component, b)); + } + + return _layout.Shapes[component].PType switch + { + PType.I8 => Read(left, offset).CompareTo(Read(right, offset)), + PType.U8 => Read(left, offset).CompareTo(Read(right, offset)), + PType.I16 => Read(left, offset).CompareTo(Read(right, offset)), + PType.U16 => Read(left, offset).CompareTo(Read(right, offset)), + PType.I32 => Read(left, offset).CompareTo(Read(right, offset)), + PType.U32 => Read(left, offset).CompareTo(Read(right, offset)), + PType.I64 => Read(left, offset).CompareTo(Read(right, offset)), + _ => Read(left, offset).CompareTo(Read(right, offset)), + }; + } + + internal override GroupKeys Fresh() => Wrap(_tuples.Fresh()); + + internal override GroupKeys ForPart() => Wrap(_tuples.ForPart()); + + internal override GroupKeys ForTable(ArrayShelf shelf) => Wrap(_tuples.ForTable(shelf)); + + internal override GroupKeys ForSpill(ArrayShelf? shelf) => Wrap(_tuples.ForSpill(shelf)); + + internal override GroupKeys? Appending() => _tuples.Appending() is { } appending ? Wrap(appending) : null; + + internal override long Footprint => _tuples.Footprint + ((long)_block.Length * TupleLayout.Bytes) + ((long)_words.Length * TupleLayout.WordBytes); + + internal override long GrowthFor(int more) => _tuples.GrowthFor(more); + + internal override bool Squeeze(int more) => _tuples.Squeeze(more); + + internal override int NullNumber => _tuples.NullNumber; + + internal override bool Spills => _tuples.Spills; + + internal override void Release() + { + _tuples.Release(); + Count = 0; + } + + internal override void Reserve(int groups) => _tuples.Reserve(groups); + + internal override void ReserveAllNew(int groups) => _tuples.ReserveAllNew(groups); + + internal override void Keep(ReadOnlySpan groups) + { + _tuples.Keep(groups); + Count = _tuples.Count; + Renumbered(); + } + + internal override void Carry(ReadOnlySpan groups) + { + _tuples.Carry(groups); + Count = _tuples.Count; + Renumbered(); + } + + internal override void MergeInto(GroupKeys target, ReadOnlySpan groups, Span map) + { + TupleKeys into = (TupleKeys)target; + _tuples.MergeInto(into._tuples, groups, map); + into.Count = into._tuples.Count; + } + + internal override void Parts(ulong seed, int shift, Span parts) => _tuples.Parts(seed, shift, parts); + + internal override void Hashes(Span hashes) => _tuples.Hashes(hashes); + + internal override void WriteKeys(ReadOnlySpan groups, SpillBuffer buffer) => _tuples.WriteKeys(groups, buffer); + + internal override void ReadKeys(ref SpillReader reader, Span groups) + { + _tuples.ReadKeys(ref reader, groups); + Count = _tuples.Count; + } + + internal override int EntryBytes => _tuples.EntryBytes; + + internal override void Scatter(ReadOnlySpan records, LaneCore lane) => _tuples.Scatter(records, lane); + + internal override void CountParts(Span counts, LaneCore lane) => _tuples.CountParts(counts, lane); + + internal override int CopyEntries(ReadOnlySpan records, EntryShape shape, int from, Span entries, out int written) => + _tuples.CopyEntries(records, shape, from, entries, out written); + + internal override void TablesOf(PartBatch batch, EntryShape shape, int shift, int mask, Span tables) => _tuples.TablesOf(batch, shape, shift, mask, tables); + + internal override ulong HashAt(PartBatch batch, EntryShape shape, int entry) => _tuples.HashAt(batch, shape, entry); + + internal override void GroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan entries, Span groups, Span scratch) + { + _tuples.GroupsOf(batch, shape, entries, groups, scratch); + Count = _tuples.Count; + } + + internal override int[] Order(bool sorted) => Identity(Count); + + internal override bool Orders(int component) => true; + + internal override int CompareKeys(int a, int b, int component) => Compare(_tuples.KeyAt(a), _tuples.KeyAt(b), component); + + internal override int CompareKeys(GroupKeys other, int a, int b, int component) => + Compare(_tuples.KeyAt(a), ((TupleKeys)other)._tuples.KeyAt(b), component); + + internal override Func Reader(int component) + { + ColumnShape shape = _layout.Shapes[component]; + int offset = _layout.Offsets[component]; + if (_layout.Text(component)) + { + byte[] buffer = new byte[16]; + return group => + { + KeyTuple tuple = _tuples.KeyAt(group); + return IsNull(tuple, component) ? default! : StorageValues.BytesToClr(TextOf(tuple, component, buffer), shape); + }; + } + + return shape.PType switch + { + PType.I8 => IntegerReader(component, offset, shape), + PType.U8 => IntegerReader(component, offset, shape), + PType.I16 => IntegerReader(component, offset, shape), + PType.U16 => IntegerReader(component, offset, shape), + PType.I32 => IntegerReader(component, offset, shape), + PType.U32 => IntegerReader(component, offset, shape), + PType.I64 => IntegerReader(component, offset, shape), + _ => IntegerReader(component, offset, shape), + }; + } + + private Func IntegerReader(int component, int offset, ColumnShape shape) + where TValue : unmanaged => + group => + { + KeyTuple tuple = _tuples.KeyAt(group); + return IsNull(tuple, component) ? default! : StorageValues.ToClr(Read(tuple, offset), shape); + }; + + internal override void Append(int component, ColumnStore store, ReadOnlySpan groups) + { + int offset = _layout.Offsets[component]; + if (_layout.Text(component)) + { + // The texts came from the column the reader checked as it decoded it, as a table of bytes appends them. + VarBinStore leaf = (VarBinStore)store.Leaf; + Span buffer = stackalloc byte[16]; + foreach (int group in groups) + { + KeyTuple tuple = _tuples.KeyAt(group); + if (IsNull(tuple, component)) + { + KeyStores.AppendNull(store); + } + else + { + leaf.AppendValidated(TextOf(tuple, component, buffer)); + } + } + + return; + } + + switch (_layout.Shapes[component].PType) + { + case PType.I8: + AppendIntegers(component, offset, store, groups); + break; + case PType.U8: + AppendIntegers(component, offset, store, groups); + break; + case PType.I16: + AppendIntegers(component, offset, store, groups); + break; + case PType.U16: + AppendIntegers(component, offset, store, groups); + break; + case PType.I32: + AppendIntegers(component, offset, store, groups); + break; + case PType.U32: + AppendIntegers(component, offset, store, groups); + break; + case PType.I64: + AppendIntegers(component, offset, store, groups); + break; + default: + AppendIntegers(component, offset, store, groups); + break; + } + } + + private void AppendIntegers(int component, int offset, ColumnStore store, ReadOnlySpan groups) + where TValue : unmanaged + { + foreach (int group in groups) + { + KeyTuple tuple = _tuples.KeyAt(group); + if (IsNull(tuple, component)) + { + KeyStores.AppendNull(store); + } + else + { + KeyStores.AppendFixed(store, Read(tuple, offset), StorageKind.Primitive); + } + } + } +} diff --git a/tests/Vorticity.Tests/Aggregation/GroupSpillTests.cs b/tests/Vorticity.Tests/Aggregation/GroupSpillTests.cs index 7a66b9886..007cd86dd 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupSpillTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupSpillTests.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Vorticity.Aggregating; using Xunit; namespace Vorticity.Tests.Aggregation; @@ -12,8 +13,9 @@ namespace Vorticity.Tests.Aggregation; /// The spill of the lanes' tables: a group by whose budget cannot hold its lanes' tables, on keys or /// states the core cannot take (a text, a composite holding one, a text's extremes, distinct counts), /// writes each table to the scratch when the budget holds it no more, then reads the runs back part by -/// part, exact, every reservation and every file given back. Budgets are set against the result: what one -/// lane holds once it has met every row. +/// part, exact, every reservation and every file given back; and a key of six columns as its tuple, which +/// the core takes and spills by its parts. Budgets are set against the result: what one lane holds once it +/// has met every row. /// [Collection(nameof(CorePressureCollection))] public sealed partial class GroupSpillTests @@ -64,16 +66,44 @@ public async Task AKeyOfSixPartsSpillsAndEndsExact(int degree, int percent) // Two texts and four integers, their numbers in a word of 256 bits: each part's values written // by its index, the words packed again as the runs are read back. (_, Row[] rows) = await Fixture.Async; - Dictionary<(string, int, int, string, int, long), SixCount> expected = rows.GroupBy(r => (r.Name, r.Key, r.Small, r.Text, r.Wide, r.Value)) - .ToDictionary(g => g.Key, g => new SixCount(g.Key.Name, g.Key.Key, g.Key.Small, g.Key.Text, g.Key.Wide, g.Key.Value, g.Count())); await ExactAsync( degree, percent, - file => file.Scan().GroupBy(r => (r.Name, r.Key, r.Small, r.Text, r.Wide, r.Value)).Select(g => (g.Key.Name, g.Key.Key, g.Key.Small, g.Key.Text, g.Key.Wide, g.Key.Value, g.Count())), + file => + { + Vorticity.Aggregation grouped = SixColumns(file); + grouped.Plan.Tuples = false; + return grouped; + }, (SixCount count) => (count.Name, count.Key, count.Small, count.Text, count.Wide, count.Value), - expected); + SixCounts(rows)); + } + + [Theory] + [InlineData(1, 200)] + [InlineData(1, 10)] + [InlineData(4, 50)] + [InlineData(14, 10)] + public async Task AKeyOfSixValuesSpillsAsItsTuplesAndEndsExact(int degree, int percent) + { + // The same six columns as the tuple of their values, a key of 64 bytes the core takes: the governor + // turns to the core, whose parts go to the scratch and come back. + (_, Row[] rows) = await Fixture.Async; + AggregationRun run = await ExactAsync(degree, percent, SixColumns, (SixCount count) => (count.Name, count.Key, count.Small, count.Text, count.Wide, count.Value), SixCounts(rows)); + if (percent <= 50) + { + Assert.NotNull(run.Core); + } } + /// A count by six columns: two texts and four integers. + private static Vorticity.Aggregation SixColumns(VortexFile file) => + file.Scan().GroupBy(r => (r.Name, r.Key, r.Small, r.Text, r.Wide, r.Value)).Select(g => (g.Key.Name, g.Key.Key, g.Key.Small, g.Key.Text, g.Key.Wide, g.Key.Value, g.Count())); + + private static Dictionary<(string, int, int, string, int, long), SixCount> SixCounts(Row[] rows) => + rows.GroupBy(r => (r.Name, r.Key, r.Small, r.Text, r.Wide, r.Value)) + .ToDictionary(g => g.Key, g => new SixCount(g.Key.Name, g.Key.Key, g.Key.Small, g.Key.Text, g.Key.Wide, g.Key.Value, g.Count())); + [Theory] [MemberData(nameof(Budgets))] public async Task ATextsExtremesByAnIntegerSpillAndEndExact(int degree, int percent) @@ -611,9 +641,10 @@ public async Task AScratchBudgetTheSpillPassesFailsTheQueryCleanly() /// /// The query under a budget of of its result, at : /// every group's answer, written to the scratch under half and a tenth of it, nothing at one lane under - /// twice it; the peak within 6 % of the ceiling, nothing reserved and no file left after. + /// twice it; the peak of the lanes' tables within 6 % of the ceiling, nothing reserved and no file left + /// after. The run, for what the test checks more. /// - private static async Task ExactAsync( + private static async Task ExactAsync( int degree, int percent, Func query, Func keyOf, Dictionary expected) where TResult : IVortexRecord where TKey : notnull @@ -639,7 +670,9 @@ private static async Task ExactAsync( Assert.Equal(answer, read[key]); } - int runs = grouped.Plan.LastRun!.SpilledRuns; + // The lanes' runs, or the core's parts when the governor turned to it. + AggregationRun run = grouped.Plan.LastRun!; + int runs = run.SpilledRuns + (run.Core?.SpilledParts ?? 0); if (percent <= 50) { Assert.True(runs > 0, $"no run written under {budget.CeilingBytes:N0} bytes of {result:N0}"); @@ -649,9 +682,16 @@ private static async Task ExactAsync( Assert.Equal(0, runs); } - Assert.True(budget.PeakBytes <= budget.CeilingBytes * 106 / 100, $"peak {budget.PeakBytes:N0} of {budget.CeilingBytes:N0}"); + // The lanes' tables keep within their budget; the core, under pressure, takes past it what a + // stack asks more, and gives everything back (CorePressureTests). + if (run.Core is null) + { + Assert.True(budget.PeakBytes <= budget.CeilingBytes * 106 / 100, $"peak {budget.PeakBytes:N0} of {budget.CeilingBytes:N0}"); + } + Assert.Equal(0, budget.ReservedBytes); Assert.Empty(Directory.EnumerateFileSystemEntries(scratch)); + return run; } finally { diff --git a/tests/Vorticity.Tests/Aggregation/TupleKeysTests.cs b/tests/Vorticity.Tests/Aggregation/TupleKeysTests.cs new file mode 100644 index 000000000..46b2aa727 --- /dev/null +++ b/tests/Vorticity.Tests/Aggregation/TupleKeysTests.cs @@ -0,0 +1,302 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Vorticity.Aggregating; +using Xunit; + +namespace Vorticity.Tests.Aggregation; + +/// +/// A key of five to eight text and integer columns as the tuple of its values: it groups as its values, +/// texts short, long and empty, nulls in texts and integers, integers of every width and sign, on one +/// lane, four and fourteen, under a filter, in the lanes' tables and in the core; its columns come back in +/// order; a key a tuple cannot hold keeps the numbers of its columns. +/// +public sealed partial class TupleKeysTests +{ + private const int Rows = 60_000; + + public static TheoryData Cases => new TheoryData + { + { 1, false, false }, + { 4, true, false }, + { 14, false, false }, + { 1, false, true }, + { 4, true, true }, + { 14, false, true }, + }; + + [Theory] + [MemberData(nameof(Cases))] + public async Task ATupleKeyGroupsAsItsValues(int degree, bool filtered, bool core) + { + Row[] rows = MakeRows(); + string path = await WriteAsync(rows); + try + { + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = degree); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + IEnumerable kept = filtered ? rows.Where(r => r.Value > 20L) : rows; + Scan Scan() => filtered ? file.Scan().Where(r => r.Value > 20L) : file.Scan(); + + // Two texts, short, long, empty and null, and four integers, two of them nullable. + AssertGroups( + kept.GroupBy(r => (r.Name, r.Code, r.Int, r.Long, r.Short, r.Tiny)) + .Select(g => $"{Text(g.Key.Name)}|{g.Key.Code}|{g.Key.Int}|{Text(g.Key.Long)}|{Text(g.Key.Short)}|{g.Key.Tiny}|{g.Count()}|{g.Sum(r => r.Value)}"), + (await ListAsync(Scan().GroupBy(r => (r.Name, r.Code, r.Int, r.Long, r.Short, r.Tiny)).Select(g => (g.Key.Name, g.Key.Code, g.Key.Int, g.Key.Long, g.Key.Short, g.Key.Tiny, g.Count(), g.Sum(r => r.Value))), core)) + .Select(g => $"{Text(g.Name)}|{g.Code}|{g.Int}|{Text(g.Long)}|{Text(g.Short)}|{g.Tiny}|{g.Count}|{g.Sum}")); + + // Eight integers, every width, signed and not, the high bit of the unsigned ones set. + AssertGroups( + kept.GroupBy(r => (r.Tiny, r.Byte, r.Short, r.UShort, r.Int, r.UInt, r.Long, r.ULong)) + .Select(g => $"{g.Key.Tiny}|{g.Key.Byte}|{Text(g.Key.Short)}|{g.Key.UShort}|{g.Key.Int}|{g.Key.UInt}|{Text(g.Key.Long)}|{g.Key.ULong}|{g.Count()}"), + (await ListAsync(Scan().GroupBy(r => (r.Tiny, r.Byte, r.Short, r.UShort, r.Int, r.UInt, r.Long, r.ULong)).Select(g => (g.Key.Tiny, g.Key.Byte, g.Key.Short, g.Key.UShort, g.Key.Int, g.Key.UInt, g.Key.Long, g.Key.ULong, g.Count())), core)) + .Select(g => $"{g.Tiny}|{g.Byte}|{Text(g.Short)}|{g.UShort}|{g.Int}|{g.UInt}|{Text(g.Long)}|{g.ULong}|{g.Count}")); + + // Three texts, one always longer than a word: every lane numbers the long ones alike. + AssertGroups( + kept.GroupBy(r => (r.Name, r.Code, r.Label, r.Byte, r.UShort)).Select(g => $"{Text(g.Key.Name)}|{g.Key.Code}|{g.Key.Label}|{g.Key.Byte}|{g.Key.UShort}|{g.Count()}"), + (await ListAsync(Scan().GroupBy(r => (r.Name, r.Code, r.Label, r.Byte, r.UShort)).Select(g => (g.Key.Name, g.Key.Code, g.Key.Label, g.Key.Byte, g.Key.UShort, g.Count())), core)) + .Select(g => $"{Text(g.Name)}|{g.Code}|{g.Label}|{g.Byte}|{g.UShort}|{g.Count}")); + } + finally + { + System.IO.File.Delete(path); + } + } + + // The key's columns come back in order, each as its column orders it, nulls last both ways. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ATupleKeyOrdersByItsComponents(bool core) + { + Row[] rows = MakeRows(); + string path = await WriteAsync(rows); + try + { + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = 4); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + List ordered = await ListAsync( + file.Scan().GroupBy(r => (r.Name, r.Code, r.Int, r.Long, r.Short, r.Tiny)) + .OrderBy(g => g.Key.Name).ThenByDescending(g => g.Key.Long).ThenBy(g => g.Key.Code).ThenBy(g => g.Key.Int).ThenByDescending(g => g.Key.Short).ThenBy(g => g.Key.Tiny) + .Select(g => (g.Key.Name, g.Key.Code, g.Key.Int, g.Key.Long, g.Key.Short, g.Key.Tiny, g.Count(), g.Sum(r => r.Value))), + core); + Comparer texts = Comparer.Create((a, b) => a is null || b is null ? (a is null).CompareTo(b is null) : string.CompareOrdinal(a, b)); + Comparer longsDown = Comparer.Create((a, b) => a is null || b is null ? (a is null).CompareTo(b is null) : b.Value.CompareTo(a.Value)); + Comparer shortsDown = Comparer.Create((a, b) => a is null || b is null ? (a is null).CompareTo(b is null) : b.Value.CompareTo(a.Value)); + List expected = + [ + .. rows.GroupBy(r => (r.Name, r.Code, r.Int, r.Long, r.Short, r.Tiny)) + .OrderBy(g => g.Key.Name, texts).ThenBy(g => g.Key.Long, longsDown).ThenBy(g => g.Key.Code, StringComparer.Ordinal).ThenBy(g => g.Key.Int).ThenBy(g => g.Key.Short, shortsDown).ThenBy(g => g.Key.Tiny) + .Select(g => new Six(g.Key.Name, g.Key.Code, g.Key.Int, g.Key.Long, g.Key.Short, g.Key.Tiny, g.Count(), g.Sum(r => r.Value))), + ]; + Assert.Equal(expected, ordered); + } + finally + { + System.IO.File.Delete(path); + } + } + + // A run numbers its long texts once for all its lanes, the 30 labels and the 80 long names, under its + // memory, which it gives back with its result. + [Fact] + public async Task ARunNumbersItsLongTextsOnceForAllItsLanes() + { + Row[] rows = MakeRows(); + string path = await WriteAsync(rows); + try + { + QueryMemoryBudget budget = new QueryMemoryBudget(1L << 30); + await using VortexSession session = VortexSession.Create(options => + { + options.MaxDegreeOfParallelism = 4; + options.MemoryBudget = budget; + }); + + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + Vorticity.Aggregation grouped = file.Scan().GroupBy(r => (r.Name, r.Code, r.Label, r.Byte, r.UShort)) + .Select(g => (g.Key.Name, g.Key.Code, g.Key.Label, g.Key.Byte, g.Key.UShort, g.Count())); + grouped.Plan.CoreOnNew = false; + TupleLayout[] layouts = []; + grouped.Plan.Watch = partitions => layouts = [.. partitions.Select(partition => ((TupleKeys)partition.Keys!).Layout)]; + long groups = 0; + await foreach (Texts group in grouped.As().ToRecordsAsync(Ct)) + { + groups++; + } + + Assert.Equal(rows.GroupBy(r => (r.Name, r.Code, r.Label, r.Byte, r.UShort)).Count(), groups); + Assert.True(layouts.Length > 1, $"{layouts.Length} lanes"); + Assert.Single(layouts.Distinct()); + Assert.Equal(110, layouts[0].LongTexts); + Assert.Equal(0, budget.ReservedBytes); + } + finally + { + System.IO.File.Delete(path); + } + } + + // A float among the columns, or four texts, which pass a tuple's 63 bytes: the numbers of the columns. + [Fact] + public async Task AKeyNoTupleHoldsKeepsTheNumbersOfItsColumns() + { + Row[] rows = MakeRows(); + string path = await WriteAsync(rows); + try + { + await using VortexFile file = await VortexFile.OpenAsync(path, Ct); + List real = await ListAsync( + file.Scan().GroupBy(r => (r.Name, r.Code, r.Int, r.Tiny, r.Real)).Select(g => (g.Key.Name, g.Key.Code, g.Key.Int, g.Key.Tiny, g.Key.Real, g.Count())), + core: false, + typeof(PackedKeys)); + Assert.Equal(rows.GroupBy(r => (r.Name, r.Code, r.Int, r.Tiny, r.Real)).Count(), real.Count); + List four = await ListAsync( + file.Scan().GroupBy(r => (r.Name, r.Code, r.Label, r.Note, r.Byte)).Select(g => (g.Key.Name, g.Key.Code, g.Key.Label, g.Key.Note, g.Key.Byte, g.Count())), + core: false, + typeof(PackedKeys)); + Assert.Equal(rows.GroupBy(r => (r.Name, r.Code, r.Label, r.Note, r.Byte)).Count(), four.Count); + } + finally + { + System.IO.File.Delete(path); + } + } + + private static void AssertGroups(IEnumerable expected, IEnumerable actual) + { + List left = [.. expected.Order(StringComparer.Ordinal)]; + List right = [.. actual.Order(StringComparer.Ordinal)]; + Assert.True(left.Count > 100, $"{left.Count} groups"); + Assert.Equal(left, right); + } + + /// + /// The groups: in the core, its caches tiny so that every batch flushes and its tables split, when + /// ; in the lanes' tables of (tuples by default) otherwise. + /// + private static async Task> ListAsync(Vorticity.Aggregation grouped, bool core, Type? keys = null) + where T : IVortexRecord + { + AggregationPlan plan = grouped.Plan; + GroupKeys?[] lanes = []; + if (core) + { + plan.Core = true; + plan.CoreLanes = 1; + plan.CoreCapacity = 96; + plan.CoreFloor = 16; + plan.CoreTableGroups = 40; + plan.CoreBatchEntries = 8; + } + else + { + plan.CoreOnNew = false; + plan.Watch = partitions => lanes = [.. partitions.Select(partition => partition.Keys)]; + } + + List list = []; + await foreach (T group in grouped.As().ToRecordsAsync(Ct)) + { + list.Add(group); + } + + if (core) + { + Assert.NotNull(plan.LastRun!.Core); + } + else + { + Assert.Null(plan.LastRun!.Core); + Assert.NotEmpty(lanes); + Assert.All(lanes, partitionKeys => Assert.IsType(keys ?? typeof(TupleKeys), partitionKeys)); + } + + return list; + } + + private static string Text(string? value) => value ?? "null"; + + private static string Text(short? value) => value?.ToString(CultureInfo.InvariantCulture) ?? "null"; + + private static string Text(long? value) => value?.ToString(CultureInfo.InvariantCulture) ?? "null"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static ulong Mix(ulong x) + { + x = (x ^ (x >> 30)) * 0xBF58_476D_1CE4_E5B9UL; + x = (x ^ (x >> 27)) * 0x94D0_49BB_1331_11EBUL; + return x ^ (x >> 31); + } + + /// + /// Names of 400, a fifth longer than a word, null one row in 17; a code and a label of the name, the + /// code empty for some, the label always long; a fourth text; integers of a few values each, of every + /// width, the unsigned ones past their signed range, two nullable; a float. + /// + private static Row[] MakeRows() + { + Row[] rows = new Row[Rows]; + for (int row = 0; row < Rows; row++) + { + ulong mix = Mix((ulong)row); + int k = (int)(mix % 400); + string? name = row % 17 == 0 ? null : k % 5 == 0 ? $"a-longer-name-{k:D5}" : $"n{k}"; + string code = k % 41 == 40 ? "" : $"c{k % 41}"; + string label = $"label-of-some-length-{k % 30:D4}"; + string note = $"note-{(mix >> 9) % 3}"; + sbyte tiny = (sbyte)((int)((mix >> 12) % 3) - 1); + byte small = (byte)((mix >> 15) % 5 * 60); + short? signed = row % 11 == 0 ? null : (short)((int)((mix >> 18) % 4) - 2); + ushort wide = (ushort)((mix >> 21) % 6 * 13_000); + int integer = ((int)((mix >> 24) % 5) * 1_000_003) - 2_000_000; + uint unsigned = ((mix >> 27) % 3) switch { 0 => 7u, 1 => uint.MaxValue - 5, _ => 2_147_483_650u }; + long? large = row % 13 == 0 ? null : ((long)((mix >> 30) % 3) - 1) * (long.MaxValue / 3); + ulong huge = ((mix >> 33) % 4) * 0x4000_0000_0000_0001UL; + double real = (double)((mix >> 36) % 4) / 2; + rows[row] = new Row(name, code, label, note, tiny, small, signed, wide, integer, unsigned, large, huge, real, (long)((mix >> 40) % 100)); + } + + return rows; + } + + private static async Task WriteAsync(Row[] rows) + { + string directory = Path.Combine(AppContext.BaseDirectory, "tuple-keys"); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, $"rows-{Environment.ProcessId}-{Guid.NewGuid():N}.vortex"); + await using (VortexFileWriter writer = VortexSession.Default.CreateWriter(path, new VortexWriteOptions { RowBlockSize = 8_192, ChunkTargetBytes = 1 << 16 })) + { + await writer.WriteAsync(rows, Ct); + await writer.CompleteAsync(Ct); + } + + return path; + } + + [VortexRecord] + public partial record struct Row( + string? Name, string Code, string Label, string Note, sbyte Tiny, byte Byte, short? Short, ushort UShort, int Int, uint UInt, long? Long, ulong ULong, double Real, long Value); + + [VortexRecord] + public partial record struct Six(string? Name, string Code, int Int, long? Long, short? Short, sbyte Tiny, long Count, long Sum); + + [VortexRecord] + public partial record struct Eight(sbyte Tiny, byte Byte, short? Short, ushort UShort, int Int, uint UInt, long? Long, ulong ULong, long Count); + + [VortexRecord] + public partial record struct Texts(string? Name, string Code, string Label, byte Byte, ushort UShort, long Count); + + [VortexRecord] + public partial record struct WithReal(string? Name, string Code, int Int, sbyte Tiny, double Real, long Count); + + [VortexRecord] + public partial record struct FourTexts(string? Name, string Code, string Label, string Note, byte Byte, long Count); +} From e1a72001a31b93ec9e1932cc15aff184bbf33fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 20:17:32 +0200 Subject: [PATCH 21/56] perf(query): a tuple's columns come out where they lie, a short text as its view The output of a key of tuples read each group's tuple whole, 64 bytes copied per value, then appended an integer through KeyStores.AppendFixed and a text by its bytes, copied out of its word and back into a view: 37 % of db-benchmark's q10 at one thread, ten million groups of three texts and three integers, against 21 % for the numbers it replaced. The keys are now read in place (FixedKeys.KeyRef), an integer column appended into its FixedStore as FixedKeys appends its own, and a text of 12 bytes or less written as its view (VarBinStore.AppendInline): its word, the length then the bytes zero-padded, is the canonical view. A long text comes out of the run's list, read without the lock once it is in it. ShortTextKeys writes its words the same way. runners.sh alt against the runner of 8a607799, warm: group-db-q10 x0.771 at one thread (889 -> 687 ms, cycles x0.767), x0.923 at fourteen (noisy, cycles x1.024); group-db-q1 x1.000 and x1.030 (0.96-1.07), group-db-q3 x0.998 and x1.017. Against ab37ead9 alone, q10 at one thread was x1.028: the tuples paid off only once their output did. --- src/Vorticity/Aggregation/GroupKeys.cs | 3 + src/Vorticity/Aggregation/ShortTextKeys.cs | 24 ++++- src/Vorticity/Aggregation/TupleKeys.cs | 107 ++++++++++++------- src/Vorticity/Writing/Builders/LeafStores.cs | 17 +++ 4 files changed, 110 insertions(+), 41 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index fd87aaddd..5e9a9a586 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -2325,6 +2325,9 @@ internal int NullGroup() /// The key of group ; the null group's is the default value. internal TValue KeyAt(int group) => _keys[group]; + /// The key of where it lies: a wide one read in place rather than copied. + internal ref readonly TValue KeyRef(int group) => ref _keys[group]; + internal override bool Ascending(int from, int to) { if (to - from < 2) diff --git a/src/Vorticity/Aggregation/ShortTextKeys.cs b/src/Vorticity/Aggregation/ShortTextKeys.cs index 407965217..8663f385a 100644 --- a/src/Vorticity/Aggregation/ShortTextKeys.cs +++ b/src/Vorticity/Aggregation/ShortTextKeys.cs @@ -688,10 +688,28 @@ internal override Func Reader(int component) internal override void Append(int component, ColumnStore store, ReadOnlySpan groups) { - // The keys came from the column the reader checked as it decoded it, as a table of bytes appends them. + // The keys came from the column the reader checked as it decoded it, as a table of bytes appends them; + // a word is its value's view. VarBinStore leaf = (VarBinStore)store.Leaf; - Span buffer = stackalloc byte[16]; int nullGroup = NullNumber; + if (_words is { } words) + { + foreach (int group in groups) + { + if (group == nullGroup) + { + KeyStores.AppendNull(store); + } + else + { + ref readonly TextWord word = ref words.KeyRef(group); + leaf.AppendInline(word.Low, word.High); + } + } + + return; + } + foreach (int group in groups) { if (group == nullGroup) @@ -700,7 +718,7 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpan Volatile.Read(ref _longCount); /// The bytes of the text a word holds, through of 16 bytes for a short one. - internal ReadOnlySpan TextOf(TextWord word, Span buffer) + internal ReadOnlySpan TextOf(TextWord word, Span buffer) => IsLong(word) ? LongText(word) : ShortTextKeys.BytesOf(word, buffer); + + /// Whether a word holds a long text's mark and number rather than a text. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool IsLong(TextWord word) => (uint)word.Low == LongMark; + + /// + /// The long text whose number a word holds. Read without the lock: a text is in the list before its + /// number is handed out, and the list a reader sees holds every number below its length, a longer + /// one being copied from it before it replaces it; a number past the list read is looked up again + /// under the lock. + /// + internal byte[] LongText(TextWord word) { - if ((uint)word.Low != LongMark) + int id = (int)(word.Low >> 32); + byte[][] longs = Volatile.Read(ref _longs); + if ((uint)id < (uint)longs.Length && longs[id] is { } text) { - return ShortTextKeys.BytesOf(word, buffer); + return text; } lock (_gate) { - return _longs[(int)(word.Low >> 32)]; + return _longs[id]; } } @@ -413,24 +427,29 @@ private static void Integers(CanonicalArena arena, int node, int start, int e } } - /// Whether column of is null. - private static bool IsNull(KeyTuple tuple, int component) => - ((Unsafe.Add(ref Unsafe.As(ref tuple), TupleLayout.FlagsAt) >> component) & 1) != 0; + /// The bytes of group 's tuple, where it lies: a tuple copied to be read took a third of q10's output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ref byte TupleOf(int group) => ref Unsafe.As(ref Unsafe.AsRef(in _tuples.KeyRef(group))); + + /// Whether column of a tuple is null. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsNull(ref byte tuple, int component) => ((Unsafe.Add(ref tuple, TupleLayout.FlagsAt) >> component) & 1) != 0; /// The value of a column of a tuple at byte , as its bytes. - private static T Read(KeyTuple tuple, int offset) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T Read(ref byte tuple, int offset) where T : unmanaged => - Unsafe.ReadUnaligned(ref Unsafe.Add(ref Unsafe.As(ref tuple), offset)); + Unsafe.ReadUnaligned(ref Unsafe.Add(ref tuple, offset)); /// The bytes of text column of a tuple, through of 16 bytes. - private ReadOnlySpan TextOf(KeyTuple tuple, int component, Span buffer) => - _layout.TextOf(Read(tuple, _layout.Offsets[component]), buffer); + private ReadOnlySpan TextOf(ref byte tuple, int component, Span buffer) => + _layout.TextOf(Read(ref tuple, _layout.Offsets[component]), buffer); /// Column of two tuples, by its type: a text by its bytes, an integer by its value, a null last. - private int Compare(KeyTuple left, KeyTuple right, int component) + private int Compare(ref byte left, ref byte right, int component) { - bool leftNull = IsNull(left, component); - bool rightNull = IsNull(right, component); + bool leftNull = IsNull(ref left, component); + bool rightNull = IsNull(ref right, component); if (leftNull || rightNull) { return leftNull == rightNull ? 0 : leftNull ? 1 : -1; @@ -441,19 +460,19 @@ private int Compare(KeyTuple left, KeyTuple right, int component) { Span a = stackalloc byte[16]; Span b = stackalloc byte[16]; - return TextOf(left, component, a).SequenceCompareTo(TextOf(right, component, b)); + return TextOf(ref left, component, a).SequenceCompareTo(TextOf(ref right, component, b)); } return _layout.Shapes[component].PType switch { - PType.I8 => Read(left, offset).CompareTo(Read(right, offset)), - PType.U8 => Read(left, offset).CompareTo(Read(right, offset)), - PType.I16 => Read(left, offset).CompareTo(Read(right, offset)), - PType.U16 => Read(left, offset).CompareTo(Read(right, offset)), - PType.I32 => Read(left, offset).CompareTo(Read(right, offset)), - PType.U32 => Read(left, offset).CompareTo(Read(right, offset)), - PType.I64 => Read(left, offset).CompareTo(Read(right, offset)), - _ => Read(left, offset).CompareTo(Read(right, offset)), + PType.I8 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + PType.U8 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + PType.I16 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + PType.U16 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + PType.I32 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + PType.U32 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + PType.I64 => Read(ref left, offset).CompareTo(Read(ref right, offset)), + _ => Read(ref left, offset).CompareTo(Read(ref right, offset)), }; } @@ -543,10 +562,9 @@ internal override void GroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan< internal override bool Orders(int component) => true; - internal override int CompareKeys(int a, int b, int component) => Compare(_tuples.KeyAt(a), _tuples.KeyAt(b), component); + internal override int CompareKeys(int a, int b, int component) => Compare(ref TupleOf(a), ref TupleOf(b), component); - internal override int CompareKeys(GroupKeys other, int a, int b, int component) => - Compare(_tuples.KeyAt(a), ((TupleKeys)other)._tuples.KeyAt(b), component); + internal override int CompareKeys(GroupKeys other, int a, int b, int component) => Compare(ref TupleOf(a), ref ((TupleKeys)other).TupleOf(b), component); internal override Func Reader(int component) { @@ -557,8 +575,8 @@ internal override Func Reader(int component) byte[] buffer = new byte[16]; return group => { - KeyTuple tuple = _tuples.KeyAt(group); - return IsNull(tuple, component) ? default! : StorageValues.BytesToClr(TextOf(tuple, component, buffer), shape); + ref byte tuple = ref TupleOf(group); + return IsNull(ref tuple, component) ? default! : StorageValues.BytesToClr(TextOf(ref tuple, component, buffer), shape); }; } @@ -579,8 +597,8 @@ private Func IntegerReader(int component, int offset, ColumnS where TValue : unmanaged => group => { - KeyTuple tuple = _tuples.KeyAt(group); - return IsNull(tuple, component) ? default! : StorageValues.ToClr(Read(tuple, offset), shape); + ref byte tuple = ref TupleOf(group); + return IsNull(ref tuple, component) ? default! : StorageValues.ToClr(Read(ref tuple, offset), shape); }; internal override void Append(int component, ColumnStore store, ReadOnlySpan groups) @@ -588,19 +606,26 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpan buffer = stackalloc byte[16]; foreach (int group in groups) { - KeyTuple tuple = _tuples.KeyAt(group); - if (IsNull(tuple, component)) + ref byte tuple = ref TupleOf(group); + if (IsNull(ref tuple, component)) { KeyStores.AppendNull(store); + continue; + } + + TextWord word = Read(ref tuple, offset); + if (TupleLayout.IsLong(word)) + { + leaf.AppendValidated(_layout.LongText(word)); } else { - leaf.AppendValidated(TextOf(tuple, component, buffer)); + leaf.AppendInline(word.Low, word.High); } } @@ -636,19 +661,25 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpanAn integer column's values, read in place in their tuples: into the column's values at once when they are as wide, as appends its own. private void AppendIntegers(int component, int offset, ColumnStore store, ReadOnlySpan groups) where TValue : unmanaged { + FixedStore? leaf = store.Leaf is FixedStore fixedStore && fixedStore.Width == Unsafe.SizeOf() ? fixedStore : null; foreach (int group in groups) { - KeyTuple tuple = _tuples.KeyAt(group); - if (IsNull(tuple, component)) + ref byte tuple = ref TupleOf(group); + if (IsNull(ref tuple, component)) { KeyStores.AppendNull(store); } + else if (leaf is not null) + { + leaf.Append(Read(ref tuple, offset)); + } else { - KeyStores.AppendFixed(store, Read(tuple, offset), StorageKind.Primitive); + KeyStores.AppendFixed(store, Read(ref tuple, offset), StorageKind.Primitive); } } } diff --git a/src/Vorticity/Writing/Builders/LeafStores.cs b/src/Vorticity/Writing/Builders/LeafStores.cs index de39c7358..76ae8f460 100644 --- a/src/Vorticity/Writing/Builders/LeafStores.cs +++ b/src/Vorticity/Writing/Builders/LeafStores.cs @@ -441,6 +441,23 @@ internal void AppendValidated(ReadOnlySpan value) Count++; } + /// + /// A value of twelve bytes or fewer, read already, as its view: its length in the low 32 bits of + /// , its bytes after, zero past them, the two words a group's key holds it as. + /// Copying its bytes out of the word to write them back into a view took a third of the output of + /// db-benchmark's q10, three texts in ten million groups. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AppendInline(ulong low, ulong high) + { + System.Diagnostics.Debug.Assert((uint)low <= MaxInline, "A view of a longer value points into a data buffer."); + EnsureViews(); + ref byte view = ref _views[(long)Count * ViewSize]; + Unsafe.WriteUnaligned(ref view, low); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref view, sizeof(ulong)), high); + Count++; + } + internal void Append(ReadOnlySpan text) { EnsureData((text.Length + 1) * 3); From dd8c279889ef40d4e15938d8bdeb4f9dc93dffab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 20:17:52 +0200 Subject: [PATCH 22/56] perf(query): a key wider than 16 bytes in slots of a hash and a group from the first FixedKeys holds a key wider than a word in slots that hold it until it has 4 096 groups, then in slots of its hash and its group (WideKeyTable). The core's sub-tables never reach that: a few thousand groups each, a tuple of 64 bytes (TupleKeys) took more than a line a slot, and every probe compared whole keys, 22 % of q10's cycles at fourteen lanes. A key wider than 16 bytes now goes to the slots of a hash and a group from its first group. runners.sh alt against the runner of 38572ad3's tree, warm: group-db-q10 x0.800 at fourteen lanes (286.0 -> 228.8 ms, cycles x0.781), x1.004 at one. --- src/Vorticity/Aggregation/GroupKeys.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index 5e9a9a586..aad6994df 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -608,7 +608,7 @@ internal sealed class FixedKeys : GroupKeys // groups one whose slots hold a hash and a group (Condense), the other then left empty. private KeyTable _index = new KeyTable(); private WideKeyTable _wide = new WideKeyTable(); - private bool _compact; + private bool _compact = CompactAlways; private TValue[] _keys = new TValue[16]; private int _null = -1; private TValue[] _values = []; @@ -782,6 +782,18 @@ private bool Compact get => Wide && _compact; } + /// + /// Whether the groups are in the table of a hash and a group from the first: a key wider than 16 bytes, + /// a tuple of values (). In slots that held it, more than a line each, the core's + /// sub-tables, a few thousand groups each, compared whole keys at every probe: 22 % of q10's cycles at + /// fourteen lanes, which took ×0.80 the time without them (2026-10-09). + /// + private static bool CompactAlways + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Unsafe.SizeOf() > 2 * sizeof(ulong); + } + /// The first pass of the index the groups are in (). [MethodImpl(MethodImplOptions.AggressiveInlining)] private int FindAtHome(ReadOnlySpan keys, Span groups, Span homes, int ahead, out bool missed) => @@ -1804,7 +1816,7 @@ internal override void Release() if (Wide) { _wide.Release(); - _compact = false; + _compact = CompactAlways; } _shelf?.Give(_keys); From 70a027bc9646055ca536a82c38b3b85f9234968f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 20:21:38 +0200 Subject: [PATCH 23/56] perf(query): the core counts a wide key's group in slots of its hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GroupCore sizes its sub-tables, its lanes' caches and α by a group's bytes, its slot counted as one that holds the key, 5/3 of the key and a number. A key wider than 16 bytes is in slots of its hash and its group since b10b1283, 8 bytes four tenths full: a tuple of 64 bytes was counted 113 bytes of slot for 20, and q10's parts split into twice the sub-tables their bytes held, about five entries of a batch each. runners.sh alt against the runner of b10b1283, warm: group-db-q10 x0.870 at fourteen lanes (232.9 -> 204.4 ms, cycles x0.893), its sub-tables 8 192 -> 4 096. --- src/Vorticity/Aggregation/GroupCore.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index b9925bf40..24e4be7c5 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -468,8 +468,14 @@ internal AggregationPartition RowPartition() => /// Whether slot folds rows, and so merges. internal bool Folds(int slot) => _inputs[slot] != AggregationPartition.Settled; - /// A group's bytes in a table: its slot at the table's load, its key, its record. - private static long GroupBytes(int keyBytes, int recordBytes) => ((keyBytes + sizeof(int)) * 5L / 3) + keyBytes + recordBytes; + /// + /// A group's bytes in a table: its slot at the table's load, its key, its record. A key wider than 16 + /// bytes is in slots of its hash and its group, 8 bytes four tenths full (), + /// not in slots that hold it: a tuple of 64 bytes counted so took sub-tables of half the groups they + /// held in their bytes. + /// + private static long GroupBytes(int keyBytes, int recordBytes) => + (keyBytes > 2 * sizeof(ulong) ? sizeof(ulong) * 10L / 4 : (keyBytes + sizeof(int)) * 5L / 3) + keyBytes + recordBytes; /// Marks the core as the query's state: a lane's cache filled. internal void Engage() From 5bfe895dc8a58411315d389c914841b65c4dd446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 20:48:14 +0200 Subject: [PATCH 24/56] perf(query): a lane rents the group of each row from the shared pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every lane of every query grew its own array of each row's group, 512 KB for the batches of 131 072 rows a lane reads, then dropped it with its tables: at fourteen lanes, 7.3 of the 8.2 MB a count and a sum by 10³ keys allocated, on the large object heap, and the collections that came with it. The partition now rents it from the shared pool, cleared as a new array starts, and gives it back with its groups (Release). The lane's admission counts it, a batch's scratch. runners.sh alt against the runner of 1ac8e366, warm, at fourteen lanes (a busy machine, wide intervals): group-total-k3 on 4M rows x0.662 (2.01 -> 1.17 ms, cycles x0.795), group-db-q4 on 1M x0.686 (cycles x0.834), group-db-q1 on 1M x0.748 (cycles x0.884); unchanged at one lane. k3 at fourteen lanes allocates 1.37 MB a query, against 8.2. --- .../Aggregation/AggregationEngine.cs | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index c3f8d35bf..b2303e6c4 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -722,6 +722,11 @@ internal sealed class AggregationPartition private readonly int[] _filterOf; private readonly FilterMasks? _masks; private readonly GroupRanges _ranges = new GroupRanges(); + + // The group of each row of a batch, rented from the shared pool and given back with the partition's + // groups (Release): a lane's of 512 KB, made anew by every lane of every query, were 89 % of what a + // count and a sum by 10³ keys allocated at fourteen lanes, 7.3 of 8.2 MB a query. The lane's admission + // counts it (a batch's scratch). private int[] _rowGroups = []; private long _batch; @@ -1565,6 +1570,29 @@ private void Foretell() /// internal long Footprint => (Keys?.Footprint ?? 0) + (_componentKeys?.Footprint ?? 0) + AggregateSlot.FootprintOf(Slots); + /// + /// Room for the group of each of rows, the one held before given back: rented + /// cleared, as a new array starts, since a row a selection leaves out keeps what the array held, and + /// another query's groups would be past this one's. + /// + private void RentRowGroups(int rows) + { + GiveBackRowGroups(); + int[] rented = ArrayPool.Shared.Rent(rows); + Array.Clear(rented); + _rowGroups = rented; + } + + /// The groups of the rows given back to the shared pool, once nothing folds into the partition. + private void GiveBackRowGroups() + { + if (_rowGroups.Length > 0) + { + ArrayPool.Shared.Return(_rowGroups); + _rowGroups = []; + } + } + /// /// Drops the partition's groups, its keys, states and scratch, once another holds them: a lane's /// tables die with the merge, whatever still holds the partition. Frames of the pass do: a @@ -1578,7 +1606,7 @@ internal void Release() Keys = null; Slots = []; Records = null; - _rowGroups = []; + GiveBackRowGroups(); _narrowed = []; _componentOf = []; _componentRows = []; @@ -2457,7 +2485,11 @@ private void Fold(RecordBatch batch) } } - Scratch.Grow(ref _rowGroups, rows); + if (_rowGroups.Length < rows) + { + RentRowGroups(rows); + } + _ranges.Clear(); int before = Keys.Count; bool ranged = Keys.Assign(arena, _nodes.AsSpan(0, _keyCount), rows, selection, _rowGroups, _ranges); From 0db1536640e7db0ba6c83d254231a4604397d57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 21:01:09 +0200 Subject: [PATCH 25/56] feat(bench): duckdb.sh times DuckDB and the runner in alternated pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comparison ran every DuckDB query, then every one of ours: a perturbation of the machine biased one side unseen, and the published q10 at fourteen threads gave DuckDB 257 ms where it took 102 the same day. bench/duckdb.sh now keeps one DuckDB process a file open, driven through a FIFO, its table loaded once, and times a block of DuckDB's and a process of the runner in turn, ABBA, three pairs and then more while the 95 % interval of the median speedup is wider than ±5 % (eight at most); 10 rounds a block, 3 dropped, or 5 and 2 past 100 ms. Each row runs on two bases: our runner with its file kept mapped against DuckDB's table in memory, mapping it anew against DuckDB's vortex reader. DuckDB's time is its JSON profile's latency under 20 ms, to the nanosecond, and its timer's past that: on q10 at fourteen threads the profile cost 2 % (median 107 ms against 105). The CPU of both sides stands beside it. The files are read once before anything is timed; the canaries, controls and replays are bench/runners.sh's; everything is kept under bench/.runs/duckdb-/. --publish builds the page's table from two sessions taken at different hours, a row published when they agree within 10 %. Two sessions of --only db-q4,db-q10 --threads 14 against the runner of 05de5a54's tree, a minute apart: q4 1.73x and 1.73x warm, 2.88x and 2.81x cold; q10 0.52x and 0.50x warm, 0.61x and 0.61x cold; the same answers. --- bench/README.md | 2 +- bench/duckdb.sh | 545 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 461 insertions(+), 86 deletions(-) diff --git a/bench/README.md b/bench/README.md index 76eec48cd..d484ed85d 100644 --- a/bench/README.md +++ b/bench/README.md @@ -59,7 +59,7 @@ finds. | who ran, and on what, when a scenario has lanes | `bench/profile.sh cycles --threads all` | 20 s | the cycles above, on as many lanes as the report's all-core rows, plus `threads.md` from `bench/profile/threads.py`: each thread's share, how many threads ran in each millisecond, and the hottest functions of the caller's thread and of the pool's workers. The profiler records fewer of each thread's cycles as more of them run: take shares from it, and the process's processor time from the runner's `cpu_ms` | | where a group by spends its cycles | `bench/profile.sh cycles group-total-k6+core ~/.cache/vorticity/queries/spread-random-4000000.vortex 0 --threads 14` | 30 s | the cycles above for a group by of the high-cardinality bench (`group--`, the plan's switches after `+`: `GroupScenarios.cs` in the runner), over the spread files `vortex-queries --matrix small` writes; the runner prints the engine's counts, the core's with it. The bench compiles as it goes, and Instruments names no frame of a method compiled at run time | | a change to the engine, against the last change to the same part | `bench/runners.sh keep ` once, then `bench/runners.sh alt HEAD --threads 1,14 [--warm] group--[@file] …` | 1 min for three scenarios at one lane | the **Native AOT runner** of a commit published once and kept under `~/.cache/vorticity/runners/`, and two kept runners run in turns, a process each, in pairs whose order alternates: three pairs, then more while the 95 % interval of the median ratio is wider than ±3 % (eight at most), the median of the rounds after the first: the ratio of the two binaries, which a loaded machine moves less than it moves a number of milliseconds taken another day. Two canary runs before and after each scenario, against the session's first, catch what other processes did meanwhile; a scenario framed by a canary 5 % off, or during which swap grew, runs again once. The table prints the interval, the pairs, the ratio of the fastest rounds (a gap of 10 % from the median marks the scenario noisy), the ratio of the rounds' cycles, the load and the processes that were busy. `--warm` keeps each side's file mapped (`VORTICITY_RUNNER_WARM=1`). Without a commit, `keep` publishes the working tree's tracked changes. Every run is kept with its time of day in `bench/.ab/runners-/runs.tsv` | -| our group bys against DuckDB's | `bench/duckdb.sh [--set hc\|db\|all] [--only ] [--rows N] [--threads 1,14]` | 4 min for both sets, at 1 and 14 threads, its files written; a minute for the rows `--only` names | the same files, the bench's own at edition core2025.10, read by the Native AOT runner, by DuckDB's vortex extension and from a DuckDB table in memory, at the same threads: the median of the last 7 of 10 runs, each answer held to the others. `hc` is the high-cardinality table (keys of 10³ to 10⁷ values), `db` db-benchmark's group by, the queries our native aggregates cover; `--only db-q4,total-k7` keeps the rows named by the runner's scenario without `group-`. Needs `duckdb` and its vortex extension; informs, gates nothing | +| our group bys against DuckDB's | `bench/duckdb.sh [--set hc\|db\|all] [--only ] [--rows N] [--threads 1,14] [--bases both\|warm\|cold]`, then `bench/duckdb.sh --publish ` | 15 to 25 min a session for both sets at 1 and 14 threads, its files written; 30 s for `--only db-q4,db-q10 --threads 14` | the same files, the bench's own at edition core2025.10, read once before anything is timed, in **alternated pairs**: one DuckDB process a file kept open and driven through a FIFO, its table loaded once, and the Native AOT runner, a block of each in turn, ABBA, three pairs and then more while the 95 % interval of the median speedup is wider than ±5 % (eight at most); 10 rounds a block, 3 dropped, or 5 and 2 past 100 ms. Two bases: our runner with its file kept mapped against DuckDB's table in memory (warm), mapping it anew against DuckDB's vortex reader (cold). DuckDB's time is its JSON profile's latency under 20 ms (to the nanosecond), its timer's past that (the profile cost 2 % on q10); the CPU of both sides beside it. The canaries, controls and replays of `bench/runners.sh`; each answer held to the others. Everything is kept under `bench/.runs/duckdb-/`; `--publish` builds the page's table from two sessions taken at different hours, a row published when they agree within 10 %. `hc` is the high-cardinality table (keys of 10³ to 10⁷ values), `db` db-benchmark's group by, the queries our native aggregates cover; `--only db-q4,total-k7` keeps the rows named by the runner's scenario without `group-`. Needs `duckdb` and its vortex extension; informs, gates nothing | | the runner, its file kept mapped, and what a round cost | `VORTICITY_RUNNER_WARM=1 Vorticity.Benchmarks.Runner --scenario … --repeat N` | — | by default each round maps its file anew, as the reference maps it at each call; `VORTICITY_RUNNER_WARM=1` keeps the session's default of 64 mappings, the file already in memory, which is what a comparison against a table loaded beforehand needs. Each round line carries `cpu_us`, `instructions` and `cycles`, the process's on every thread, read with `proc_pid_rusage` as a user (macOS; zero elsewhere): a time that moves while the cycles do not was moved by the machine | | a ratchet | `dotnet test Vorticity.slnx -c Release` | ~1 min | the suite plus the ten allocation, count, budget and work ratchets | | a change, and only the tests it can affect | `dotnet run -c Release --project tools/testimpact -- run` (after `-- map` once) | seconds | the test classes that ran a method the working tree changed against `HEAD` (`--base REF` for another), from a map of what each class runs: every class in a process of its own with inlining off, the JIT's list of compiled methods recorded (35 s). A constant, a build file or the source generator runs everything; a new class runs; a comment or a `using` runs nothing | diff --git a/bench/duckdb.sh b/bench/duckdb.sh index 6c25564ac..9f1336123 100755 --- a/bench/duckdb.sh +++ b/bench/duckdb.sh @@ -1,23 +1,55 @@ #!/usr/bin/env bash -# Our group bys against DuckDB's, on the same files: DuckDB through its vortex extension and on a native -# table already in memory, our side the Native AOT runner, at the same threads. +# Our group bys against DuckDB's, on the same files, in alternated pairs: DuckDB through its vortex +# extension and on a native table already in memory, our side the Native AOT runner, at the same threads. # # bench/duckdb.sh [--set hc|db|all] [--only ] [--rows N] [--db-rows N] [--threads N[,M…]] -# [--runner |] [--rounds N] [--rest S] +# [--runner |] [--bases both|warm|cold] [--target P] [--most N] [--rest S] +# [--no-canary] +# bench/duckdb.sh --publish [] # # bench/duckdb.sh PR #43's table: 40M rows, 1 and 14 threads # bench/duckdb.sh --set db --db-rows 10000000 db-benchmark's group by at 10^7 rows -# bench/duckdb.sh --rows 4000000 --threads 1 a quick look # bench/duckdb.sh --set all --only db-q4,total-k7 --threads 14 # the rows a change touches, and no others: the # runner's scenario names without "group-" (total-k3, # range-k6, strided-k7, db-q10…) +# bench/duckdb.sh --publish bench/.runs/duckdb-20261010-0900 bench/.runs/duckdb-20261010-1500 +# the page's table, from two sessions at different hours # # THE SAME WORK ON BOTH SIDES. Each query is a group by whose result is aggregated once more: DuckDB's outer # query sums every column the group by aggregates, and the runner sums the same columns as it reads the # groups (GroupScenarios.Checksum). The two sums are the same answer, and the table says when they are -# not. Each figure is the median of the last 7 of 10 runs (--rounds), each side's process started once a -# query; DuckDB's timer counts whole milliseconds. The native table's time excludes its loading. +# not. +# +# TWO BASES. Warm: our runner with its file kept mapped (VORTICITY_RUNNER_WARM=1) against DuckDB's native +# table, loaded before anything is timed. Cold: our runner mapping its file anew each round against +# DuckDB's vortex reader. A comparison of a table in memory with a file mapped anew measured page faults, +# not engines. +# +# PAIRS, NOT ONE SIDE THEN THE OTHER. One DuckDB process a file, kept open and driven through a FIFO, its +# table loaded once. For each row, degree and base, a block of DuckDB's and a process of our runner in +# turn, ABBA (DuckDB then us, us then DuckDB…), which cancels a drift of the machine: run all of one side +# then all of the other, a perturbation biased one side unseen (the published q10 at fourteen threads, +# 257 ms for DuckDB where it took 102 the same day). A block is 10 rounds, the first 3 dropped, or 5, the +# first 2 dropped, once a row's first block passes 100 ms. A pair's speedup is DuckDB's median over ours; +# a row's, the median of its pairs', 3 pairs and then one more while the 95 % interval of that median (a +# bootstrap of the pairs) is wider than ±5 % (--target), 8 at most (--most). DuckDB's time is its JSON +# profile's latency, to the nanosecond, a profile a round (its timer counts whole milliseconds); its CPU, +# the profile's cpu_time. Ours, the runner's work_us and cpu_us. +# +# THE MACHINE, as bench/runners.sh watches it, all of it as a user: before and after each row and base, +# two canaries of our runner with their files kept mapped (a scan of one lane, a count and sum of 10^3 +# keys on 14 lanes), the first values of the session their standard; the busiest other process, which +# holds a row back while it takes more than half a core (30 s at most); the swap, the free memory and the +# power source. A row framed by a canary more than 5 % off whose pairs did not hold the target, or during +# which the swap grew, is run again once. THE FILES are read once before anything is timed, so that both +# engines find them in the page cache. +# +# A SESSION keeps everything it ran under bench/.runs/duckdb-/ (ignored by git): runs.tsv, each +# block, canary and control with its time of day; session.tsv, each row's result; DuckDB's output and +# profiles. --publish takes two sessions run at different hours and prints the page's table: a row is +# published when the two agree within 10 %, a third session deciding otherwise, beside its pairs' range +# and count, then the sessions' canaries and controls. # # THE FILES are the bench's own, written by its generators (bench/Vorticity.Benchmarks.Queries --fixture), # canonical, at edition core2025.10, the one the extension reads (it does not know vortex.zoned); both @@ -32,22 +64,28 @@ # DuckDB is a workstation dependency, as cargo is for the comparison with Rust: duckdb on the PATH (or # DUCKDB), its vortex extension installed (INSTALL vortex). Never in CI. These figures inform; they gate # nothing (the plan's steps are judged on their ratchets). A figure for publication is taken on a machine -# at rest: docs/guide/benchmarks-duckdb.md. +# at rest, twice: docs/guide/benchmarks-duckdb.md. set -uo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" duckdb="${DUCKDB:-$(command -v duckdb || echo /opt/homebrew/bin/duckdb)}" +corpus="${VORTICITY_QUERIES_CORPUS:-$HOME/.cache/vorticity/queries}" set_name="hc" only="" rows=40000000 db_rows=10000000 threads="1,14" runner="HEAD" -rounds=10 +bases="both" +target=5 +most=8 rest=0 +canary=1 +publish=() usage() { - echo "usage: bench/duckdb.sh [--set hc|db|all] [--only ] [--rows N] [--db-rows N] [--threads N[,M…]] [--runner |] [--rounds N] [--rest S]" >&2 + echo "usage: bench/duckdb.sh [--set hc|db|all] [--only ] [--rows N] [--db-rows N] [--threads N[,M…]] [--runner |] [--bases both|warm|cold] [--target P] [--most N] [--rest S] [--no-canary]" >&2 + echo " bench/duckdb.sh --publish []" >&2 exit 2 } @@ -59,15 +97,108 @@ while [ $# -gt 0 ]; do --db-rows) db_rows="$2"; shift 2 ;; --threads) threads="$2"; shift 2 ;; --runner) runner="$2"; shift 2 ;; - --rounds) rounds="$2"; shift 2 ;; + --bases) bases="$2"; shift 2 ;; + --target) target="$2"; shift 2 ;; + --most) most="$2"; shift 2 ;; --rest) rest="$2"; shift 2 ;; + --no-canary) canary=0; shift ;; + --publish) shift; while [ $# -gt 0 ]; do publish+=("$1"); shift; done ;; *) usage ;; esac done +# The awk functions the decisions and the tables share: a median, and the 95 % interval of a median by a +# bootstrap of 2 000 draws, seeded so that the same ratios give the same interval (as bench/runners.sh). +stats=' + function sortn(a, n, gap, i, j, t) { + for (gap = int(n / 2); gap > 0; gap = int(gap / 2)) + for (i = gap; i < n; i++) { t = a[i]; for (j = i; j >= gap && a[j - gap] + 0 > t + 0; j -= gap) a[j] = a[j - gap]; a[j] = t } + } + function median(a, n, c, i) { + for (i = 0; i < n; i++) c[i] = a[i] + sortn(c, n) + return n % 2 ? c[int(n / 2)] : (c[n / 2 - 1] + c[n / 2]) / 2 + } + function interval(r, n, b, i, s, m) { + srand(7) + for (b = 0; b < 2000; b++) { for (i = 0; i < n; i++) s[i] = r[int(rand() * n)]; m[b] = median(s, n) } + sortn(m, 2000) + low = m[49]; high = m[1950] + }' + +# --publish: the page's table from two or three sessions' session.tsv. A row of a base is published when +# two sessions agree within 10 %: the mean of their speedups, the widest of their 95 % intervals, their +# pairs summed; none agreeing, it is marked and its speedups printed. +if [ ${#publish[@]} -gt 0 ]; then + [ ${#publish[@]} -ge 2 ] || usage + files=() + for session in "${publish[@]}"; do + [ -f "$session/session.tsv" ] || { echo "no session.tsv in $session" >&2; exit 2; } + files+=("$session/session.tsv") + done + + echo "# Our group bys against DuckDB's" + echo + for session in "${publish[@]}"; do + echo "- session $(basename "$session"): $(head -1 "$session/about.txt" 2> /dev/null)" + done + + echo "- a speedup is DuckDB's time over ours, above 1.00x we are faster: the mean of the two sessions that agreed within 10 %, in brackets the widest of their 95 % intervals and their pairs" + echo + echo "| query | rows | threads | ours warm (ms) | DuckDB native table (ms) | speedup | ours cold (ms) | DuckDB Vortex reader (ms) | speedup | answer |" + echo "|---|---:|---:|---:|---:|---:|---:|---:|---:|---|" + awk -F '\t' ' + FNR == 1 { s++ } + { + key = $1 SUBSEP $2 SUBSEP $3 + if (!(key in seen)) { seen[key] = 1; order[++n] = key; label[key] = $1; rows[key] = $2; degree[key] = $3 } + k = key SUBSEP $4; c = count[k]++ + ours[k, c] = $5; duck[k, c] = $6; sp[k, c] = $7; lo[k, c] = $8; hi[k, c] = $9; pairs[k, c] = $10 + if ($12 != "same") answer[key] = "DIFFERENT" + } + function cell(k, i, j, best, bi, bj, d, m) { + best = -1 + for (i = 0; i < count[k]; i++) + for (j = i + 1; j < count[k]; j++) { + d = sp[k, i] / sp[k, j] - 1; if (d < 0) d = -d + if (d <= 0.10 && (best < 0 || d < best)) { best = d; bi = i; bj = j } + } + if (count[k] == 0) return "| | |" + if (best < 0) { + m = "" + for (i = 0; i < count[k]; i++) m = m sprintf("%s%.2fx", (i ? ", " : ""), sp[k, i]) + return sprintf("%.1f | %.1f | unsettled: %s", ours[k, 0], duck[k, 0], m) + } + return sprintf("%.1f | %.1f | %.2fx [%.2f–%.2f, %d pairs]", (ours[k, bi] + ours[k, bj]) / 2, (duck[k, bi] + duck[k, bj]) / 2, + (sp[k, bi] + sp[k, bj]) / 2, (lo[k, bi] < lo[k, bj] ? lo[k, bi] : lo[k, bj]), (hi[k, bi] > hi[k, bj] ? hi[k, bi] : hi[k, bj]), pairs[k, bi] + pairs[k, bj]) + } + END { + for (i = 1; i <= n; i++) { + key = order[i] + printf "| %s | %s | %s | %s | %s | %s |\n", label[key], rows[key], degree[key], cell(key SUBSEP "warm"), cell(key SUBSEP "cold"), (key in answer) ? answer[key] : "same" + } + }' "${files[@]}" + + echo + echo "The machine during each session (its canaries' spread around their standard, the controls' worst):" + echo + for session in "${publish[@]}"; do + awk -F '\t' -v name="$(basename "$session")" ' + $1 == "canary" { scan = $7 + 0; lanes = $8 + 0; split($9, st, "/") + d = scan / st[1] - 1; if (d < 0) d = -d; if (d > worst) worst = d + d = lanes / st[2] - 1; if (d < 0) d = -d; if (d > worst) worst = d; canaries++ } + $1 == "control" { if ($8 + 0 > swap) swap = $8 + 0; if ($5 == "before") waited += $6; if ($10 != "ac") battery = 1 } + $1 == "replay" { replays++ } + END { printf "- %s: %d canaries, at most %.1f %% off their standard; %d rows replayed; %d s waited on another process; swap at most %s MB; %s\n", + name, canaries, 100 * worst, replays, waited, swap, battery ? "on battery at times" : "on mains power" }' "$session/runs.tsv" + done + + exit 0 +fi + [ -x "$duckdb" ] || { echo "no duckdb at '$duckdb': install it, or set DUCKDB" >&2; exit 2; } "$duckdb" -c "LOAD vortex;" > /dev/null 2>&1 || { echo "duckdb cannot load its vortex extension: run INSTALL vortex in it" >&2; exit 2; } -[ "$rounds" -ge 7 ] || { echo "--rounds must be 7 or more: the median is of the last 7" >&2; exit 2; } +case "$bases" in both) base_list="warm cold" ;; warm|cold) base_list="$bases" ;; *) usage ;; esac # The runner: a directory, or a commit's, kept by bench/runners.sh. if [ -d "$runner" ]; then @@ -83,9 +214,6 @@ fixture() { dotnet run -c Release --project "$root/bench/Vorticity.Benchmarks.Queries" -- --fixture "$1" 2> /dev/null | tail -1 } -# The median of the last 7 of the numbers on stdin, one a line. -median7() { tail -7 | sort -n | sed -n 4p; } - # The rows of the comparison: a label, the runner's scenario, the fixture, and DuckDB's query over `x`. queries=() if [ "$set_name" = "hc" ] || [ "$set_name" = "all" ]; then @@ -146,93 +274,340 @@ for entry in "${queries[@]}"; do paths+=("$path") done -# DuckDB's side, a process a file: its view over the extension and its native table, then each query at -# each degree, `rounds` times on each, each block marked. Prints " internal void Judge(RecordBatch batch) { + if (TurnOnNew && !_sampled && batch.SelectedRows > 0) + { + JudgeSample(batch); + } + if (!TurnOnSpread || _spreadJudged || Keys is not { NumberedByValue: true } keys || batch.RowCount == 0 || batch.SelectedRows == 0) { return; @@ -1344,6 +1349,80 @@ internal void Judge(RecordBatch batch) } } + /// The rows of a lane's first batch whose keys judge a hashed key before the batch is folded (). + internal const int SampledRows = 8_192; + + // Whether the lane judged its key on a sample of its first batch (JudgeSample). + private bool _sampled; + + /// + /// Judges a hashed key on the lane's first batch, before a row of it is folded: the groups its first + /// rows make in the lane's own table, no value folded, and the values of a + /// uniform key that makes as many (). Past , every + /// lane turns to the core with its table nearly empty, those groups' states still empty. Folded first, + /// the batch grew each lane's table to its rows, nearly all new, its arrays doubled some fourteen times, + /// then emptied into the core and dropped: 400 of the 490 MB a hashed key of 3.3M values over 4M rows + /// allocated a query at fourteen lanes (2026-10-09). A key of fewer values keeps its table, the batch's + /// rows finding there the groups the sample made, and is judged on its first rows as before. 8 192 rows + /// tell 10⁵ values from 10⁶: about 335 repeats against 34. + /// + private void JudgeSample(RecordBatch batch) + { + _sampled = true; + int rows = batch.RowCount; + if (rows < SampledRows || Keys is not { NumberedByValue: false, Count: 0, EntryBytes: > 0 } keys || Pressure is not { Outgrown: false } pressure) + { + return; + } + + // The rows a filter keeps among the first ones: a filter that keeps few leaves the key to its first rows. + ReadOnlySpan selection = batch.SelectionWords; + int sampled = SampledRows; + if (!selection.IsEmpty) + { + selection = selection[..(SampledRows >> 6)]; + sampled = 0; + foreach (ulong word in selection) + { + sampled += BitOperations.PopCount(word); + } + + if (sampled < SampledRows / 2) + { + return; + } + } + + CanonicalArena arena = batch.Arena; + for (int c = 0; c < _keyCount; c++) + { + int node = FilterEvaluator.Resolve(arena, batch.RootIndex, _columns[c].Field, rows); + while (arena.RecordRef(node).Kind == CanonicalKind.Extension) + { + node = arena.GetNode(node).StorageIndex; + } + + _nodes[c] = node; + } + + if (_rowGroups.Length < SampledRows) + { + RentRowGroups(SampledRows); + } + + _ranges.Clear(); + keys.Assign(arena, _nodes.AsSpan(0, _keyCount), SampledRows, selection, _rowGroups, _ranges); + for (int i = 0; i < Slots.Length; i++) + { + Slots[i].EnsureGroups(keys.Count); + } + + if (EstimatedValues(sampled, keys.Count) >= TurnValues) + { + pressure.Outgrew(CoreReason.FirstRows, _rowsFolded); + } + } + /// /// The groups, nulls included, the statistics bound the key of a streaming group by to at most for /// its slots to take the shape of few groups (): a distinct diff --git a/tests/Vorticity.Tests/Aggregation/FanTests.cs b/tests/Vorticity.Tests/Aggregation/FanTests.cs index 659e6153b..4c0d346c1 100644 --- a/tests/Vorticity.Tests/Aggregation/FanTests.cs +++ b/tests/Vorticity.Tests/Aggregation/FanTests.cs @@ -12,7 +12,7 @@ namespace Vorticity.Tests.Aggregation; /// A key whose cardinality changes as the rows go: a quarter of the rows of few values, then many, or /// the other way round. The lanes start across the source, lane i at i / N of its ranges, and the first /// lane whose rows show many values turns the query to the core: an integer key on its first batch's -/// spread, a hashed one on its first rows' new groups. Few values then many turns on the lanes that +/// spread, a hashed one on the groups a sample of its first batch makes. Few values then many turns on the lanes that /// start past the first quarter, which without the fan may all start in it; many then few turns on the /// first lane, fan or not. The statistics say why and when. /// @@ -58,9 +58,9 @@ public async Task TheFirstLaneThatMeetsManyValuesTurnsTheQuery(string shape, boo Assert.Equal(expected, grouping.CoreReason); - // The first batch's spread turns a lane before it folds a row; the first rows' new groups, - // once it has folded them. - Assert.True(expected == GroupCoreReason.Spread ? grouping.TurnedAfterRows == 0 : grouping.TurnedAfterRows > 0, $"turned after {grouping.TurnedAfterRows} rows"); + // The first batch turns a lane before it folds a row: an integer key by its spread, a hashed one + // by the groups a sample of its rows makes. + Assert.True(grouping.TurnedAfterRows == 0, $"turned after {grouping.TurnedAfterRows} rows"); } finally { From 88981a5956c86dcf5dfab11a4dc20f298cfb977e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 23:01:21 +0200 Subject: [PATCH 28/56] bench: the runner prints each round's collections and the time they held A round's line now ends with gc=//, the collections it called, and gc_paused_us, the time they held every thread (GC.GetTotalPauseDuration): the stops a lane cannot work through. At fourteen lanes db-benchmark's q10 called 7 to 8 gen0 and 4 to 5 gen1 collections a round, 17 to 25 ms of a 150 to 190 ms round; the strided key of 10^7 values 4 to 5 and 2 to 3, about 10 ms. --- bench/Vorticity.Benchmarks.Runner/Program.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bench/Vorticity.Benchmarks.Runner/Program.cs b/bench/Vorticity.Benchmarks.Runner/Program.cs index a18c8ba4b..b1de5ff6a 100644 --- a/bench/Vorticity.Benchmarks.Runner/Program.cs +++ b/bench/Vorticity.Benchmarks.Runner/Program.cs @@ -92,6 +92,11 @@ private static async Task Main(string[] args) long[] workMicros = new long[repeat]; long[] allocated = new long[repeat]; ProcessCounters[] spent = new ProcessCounters[repeat]; + + // The collections of each generation a round called, and the time they held every thread: the + // stops a lane cannot work through. + int[] collections = new int[repeat * 3]; + long[] pausedMicros = new long[repeat]; long before = AllocatedSoFar(); for (int round = 0; round < repeat; round++) { @@ -99,11 +104,19 @@ private static async Task Main(string[] args) // is the work, and the process start is a property of the build that the parent times // apart. The processor's counters are read outside the timed span, on every thread of the // process: what the round cost the machine beside how long it took. + int gen0 = GC.CollectionCount(0); + int gen1 = GC.CollectionCount(1); + int gen2 = GC.CollectionCount(2); + TimeSpan paused = GC.GetTotalPauseDuration(); ProcessCounters counted = ProcessCost.ReadCounters(); long started = Stopwatch.GetTimestamp(); delivered[round] = await scenario(args[2]).ConfigureAwait(false); workMicros[round] = (long)(Stopwatch.GetElapsedTime(started).TotalMilliseconds * 1000); spent[round] = ProcessCost.ReadCounters().Since(counted); + pausedMicros[round] = (long)(GC.GetTotalPauseDuration() - paused).TotalMicroseconds; + collections[(3 * round) + 0] = GC.CollectionCount(0) - gen0; + collections[(3 * round) + 1] = GC.CollectionCount(1) - gen1; + collections[(3 * round) + 2] = GC.CollectionCount(2) - gen2; long after = AllocatedSoFar(); allocated[round] = after - before; before = after; @@ -115,7 +128,7 @@ private static async Task Main(string[] args) { Console.WriteLine(string.Create( CultureInfo.InvariantCulture, - $"round={round} rows={delivered[round]} work_us={workMicros[round]} allocated_bytes={allocated[round]} cpu_us={spent[round].CpuNanoseconds / 1000} instructions={spent[round].Instructions} cycles={spent[round].Cycles}")); + $"round={round} rows={delivered[round]} work_us={workMicros[round]} allocated_bytes={allocated[round]} cpu_us={spent[round].CpuNanoseconds / 1000} instructions={spent[round].Instructions} cycles={spent[round].Cycles} gc={collections[3 * round]}/{collections[(3 * round) + 1]}/{collections[(3 * round) + 2]} gc_paused_us={pausedMicros[round]}")); } } From d1758ed7013ecf72a71288cb55b57ab4f565dd63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 23:30:04 +0200 Subject: [PATCH 29/56] test(query): a warm group by's allocations held against a ratchet What a group by allocates once its process is warm, a whole query from its plan to its last result, on every thread: its lanes run where the pool schedules them and the core's parts apply on any of them, and what they allocate is what stops every lane. A collection of the workstation collector suspends every thread and collects on one; at fourteen lanes the stops were a fifth of s7 and q10 on 4M rows. Ten axes, a path each at one lane and at four: integers numbered by value, short texts, hashed integers (the core turned to on the first rows), integers spread over 2*10^6 (the core by pages), six keys as tuples. Each checks the path it took, and its floor over five warm queries against a ceiling just above it, in AllocationCollection, which runs alone. Today: 0.10 to 0.36 MB for the small integers, 0.5 to 1.2 MB for the texts, 33 to 164 MB for the keys of many groups. The ceilings come down as the zero-allocation work removes each site. --- .../Aggregation/GroupAllocationTests.cs | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs new file mode 100644 index 000000000..ab7c0299c --- /dev/null +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -0,0 +1,247 @@ +// What a group by allocates once its process is warm, held against a ratchet: a whole query, from its +// plan to its last result, on every thread. +// +// WHY EVERY THREAD. A group by folds on lanes the pool schedules and applies the core's parts on any +// of them: the bytes the caller's thread allocates are a fraction of what the query costs. And what +// the rest allocates is what stops every lane: a collection of the workstation collector suspends +// every thread and collects on one. At fourteen lanes the stops took a fifth of the time of s7 and +// q10 on 4·10⁶ rows (PLAN-DUCKDB.md, D9, 2026-10-09). So the figure is the process's, measured in +// AllocationCollection, which runs alone; D9 drives every axis to zero. +// +// WHY THE FLOOR OF SEVERAL QUERIES. The lanes take ranges from a queue in the order the pool runs +// them: the tables each grows, the sub-tables the core splits and the batches its lanes fill move a +// little from one query to the next. The floor of several queries is what a query cannot avoid; a +// cost paid on every one still raises it. +// +// THE CEILINGS sit just above the floor measured. LOWER ONE BY HAND in the commit that earns it, and +// never raise one without saying in the commit what grew and why. +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Vorticity.Aggregating; +using Vorticity.Tests.Scan; +using Xunit; + +namespace Vorticity.Tests.Aggregation; + +[Collection(nameof(AllocationCollection))] +public sealed partial class GroupAllocationTests +{ + /// The rows of the file: enough for four lanes to judge their first rows (). + private const int Rows = 1_000_000; + + /// Queries before any measurement: the JIT, its tiers, the pools' first rents and the process's shelf. + private const int WarmUp = 4; + + /// Queries measured, of which the floor is the answer. + private const int Runs = 5; + + private static readonly string[] Names = Texts("name", 1_000); + + private static readonly string[] Labels = Texts("label", 13); + + private static readonly Lazy> Written = new Lazy>(WriteAsync); + + /// + /// The axes: a path of the engine each, at one lane and at four, the core asked from four lanes + /// () where a key makes it turn. The ceiling is a warm + /// query's bytes on every thread. + /// + /// + /// At one lane a ceiling is the floor rounded up to 512 bytes; at four, the lanes' schedule moves + /// the core's splits and batches by a few hundred kilobytes from one run to the next, so a ceiling + /// is the floor and 2 % (1 % for six keys at one lane), rounded up to 64 KiB. + /// + private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceiling, Func> Query)[] Axes = + [ + // A thousand integers numbered by their value, a table a lane. + ("small integers", 1, GroupCoreReason.None, 360_448, SmallAsync), + ("small integers", 4, GroupCoreReason.None, 561_152, SmallAsync), + + // A thousand short texts, hashed as their words. + ("short texts", 1, GroupCoreReason.None, 521_728, NamesAsync), + ("short texts", 4, GroupCoreReason.None, 1_159_168, NamesAsync), + + // A long integer a row, hashed: at four lanes, a lane's first rows are all new groups. + ("hashed integers", 1, GroupCoreReason.None, 82_157_056, WideAsync), + ("hashed integers", 4, GroupCoreReason.FirstRows, 51_249_152, WideAsync), + + // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. + ("spread integers", 1, GroupCoreReason.None, 58_423_808, DenseAsync), + ("spread integers", 4, GroupCoreReason.Spread, 34_275_328, DenseAsync), + + // Six keys, three texts and three integers, nearly a group a row: tuples of their values. + ("six keys", 1, GroupCoreReason.None, 165_806_080, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 120_389_632, SixAsync), + ]; + + [Fact] + public async Task EveryWarmGroupByAllocatesWithinItsCeiling() + { + ReleaseOnlyCeilings.Require(); + string path = await Written.Value; + StringBuilder report = new StringBuilder("GROUP BY ALLOCATIONS: a warm query on every thread, floor of ") + .Append(Runs.ToString(CultureInfo.InvariantCulture)) + .Append(" after ") + .Append(WarmUp.ToString(CultureInfo.InvariantCulture)) + .Append('\n'); + + List failures = []; + foreach ((string axis, int lanes, GroupCoreReason expected, long ceiling, Func> query) in Axes) + { + await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = lanes); + await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); + for (int i = 0; i < WarmUp; i++) + { + await query(file, lanes); + } + + long floor = long.MaxValue; + GroupCoreReason took = expected; + for (int i = 0; i < Runs && floor > 0; i++) + { + long before = GC.GetTotalAllocatedBytes(precise: true); + (long rows, took) = await query(file, lanes); + long allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + Assert.Equal((long)Rows, rows); + floor = Math.Min(floor, allocated); + } + + string name = string.Create(CultureInfo.InvariantCulture, $"{axis}, {lanes} lane{(lanes > 1 ? "s" : "")}"); + report.Append(" ") + .Append(name.PadRight(28)) + .Append(floor.ToString("N0", CultureInfo.InvariantCulture).PadLeft(14)) + .Append(" B ceiling ") + .Append(ceiling.ToString("N0", CultureInfo.InvariantCulture).PadLeft(14)) + .Append(" headroom ") + .Append((ceiling - floor).ToString("N0", CultureInfo.InvariantCulture).PadLeft(12)) + .Append(" ") + .Append(took) + .Append('\n'); + if (floor > ceiling) + { + failures.Add(string.Create(CultureInfo.InvariantCulture, $"{name} allocated {floor:N0} B against a ceiling of {ceiling:N0} B")); + } + + // An axis measures one path: a key that no longer turns, or turns for another reason, measures another. + if (took != expected) + { + failures.Add($"{name} took the path {took}, where the axis measures {expected}"); + } + } + + Console.Out.Write(report.ToString()); + + // Reported together, as the read paths are: a change that moves one axis usually moves several. + Assert.True(failures.Count == 0, string.Join("\n", failures) + "\n" + report); + } + + private static Task<(long, GroupCoreReason)> SmallAsync(VortexFile file, int lanes) => + CountAsync(file.Scan().GroupBy(r => r.Small).Select(g => (g.Key, g.Count(), g.Sum(r => r.Value))), lanes, 1); + + private static Task<(long, GroupCoreReason)> NamesAsync(VortexFile file, int lanes) => + CountAsync(file.Scan().GroupBy(r => r.Name).Select(g => (g.Key, g.Count(), g.Sum(r => r.Value))), lanes, 1); + + private static Task<(long, GroupCoreReason)> WideAsync(VortexFile file, int lanes) => + CountAsync(file.Scan().GroupBy(r => r.Wide).Select(g => (g.Key, g.Count(), g.Sum(r => r.Value))), lanes, 1); + + private static Task<(long, GroupCoreReason)> DenseAsync(VortexFile file, int lanes) => + CountAsync(file.Scan().GroupBy(r => r.Dense).Select(g => (g.Key, g.Count(), g.Sum(r => r.Value))), lanes, 1); + + private static Task<(long, GroupCoreReason)> SixAsync(VortexFile file, int lanes) => + CountAsync( + file.Scan().GroupBy(r => (r.Name, r.Label, r.City, r.Small, r.Dense, r.Other)) + .Select(g => (g.Key.Item1, g.Key.Item2, g.Key.Item3, g.Key.Item4, g.Key.Item5, g.Key.Item6, g.Count(), g.Sum(r => r.Value))), + lanes, + 6); + + /// + /// Runs and reads its result as the batches it is, its counts summed: the + /// rows the groups hold, and the path the groups took. + /// + /// The column of the counts. + private static async Task<(long, GroupCoreReason)> CountAsync(Vorticity.Aggregation query, int lanes, int counted) + where TRecord : IVortexRecord + { + query.Plan.CoreLanes = lanes; + long rows = 0; + await foreach (Columns batch in query.As()) + { + rows += Sum(batch.Column(counted).Values); + } + + return (rows, query.Statistics.Grouping!.CoreReason); + } + + private static long Sum(ReadOnlySpan values) + { + long sum = 0; + foreach (long value in values) + { + sum += value; + } + + return sum; + } + + private static async Task WriteAsync() + { + // One per run, named by the process, gone when the run ends. + string directory = Path.Combine(AppContext.BaseDirectory, "group-allocations"); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, $"rows-{Environment.ProcessId}.vortex"); + AppDomain.CurrentDomain.ProcessExit += (_, _) => System.IO.File.Delete(path); + + Row[] rows = new Row[Rows]; + for (int row = 0; row < Rows; row++) + { + ulong mixed = (ulong)row * 0x9E37_79B9_7F4A_7C15UL; + rows[row] = new Row( + row % 1_000, + Names[(int)((mixed >> 40) % 1_000)], + (long)(mixed & long.MaxValue), + (int)((mixed >> 24) % (2 * Rows)), + row % 7, + Labels[row % 13], + Names[row % 97], + row); + } + + await using VortexFileWriter writer = VortexSession.Default.CreateWriter(path); + await writer.WriteAsync(rows, CancellationToken.None); + await writer.CompleteAsync(CancellationToken.None); + return path; + } + + private static string[] Texts(string prefix, int count) + { + string[] texts = new string[count]; + for (int i = 0; i < count; i++) + { + texts[i] = string.Create(CultureInfo.InvariantCulture, $"{prefix}-{i}"); + } + + return texts; + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [VortexRecord] + public partial record struct Row(int Small, string Name, long Wide, int Dense, int Other, string Label, string City, long Value); + + [VortexRecord] + public partial record struct IntTotal(int Key, long Count, long Sum); + + [VortexRecord] + public partial record struct NameTotal(string Key, long Count, long Sum); + + [VortexRecord] + public partial record struct WideTotal(long Key, long Count, long Sum); + + [VortexRecord] + public partial record struct SixTotal(string Name, string Label, string City, int Small, int Dense, int Other, long Count, long Sum); +} From 1fa52f98c013ad01128be59594f107d113f62c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 23:39:08 +0200 Subject: [PATCH 30/56] perf(query): a part's order is lent by a shelf, not made A core's part built its batches from the order of its groups, 0 to n - 1, made new for every part: 256 arrays of 50 to 60 kB a query on 4M rows, just under the large object heap, 16 MB on db-benchmark's q10 and 13 MB on the strided key of 10^7 values at fourteen lanes, a third of what their queries allocated on the small object heap, which is what calls the collections that stop every lane. The order is now lent by the core's shelf, a power of two long, so that the parts of a query take the same few arrays again and the next query those its shelf left to the process's (AggregationEngine.LentOrder), and given back once the part is built; an order whose build failed is let go, a ranking it started may still read it. A merge in parts lends its orders from the process's shelf the same way. An outcome counts the groups it delivers apart from its order's length (Count), and makes no array for chosen rows when there are none. The runner at fourteen lanes, warm, against 3e0d7957: the strided key on 4M x0.886 (271 MB a query against 284, 3.6 collections against 5.6, 4.5 ms stopped against 6.0), q10 on 4M x0.954 (693 MB, 5.8 collections against 7.8, 10.4 ms against 12.9); the cycles x0.99, the time gained is the stops. GroupAllocationTests: the core's axes 3 to 4 MB lower. --- .../Aggregation/AggregationEngine.cs | 50 ++++++++++++++++--- src/Vorticity/Aggregation/GroupBatches.cs | 2 +- src/Vorticity/Aggregation/GroupCore.cs | 22 ++++++-- src/Vorticity/Aggregation/MergedParts.cs | 8 ++- .../Aggregation/GroupAllocationTests.cs | 9 ++-- 5 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index d704b6537..b3841c2db 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -621,13 +621,15 @@ internal sealed class AggregationOutcome // The means read from the slots of sums, by the sum's slot, made on the first read. private AggregateSlot?[]? _views; - internal AggregationOutcome(AggregationPlan plan, AggregateSlot[] slots, GroupKeys? keys, int[] order) + /// The groups of in their states, delivered in the first of , every one by default. + internal AggregationOutcome(AggregationPlan plan, AggregateSlot[] slots, GroupKeys? keys, int[] order, int count = -1) { _plan = plan; _slots = slots; Keys = keys; Order = order; - _chosen = new ChosenValues[plan.Chosen.Length]; + Count = count < 0 ? order.Length : count; + _chosen = plan.Chosen.Length == 0 ? [] : new ChosenValues[plan.Chosen.Length]; for (int c = 0; c < _chosen.Length; c++) { _chosen[c] = plan.Chosen[c].CreateValues(); @@ -658,9 +660,15 @@ internal void Delivered() internal GroupKeys? Keys { get; } - /// The groups in delivery order; the one group of a scalar aggregation. + /// + /// The groups in delivery order, its first ; the one group of a scalar aggregation. + /// An order lent by a shelf is longer. + /// internal int[] Order { get; } + /// The groups delivered: the first ones of . + internal int Count { get; } + internal AggregateSlot SlotOf(IAggregateNode node) { int index = _plan.IndexOf(node); @@ -3543,18 +3551,44 @@ private static async ValueTask SpilledAsync( /// copy shuffled by a draw the process's seed and the groups' count make: the same for two identical /// queries of a process, and unlike the merge's. /// - internal static int[] Shuffled(int[] order) + internal static int[] Shuffled(int[] order) => Shuffled(order, order.Length); + + /// of the first groups of an order a shelf lent, longer than they are: a copy of them alone. + internal static int[] Shuffled(int[] order, int count) { - if (!AggregationPlan.ShuffledOrder || order.Length < 2) + if (!AggregationPlan.ShuffledOrder || count < 2) { return order; } - int[] shuffled = [.. order]; - new Random(unchecked((int)(MergeHash.Seed ^ (ulong)order.Length))).Shuffle(shuffled); + int[] shuffled = order.AsSpan(0, count).ToArray(); + new Random(unchecked((int)(MergeHash.Seed ^ (ulong)count))).Shuffle(shuffled); return shuffled; } + /// + /// The groups 0 to less one in their order, in an array of + /// 's a power of two long, so that the parts of a query take the same + /// arrays again, and the next query those its shelf handed to the process's: given back once the + /// groups are built into their batches. Taken past the budget when it must: it is held only while + /// its part is built, and the part's keys and states hold several times its bytes. + /// + internal static int[] LentOrder(ArrayShelf shelf, int count) + { + if (count == 0) + { + return []; + } + + int[] order = shelf.Take(Scratch.Capacity(count, 0), zeroed: false, overdraw: true); + for (int g = 0; g < count; g++) + { + order[g] = g; + } + + return order; + } + /// /// The result keeps of its query's memory, what its groups hold, and the /// order they are delivered in, until it is delivered; the rest is given back. @@ -3563,7 +3597,7 @@ private static AggregationOutcome Counted(AggregationOutcome outcome, QueryMemor { memory.Keep(bytes + ((long)outcome.Order.Length * sizeof(int))); outcome.Memory = memory; - outcome.Plan.LastGroups = outcome.Order.Length; + outcome.Plan.LastGroups = outcome.Count; outcome.Plan.LastPeakBytes = memory.Peak; return outcome; } diff --git a/src/Vorticity/Aggregation/GroupBatches.cs b/src/Vorticity/Aggregation/GroupBatches.cs index 504f05f0f..7a9995db0 100644 --- a/src/Vorticity/Aggregation/GroupBatches.cs +++ b/src/Vorticity/Aggregation/GroupBatches.cs @@ -808,7 +808,7 @@ internal static int ChosenReader(GroupOperator[] operators) bool chosen = query.Plan.Chosen.Length > 0; int reader = chosen ? Math.Min(query.ChosenReader, operators) : operators; int degree = spec.Options.DegreeOfParallelism > 0 ? spec.Options.DegreeOfParallelism : query.Session.Options.MaxDegreeOfParallelism; - (int[] groups, int count) = await OperatorsAsync(query, outcome, 0, reader, outcome.Order, outcome.Order.Length, degree, cancellationToken).ConfigureAwait(false); + (int[] groups, int count) = await OperatorsAsync(query, outcome, 0, reader, outcome.Order, outcome.Count, degree, cancellationToken).ConfigureAwait(false); if (!chosen) { return (groups, count); diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 24e4be7c5..a499715a7 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -1115,14 +1115,30 @@ internal async ValueTask BuildAsync(CorePart part, PartBuilder build { (GroupKeys keys, AggregateSlot[] slots) = Joined(part); Held(keys.Count); + int count = keys.Count; + int[] order = AggregationEngine.LentOrder(_shelf, count); + bool built = false; try { - AggregationOutcome outcome = new AggregationOutcome(_plan, slots, keys, AggregationEngine.Shuffled(keys.Order(sorted: false))) { Memory = _memory }; - return await builder.BuildAsync(outcome, cancellationToken).ConfigureAwait(false); + AggregationOutcome outcome = new AggregationOutcome(_plan, slots, keys, AggregationEngine.Shuffled(order, count), count) { Memory = _memory }; + PartResult result = await builder.BuildAsync(outcome, cancellationToken).ConfigureAwait(false); + built = true; + return result; } finally { - Held(-keys.Count); + Held(-count); + + // An order whose build failed may still be read by a ranking the build started: let go, not lent again. + if (built) + { + _shelf.Give(order); + } + else + { + _shelf.Drop(order); + } + Let(part); } } diff --git a/src/Vorticity/Aggregation/MergedParts.cs b/src/Vorticity/Aggregation/MergedParts.cs index 6238db6fa..b6d134249 100644 --- a/src/Vorticity/Aggregation/MergedParts.cs +++ b/src/Vorticity/Aggregation/MergedParts.cs @@ -424,10 +424,16 @@ private async ValueTask BuildAsync(int part, CancellationToken cancellati } Built built = new Built(reserved, measured); + + // The part's order lent by the process's shelf, which counts nothing, as the order the part made + // did not; given back once the part is built. + int count = keys.Count; + int[] order = AggregationEngine.LentOrder(ArrayShelf.Retained, count); try { - AggregationOutcome outcome = new AggregationOutcome(_plan, slots, keys, AggregationEngine.Shuffled(keys.Order(sorted: false))) { Memory = _memory }; + AggregationOutcome outcome = new AggregationOutcome(_plan, slots, keys, AggregationEngine.Shuffled(order, count), count) { Memory = _memory }; built.Result = await _builder.BuildAsync(outcome, cancellationToken).ConfigureAwait(false); + ArrayShelf.Retained.Give(order); return built; } catch diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index ab7c0299c..91229ffc3 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -68,15 +68,18 @@ private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceil // A long integer a row, hashed: at four lanes, a lane's first rows are all new groups. ("hashed integers", 1, GroupCoreReason.None, 82_157_056, WideAsync), - ("hashed integers", 4, GroupCoreReason.FirstRows, 51_249_152, WideAsync), + // + // On this axis and the core's two others, each part is built in an order its shelf lends: 3 to + // 4 MB less, four bytes a group. + ("hashed integers", 4, GroupCoreReason.FirstRows, 48_758_784, WideAsync), // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. ("spread integers", 1, GroupCoreReason.None, 58_423_808, DenseAsync), - ("spread integers", 4, GroupCoreReason.Spread, 34_275_328, DenseAsync), + ("spread integers", 4, GroupCoreReason.Spread, 29_949_952, DenseAsync), // Six keys, three texts and three integers, nearly a group a row: tuples of their values. ("six keys", 1, GroupCoreReason.None, 165_806_080, SixAsync), - ("six keys", 4, GroupCoreReason.FirstRows, 120_389_632, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 115_867_648, SixAsync), ]; [Fact] From 45c59208c044c24377f9c157495267a8a5ecfaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Fri, 9 Oct 2026 23:59:16 +0200 Subject: [PATCH 31/56] perf(query): a lane's arrays come from the process's shelf and go back to it A lane's shelf handed out new arrays and left every one its tables gave back to the next collection: each query grew its lanes' tables again, its caches to a batch's keys, the partition it bypasses a cache with to a batch's rows. On the strided key of 10^7 values over 4M rows that was some 226 of the 255 MB a query allocated at fourteen lanes. - A lane's shelf takes its arrays from the process's shelf and gives them back to it, its bytes still reserved and given back under the query's memory as before. - A partition gives its keys' and records' arrays back when it dies (Release, LetGo) and when it turns to the core, the partition its lane bypassed its cache with too. That partition takes its arrays from a shelf of its own, counted by no query as it never was. - The group of each row comes from the process's shelf: the shared array pool kept one array a thread, which the lane of the next query, on another thread, did not find. - The process's shelf, past its quarter of a gigabyte, lets go of the lengths taken or given least recently: full of an earlier query's lengths, it kept them while every query went on taking from it, and the arrays of the queries running then could not come in. A query's shelf hands its arrays to it with no list made first. An array a shelf lends again must be dead where it was given: the tests now poison every array given back (ArrayShelf.PoisonsGiven, set by a module initializer in both test projects), so that a table reading one after giving it fails there. The suite and the conformance tests pass under it. GroupAllocationTests, measured without the tests' shuffle of each result's order: the core's axes 8 to 17 times lower (hashed integers 47.8 -> 3.9 MB, spread integers 29.3 -> 2.8, six keys 113.6 -> 6.7), a lane alone's a quarter to two fifths lower, the rest its table, which is the result. --- .../Aggregation/AggregationEngine.cs | 50 +++++-- src/Vorticity/Aggregation/ArrayShelf.cs | 134 ++++++++++++++---- src/Vorticity/Aggregation/GroupCore.cs | 13 +- src/Vorticity/Aggregation/GroupKeys.cs | 8 +- src/Vorticity/Aggregation/RawKeys.cs | 2 +- src/Vorticity/Aggregation/TupleKeys.cs | 2 +- .../Vorticity.Conformance/PoisonedShelves.cs | 15 ++ .../Aggregation/GroupAllocationTests.cs | 49 +++++-- tests/Vorticity.Tests/PoisonedShelves.cs | 15 ++ 9 files changed, 227 insertions(+), 61 deletions(-) create mode 100644 tests/Vorticity.Conformance/PoisonedShelves.cs create mode 100644 tests/Vorticity.Tests/PoisonedShelves.cs diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index b3841c2db..e1557718d 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -797,12 +797,16 @@ internal sealed class AggregationPartition private int _settledEnd; private ZoneSettling.Scratch? _settledScratch; + /// + /// A partition of 's groups: under , its arrays from a + /// lane's shelf, or from , a shelf of its own counted by no query. + /// internal AggregationPartition( AggregationPlan plan, AggregateSlot?[] settled, ColumnShape[] columns, int[] inputs, bool sorted, int streaming = -1, ScanSource? source = null, KeyFacts? facts = null, - GroupKeys? keys = null, QueryMemory? memory = null) + GroupKeys? keys = null, QueryMemory? memory = null, ArrayShelf? arrays = null) { Memory = memory; - _arrays = memory is null ? null : new ArrayShelf(memory); + _arrays = arrays ?? (memory is null ? null : new ArrayShelf(memory)); _turnAt = plan.CoreTurnAt; _plan = plan; _settledSlots = settled; @@ -1658,24 +1662,24 @@ private void Foretell() internal long Footprint => (Keys?.Footprint ?? 0) + (_componentKeys?.Footprint ?? 0) + AggregateSlot.FootprintOf(Slots); /// - /// Room for the group of each of rows, the one held before given back: rented - /// cleared, as a new array starts, since a row a selection leaves out keeps what the array held, and - /// another query's groups would be past this one's. + /// Room for the group of each of rows, the one held before given back: taken + /// from the process's shelf a power of two long, cleared, as a new array starts, since a row a + /// selection leaves out keeps what the array held, and another query's groups would be past this + /// one's. The shared array pool kept an array for each thread that gave one back, which the lane of + /// the next query, on another thread, did not find: half a megabyte a query at fourteen lanes. /// private void RentRowGroups(int rows) { GiveBackRowGroups(); - int[] rented = ArrayPool.Shared.Rent(rows); - Array.Clear(rented); - _rowGroups = rented; + _rowGroups = ArrayShelf.Retained.Take((int)BitOperations.RoundUpToPowerOf2((uint)Math.Max(rows, 16)), zeroed: true); } - /// The groups of the rows given back to the shared pool, once nothing folds into the partition. + /// The groups of the rows given back to the process's shelf, once nothing folds into the partition. private void GiveBackRowGroups() { if (_rowGroups.Length > 0) { - ArrayPool.Shared.Return(_rowGroups); + ArrayShelf.Retained.Give(_rowGroups); _rowGroups = []; } } @@ -1690,6 +1694,7 @@ private void GiveBackRowGroups() /// internal void Release() { + ReturnArrays(); Keys = null; Slots = []; Records = null; @@ -1702,16 +1707,34 @@ internal void Release() } /// - /// Lets the partition's tables go and gives back what it held of its query's memory, the arrays - /// left to the next collection: a range a stream followed into the - /// partition before it, whose groups now live there. + /// Lets the partition's tables go and gives back what it held of its query's memory: a range a + /// stream followed into the partition before it, whose groups now live there. /// internal void LetGo() { + ReturnArrays(); GiveBack(); Release(); } + /// + /// The arrays of the partition's keys and records back to the shelf they came from, a lane's, which + /// hands them to the process's for the next table of their length, this query's or the next one's: + /// a lane's tables were left to the next collection, every query growing its own again. Before what + /// the partition held of its query's memory is given back, the shelf counting each out; its tables + /// are empty after. + /// + private void ReturnArrays() + { + Keys?.Release(); + Records?.Release(); + + // The partition the lane bypassed its cache with, and a shelf of the partition's own, which hands + // what it holds to the process's; a lane's holds nothing. + _core?.Rows?.Release(); + _arrays?.Clear(); + } + /// What the partition's tables held of its query's memory given back, the tables themselves left to the next collection. private void GiveBack() { @@ -1767,6 +1790,7 @@ internal void TurnTo(GroupCore core) } int[]? numbers = null; cache.MergeFrom(this, ref numbers, apart: null); + ReturnArrays(); GiveBack(); lane.Turning = false; Keys = cache.Keys; diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index a71568538..f733d6633 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; using Vorticity.Buffers; @@ -43,6 +44,9 @@ internal sealed class ArrayShelf : ISweptAfterCollections private long _held; private bool _used; + // Counts the takes and gives under the lock: when each pile was last used. + private long _clock; + // A shelf under a query's memory; for a lane's, the bytes of the arrays it handed out that its // tables hold, and what it reserved ahead of the next ones. private readonly QueryMemory? _memory; @@ -66,11 +70,12 @@ internal ArrayShelf(QueryMemory memory, bool pooled) /// /// A lane's shelf under its query's memory: an array it - /// hands out is reserved before it is allocated, and given back when its table lets it go, which - /// leaves it to the next collection; it keeps none. A table that doubles reserves its new arrays + /// hands out is reserved before it is taken, from the process's shelf or new, and given back when + /// its table lets it go, to the process's shelf, which keeps it for the next table of its length, + /// this query's or the next one's; it keeps none itself. A table that doubles reserves its new arrays /// while it still holds the old ones: what the copy holds, no more. It reserves /// at a time, and keeps up to twice that of what its tables give back; one lane uses it at a time, - /// and it takes no lock. + /// and it takes no lock but the process's shelf's. /// internal ArrayShelf(QueryMemory memory) : this(parent: null, budget: 0) => _memory = memory; @@ -216,17 +221,21 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) Handed++; HandedBytes += bytes; - return zeroed ? new T[length] : GC.AllocateUninitializedArray(length); + return Retained.Take(length, zeroed); } Array? found = null; lock (_gate) { _used = true; - if (_piles.TryGetValue((typeof(T), length), out Pile? pile) && pile.Arrays.Count > 0) + if (_piles.TryGetValue((typeof(T), length), out Pile? pile)) { - found = pile.Arrays.Pop(); - _held -= (long)length * pile.ElementBytes; + pile.Used = ++_clock; + if (pile.Arrays.Count > 0) + { + found = pile.Arrays.Pop(); + _held -= (long)length * pile.ElementBytes; + } } } @@ -263,7 +272,7 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) /// /// Puts an array nothing holds any more back on the shelf; past the shelf's budget, lets it go. - /// Under a query's memory, its bytes given back, and left to the next collection. + /// A lane's shelf gives its bytes back to the query's memory and the array to the process's shelf. /// internal void Give(T[] array) { @@ -274,9 +283,15 @@ internal void Give(T[] array) Unreserve(_memory!, bytes); } + Retained.Give(array); return; } + if (PoisonsGiven) + { + Poison(array); + } + // Let go past the shelf's budget, or while the shelf lets go of what it is given: it leaves the query's count. if (Drops || !Give(typeof(T), array, Unsafe.SizeOf())) { @@ -284,6 +299,24 @@ internal void Give(T[] array) } } + /// + /// Whether an array given back is filled with a pattern no table writes before it is kept: a table + /// that reads an array after giving it reads garbage at once, rather than another table's groups + /// once a shelf lends it again. The tests' switch. + /// + internal static bool PoisonsGiven { get; set; } + + private static void Poison(T[] array) + { + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + Array.Clear(array); + return; + } + + MemoryMarshal.CreateSpan(ref Unsafe.As(ref MemoryMarshal.GetArrayDataReference(array)), array.Length * Unsafe.SizeOf()).Fill(0xDB); + } + /// /// Whether a query's shelf lets go of what it is given rather than keep it for the next to ask: /// the core delivering its parts one after the other once it spilled, each given back once @@ -417,35 +450,35 @@ private void Unreserve(QueryMemory memory, long bytes) /// internal void Clear() { - List<(Type Type, Pile Pile)> piles = []; + // Each leaves the query's count, kept by the process's shelf or let go: handed under this shelf's + // lock, then the process's, in that order only, with no list of them made first. lock (_gate) { - foreach (((Type type, int _), Pile pile) in _piles) + if (_parent is not null) { - piles.Add((type, pile)); + foreach (KeyValuePair<(Type Type, int Length), Pile> entry in _piles) + { + Pile pile = entry.Value; + while (pile.Arrays.Count > 0) + { + Array array = pile.Arrays.Pop(); + _parent.Give(entry.Key.Type, array, pile.ElementBytes); + Leave((long)array.Length * pile.ElementBytes); + } + } } _piles.Clear(); _held = 0; } - - if (_parent is null) - { - return; - } - - // Each leaves the query's count, kept by the process's shelf or let go. - foreach ((Type type, Pile pile) in piles) - { - foreach (Array array in pile.Arrays) - { - _parent.Give(type, array, pile.ElementBytes); - Leave((long)array.Length * pile.ElementBytes); - } - } } - /// An array of elements of , each, on the shelf, within its budget: whether it kept it. + /// + /// An array of elements of , each, on the shelf, + /// within its budget: whether it kept it. Past the budget, the arrays of the lengths taken or given + /// least recently go first: a shelf full of another query's lengths kept them while every query + /// went on taking from it, and the arrays of the queries running now could not come in. + /// private bool Give(Type type, Array array, int elementBytes) { int length = array.Length; @@ -457,7 +490,7 @@ private bool Give(Type type, Array array, int elementBytes) long bytes = (long)length * elementBytes; lock (_gate) { - if (_held + bytes > _budget) + if (_held + bytes > _budget && (bytes > _budget || !Evict(_held + bytes - _budget, (type, length)))) { return false; } @@ -469,11 +502,53 @@ private bool Give(Type type, Array array, int elementBytes) } pile.Arrays.Push(array); + pile.Used = ++_clock; _held += bytes; return true; } } + /// + /// Lets go of at least of the arrays on the piles taken from or given to least + /// recently, never 's, under the lock: whether it could. A pile emptied goes, + /// so that the lengths a process once met do not pile up. + /// + private bool Evict(long bytes, (Type Type, int Length) kept) + { + while (bytes > 0) + { + Pile? oldest = null; + (Type Type, int Length) key = default; + foreach (KeyValuePair<(Type Type, int Length), Pile> entry in _piles) + { + if (entry.Value.Arrays.Count > 0 && entry.Key != kept && (oldest is null || entry.Value.Used < oldest.Used)) + { + oldest = entry.Value; + key = entry.Key; + } + } + + if (oldest is null) + { + return false; + } + + while (bytes > 0 && oldest.Arrays.Count > 0) + { + long freed = (long)oldest.Arrays.Pop().Length * oldest.ElementBytes; + _held -= freed; + bytes -= freed; + } + + if (oldest.Arrays.Count == 0) + { + _piles.Remove(key); + } + } + + return true; + } + /// After a collection: the process's shelf emptied when no query took from it since the last sweep, or under a high memory load. void ISweptAfterCollections.Sweep() { @@ -497,5 +572,8 @@ private sealed class Pile(int elementBytes) internal Stack Arrays { get; } = new Stack(); internal int ElementBytes { get; } = elementBytes; + + /// When an array was last taken from the pile or given to it, on the shelf's clock: the piles past the budget let go first. + internal long Used { get; set; } } } diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index a499715a7..788aa6dbd 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -448,9 +448,16 @@ private static bool Holds( internal AggregationPartition Cache(QueryMemory? memory) => new AggregationPartition(_plan, _settled, _columns, _inputs, sorted: false, source: _source, facts: _facts, memory: memory); - /// A partition that folds each row of a batch into a group of its own, which a lane bypassing its cache flushes after each batch. - internal AggregationPartition RowPartition() => - new AggregationPartition(_plan, _settled, _columns, _inputs, sorted: false, source: _source, keys: Kind.Appending()); + /// + /// A partition that folds each row of a batch into a group of its own, which a lane bypassing its cache + /// flushes after each batch. Counted by no query's memory, as it always was; its arrays from a shelf + /// of its own, which takes them from the process's and hands them back when the partition is let go. + /// + internal AggregationPartition RowPartition() + { + ArrayShelf arrays = new ArrayShelf(); + return new AggregationPartition(_plan, _settled, _columns, _inputs, sorted: false, source: _source, keys: Kind.Appending(arrays), arrays: arrays); + } /// Counts rows a lane folded apart from its cache. internal void Bypassed(int rows) => Interlocked.Add(ref _bypassed, rows); diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index aad6994df..d5a5c3932 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -222,10 +222,10 @@ internal virtual void Reserve(int groups) /// /// An index of the same kind that looks no key up: each row's key a group of its own, but a row /// whose key is the row before's, a run's or a code's; the null group one. What a lane folds its - /// rows into when its cache finds too few of its keys, the bypass. - /// Null for a key that does not travel in batches. + /// rows into when its cache finds too few of its keys, the bypass; its arrays from + /// . Null for a key that does not travel in batches. /// - internal virtual GroupKeys? Appending() => null; + internal virtual GroupKeys? Appending(ArrayShelf? shelf) => null; /// /// Copies every group but the null one into an entry of the lane's batch of the part its key falls @@ -2066,7 +2066,7 @@ internal override long GrowthFor(int more) return bytes + (Math.Min(more, left) << PageBits) * sizeof(int); } - internal override GroupKeys? Appending() => new FixedKeys(_shape, sorted: false, appending: true); + internal override GroupKeys? Appending(ArrayShelf? shelf) => new FixedKeys(_shape, sorted: false, appending: true, shelf: shelf); internal override void Scatter(ReadOnlySpan records, LaneCore lane) => EntryKeys.Scatter(_keys.AsSpan(0, Count), _null, records, lane); diff --git a/src/Vorticity/Aggregation/RawKeys.cs b/src/Vorticity/Aggregation/RawKeys.cs index 13e593581..00b9cd261 100644 --- a/src/Vorticity/Aggregation/RawKeys.cs +++ b/src/Vorticity/Aggregation/RawKeys.cs @@ -326,7 +326,7 @@ internal override void ReadKeys(ref SpillReader reader, Span groups) /// The word: a tuple's nulls are bits of it, so no group is apart from the others. internal override int EntryBytes => Unsafe.SizeOf(); - internal override GroupKeys? Appending() => new RawKeys(_layout, appending: true); + internal override GroupKeys? Appending(ArrayShelf? shelf) => new RawKeys(_layout, appending: true, shelf: shelf); internal override void Scatter(ReadOnlySpan records, LaneCore lane) => EntryKeys.Scatter(_keys.AsSpan(0, Count), -1, records, lane); diff --git a/src/Vorticity/Aggregation/TupleKeys.cs b/src/Vorticity/Aggregation/TupleKeys.cs index 437449b98..64fd527c9 100644 --- a/src/Vorticity/Aggregation/TupleKeys.cs +++ b/src/Vorticity/Aggregation/TupleKeys.cs @@ -484,7 +484,7 @@ private int Compare(ref byte left, ref byte right, int component) internal override GroupKeys ForSpill(ArrayShelf? shelf) => Wrap(_tuples.ForSpill(shelf)); - internal override GroupKeys? Appending() => _tuples.Appending() is { } appending ? Wrap(appending) : null; + internal override GroupKeys? Appending(ArrayShelf? shelf) => _tuples.Appending(shelf) is { } appending ? Wrap(appending) : null; internal override long Footprint => _tuples.Footprint + ((long)_block.Length * TupleLayout.Bytes) + ((long)_words.Length * TupleLayout.WordBytes); diff --git a/tests/Vorticity.Conformance/PoisonedShelves.cs b/tests/Vorticity.Conformance/PoisonedShelves.cs new file mode 100644 index 000000000..d0cee038d --- /dev/null +++ b/tests/Vorticity.Conformance/PoisonedShelves.cs @@ -0,0 +1,15 @@ +using System.Runtime.CompilerServices; +using Vorticity.Aggregating; + +namespace Vorticity.Conformance; + +/// +/// Every array a group by gives back to a shelf is poisoned throughout these tests: the shelves lend +/// them again, to another table of the query or of the next one, so a table that still read one after +/// giving it would read another's groups in production; here it reads garbage, and its test fails. +/// +internal static class PoisonedShelves +{ + [ModuleInitializer] + internal static void Poison() => ArrayShelf.PoisonsGiven = true; +} diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index 91229ffc3..d03801205 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -52,34 +52,45 @@ public sealed partial class GroupAllocationTests /// query's bytes on every thread. /// /// + /// /// At one lane a ceiling is the floor rounded up to 512 bytes; at four, the lanes' schedule moves /// the core's splits and batches by a few hundred kilobytes from one run to the next, so a ceiling - /// is the floor and 2 % (1 % for six keys at one lane), rounded up to 64 KiB. + /// is the floor and 2 %, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too). + /// + /// + /// Measured without the shuffle of every result's order the tests make: 3 to 4 MB less on the + /// core's axes. A lane's arrays come from the process's shelf and go back to it, as they grow and + /// when its tables die, its cache's and the partition it bypasses it with, where they were left to + /// the next collection: the core's axes 8 to 17 times lower; a lane alone's a quarter to two fifths + /// lower, the rest its table, which is the result; and the group of each row is no longer rented + /// from the shared array pool, whose array kept by one thread the lane of the next query, on + /// another, did not find. + /// /// private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceiling, Func> Query)[] Axes = [ // A thousand integers numbered by their value, a table a lane. - ("small integers", 1, GroupCoreReason.None, 360_448, SmallAsync), - ("small integers", 4, GroupCoreReason.None, 561_152, SmallAsync), + ("small integers", 1, GroupCoreReason.None, 355_840, SmallAsync), + ("small integers", 4, GroupCoreReason.None, 495_616, SmallAsync), // A thousand short texts, hashed as their words. - ("short texts", 1, GroupCoreReason.None, 521_728, NamesAsync), - ("short texts", 4, GroupCoreReason.None, 1_159_168, NamesAsync), + ("short texts", 1, GroupCoreReason.None, 434_176, NamesAsync), + ("short texts", 4, GroupCoreReason.None, 536_576, NamesAsync), // A long integer a row, hashed: at four lanes, a lane's first rows are all new groups. - ("hashed integers", 1, GroupCoreReason.None, 82_157_056, WideAsync), + ("hashed integers", 1, GroupCoreReason.None, 58_998_272, WideAsync), // // On this axis and the core's two others, each part is built in an order its shelf lends: 3 to // 4 MB less, four bytes a group. - ("hashed integers", 4, GroupCoreReason.FirstRows, 48_758_784, WideAsync), + ("hashed integers", 4, GroupCoreReason.FirstRows, 4_063_232, WideAsync), // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. - ("spread integers", 1, GroupCoreReason.None, 58_423_808, DenseAsync), - ("spread integers", 4, GroupCoreReason.Spread, 29_949_952, DenseAsync), + ("spread integers", 1, GroupCoreReason.None, 34_865_152, DenseAsync), + ("spread integers", 4, GroupCoreReason.Spread, 2_883_584, DenseAsync), // Six keys, three texts and three integers, nearly a group a row: tuples of their values. - ("six keys", 1, GroupCoreReason.None, 165_806_080, SixAsync), - ("six keys", 4, GroupCoreReason.FirstRows, 115_867_648, SixAsync), + ("six keys", 1, GroupCoreReason.None, 107_937_792, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 6_815_744, SixAsync), ]; [Fact] @@ -93,6 +104,22 @@ public async Task EveryWarmGroupByAllocatesWithinItsCeiling() .Append(WarmUp.ToString(CultureInfo.InvariantCulture)) .Append('\n'); + // The tests shuffle every result's order (ShuffledOrder), a copy of it a part, which a query + // outside them never makes: measured without, as alone in its collection nothing else runs. + bool shuffled = AggregationPlan.ShuffledOrder; + AggregationPlan.ShuffledOrder = false; + try + { + await MeasureAsync(path, report); + } + finally + { + AggregationPlan.ShuffledOrder = shuffled; + } + } + + private static async Task MeasureAsync(string path, StringBuilder report) + { List failures = []; foreach ((string axis, int lanes, GroupCoreReason expected, long ceiling, Func> query) in Axes) { diff --git a/tests/Vorticity.Tests/PoisonedShelves.cs b/tests/Vorticity.Tests/PoisonedShelves.cs new file mode 100644 index 000000000..434154ac9 --- /dev/null +++ b/tests/Vorticity.Tests/PoisonedShelves.cs @@ -0,0 +1,15 @@ +using System.Runtime.CompilerServices; +using Vorticity.Aggregating; + +namespace Vorticity.Tests; + +/// +/// Every array a group by gives back to a shelf is poisoned throughout these tests: the shelves lend +/// them again, to another table of the query or of the next one, so a table that still read one after +/// giving it would read another's groups in production; here it reads garbage, and its test fails. +/// +internal static class PoisonedShelves +{ + [ModuleInitializer] + internal static void Poison() => ArrayShelf.PoisonsGiven = true; +} From ea927ee57d63c21711c0afb0054178680db2968e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 00:10:42 +0200 Subject: [PATCH 32/56] perf(query): the process's shelf finds its oldest pile without a scan Past its budget the process's shelf let go of the arrays of the length taken or given least recently, found by reading every pile, under its lock, at every array given back: on the strided key of 10^7 values and on db-benchmark's q10 at fourteen lanes the shelf is full, and each give read every length the query met. The piles are now linked from the one used most recently to the one least recently, moved to the front as an array is taken or given: the oldest is the last. GroupAllocationTests: a ceiling of the core's axes is the floor and 5 % (2 % was under the floor the full suite measured, the process's shelf holding what the suite's other tests left). --- src/Vorticity/Aggregation/ArrayShelf.cs | 128 ++++++++++++------ .../Aggregation/GroupAllocationTests.cs | 11 +- 2 files changed, 95 insertions(+), 44 deletions(-) diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index f733d6633..c83062fcd 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -44,8 +44,10 @@ internal sealed class ArrayShelf : ISweptAfterCollections private long _held; private bool _used; - // Counts the takes and gives under the lock: when each pile was last used. - private long _clock; + // The piles from the one taken from or given to most recently to the one least recently, linked: + // past its budget, the shelf lets go of the oldest first. + private Pile? _newest; + private Pile? _oldest; // A shelf under a query's memory; for a lane's, the bytes of the arrays it handed out that its // tables hold, and what it reserved ahead of the next ones. @@ -230,7 +232,7 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) _used = true; if (_piles.TryGetValue((typeof(T), length), out Pile? pile)) { - pile.Used = ++_clock; + Touch(pile); if (pile.Arrays.Count > 0) { found = pile.Arrays.Pop(); @@ -468,8 +470,7 @@ internal void Clear() } } - _piles.Clear(); - _held = 0; + Empty(); } } @@ -490,19 +491,19 @@ private bool Give(Type type, Array array, int elementBytes) long bytes = (long)length * elementBytes; lock (_gate) { - if (_held + bytes > _budget && (bytes > _budget || !Evict(_held + bytes - _budget, (type, length)))) + if (!_piles.TryGetValue((type, length), out Pile? pile)) { - return false; + pile = new Pile((type, length), elementBytes); + _piles.Add(pile.Key, pile); } - if (!_piles.TryGetValue((type, length), out Pile? pile)) + Touch(pile); + if (_held + bytes > _budget && (bytes > _budget || !Evict(_held + bytes - _budget, pile))) { - pile = new Pile(elementBytes); - _piles.Add((type, length), pile); + return false; } pile.Arrays.Push(array); - pile.Used = ++_clock; _held += bytes; return true; } @@ -513,40 +514,85 @@ private bool Give(Type type, Array array, int elementBytes) /// recently, never 's, under the lock: whether it could. A pile emptied goes, /// so that the lengths a process once met do not pile up. /// - private bool Evict(long bytes, (Type Type, int Length) kept) + private bool Evict(long bytes, Pile kept) { - while (bytes > 0) + Pile? pile = _oldest; + while (bytes > 0 && pile is not null) { - Pile? oldest = null; - (Type Type, int Length) key = default; - foreach (KeyValuePair<(Type Type, int Length), Pile> entry in _piles) + Pile? newer = pile.Newer; + if (pile != kept) { - if (entry.Value.Arrays.Count > 0 && entry.Key != kept && (oldest is null || entry.Value.Used < oldest.Used)) + while (bytes > 0 && pile.Arrays.Count > 0) { - oldest = entry.Value; - key = entry.Key; + long freed = (long)pile.Arrays.Pop().Length * pile.ElementBytes; + _held -= freed; + bytes -= freed; } - } - if (oldest is null) - { - return false; + if (pile.Arrays.Count == 0) + { + Unlink(pile); + _piles.Remove(pile.Key); + } } - while (bytes > 0 && oldest.Arrays.Count > 0) - { - long freed = (long)oldest.Arrays.Pop().Length * oldest.ElementBytes; - _held -= freed; - bytes -= freed; - } + pile = newer; + } - if (oldest.Arrays.Count == 0) - { - _piles.Remove(key); - } + return bytes <= 0; + } + + /// Makes the one used most recently, under the lock. + private void Touch(Pile pile) + { + if (_newest == pile) + { + return; + } + + Unlink(pile); + pile.Older = _newest; + if (_newest is not null) + { + _newest.Newer = pile; + } + + _newest = pile; + _oldest ??= pile; + } + + /// Takes out of the piles' order, under the lock; nothing for one not in it. + private void Unlink(Pile pile) + { + if (pile.Newer is { } newer) + { + newer.Older = pile.Older; + } + else if (_newest == pile) + { + _newest = pile.Older; } - return true; + if (pile.Older is { } older) + { + older.Newer = pile.Newer; + } + else if (_oldest == pile) + { + _oldest = pile.Newer; + } + + pile.Newer = null; + pile.Older = null; + } + + /// Every pile gone, under the lock. + private void Empty() + { + _piles.Clear(); + _newest = null; + _oldest = null; + _held = 0; } /// After a collection: the process's shelf emptied when no query took from it since the last sweep, or under a high memory load. @@ -558,8 +604,7 @@ void ISweptAfterCollections.Sweep() { if (!_used || pressed) { - _piles.Clear(); - _held = 0; + Empty(); } _used = false; @@ -567,13 +612,18 @@ void ISweptAfterCollections.Sweep() } /// The arrays of one type and length, and the bytes of an element. - private sealed class Pile(int elementBytes) + private sealed class Pile((Type Type, int Length) key, int elementBytes) { + internal (Type Type, int Length) Key { get; } = key; + internal Stack Arrays { get; } = new Stack(); internal int ElementBytes { get; } = elementBytes; - /// When an array was last taken from the pile or given to it, on the shelf's clock: the piles past the budget let go first. - internal long Used { get; set; } + /// The pile used next more recently, null for the newest. + internal Pile? Newer { get; set; } + + /// The pile used next less recently, null for the oldest. + internal Pile? Older { get; set; } } } diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index d03801205..905c0ec2b 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -54,8 +54,9 @@ public sealed partial class GroupAllocationTests /// /// /// At one lane a ceiling is the floor rounded up to 512 bytes; at four, the lanes' schedule moves - /// the core's splits and batches by a few hundred kilobytes from one run to the next, so a ceiling - /// is the floor and 2 %, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too). + /// the core's splits and batches by a few hundred kilobytes from one run to the next, and the + /// process's shelf holds what the suite's other tests left, so a ceiling of the core is the floor + /// and 5 %, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too). /// /// /// Measured without the shuffle of every result's order the tests make: 3 to 4 MB less on the @@ -82,15 +83,15 @@ private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceil // // On this axis and the core's two others, each part is built in an order its shelf lends: 3 to // 4 MB less, four bytes a group. - ("hashed integers", 4, GroupCoreReason.FirstRows, 4_063_232, WideAsync), + ("hashed integers", 4, GroupCoreReason.FirstRows, 4_194_304, WideAsync), // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. ("spread integers", 1, GroupCoreReason.None, 34_865_152, DenseAsync), - ("spread integers", 4, GroupCoreReason.Spread, 2_883_584, DenseAsync), + ("spread integers", 4, GroupCoreReason.Spread, 2_949_120, DenseAsync), // Six keys, three texts and three integers, nearly a group a row: tuples of their values. ("six keys", 1, GroupCoreReason.None, 107_937_792, SixAsync), - ("six keys", 4, GroupCoreReason.FirstRows, 6_815_744, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 7_208_960, SixAsync), ]; [Fact] From 2e1e4f6a1d9d1be1a1a05f704e6539ab3e6f4ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 00:26:52 +0200 Subject: [PATCH 33/56] perf(query): the process's shelf keeps its small arrays apart Past its budget the process's shelf let go of the arrays used least recently whatever their size: on the strided key of 10^7 values over 4M rows at fourteen lanes, whose arrays the quarter of a gigabyte does not hold, every query lost small arrays to the large ones and allocated them again on the small object heap, 18 MB a query, whose collections stop every lane and copy what lives there; a large array missed costs a page fault. The shelf now keeps the arrays under 85 000 bytes apart, under a quarter of its budget more, each kind in its own order, and lets go of the oldest of the kind it takes in. A pile emptied stays, its next array a push, until the shelf holds 4 096 of them: a pile let go each query was made again at the next, its stack with it. Runners alternated against e379105b, warm, at fourteen lanes, the machine busy: the strided key on 4M x0.899 (139 MB a query, 1.2 collections against 5.6 at 3e0d7957, 2.2 ms stopped against 6.0), k7 on 4M x0.960 (78 MB, 0.8 collections), q10 on 4M x1.028, whose arrays the shelf holds a third of. --- src/Vorticity/Aggregation/ArrayShelf.cs | 134 ++++++++++++++++-------- 1 file changed, 90 insertions(+), 44 deletions(-) diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index c83062fcd..cfa24c401 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -37,17 +37,29 @@ internal sealed class ArrayShelf : ISweptAfterCollections /// private const long Ahead = 256 * 1024; + /// The bytes from which an array lies on the large object heap. + private const long LargeBytes = 85_000; + + /// The piles a shelf keeps once emptied: past them, a pile its budget empties goes. + private const int MostPiles = 4096; + private readonly Lock _gate = new Lock(); private readonly Dictionary<(Type Type, int Length), Pile> _piles = []; private readonly ArrayShelf? _parent; + + // What the shelf keeps of large arrays, and apart of small ones: a small array missed is allocated on + // the small object heap, whose collections stop every lane and copy what lives there; a large one, + // a page fault. The bytes it holds, and of them the small arrays'. private readonly long _budget; + private readonly long _smallBudget; private long _held; + private long _heldSmall; private bool _used; - // The piles from the one taken from or given to most recently to the one least recently, linked: - // past its budget, the shelf lets go of the oldest first. - private Pile? _newest; - private Pile? _oldest; + // The piles of each kind from the one taken from or given to most recently to the one least + // recently, linked: past its budget, the shelf lets go of the oldest first. + private PileOrder _large; + private PileOrder _small; // A shelf under a query's memory; for a lane's, the bytes of the arrays it handed out that its // tables hold, and what it reserved ahead of the next ones. @@ -57,7 +69,7 @@ internal sealed class ArrayShelf : ISweptAfterCollections /// A query's shelf, which takes from the process's when it holds nothing of a length, and hands it what it holds at the end. internal ArrayShelf() - : this(Retained, long.MaxValue) + : this(Retained, long.MaxValue, long.MaxValue) { } @@ -68,7 +80,7 @@ internal ArrayShelf() /// exact, each taken under no lock of its own: the lanes that apply bursts meet on it. /// internal ArrayShelf(QueryMemory memory, bool pooled) - : this(Retained, pooled ? long.MaxValue : 0) => _memory = memory; + : this(Retained, pooled ? long.MaxValue : 0, pooled ? long.MaxValue : 0) => _memory = memory; /// /// A lane's shelf under its query's memory: an array it @@ -80,7 +92,7 @@ internal ArrayShelf(QueryMemory memory, bool pooled) /// and it takes no lock but the process's shelf's. /// internal ArrayShelf(QueryMemory memory) - : this(parent: null, budget: 0) => _memory = memory; + : this(parent: null, budget: 0, smallBudget: 0) => _memory = memory; /// Whether the shelf is a lane's: under a query's memory, with no pile and no process's shelf behind it. private bool Lane => _memory is not null && _parent is null; @@ -88,23 +100,26 @@ internal ArrayShelf(QueryMemory memory) /// The query's memory the shelf counts its arrays under: its run's; null for a shelf that counts nothing. internal QueryMemory? Memory => _memory; - private ArrayShelf(ArrayShelf? parent, long budget) + private ArrayShelf(ArrayShelf? parent, long budget, long smallBudget) { _parent = parent; _budget = budget; + _smallBudget = smallBudget; } /// - /// The process's shelf: what queries left, up to a quarter of a gigabyte or a sixteenth of the - /// memory the process may use, whichever is less, for the next query not to touch fresh memory - /// again. Swept after collections: emptied when no query took from it since the last, or when the - /// machine's memory load is high. + /// The process's shelf: what queries left, for the next query not to touch fresh memory again: of + /// large arrays, up to a quarter of a gigabyte or a sixteenth of the memory the process may use, + /// whichever is less; of small ones, a quarter of that again, apart, so that a query whose large + /// arrays its budget does not hold still finds its small ones. Swept after collections: emptied when + /// no query took from it since the last, or when the machine's memory load is high. /// internal static ArrayShelf Retained { get; } = NewRetained(); private static ArrayShelf NewRetained() { - ArrayShelf shelf = new ArrayShelf(parent: null, Math.Min(256L << 20, GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 16)); + long budget = Math.Min(256L << 20, GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 16); + ArrayShelf shelf = new ArrayShelf(parent: null, budget, budget / 4); CollectionSweeper.Register(shelf); return shelf; } @@ -136,8 +151,8 @@ internal long Held /// The bytes a lane's tables copied from an array into the one that replaced it (). internal long CopiedBytes { get; private set; } - /// The most bytes the shelf keeps on its piles; past it, what it is given goes. - internal long Budget => _budget; + /// The most bytes the shelf keeps on its piles, its large arrays' and its small ones'; past it, what it is given goes. + internal long Budget => _budget > long.MaxValue - _smallBudget ? long.MaxValue : _budget + _smallBudget; /// /// Whether a lane's shelf takes an array its budget refuses all the same, counted past the ceiling: @@ -236,7 +251,7 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) if (pile.Arrays.Count > 0) { found = pile.Arrays.Pop(); - _held -= (long)length * pile.ElementBytes; + Hold(pile, -pile.Bytes); } } } @@ -488,7 +503,6 @@ private bool Give(Type type, Array array, int elementBytes) return true; } - long bytes = (long)length * elementBytes; lock (_gate) { if (!_piles.TryGetValue((type, length), out Pile? pile)) @@ -498,25 +512,39 @@ private bool Give(Type type, Array array, int elementBytes) } Touch(pile); - if (_held + bytes > _budget && (bytes > _budget || !Evict(_held + bytes - _budget, pile))) + long budget = pile.Small ? _smallBudget : _budget; + long held = pile.Small ? _heldSmall : _held - _heldSmall; + if (held + pile.Bytes > budget && (pile.Bytes > budget || !Evict(held + pile.Bytes - budget, pile))) { return false; } pile.Arrays.Push(array); - _held += bytes; + Hold(pile, pile.Bytes); return true; } } + /// The bytes the shelf holds moved by , an array of 's taken or given. + private void Hold(Pile pile, long bytes) + { + _held += bytes; + if (pile.Small) + { + _heldSmall += bytes; + } + } + /// - /// Lets go of at least of the arrays on the piles taken from or given to least - /// recently, never 's, under the lock: whether it could. A pile emptied goes, - /// so that the lengths a process once met do not pile up. + /// Lets go of at least of the arrays on the piles of 's + /// kind taken from or given to least recently, never 's, under the lock: + /// whether it could. Past , a pile emptied goes, so that the lengths a process + /// once met do not pile up; short of them it stays, its next array a push. /// private bool Evict(long bytes, Pile kept) { - Pile? pile = _oldest; + ref PileOrder order = ref OrderOf(kept); + Pile? pile = order.Oldest; while (bytes > 0 && pile is not null) { Pile? newer = pile.Newer; @@ -524,14 +552,14 @@ private bool Evict(long bytes, Pile kept) { while (bytes > 0 && pile.Arrays.Count > 0) { - long freed = (long)pile.Arrays.Pop().Length * pile.ElementBytes; - _held -= freed; - bytes -= freed; + pile.Arrays.Pop(); + Hold(pile, -pile.Bytes); + bytes -= pile.Bytes; } - if (pile.Arrays.Count == 0) + if (pile.Arrays.Count == 0 && _piles.Count > MostPiles) { - Unlink(pile); + Unlink(ref order, pile); _piles.Remove(pile.Key); } } @@ -542,44 +570,48 @@ private bool Evict(long bytes, Pile kept) return bytes <= 0; } - /// Makes the one used most recently, under the lock. + /// The order of the piles of 's kind. + private ref PileOrder OrderOf(Pile pile) => ref pile.Small ? ref _small : ref _large; + + /// Makes the one of its kind used most recently, under the lock. private void Touch(Pile pile) { - if (_newest == pile) + ref PileOrder order = ref OrderOf(pile); + if (order.Newest == pile) { return; } - Unlink(pile); - pile.Older = _newest; - if (_newest is not null) + Unlink(ref order, pile); + pile.Older = order.Newest; + if (order.Newest is not null) { - _newest.Newer = pile; + order.Newest.Newer = pile; } - _newest = pile; - _oldest ??= pile; + order.Newest = pile; + order.Oldest ??= pile; } - /// Takes out of the piles' order, under the lock; nothing for one not in it. - private void Unlink(Pile pile) + /// Takes out of , under the lock; nothing for one not in it. + private static void Unlink(ref PileOrder order, Pile pile) { if (pile.Newer is { } newer) { newer.Older = pile.Older; } - else if (_newest == pile) + else if (order.Newest == pile) { - _newest = pile.Older; + order.Newest = pile.Older; } if (pile.Older is { } older) { older.Newer = pile.Newer; } - else if (_oldest == pile) + else if (order.Oldest == pile) { - _oldest = pile.Newer; + order.Oldest = pile.Newer; } pile.Newer = null; @@ -590,9 +622,10 @@ private void Unlink(Pile pile) private void Empty() { _piles.Clear(); - _newest = null; - _oldest = null; + _large = default; + _small = default; _held = 0; + _heldSmall = 0; } /// After a collection: the process's shelf emptied when no query took from it since the last sweep, or under a high memory load. @@ -611,6 +644,13 @@ void ISweptAfterCollections.Sweep() } } + /// The newest and the oldest pile of a kind, linked through them. + private struct PileOrder + { + internal Pile? Newest; + internal Pile? Oldest; + } + /// The arrays of one type and length, and the bytes of an element. private sealed class Pile((Type Type, int Length) key, int elementBytes) { @@ -620,6 +660,12 @@ private sealed class Pile((Type Type, int Length) key, int elementBytes) internal int ElementBytes { get; } = elementBytes; + /// The bytes of each of its arrays. + internal long Bytes { get; } = (long)key.Length * elementBytes; + + /// Whether its arrays lie on the small object heap. + internal bool Small => Bytes < LargeBytes; + /// The pile used next more recently, null for the newest. internal Pile? Newer { get; set; } From 9f07ddd668f6851b0e5f772ce569843012124074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 00:33:16 +0200 Subject: [PATCH 34/56] perf(query): a lane that turns with a small table empties it as a cache A lane turning to the core emptied its table into a batch of its own on every part, exactly its part's share: given back as the part applies it, where batches the lanes fill again stay held to the end, which is what a lane turned under pressure needs. A lane that turns on its first rows, judged on a sample of 8 192, has a table of a few thousand groups, and made 256 arrays of a few entries each, of lengths no other batch takes again: 3 690 of them a query on the strided key of 10^7 values over 4M rows at fourteen lanes, with their batches. A table no larger than a cache now goes into the lane's open batches, as its cache's flushes do. The runner at fourteen lanes, warm: the strided key on 4M 133 MB a query against 139, 1.0 collection against 1.2, 1.97 ms stopped against 2.24. GroupAllocationTests: hashed integers at four lanes 3.9 -> 3.0 MB. --- src/Vorticity/Aggregation/GroupCore.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 788aa6dbd..acfc6f24e 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -2017,9 +2017,19 @@ private PartBatch Take() /// turning to the core when its budget cannot let the table grow. Its groups counted /// by part first, each part's share goes into a batch of its own, exactly its size, deposited whole: /// given back as its part applies it, where batches the lanes fill again would stay held to the end. + /// A table no larger than a cache goes into the lane's open batches, as its cache's flushes do: the + /// table of a lane that turned on its first rows, judged on a sample, made a batch of its own on + /// every part, an array of its length that no other batch takes again, 3 600 of them a query at + /// fourteen lanes. /// internal void Empty(AggregationPartition table) { + if (table.Keys!.Count <= _core.Capacity) + { + Flush(table); + return; + } + Span counts = stackalloc int[GroupCore.PartCount]; counts.Clear(); table.Keys!.CountParts(counts, this); From 3d3b70e97052c016df5d8cafa765176fb42678b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 00:43:09 +0200 Subject: [PATCH 35/56] perf(query): a sub-table makes no array before its first group Each of a core's sub-tables made, as it was made, the first array of its keys, sixteen of them, outside any shelf, and laid its records out again, four objects the same for every sub-table of the core: on db-benchmark's q10 over 4M rows at fourteen lanes, 3 840 sub-tables, 3 885 arrays of a kilobyte of tuples on the small object heap, which live to the end of the query, so that each collection copied them. The keys' array is now made at the first group, from the shelf when the keys have one, and a core lays its records out once, for every sub-table (AggregationPartition.NewSlots takes the layout). The first array a lane's keys take now comes from its shelf, which counts it: WorkCounterTests counts one array more on each query whose keys are fixed, its sixteen keys' bytes with it. The runner at fourteen lanes, warm: q10 on 4M stopped 5.6 ms a query against 9.9, 3.4 collections against 4.6, 615 MB against 631; the strided key 1.5 ms against 2.0. --- .../Aggregation/AggregationEngine.cs | 15 ++++++++------ src/Vorticity/Aggregation/GroupCore.cs | 6 +++++- src/Vorticity/Aggregation/GroupKeys.cs | 11 ++++++++-- .../Aggregation/WorkCounterTests.cs | 20 +++++++++---------- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index e1557718d..4957e3424 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -2179,9 +2179,12 @@ internal static AggregateSlot[] NewSlots(AggregationPlan plan, AggregateSlot?[] /// /// A slot for each aggregate of the plan, the settled one or a new one, the new ones whose states - /// hold no reference sharing , a record a group. + /// hold no reference sharing , a record a group, laid out as + /// says when the caller knows it already: a core's, the same for each of + /// its sub-tables, whose four objects each made again. /// - internal static AggregateSlot[] NewSlots(AggregationPlan plan, AggregateSlot?[] settled, ScanSource? source, out GroupRecords? records, ArrayShelf? shelf = null) + internal static AggregateSlot[] NewSlots( + AggregationPlan plan, AggregateSlot?[] settled, ScanSource? source, out GroupRecords? records, ArrayShelf? shelf = null, RecordLayout? layout = null) { AggregateSlot[] slots = new AggregateSlot[settled.Length]; for (int i = 0; i < settled.Length; i++) @@ -2200,14 +2203,14 @@ internal static AggregateSlot[] NewSlots(AggregationPlan plan, AggregateSlot?[] } records = null; - if (RecordLayout.Of(slots) is { } layout) + if ((layout ?? RecordLayout.Of(slots)) is { } laid) { - records = new GroupRecords(layout, shelf); + records = new GroupRecords(laid, shelf); for (int i = 0; i < slots.Length; i++) { - if (layout.Offsets[i] >= 0) + if (laid.Offsets[i] >= 0) { - slots[i].Bind(records, layout.Offsets[i]); + slots[i].Bind(records, laid.Offsets[i]); } } } diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index acfc6f24e..e7ff00633 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -116,6 +116,9 @@ internal sealed partial class GroupCore private readonly AggregationPlan _plan; private readonly AggregateSlot?[] _settled; + + // How a record lays its states out, the same for every sub-table. + private readonly RecordLayout? _layout; private readonly ColumnShape[] _columns; private readonly int[] _inputs; private readonly ScanSource? _source; @@ -186,6 +189,7 @@ private GroupCore( Kind = kind; int keyBytes = kind.EntryBytes; Shape = EntryShape.Of(layout, keyBytes); + _layout = layout; long groupBytes = GroupBytes(keyBytes, (layout?.Stride ?? 0) * sizeof(ulong)); _groupBytes = groupBytes; int entryBytes = Shape.Words * sizeof(ulong); @@ -578,7 +582,7 @@ internal void GiveShared(PartBatch first, PartBatch last, int count) /// A sub-table of a part, empty, holding the groups whose hashes share bits past the part's. internal SubTable NewTable(int depth) { - AggregateSlot[] slots = AggregationPartition.NewSlots(_plan, _settled, _source, out GroupRecords? records, _shelf); + AggregateSlot[] slots = AggregationPartition.NewSlots(_plan, _settled, _source, out GroupRecords? records, _shelf, _layout); return new SubTable(Kind.ForTable(_shelf), slots, records, depth); } diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index d5a5c3932..da80b8cad 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -592,6 +592,9 @@ internal sealed class FixedKeys : GroupKeys /// The span of values a table of groups covers whatever the rows: 2^16, a quarter of a megabyte. internal const long DirectValues = 1 << 16; + /// The keys the keys' first array holds, made at the first group. + private const int FirstKeys = 16; + /// The values a row of the source a table of groups may span past : four, the budget of a numbering by value. internal const long DirectPerRow = 4; @@ -609,7 +612,11 @@ internal sealed class FixedKeys : GroupKeys private KeyTable _index = new KeyTable(); private WideKeyTable _wide = new WideKeyTable(); private bool _compact = CompactAlways; - private TValue[] _keys = new TValue[16]; + + // The keys, by group: none until the first group, then from the shelf when there is one. Each of a + // core's sub-tables made its first array of sixteen at once, outside any shelf: 3 885 arrays a query + // of six keys over 4M rows at fourteen lanes, 4 MB on the small object heap. + private TValue[] _keys = []; private int _null = -1; private TValue[] _values = []; private ValuesCache _entries; @@ -2362,7 +2369,7 @@ private int Add(TValue value) { if (Count == _keys.Length) { - Grow(DoubledUpTo(Count, _doublesUpTo)); + Grow(Count == 0 ? FirstKeys : DoubledUpTo(Count, _doublesUpTo)); } _keys[Count] = value; diff --git a/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs b/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs index 3a2dd9397..6b6dd08b9 100644 --- a/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs +++ b/tests/Vorticity.Tests/Aggregation/WorkCounterTests.cs @@ -30,18 +30,18 @@ public sealed partial class WorkCounterTests private static readonly string[] Expected = [ "count and sum by 10^3 values: result 1501672601, rows 300000, lanes 1, lane groups 1000, arrays 3 of 36824 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 1310720 B", - "count and sum by 10^5 values: result 1501672601, rows 300000, lanes 1, lane groups 95013, arrays 19 of 3653560 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 4177976 B", - "count and sum by 10^6 values: result 1501672601, rows 300000, lanes 1, lane groups 259343, arrays 35 of 12566512 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 12566640 B", - "count and sum by an ordered key: result 1501672601, rows 300000, lanes 1, lane groups 1, arrays 51 of 3720508 B, copied 0 B, blocks 4/0/0, merge 0 parts 0 entries, peak 3606448 B", - "count and sum by a strided long: result 1501672601, rows 300000, lanes 1, lane groups 259147, arrays 78 of 27797100 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 17680988 B", - "count and sum by 10^6 values, the core: result 1501672601, rows 300000, lanes 1, lane groups 37176, arrays 72 of 18380500 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, core: flushes 2 of 267855, bypassed 0, applied 267855 made 259343", - "distinct count by 10^5 values: result 299955, rows 300000, lanes 1, lane groups 95013, arrays 65 of 30391360 B, copied 8388352 B, blocks 0/0/4, merge 0 parts 0 entries, peak 19906560 B", + "count and sum by 10^5 values: result 1501672601, rows 300000, lanes 1, lane groups 95013, arrays 20 of 3653624 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 4177976 B", + "count and sum by 10^6 values: result 1501672601, rows 300000, lanes 1, lane groups 259343, arrays 36 of 12566576 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 12566640 B", + "count and sum by an ordered key: result 1501672601, rows 300000, lanes 1, lane groups 1, arrays 52 of 3720572 B, copied 0 B, blocks 4/0/0, merge 0 parts 0 entries, peak 3606448 B", + "count and sum by a strided long: result 1501672601, rows 300000, lanes 1, lane groups 259147, arrays 79 of 27797228 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 17680988 B", + "count and sum by 10^6 values, the core: result 1501672601, rows 300000, lanes 1, lane groups 37176, arrays 73 of 18380564 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, core: flushes 2 of 267855, bypassed 0, applied 267855 made 259343", + "distinct count by 10^5 values: result 299955, rows 300000, lanes 1, lane groups 95013, arrays 66 of 30391424 B, copied 8388352 B, blocks 0/0/4, merge 0 parts 0 entries, peak 19906560 B", "distinct values of 10^6: result 129508936719, rows 0, lanes 0, lane groups 0, arrays 0 of 0 B, copied 0 B, blocks 0/0/0, merge parts 0 entries, peak 0 B", "count by a text: result 200000, rows 200000, lanes 1, lane groups 49081, arrays 50 of 7730472 B, copied 1833792 B, blocks 0/12/1, merge 0 parts 0 entries, peak 6029368 B", - "count by a text and an int: result 200000, rows 200000, lanes 1, lane groups 196058, arrays 94 of 14492304 B, copied 2357952 B, blocks 0/0/0, merge 0 parts 0 entries, peak 12126112 B", - "count by a name, each once then all again: result 200000, rows 200000, lanes 1, lane groups 100000, arrays 70 of 11901724 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 7947288 B", - "least and greatest text by 10^3 values: result 56000, rows 200000, lanes 1, lane groups 1000, arrays 21 of 171968 B, copied 0 B, blocks 0/0/13, merge 0 parts 0 entries, peak 1310720 B", - "distinct users by day: result 299552, rows 300000, lanes 1, lane groups 1, arrays 370 of 2940924 B, copied 0 B, blocks 4/0/0, merge 0 parts 0 entries, peak 2621440 B", + "count by a text and an int: result 200000, rows 200000, lanes 1, lane groups 196058, arrays 95 of 14492368 B, copied 2357952 B, blocks 0/0/0, merge 0 parts 0 entries, peak 12126112 B", + "count by a name, each once then all again: result 200000, rows 200000, lanes 1, lane groups 100000, arrays 71 of 11901980 B, copied 0 B, blocks 0/0/4, merge 0 parts 0 entries, peak 7947288 B", + "least and greatest text by 10^3 values: result 56000, rows 200000, lanes 1, lane groups 1000, arrays 22 of 172032 B, copied 0 B, blocks 0/0/13, merge 0 parts 0 entries, peak 1310720 B", + "distinct users by day: result 299552, rows 300000, lanes 1, lane groups 1, arrays 371 of 2940988 B, copied 0 B, blocks 4/0/0, merge 0 parts 0 entries, peak 2621440 B", ]; [Fact] From 53d11f9132a0abdfd3ce5d84080a3e56eb3ede14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 00:52:17 +0200 Subject: [PATCH 36/56] perf(query): scratch comes from the process's shelf and goes back to it A slot's, a key's or an applier's scratch grew from nothing at every query and was left to the next collection: the two passes' homes and rows left over, a tuple's and a text's words, the entries an applier of the core cuts by sub-table, its map, its bits. On db-benchmark's q10 over 4M rows at fourteen lanes that was 5.6 MB a query on the small object heap. Scratch.Grow now takes its array from the process's shelf, a power of two long, and gives it the one it replaces; keys give theirs back when they are released, an applier when its worker or its lane is done (CoreApplier.Release, LaneCore.Release with the partition its lane bypassed its cache with), its batches taken in arrays of the shelf too. An array of references given to a shelf is cleared, whatever kept it would keep the objects of the query that gave it. GroupAllocationTests: hashed integers at four lanes 3.1 -> 1.3 MB, spread integers 2.8 -> 1.4, six keys 6.2 -> 4.6. --- .../Aggregation/AggregationEngine.cs | 6 +-- src/Vorticity/Aggregation/ArrayShelf.cs | 5 +++ src/Vorticity/Aggregation/GroupCore.cs | 37 ++++++++++++++++++- src/Vorticity/Aggregation/GroupKeys.cs | 5 +++ src/Vorticity/Aggregation/Kernels.cs | 17 ++++++++- src/Vorticity/Aggregation/ShortTextKeys.cs | 1 + src/Vorticity/Aggregation/TupleKeys.cs | 2 + .../Aggregation/GroupAllocationTests.cs | 17 ++++++--- 8 files changed, 77 insertions(+), 13 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 4957e3424..af35587fd 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -1729,9 +1729,9 @@ private void ReturnArrays() Keys?.Release(); Records?.Release(); - // The partition the lane bypassed its cache with, and a shelf of the partition's own, which hands - // what it holds to the process's; a lane's holds nothing. - _core?.Rows?.Release(); + // The lane's side of the core, its applier and the partition it bypassed its cache with, and a + // shelf of the partition's own, which hands what it holds to the process's; a lane's holds nothing. + _core?.Release(); _arrays?.Clear(); } diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index cfa24c401..c379d0a53 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -304,10 +304,15 @@ internal void Give(T[] array) return; } + // An array of references kept would keep what it points to: the objects of the query that gave it. if (PoisonsGiven) { Poison(array); } + else if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + Array.Clear(array); + } // Let go past the shelf's budget, or while the shelf lets go of what it is given: it leaves the query's count. if (Drops || !Give(typeof(T), array, Unsafe.SizeOf())) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index e7ff00633..7649316a4 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -1116,6 +1116,8 @@ private async ValueTask PublishPartsAsync(int[] queue, ChannelWriter // thread of it: each worker queues itself behind it after each part. await Task.Yield(); } + + applier.Release(); } /// @@ -1290,6 +1292,8 @@ private async ValueTask ApplyPartsAsync(int[] queue, CancellationToken cancellat cancellationToken.ThrowIfCancellationRequested(); await ApplyPartAsync(_parts[index], applier, cancellationToken).ConfigureAwait(false); } + + applier.Release(); } /// A part's whole stack applied; a part spilled, or one whose budget cannot take its stack's groups once it spills, writes the stack to the scratch instead. @@ -1892,6 +1896,14 @@ internal int PartOf(TKey key) /// What the lane applies a part's stack with: made at its first burst. internal CoreApplier Applier => _applier ??= new CoreApplier(_core, this); + /// The lane's applier and the partition it bypassed its cache with given back, the lane's partition let go. + internal void Release() + { + _applier?.Release(); + _applier = null; + Rows?.Release(); + } + /// The partition the lane folds its rows into while it bypasses its cache, a group a row; null before it does. internal AggregationPartition? Rows { get; private set; } @@ -2120,7 +2132,7 @@ internal sealed class CoreApplier private readonly AggregateSlot[] _entries; private readonly GroupRecords? _view; private readonly int _keyBytes; - private PartBatch[] _taken = new PartBatch[16]; + private PartBatch[] _taken = []; private int[] _map = []; private ulong[] _keys = []; private int[] _tableOf = []; @@ -2160,6 +2172,24 @@ internal CoreApplier(GroupCore core, LaneCore? lane) } } + /// The applier's scratch back to the process's shelf, its worker or its lane done with it: the next query's appliers grow into it. + internal void Release() + { + Scratch.Return(ref _taken); + Scratch.Return(ref _map); + Scratch.Return(ref _keys); + Scratch.Return(ref _tableOf); + Scratch.Return(ref _starts); + Scratch.Return(ref _next); + Scratch.Return(ref _entryAt); + Scratch.Return(ref _batchAt); + Scratch.Return(ref _bits); + Scratch.Return(ref _clear); + Scratch.Return(ref _set); + Scratch.Return(ref _setEntries); + Scratch.Return(ref _setBatches); + } + /// /// Applies a stack of batches to the part: every entry's key found or added in its sub-table, its /// record merged into its group's, a sub-table at a time; then the sub-tables past their bound @@ -2174,7 +2204,10 @@ internal int Apply(CorePart part, PartBatch? stack, bool burst) { if (count == _taken.Length) { - Array.Resize(ref _taken, count * 2); + PartBatch[] grown = ArrayShelf.Retained.Take(Math.Max(16, count * 2), zeroed: false); + _taken.AsSpan(0, count).CopyTo(grown); + ArrayShelf.Retained.Give(_taken); + _taken = grown; } _taken[count++] = batch; diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index da80b8cad..a537e7375 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -1836,6 +1836,11 @@ internal override void Release() _shelf?.Give(_metBits); _metBits = null; } + + // The scratch of the two passes and of the values widened, from the process's shelf, back to it. + Scratch.Return(ref _homes); + Scratch.Return(ref _left); + Scratch.Return(ref _values); } internal override int NullNumber => _null; diff --git a/src/Vorticity/Aggregation/Kernels.cs b/src/Vorticity/Aggregation/Kernels.cs index c548f6734..44ea26077 100644 --- a/src/Vorticity/Aggregation/Kernels.cs +++ b/src/Vorticity/Aggregation/Kernels.cs @@ -705,7 +705,11 @@ internal static void Unclip(ulong[] scratch, int start, int end) } } -/// Buffers a slot keeps from one batch to the next, grown and never shrunk, so a batch allocates nothing once warm. +/// +/// Buffers a slot keeps from one batch to the next, grown and never shrunk, so a batch allocates nothing +/// once warm; taken from the process's shelf, a power of two long, and the one they replace given back +/// to it, so that the next query's grow into the arrays this one's left. +/// internal static class Scratch { /// Makes hold at least elements; a grown array starts empty, its old content is not copied. @@ -713,10 +717,19 @@ internal static void Grow(ref T[] array, int length) { if (array.Length < length) { - array = new T[Capacity(length, array.Length)]; + T[] old = array; + array = ArrayShelf.Retained.Take(Capacity(length, array.Length), zeroed: true); + ArrayShelf.Retained.Give(old); } } + /// Gives back to the process's shelf, its owner done with it, and leaves it empty. + internal static void Return(ref T[] array) + { + ArrayShelf.Retained.Give(array); + array = []; + } + /// /// The length an array of elements grows to so that it holds /// : twice its length at least, rounded up to a power of two. A group by diff --git a/src/Vorticity/Aggregation/ShortTextKeys.cs b/src/Vorticity/Aggregation/ShortTextKeys.cs index 8663f385a..79993dc13 100644 --- a/src/Vorticity/Aggregation/ShortTextKeys.cs +++ b/src/Vorticity/Aggregation/ShortTextKeys.cs @@ -472,6 +472,7 @@ internal override void Release() { _words?.Release(); _bytes?.Release(); + Scratch.Return(ref _block); } /// Room for groups, foretold by a lane's first rows, in the table the keys are in. diff --git a/src/Vorticity/Aggregation/TupleKeys.cs b/src/Vorticity/Aggregation/TupleKeys.cs index 64fd527c9..8050a36ef 100644 --- a/src/Vorticity/Aggregation/TupleKeys.cs +++ b/src/Vorticity/Aggregation/TupleKeys.cs @@ -500,6 +500,8 @@ internal override void Release() { _tuples.Release(); Count = 0; + Scratch.Return(ref _block); + Scratch.Return(ref _words); } internal override void Reserve(int groups) => _tuples.Reserve(groups); diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index 905c0ec2b..6f4e27168 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -55,8 +55,8 @@ public sealed partial class GroupAllocationTests /// /// At one lane a ceiling is the floor rounded up to 512 bytes; at four, the lanes' schedule moves /// the core's splits and batches by a few hundred kilobytes from one run to the next, and the - /// process's shelf holds what the suite's other tests left, so a ceiling of the core is the floor - /// and 5 %, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too). + /// process's shelf holds what the suite's other tests left, so a ceiling of the core is the highest + /// floor seen and 256 KiB, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too). /// /// /// Measured without the shuffle of every result's order the tests make: 3 to 4 MB less on the @@ -67,6 +67,11 @@ public sealed partial class GroupAllocationTests /// from the shared array pool, whose array kept by one thread the lane of the next query, on /// another, did not find. /// + /// + /// A sub-table makes no array before its first group, and lays its records out as its core does; the + /// scratch of keys and of the core's appliers comes from the process's shelf and goes back to it: + /// the core's axes a half to a third lower again. + /// /// private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceiling, Func> Query)[] Axes = [ @@ -76,22 +81,22 @@ private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceil // A thousand short texts, hashed as their words. ("short texts", 1, GroupCoreReason.None, 434_176, NamesAsync), - ("short texts", 4, GroupCoreReason.None, 536_576, NamesAsync), + ("short texts", 4, GroupCoreReason.None, 520_192, NamesAsync), // A long integer a row, hashed: at four lanes, a lane's first rows are all new groups. ("hashed integers", 1, GroupCoreReason.None, 58_998_272, WideAsync), // // On this axis and the core's two others, each part is built in an order its shelf lends: 3 to // 4 MB less, four bytes a group. - ("hashed integers", 4, GroupCoreReason.FirstRows, 4_194_304, WideAsync), + ("hashed integers", 4, GroupCoreReason.FirstRows, 1_835_008, WideAsync), // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. ("spread integers", 1, GroupCoreReason.None, 34_865_152, DenseAsync), - ("spread integers", 4, GroupCoreReason.Spread, 2_949_120, DenseAsync), + ("spread integers", 4, GroupCoreReason.Spread, 1_769_472, DenseAsync), // Six keys, three texts and three integers, nearly a group a row: tuples of their values. ("six keys", 1, GroupCoreReason.None, 107_937_792, SixAsync), - ("six keys", 4, GroupCoreReason.FirstRows, 7_208_960, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 4_980_736, SixAsync), ]; [Fact] From 09e5b6d691c578050c21d02f8f41ea81031ee652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 01:12:21 +0200 Subject: [PATCH 37/56] perf(query): the core's batches outlive their query, and the shared pool goes Three sources of objects that lived a whole query on the small object heap, copied by each collection, or that the next query made again: - A core made a batch object for every 150 or so entries a lane deposits, 29 000 on db-benchmark's q10 over 4M rows. The batches a core made are now kept by the process once its slabs go (PartBatch. Linked, Own, Keep), up to 2^17 of them, and the next core takes those first. - A lane that turns to the core with a table no larger than a sample of its first rows empties it into its open batches, as a cache does: with six keys a cache holds fewer groups than a sample, and the table still went into 2 283 arrays of exact lengths, 7.7 MB a query. - Every rent from the shared array pool in the group by now comes from the process's shelf (ArrayShelf.Rent, Return), a power of two long: the pool keeps an array for each thread that gave one back, which the next query, on another thread, did not find. A merge's parts, a key's sort, the slots' reads by part and the arrays QueryArrays counts. The runner at fourteen lanes, warm, against 3e0d7957: q10 on 4M 0.8 collections a query against 7.8, 1.2 ms stopped against 12.9, 56 ms against 71; the strided key 0.6 collections against 5.6, 21.6 ms against 29.2; k7 0.6 against 3.6. GroupAllocationTests: six keys at four lanes 4.3 -> 1.6 MB. --- .../Aggregation/AggregationEngine.cs | 5 +- src/Vorticity/Aggregation/ArrayShelf.cs | 13 ++ src/Vorticity/Aggregation/GroupCore.cs | 178 ++++++++++++++++-- src/Vorticity/Aggregation/GroupKeys.cs | 8 +- src/Vorticity/Aggregation/JoinedKeys.cs | 5 +- src/Vorticity/Aggregation/QueryMemory.cs | 15 +- src/Vorticity/Aggregation/Slots.cs | 13 +- .../Aggregation/GroupAllocationTests.cs | 2 +- 8 files changed, 195 insertions(+), 44 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index af35587fd..6dff1e256 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -1,5 +1,4 @@ using System; -using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Numerics; @@ -4112,7 +4111,7 @@ private static int Parts(int degree, int largest) private static (int[] Placed, int[] Starts) Cut(GroupKeys keys, ulong seed, int shift, int parts, bool byValue) { int count = keys.Count; - byte[] partOf = ArrayPool.Shared.Rent(count); + byte[] partOf = ArrayShelf.Rent(count); try { if (byValue) @@ -4145,7 +4144,7 @@ private static (int[] Placed, int[] Starts) Cut(GroupKeys keys, ulong seed, int } finally { - ArrayPool.Shared.Return(partOf); + ArrayShelf.Return(partOf); } } diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index c379d0a53..4723177e1 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; @@ -124,6 +125,18 @@ private static ArrayShelf NewRetained() return shelf; } + /// + /// An array of elements at least, its power of two from sixteen, from the + /// process's shelf, holding whatever it held: the shared array pool's rent, but found again on any + /// thread, where the pool kept an array for each thread that gave one back and the next query's, + /// on another thread, made a new one. + /// + internal static T[] Rent(int length) => + Retained.Take(length <= 1 << 30 ? (int)BitOperations.RoundUpToPowerOf2((uint)Math.Max(length, 16)) : length, zeroed: false); + + /// An array handed out, back to the process's shelf. + internal static void Return(T[] array) => Retained.Give(array); + /// The bytes the shelf holds. internal long Held { diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 7649316a4..0e2b03e3e 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -519,18 +519,14 @@ internal PartBatch NewSlab(LaneCore lane) { Interlocked.Add(ref _batches, SlabBatches); ulong[] slab = _shelf.Take(SlabBatches * _batchWords, zeroed: false, overdraw: lane.Pressed); + PartBatch first = PartBatch.Linked(slab, SlabBatches, _batchWords, _batchEntries); lock (_freeGate) { _slabs.Add(slab); + Made(first, SlabBatches); } - PartBatch? next = null; - for (int b = SlabBatches - 1; b >= 0; b--) - { - next = new PartBatch(slab, b * _batchWords, _batchEntries) { Next = next }; - } - - return next!; + return first; } /// @@ -542,7 +538,39 @@ internal PartBatch NewAlone(int entries) { ulong[] words = _shelf.Take(entries * Shape.Words, zeroed: false, overdraw: true); Interlocked.Add(ref _alone, words.Length * sizeof(ulong)); - return new PartBatch(words, 0, entries) { Alone = true }; + PartBatch batch = PartBatch.Own(words, entries); + lock (_freeGate) + { + Made(batch, 1); + } + + return batch; + } + + // The batches the core made, every one, which the process keeps once its slabs go: under _freeGate. + private PartBatch[] _batchesMade = []; + private int _batchesMadeCount; + + /// The batches linked from counted among those the core made, under _freeGate. + private void Made(PartBatch first, int count) + { + if (_batchesMadeCount + count > _batchesMade.Length) + { + PartBatch[] grown = ArrayShelf.Retained.Take(Aggregating.Scratch.Capacity(_batchesMadeCount + count, _batchesMade.Length), zeroed: false); + _batchesMade.AsSpan(0, _batchesMadeCount).CopyTo(grown); + ArrayShelf.Retained.Give(_batchesMade); + _batchesMade = grown; + } + + PartBatch batch = first; + for (int b = 0; b < count; b++) + { + _batchesMade[_batchesMadeCount++] = batch; + if (b + 1 < count) + { + batch = batch.Next!; + } + } } /// A batch of its own applied: its array leaves the query's count, to the next collection. @@ -924,7 +952,10 @@ internal void Merge(AggregateSlot[] from, ReadOnlySpan groups, SubTable int return (joined, joinedSlots, bytes, [.. spilled]); } - /// The slabs of the batches given to the shelf, every batch applied or spilled: no lane fills one again. + /// + /// The slabs of the batches given to the shelf, every batch applied or spilled: no lane fills one + /// again; the batches kept by the process for the next queries. + /// private void LetSlabs() { lock (_freeGate) @@ -936,6 +967,9 @@ private void LetSlabs() } _slabs.Clear(); + PartBatch.Keep(_batchesMade.AsSpan(0, _batchesMadeCount)); + _batchesMadeCount = 0; + Aggregating.Scratch.Return(ref _batchesMade); } } @@ -1440,6 +1474,16 @@ internal static EntryShape Of(RecordLayout? layout, int keyBytes) /// internal sealed class PartBatch { + // The batches cores let go of, linked by Next, for the next query's: a query of a group a row made + // one for every 150 or so of its rows, 29 000 on 4M rows of six keys, which lived to its end on + // the small object heap, copied by every collection. + private static readonly Lock s_gate = new Lock(); + private static PartBatch? s_kept; + private static int s_keptCount; + + /// The batches the process keeps for the next queries at most: some 6 MB. + private const int MostKept = 1 << 17; + internal PartBatch(ulong[] slab, int start, int capacity) { Words = slab; @@ -1448,13 +1492,13 @@ internal PartBatch(ulong[] slab, int start, int capacity) } /// The slab the entries lie in, at words each from . - internal ulong[] Words { get; } + internal ulong[] Words { get; private set; } /// The slab's word the batch's first entry starts at. - internal int Start { get; } + internal int Start { get; private set; } /// The entries the batch holds at most. - internal int Capacity { get; } + internal int Capacity { get; private set; } /// The entries it holds. internal int Count; @@ -1466,7 +1510,105 @@ internal PartBatch(ulong[] slab, int start, int capacity) /// Whether the batch is an array of its own, a part's share of a lane's table emptied into the core /// under pressure: given back to the budget once applied, never filled again. /// - internal bool Alone { get; init; } + internal bool Alone { get; private set; } + + /// + /// batches, linked by , over + /// from its start, words and entries each: those + /// the process kept first, then new ones. + /// + internal static PartBatch Linked(ulong[] slab, int count, int words, int capacity) + { + PartBatch? kept; + lock (s_gate) + { + kept = s_kept; + PartBatch? last = null; + int taken = 0; + for (PartBatch? batch = kept; batch is not null && taken < count; batch = batch.Next) + { + last = batch; + taken++; + } + + s_kept = last?.Next; + s_keptCount -= taken; + if (last is not null) + { + last.Next = null; + } + } + + PartBatch? next = null; + for (int b = count - 1; b >= 0; b--) + { + PartBatch batch; + if (kept is not null) + { + batch = kept; + kept = kept.Next; + batch.Words = slab; + batch.Start = b * words; + batch.Capacity = capacity; + batch.Count = 0; + batch.Alone = false; + } + else + { + batch = new PartBatch(slab, b * words, capacity); + } + + batch.Next = next; + next = batch; + } + + return next!; + } + + /// A batch of entries in an array of its own, : one the process kept, or a new one. + internal static PartBatch Own(ulong[] words, int entries) + { + PartBatch batch = Linked(words, 1, 0, entries); + batch.Alone = true; + return batch; + } + + /// + /// , which a query is done with, kept for the next queries as far as the + /// process keeps them, nothing of their slabs kept with them. + /// + internal static void Keep(ReadOnlySpan batches) + { + PartBatch? first = null; + PartBatch? last = null; + int count = 0; + foreach (PartBatch batch in batches) + { + batch.Words = []; + batch.Count = 0; + batch.Next = first; + first = batch; + last ??= batch; + count++; + } + + if (first is null) + { + return; + } + + lock (s_gate) + { + if (s_keptCount + count > MostKept) + { + return; + } + + last!.Next = s_kept; + s_kept = first; + s_keptCount += count; + } + } } /// @@ -2033,14 +2175,14 @@ private PartBatch Take() /// turning to the core when its budget cannot let the table grow. Its groups counted /// by part first, each part's share goes into a batch of its own, exactly its size, deposited whole: /// given back as its part applies it, where batches the lanes fill again would stay held to the end. - /// A table no larger than a cache goes into the lane's open batches, as its cache's flushes do: the - /// table of a lane that turned on its first rows, judged on a sample, made a batch of its own on - /// every part, an array of its length that no other batch takes again, 3 600 of them a query at - /// fourteen lanes. + /// A table no larger than a cache, or than a sample of a lane's first rows, goes into the lane's open + /// batches, as its cache's flushes do: the table of a lane that turned on its first rows, judged on + /// a sample, made a batch of its own on every part, an array of its length that no other batch + /// takes again, 3 600 of them a query at fourteen lanes. /// internal void Empty(AggregationPartition table) { - if (table.Keys!.Count <= _core.Capacity) + if (table.Keys!.Count <= Math.Max(_core.Capacity, AggregationPartition.SampledRows)) { Flush(table); return; diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index a537e7375..148579e71 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -856,7 +856,7 @@ internal override void SortByKey(Span groups) return; } - TValue[] keys = System.Buffers.ArrayPool.Shared.Rent(groups.Length); + TValue[] keys = ArrayShelf.Rent(groups.Length); try { Span values = keys.AsSpan(0, groups.Length); @@ -869,7 +869,7 @@ internal override void SortByKey(Span groups) } finally { - System.Buffers.ArrayPool.Shared.Return(keys); + ArrayShelf.Return(keys); } } @@ -891,7 +891,7 @@ private bool TryPlace(Span groups) } int span = (int)(most - least) + 1; - int[] places = System.Buffers.ArrayPool.Shared.Rent(span); + int[] places = ArrayShelf.Rent(span); try { Span at = places.AsSpan(0, span); @@ -912,7 +912,7 @@ private bool TryPlace(Span groups) } finally { - System.Buffers.ArrayPool.Shared.Return(places); + ArrayShelf.Return(places); } return true; diff --git a/src/Vorticity/Aggregation/JoinedKeys.cs b/src/Vorticity/Aggregation/JoinedKeys.cs index 72c98208b..6d0c34a21 100644 --- a/src/Vorticity/Aggregation/JoinedKeys.cs +++ b/src/Vorticity/Aggregation/JoinedKeys.cs @@ -1,5 +1,4 @@ using System; -using System.Buffers; using Vorticity.Arrays; using Vorticity.Writing; @@ -63,7 +62,7 @@ internal override Func Reader(int component) /// internal override void Append(int component, ColumnStore store, ReadOnlySpan groups) { - int[] local = ArrayPool.Shared.Rent(groups.Length); + int[] local = ArrayShelf.Rent(groups.Length); try { int start = 0; @@ -85,7 +84,7 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpan.Shared.Return(local); + ArrayShelf.Return(local); } } diff --git a/src/Vorticity/Aggregation/QueryMemory.cs b/src/Vorticity/Aggregation/QueryMemory.cs index 91536d649..d81d83b4a 100644 --- a/src/Vorticity/Aggregation/QueryMemory.cs +++ b/src/Vorticity/Aggregation/QueryMemory.cs @@ -1,5 +1,4 @@ using System; -using System.Buffers; using System.Globalization; using System.Numerics; using System.Runtime.CompilerServices; @@ -9,9 +8,9 @@ namespace Vorticity.Aggregating; /// /// The scratch arrays of the steps that grow with the groups — the order of a result, its top, the -/// sort of the rows a fetch reads, a lane's top — rented from the shared pool under the query's +/// sort of the rows a fetch reads, a lane's top — rented from the process's shelf under the query's /// memory: reserved before they are rented, at the length -/// the pool hands out, and given back with them. What the pool keeps of them afterwards is the rest +/// the shelf hands out, and given back with them. What the shelf keeps of them afterwards is the rest /// of the process's memory, which the process's budget reads at its next full collection. An array /// under a page goes uncounted both ways, as a shelf's. /// @@ -20,7 +19,7 @@ internal static class QueryArrays /// The least array counted: a page. private const long LeastCounted = 4096; - /// An array of elements at least, from the shared pool, reserved first under for . + /// An array of elements at least, from the process's shelf (), reserved first under for . /// The query's budget does not grant the array. internal static T[] Rent(QueryMemory? memory, int length, string what) { @@ -34,10 +33,10 @@ internal static T[] Rent(QueryMemory? memory, int length, string what) memory.Measure(bytes); } - return ArrayPool.Shared.Rent(length); + return ArrayShelf.Rent(length); } - /// An array handed out, back to the shared pool, its bytes given back to . + /// An array handed out, back to the process's shelf, its bytes given back to . internal static void Return(QueryMemory? memory, T[] array) { if (memory is not null && Bytes(array.Length) is long bytes and >= LeastCounted) @@ -46,10 +45,10 @@ internal static void Return(QueryMemory? memory, T[] array) memory.Measure(-bytes); } - ArrayPool.Shared.Return(array); + ArrayShelf.Return(array); } - /// The length the shared pool hands out for : its power of two from sixteen, past a gigabyte of elements the length itself. + /// The length the shelf hands out for : its power of two from sixteen, past a gigabyte of elements the length itself. private static int Pooled(int length) => length <= 1 << 30 ? (int)BitOperations.RoundUpToPowerOf2((uint)Math.Max(length, 16)) : length; private static long Bytes(int length) => (long)length * Unsafe.SizeOf(); diff --git a/src/Vorticity/Aggregation/Slots.cs b/src/Vorticity/Aggregation/Slots.cs index 8a804a382..15bb8bce8 100644 --- a/src/Vorticity/Aggregation/Slots.cs +++ b/src/Vorticity/Aggregation/Slots.cs @@ -1,5 +1,4 @@ using System; -using System.Buffers; using System.Numerics; using System.Runtime.CompilerServices; using System.Threading; @@ -399,7 +398,7 @@ internal sealed class JoinedSlot(AggregateSlot[] parts, int[] offsets) /// The bytes of the groups' answers, a run at a time of one part, as reads them. public void AppendBytes(VarBinStore store, ReadOnlySpan groups) { - int[] local = ArrayPool.Shared.Rent(groups.Length); + int[] local = ArrayShelf.Rent(groups.Length); try { int start = 0; @@ -421,7 +420,7 @@ public void AppendBytes(VarBinStore store, ReadOnlySpan groups) } finally { - ArrayPool.Shared.Return(local); + ArrayShelf.Return(local); } } internal override TResult Result(int group) @@ -436,7 +435,7 @@ internal override TResult Result(int group) /// internal override void Results(ReadOnlySpan groups, Span into) { - int[] local = ArrayPool.Shared.Rent(groups.Length); + int[] local = ArrayShelf.Rent(groups.Length); try { int start = 0; @@ -458,7 +457,7 @@ internal override void Results(ReadOnlySpan groups, Span into) } finally { - ArrayPool.Shared.Return(local); + ArrayShelf.Return(local); } } @@ -471,7 +470,7 @@ internal override void Results(ReadOnlySpan groups, Span into) /// The means a run at a time of one part, as reads the answers. public void Means(ReadOnlySpan groups, Span into) { - int[] local = ArrayPool.Shared.Rent(groups.Length); + int[] local = ArrayShelf.Rent(groups.Length); try { int start = 0; @@ -493,7 +492,7 @@ public void Means(ReadOnlySpan groups, Span into) } finally { - ArrayPool.Shared.Return(local); + ArrayShelf.Return(local); } } diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index 6f4e27168..3e0ee4466 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -96,7 +96,7 @@ private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceil // Six keys, three texts and three integers, nearly a group a row: tuples of their values. ("six keys", 1, GroupCoreReason.None, 107_937_792, SixAsync), - ("six keys", 4, GroupCoreReason.FirstRows, 4_980_736, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 1_900_544, SixAsync), ]; [Fact] From 6d482de222b665949505e04b9bee3c08ab4f6806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 02:08:05 +0200 Subject: [PATCH 38/56] perf(query): what a query makes once goes back to a shelf A merge borrows its map of numbers from the process's shelf unless its caller keeps it, a partition gives the group of each row back once it folds no more, a key gives its page slabs back and makes its ranges at the first, a result column's values grow through the process's shelf and go back to it, the part builder's column sets are released with their query, and the core's spread map lives on the stack. A core part restarts its directory in one place. The allocation ratchet's floors: a lane alone's small axes three to five times lower (small integers 355 to 76 KB, short texts 434 to 154 KB), the core's axes a sixth to a third (hashed integers at four lanes 1.55 to 0.97 MB, six keys 1.58 to 1.33 MB). Ceilings lowered to them, a margin per regime written in the test. --- .../Aggregation/AggregationEngine.cs | 98 ++++++++++++++----- src/Vorticity/Aggregation/GroupBatches.cs | 6 +- src/Vorticity/Aggregation/GroupCore.Spill.cs | 9 +- src/Vorticity/Aggregation/GroupCore.cs | 21 +++- src/Vorticity/Aggregation/GroupKeys.cs | 29 +++++- src/Vorticity/Aggregation/PartBatches.cs | 34 +++++-- src/Vorticity/Aggregation/ResultColumns.cs | 13 ++- .../Aggregation/GroupAllocationTests.cs | 34 ++++--- 8 files changed, 181 insertions(+), 63 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 6dff1e256..98f30279a 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -414,8 +414,23 @@ internal static ResultNode Result(Sym symbol, SymNode[] components) $"'{symbol}' is neither an aggregate nor a component of the key: a result is an aggregate of the group, or g.Key."); } - /// The slot of : the one aggregate of the plan it is the same as. - internal int IndexOf(IAggregateNode node) => Array.FindIndex(Aggregates, known => known.Identity.Equals(node.Identity)); + /// + /// The slot of : the one aggregate of the plan it is the same as. A loop, where a + /// predicate capturing the node made a closure and a delegate at each call: two a result column a + /// part of the core's result, 1 024 a query of two aggregates. + /// + internal int IndexOf(IAggregateNode node) + { + for (int i = 0; i < Aggregates.Length; i++) + { + if (Aggregates[i].Identity.Equals(node.Identity)) + { + return i; + } + } + + return -1; + } private static void Add(List aggregates, IAggregateNode aggregate) { @@ -1315,9 +1330,6 @@ internal void RangeEnded() } } - // The map of the first batch's values over the span, 4 096 bits. - private ulong[]? _spreadMap; - /// /// Judges the key on the lane's first batch, before a row of it is folded: its selected values /// placed in 4 096 bins over the span, a bit each, and compared to the B (1 − e^(−n/B)) of B bins n @@ -1347,8 +1359,9 @@ internal void Judge(RecordBatch batch) node = arena.GetNode(node).StorageIndex; } - _spreadMap ??= new ulong[4096 / 64]; - if (keys.Spread(arena, [node], batch.RowCount, batch.SelectionWords, AggregationPlan.ScatteredSpan, _spreadMap) is not { } spread || spread.Values == 0) + // The map of the first batch's values over the span, 4 096 bits, on the stack: it is read once. + Span map = stackalloc ulong[4096 / 64]; + if (keys.Spread(arena, [node], batch.RowCount, batch.SelectionWords, AggregationPlan.ScatteredSpan, map) is not { } spread || spread.Values == 0) { return; } @@ -1788,7 +1801,7 @@ internal void TurnTo(GroupCore core) arrays.Exact = true; } int[]? numbers = null; - cache.MergeFrom(this, ref numbers, apart: null); + cache.MergeFrom(this, ref numbers); ReturnArrays(); GiveBack(); lane.Turning = false; @@ -2372,6 +2385,13 @@ internal void DropUnmet() } } + /// + /// The lane folds no more rows: the group of each row back to the process's shelf now, where it went + /// back with the partition's release, which the partition a merge in series keeps as the result + /// never has: the next query's lane made another, half a megabyte. + /// + internal void FoldsNoMore() => GiveBackRowGroups(); + /// The groups the partition held at most since its top last counted them. internal int PeakGroups { get; set; } @@ -2917,33 +2937,48 @@ internal void Follow(AggregationPartition next, ref int[]? numbers) /// The numbers of the groups of a partition past those every thread shares (), the caller's. /// The slots left to merge apart, by parts of their pairs (); null for none. /// The map of the other's groups onto this one's. - internal int[] MergeFrom(AggregationPartition other, ref int[]? numbers, bool[]? apart = null) + internal int[] MergeFrom(AggregationPartition other, ref int[]? numbers, bool[]? apart) => + MergeFrom(other, ref numbers, apart, kept: true)!; + + /// + /// for a caller that keeps no map: the + /// map lent by the process's shelf and given back, where every lane merged in series made one. + /// + internal void MergeFrom(AggregationPartition other, ref int[]? numbers) => MergeFrom(other, ref numbers, apart: null, kept: false); + + private int[]? MergeFrom(AggregationPartition other, ref int[]? numbers, bool[]? apart, bool kept) { - int[] map; - if (Keys is null) + int count = Keys is null ? 1 : other.Keys!.Count; + int[] map = Keys is null ? [0] : kept ? new int[count] : ArrayShelf.Rent(count); + if (Keys is not null) { - map = [0]; - } - else - { - map = new int[other.Keys!.Count]; - other.Keys.MergeInto(Keys, Numbers.Upto(map.Length, ref numbers), map); + other.Keys!.MergeInto(Keys, Numbers.Upto(count, ref numbers), map.AsSpan(0, count)); foreach (AggregateSlot slot in Slots) { slot.EnsureGroups(Keys.Count); } } - ReadOnlySpan all = Numbers.Upto(map.Length, ref numbers); + ReadOnlySpan all = Numbers.Upto(count, ref numbers); for (int i = 0; i < Slots.Length; i++) { if (_inputs[i] != Settled && (apart is null || !apart[i])) { - Slots[i].MergeFrom(other.Slots[i], all, map); + Slots[i].MergeFrom(other.Slots[i], all, map.AsSpan(0, count)); } } - return map; + if (kept) + { + return map; + } + + if (Keys is not null) + { + ArrayShelf.Return(map); + } + + return null; } /// The input of an aggregate settled before the scan, which is never stepped. @@ -3302,6 +3337,7 @@ internal static async ValueTask RunAsync( plan.Watch?.Invoke(partitions); await RunPartitionAsync(source, pass, metrics, only, cancellationToken).ConfigureAwait(false); only.DropUnmet(); + only.FoldsNoMore(); } else { @@ -3815,7 +3851,7 @@ private static long MergeAhead(AggregationPartition[] partitions) { if (partition.Core is null && partition.Keys is not null && (!ended || partition.HasEnded) && !ReferenceEquals(partition, largest)) { - largest!.MergeFrom(partition, ref numbers, apart: null); + largest!.MergeFrom(partition, ref numbers); largest.Recount(); partition.LetGo(); partition.Gave(); @@ -3898,19 +3934,30 @@ private static long MergeAhead(AggregationPartition[] partitions) // are many, apart, by parts of the pairs taken side by side. int[]? numbers = null; bool[]? apart = partitions.Length > 1 && degree > 1 ? Paired(partitions, inputs) : null; - int[][] maps = new int[partitions.Length][]; + int[][]? maps = apart is null ? null : new int[partitions.Length][]; for (int p = 0; p < partitions.Length; p++) { - if (p != largest) + if (p == largest) + { + continue; + } + + // The maps kept for the pairs merged apart; with none, each lent and given back. + if (maps is not null) { maps[p] = biggest.MergeFrom(partitions[p], ref numbers, apart); - biggest.Recount(); } + else + { + biggest.MergeFrom(partitions[p], ref numbers); + } + + biggest.Recount(); } if (apart is not null) { - await MergePairedAsync(partitions, largest, maps, apart, degree, memory, cancellationToken).ConfigureAwait(false); + await MergePairedAsync(partitions, largest, maps!, apart, degree, memory, cancellationToken).ConfigureAwait(false); } return (biggest.Keys, biggest.Slots, partitions.Length - 1, 0, null, serial); @@ -4683,6 +4730,7 @@ private static async Task RunQueueAsync( } await partition.EndedAsync(token).ConfigureAwait(false); + partition.FoldsNoMore(); } catch { diff --git a/src/Vorticity/Aggregation/GroupBatches.cs b/src/Vorticity/Aggregation/GroupBatches.cs index 7a9995db0..e04c1f4ca 100644 --- a/src/Vorticity/Aggregation/GroupBatches.cs +++ b/src/Vorticity/Aggregation/GroupBatches.cs @@ -660,9 +660,13 @@ private void Release() _cut?.Reset(); // The outcome held in memory holds the query's memory and its spilled parts: given back last, - // then the parts' stores. + // then the parts' stores, and the scratch of the result's columns. _outcome?.Delivered(); _builder?.Release(); + foreach (ResultColumn column in _query.Columns) + { + column.Release(); + } } } diff --git a/src/Vorticity/Aggregation/GroupCore.Spill.cs b/src/Vorticity/Aggregation/GroupCore.Spill.cs index 30a9af9b0..73c748b27 100644 --- a/src/Vorticity/Aggregation/GroupCore.Spill.cs +++ b/src/Vorticity/Aggregation/GroupCore.Spill.cs @@ -113,8 +113,7 @@ private async ValueTask EvictAsync(CorePart part, CancellationToken cancellation { taken = [.. part.Tables]; part.Tables.Clear(); - part.Directory = [0]; - part.Depth = 0; + part.Restart(); Volatile.Write(ref part.Groups, 0); // Its keys written, a key new to its sub-tables may be one of them; and those it made @@ -329,8 +328,7 @@ internal async ValueTask MaterializeAsync(CorePart part, CancellationToken cance { silent = [.. part.Tables]; part.Tables.Clear(); - part.Directory = [0]; - part.Depth = 0; + part.Restart(); Volatile.Write(ref part.Groups, 0); } @@ -408,8 +406,7 @@ internal void Let(CorePart part) } part.Tables.Clear(); - part.Directory = [0]; - part.Depth = 0; + part.Restart(); Volatile.Write(ref part.Groups, 0); if (!_shelf.Drops && Volatile.Read(ref _allApplied)) { diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 0e2b03e3e..89cc56ba4 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -1741,18 +1741,35 @@ internal sealed class CorePart /// internal bool Silent; - /// Doubles the directory: one more bit of the hash, each sub-table at both places its old one gave. + /// + /// Doubles the directory: one more bit of the hash, each sub-table at both places its old one gave; + /// the directories from the process's shelf, which the next part's take again. + /// internal void Grow() { - int[] grown = new int[Directory.Length * 2]; + int[] grown = ArrayShelf.Retained.Take(Directory.Length * 2, zeroed: false); for (int j = 0; j < grown.Length; j++) { grown[j] = Directory[j >> 1]; } + ArrayShelf.Retained.Give(Directory); Directory = grown; Depth++; } + + /// The part's directory back to one place, its first sub-table's, the one it held given back to the process's shelf. + internal void Restart() + { + if (Directory.Length > 1) + { + ArrayShelf.Retained.Give(Directory); + Directory = ArrayShelf.Retained.Take(1, zeroed: false); + } + + Directory[0] = 0; + Depth = 0; + } } /// diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index 148579e71..0d4a60a30 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -15,9 +15,11 @@ namespace Vorticity.Aggregating; /// Row ranges of one batch that each belong to a single group, for the aggregates to fold a range at a time. internal sealed class GroupRanges { - private int[] _starts = new int[16]; - private int[] _ends = new int[16]; - private int[] _groups = new int[16]; + // Made at the first range: a key that is not in runs never adds one, and every partition made three + // arrays for them all the same. + private int[] _starts = []; + private int[] _ends = []; + private int[] _groups = []; internal int Count { get; private set; } @@ -48,7 +50,7 @@ internal void Add(int start, int end, int group) if (Count == _starts.Length) { - int grown = Count * 2; + int grown = Math.Max(16, Count * 2); Array.Resize(ref _starts, grown); Array.Resize(ref _ends, grown); Array.Resize(ref _groups, grown); @@ -1841,6 +1843,25 @@ internal override void Release() Scratch.Return(ref _homes); Scratch.Return(ref _left); Scratch.Return(ref _values); + + // The slabs of the pages of a key numbered by value, each given once, by the page that starts it. + if (_pages is not null) + { + for (int at = 0; at < _pages.Length; at++) + { + if (_pages[at] != Unmet && _pageStarts![at] == 0) + { + _shelf?.Give(_pages[at]); + } + + _pages[at] = Unmet; + } + + _slab = null; + _slabUsed = 0; + _slabsHeld = 0; + _slabPages = 1; + } } internal override int NullNumber => _null; diff --git a/src/Vorticity/Aggregation/PartBatches.cs b/src/Vorticity/Aggregation/PartBatches.cs index ec63f40c2..6f3a96d06 100644 --- a/src/Vorticity/Aggregation/PartBatches.cs +++ b/src/Vorticity/Aggregation/PartBatches.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -27,7 +26,10 @@ internal sealed class PartBuilder private readonly AggregationQuery _query; private readonly int _batchRows; private readonly ScanSpec _spec; - private readonly ConcurrentBag _columns = []; + + // The column sets the workers build with, one taken at a time under the gate: a concurrent bag made a + // queue and a slot for each thread that gave one back, two dozen objects a query. + private readonly Stack _columns = []; private readonly Lock _gate = new Lock(); private readonly Stack _slates = []; private DType _dtype; @@ -57,9 +59,8 @@ internal void SortRows(int order, VortexSchema schema, Func colu _until = order; _sortColumns = columns; _dtype = VortexTypes.ToDType(schema, new DTypeArena()); + _columns.Clear(); } - - _columns.Clear(); } /// @@ -72,11 +73,14 @@ internal async ValueTask BuildAsync(AggregationOutcome outcome, Canc // every group its operators keep. (int[] groups, int count) = await GroupSelection.ApplyAsync(_query, outcome, _spec, cancellationToken, windowed: false, until: _until).ConfigureAwait(false); PartResult result = new PartResult(outcome.Keys?.Count ?? 0); - if (!_columns.TryTake(out ResultColumn[]? columns)) + ResultColumn[]? columns; + lock (_gate) { - columns = _sortColumns is { } sort ? sort() : _query.NewColumns(); + _columns.TryPop(out columns); } + columns ??= _sortColumns is { } sort ? sort() : _query.NewColumns(); + try { for (int from = 0; from < count; from += _batchRows) @@ -94,7 +98,11 @@ internal async ValueTask BuildAsync(AggregationOutcome outcome, Canc } finally { - _columns.Add(columns); + lock (_gate) + { + _columns.Push(columns); + } + outcome.LetChosen(); } @@ -135,13 +143,23 @@ internal void Release() _released = true; } - // Past the flag, no store comes back to the pool: the ones on it are the last. + // Past the flag, no store comes back to the pool: the ones on it are the last. The workers are done + // building: their columns' scratch goes back to the process's shelf. foreach (PartSlate slate in _slates) { slate.Release(); } _slates.Clear(); + foreach (ResultColumn[] columns in _columns) + { + foreach (ResultColumn column in columns) + { + column.Release(); + } + } + + _columns.Clear(); } /// A store of the pool, or a new one. diff --git a/src/Vorticity/Aggregation/ResultColumns.cs b/src/Vorticity/Aggregation/ResultColumns.cs index b1b166b5e..1c1a2b649 100644 --- a/src/Vorticity/Aggregation/ResultColumns.cs +++ b/src/Vorticity/Aggregation/ResultColumns.cs @@ -35,6 +35,11 @@ protected ResultColumn(string name, VortexType type) /// Whether reads the values into an array: the column is built for an order otherwise. internal virtual bool ReadsOrder => false; + + /// The column's scratch back to the process's shelf: the result is done with it. + internal virtual void Release() + { + } } /// A component of the key, in a column of the key column's own type: its values written as the index holds them. @@ -84,15 +89,13 @@ internal override void Append(AggregationOutcome outcome, ColumnStore store, Rea return; } - if (_values.Length < groups.Length) - { - _values = new T[Scratch.Capacity(groups.Length, _values.Length)]; - } - + Scratch.Grow(ref _values, groups.Length); Read(outcome, groups, _values.AsSpan(0, groups.Length)); ResultValues.Append(store, _values, groups.Length, _record); } + internal override void Release() => Scratch.Return(ref _values); + internal override bool ReadsOrder => _record is null && ValuesOrder.Orders(); internal override ColumnOrder? OrderOf(AggregationOutcome outcome, ReadOnlySpan groups, bool descending) diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index 3e0ee4466..cca934e6d 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -53,10 +53,13 @@ public sealed partial class GroupAllocationTests /// /// /// - /// At one lane a ceiling is the floor rounded up to 512 bytes; at four, the lanes' schedule moves + /// At one lane a ceiling is the highest floor seen and 512 bytes, rounded up to 512 bytes: what the + /// process's shelf holds moves it by a hundred bytes or so; at four, the lanes' schedule moves /// the core's splits and batches by a few hundred kilobytes from one run to the next, and the /// process's shelf holds what the suite's other tests left, so a ceiling of the core is the highest - /// floor seen and 256 KiB, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too). + /// floor seen and 256 KiB, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too); + /// four lanes without the core move by a few kilobytes: the highest floor seen and 16 KiB, rounded + /// up to 4 KiB. /// /// /// Measured without the shuffle of every result's order the tests make: 3 to 4 MB less on the @@ -72,31 +75,38 @@ public sealed partial class GroupAllocationTests /// scratch of keys and of the core's appliers comes from the process's shelf and goes back to it: /// the core's axes a half to a third lower again. /// + /// + /// What each query made once goes back to a shelf too: a merge's map of numbers is borrowed, a + /// partition gives the group of each row back once it folds no more, a key's page slabs and a + /// result column's values are given back, a key's ranges and the core's spread map are made at + /// their first use or on the stack: a lane alone's small axes three to five times lower, the core's + /// axes a sixth to a third. + /// /// private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceiling, Func> Query)[] Axes = [ // A thousand integers numbered by their value, a table a lane. - ("small integers", 1, GroupCoreReason.None, 355_840, SmallAsync), - ("small integers", 4, GroupCoreReason.None, 495_616, SmallAsync), + ("small integers", 1, GroupCoreReason.None, 76_800, SmallAsync), + ("small integers", 4, GroupCoreReason.None, 163_840, SmallAsync), // A thousand short texts, hashed as their words. - ("short texts", 1, GroupCoreReason.None, 434_176, NamesAsync), - ("short texts", 4, GroupCoreReason.None, 520_192, NamesAsync), + ("short texts", 1, GroupCoreReason.None, 155_136, NamesAsync), + ("short texts", 4, GroupCoreReason.None, 237_568, NamesAsync), // A long integer a row, hashed: at four lanes, a lane's first rows are all new groups. - ("hashed integers", 1, GroupCoreReason.None, 58_998_272, WideAsync), + ("hashed integers", 1, GroupCoreReason.None, 58_327_040, WideAsync), // // On this axis and the core's two others, each part is built in an order its shelf lends: 3 to // 4 MB less, four bytes a group. - ("hashed integers", 4, GroupCoreReason.FirstRows, 1_835_008, WideAsync), + ("hashed integers", 4, GroupCoreReason.FirstRows, 1_245_184, WideAsync), // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. - ("spread integers", 1, GroupCoreReason.None, 34_865_152, DenseAsync), - ("spread integers", 4, GroupCoreReason.Spread, 1_769_472, DenseAsync), + ("spread integers", 1, GroupCoreReason.None, 33_488_896, DenseAsync), + ("spread integers", 4, GroupCoreReason.Spread, 1_245_184, DenseAsync), // Six keys, three texts and three integers, nearly a group a row: tuples of their values. - ("six keys", 1, GroupCoreReason.None, 107_937_792, SixAsync), - ("six keys", 4, GroupCoreReason.FirstRows, 1_900_544, SixAsync), + ("six keys", 1, GroupCoreReason.None, 106_561_536, SixAsync), + ("six keys", 4, GroupCoreReason.FirstRows, 1_638_400, SixAsync), ]; [Fact] From 5ddba40f4f8d21fcb14addcb346358b0bda192d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 02:24:25 +0200 Subject: [PATCH 39/56] test(io): a group by's dependent steps held clear of the timers' edge The steps a group by waits on the store depend on which timer of the counting source fires first; the ceiling was the highest seen, 11, and the whole suite's load pushed a count distinct to 12 in a few runs since 2026-10-08. Swept over eighteen runs, alone and beside a run of the aggregation tests: the group by and the order 9 to 11, the count distinct 6 to 11. The ceiling is now the highest seen and two, still far under the twenty steps a regression that waits a batch more adds. --- .../Vorticity.Tests/IO/GroupRoundTripTests.cs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs b/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs index 5bf7acc9a..f2fbccfaf 100644 --- a/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs +++ b/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs @@ -28,16 +28,23 @@ public sealed partial class GroupRoundTripTests /// Measured on 2026-10-06: 25 chunks read in 22 requests, the /// reader joining neighbours, at any degree. The steps are not exact: the scan reads ahead, and /// whether a read joins the step of the one before or opens the next depends on which timer of - /// the source fires first. Measured 8 to 11 at one lane, at 20 ms a request as at 50; the - /// ceiling is the highest, and a regression that waits on the store a batch more than now adds - /// some twenty. At four lanes, 5 or 6, which the lanes' interleaving decides, so not held. + /// the source fires first. Measured 8 to 11 at one lane, at 20 ms a request as at 50, and a + /// regression that waits on the store a batch more than now adds some twenty. At four lanes, 5 + /// or 6, which the lanes' interleaving decides, so not held. + /// + /// The ceiling was the highest, 11, and the whole suite's load pushed a count distinct to 12 + /// once in a few runs (2026-10-08 to 2026-10-10). Swept on 2026-10-10, eighteen runs alone and + /// beside a run of the aggregation tests: the group by 9 to 11, the order 9 to 11, the count + /// distinct 6 to 11. The ceiling is the highest seen and two, clear of the timers' edge and far + /// under the twenty steps of a regression. + /// /// private static readonly (string Query, int Degree, int Requests, int Steps)[] Ceilings = [ - ("group by a key of 10^4 values, count sum", 1, 22, 11), + ("group by a key of 10^4 values, count sum", 1, 22, 14), ("group by a key of 10^4 values, count sum", 4, 22, 0), - ("order by count take 10", 1, 22, 11), - ("count distinct over the scan", 1, 22, 11), + ("order by count take 10", 1, 22, 14), + ("count distinct over the scan", 1, 22, 14), ]; [Fact] From 721a2b4f615a05841bae4b61e1cc704d3a94794a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 02:24:25 +0200 Subject: [PATCH 40/56] perf(query): a result's table goes back to the shelf once delivered The table a lane alone holds as the result, or the largest one a merge in series kept, was left to the next collection once its batches were read: every query at one lane grew its whole table again from fresh memory. The outcome now keeps that partition and, once delivered, hands its arrays to the process's shelf, its lane's shelf counting nothing more since the query gave its memory back whole. A whole result's order, the identity its keys made new each time, is lent by the process's shelf and given back with it. The allocation ratchet at one lane: hashed integers 57.7 MB to 100 KB, spread integers 33.1 MB to 106 KB, six keys 105.5 MB to 108 KB, small integers 76 to 35 KB, short texts 154 to 36 KB; at four lanes without the core, small integers 147 to 107 KB and short texts 220 to 123 KB. --- .../Aggregation/AggregationEngine.cs | 72 +++++++++++++++++-- src/Vorticity/Aggregation/ArrayShelf.cs | 14 +++- .../Aggregation/GroupAllocationTests.cs | 24 ++++--- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 98f30279a..3fd898820 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -655,7 +655,21 @@ internal AggregationOutcome(AggregationPlan plan, AggregateSlot[] slots, GroupKe /// What the result holds of its query's memory budget, until it is delivered; null for a result nobody counts. internal QueryMemory? Memory { get; set; } - /// Gives back what the result held of its query's budget: delivered, it is the caller's. + /// + /// The partition whose tables are the result, a lane's alone or the largest a merge in series kept + /// as it: its arrays go back to the process's shelf once the result is delivered, for the next query. + /// + internal AggregationPartition? Kept { get; init; } + + /// Whether is lent by the process's shelf, which takes it back once the result is delivered. + internal bool OrderLent { get; init; } + + private int _givenBack; + + /// + /// Gives back what the result held of its query's budget: delivered, it is the caller's; and, once, + /// the tables and the order the shelves lent it, its batches dead with the reads that took them. + /// internal void Delivered() { Parts?.Close(); @@ -664,6 +678,15 @@ internal void Delivered() _plan.LastPeakBytes = Math.Max(_plan.LastPeakBytes, memory.Peak); memory.Dispose(); } + + if (Interlocked.Exchange(ref _givenBack, 1) == 0) + { + Kept?.Delivered(); + if (OrderLent) + { + ArrayShelf.Retained.Give(Order); + } + } } /// @@ -1718,6 +1741,17 @@ internal void Release() _followComponents = []; } + /// + /// The partition's tables, a result, delivered: their arrays back to the process's shelf for the + /// next query, which grew its own again while a collection took these; its query gave its memory + /// back whole, so its shelf counts them nowhere. + /// + internal void Delivered() + { + _arrays?.Forget(); + Release(); + } + /// /// Lets the partition's tables go and gives back what it held of its query's memory: a range a /// stream followed into the partition before it, whose groups now live there. @@ -3455,7 +3489,8 @@ internal static async ValueTask RunAsync( (GroupKeys held, AggregateSlot[] heldSlots, long heldBytes, CorePart[] spilled) = await engaged.FinishAsync(joined, lanes, cancellationToken).ConfigureAwait(false); plan.LastRun = Gathered(partitions, Stopwatch.GetTimestamp() - merging, 0, heldBytes) with { Core = engaged.Run() }; - AggregationOutcome outcome = new AggregationOutcome(plan, heldSlots, held, Shuffled(held.Order(sorted: false))); + (int[] heldOrder, bool heldLent) = LentWholeOrder(held.Count); + AggregationOutcome outcome = new AggregationOutcome(plan, heldSlots, held, heldOrder, held.Count) { OrderLent = heldLent }; if (spilled.Length == 0) { foreach (AggregationPartition partition in partitions) @@ -3495,13 +3530,16 @@ internal static async ValueTask RunAsync( plan.LastRun = (Gathered(partitions, Stopwatch.GetTimestamp() - merging, parts, mergedBytes) with { MergeEntries = entries }).Spilled(spill); spill?.Dispose(); - // The lanes' tables die with the merge, but the one a merge in series kept as the result. + // The lanes' tables die with the merge, but the one a merge in series kept as the result, + // which goes back to the process's shelf once delivered. long result = mergedBytes; + AggregationPartition? whole = null; foreach (AggregationPartition partition in tables) { if (ReferenceEquals(partition.Slots, slots)) { result = partition.Footprint; + whole = partition; } else { @@ -3510,8 +3548,14 @@ internal static async ValueTask RunAsync( } // Groups as they were first met, or part after part: an order is asked for, with OrderBy. - int[] order = keys is null ? [0] : Shuffled(keys.Order(sorted: false)); - return Counted(new AggregationOutcome(plan, slots, keys, order), memory, result); + // Lent by the process's shelf, and given back with the result. + if (keys is null) + { + return Counted(new AggregationOutcome(plan, slots, keys, [0]) { Kept = whole }, memory, result); + } + + (int[] order, bool lent) = LentWholeOrder(keys.Count); + return Counted(new AggregationOutcome(plan, slots, keys, order, keys.Count) { Kept = whole, OrderLent = lent }, memory, result); } catch { @@ -3628,6 +3672,24 @@ internal static int[] Shuffled(int[] order, int count) return shuffled; } + /// + /// The order of a whole result, the groups 0 to less one, lent by the + /// process's shelf a power of two long: the identity a result's keys made new at every query. Under + /// the tests' shuffle, the shuffled copy is the result's, and the lent one goes back at once. + /// + internal static (int[] Order, bool Lent) LentWholeOrder(int count) + { + int[] order = LentOrder(ArrayShelf.Retained, count); + int[] shuffled = Shuffled(order, count); + if (!ReferenceEquals(shuffled, order)) + { + ArrayShelf.Retained.Give(order); + return (shuffled, false); + } + + return (order, order.Length > 0); + } + /// /// The groups 0 to less one in their order, in an array of /// 's a power of two long, so that the parts of a query take the same diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index 4723177e1..6cdd09674 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -67,6 +67,7 @@ internal sealed class ArrayShelf : ISweptAfterCollections private readonly QueryMemory? _memory; private long _out; private long _credit; + private bool _forgotten; /// A query's shelf, which takes from the process's when it holds nothing of a length, and hands it what it holds at the end. internal ArrayShelf() @@ -308,7 +309,7 @@ internal void Give(T[] array) { if (Lane) { - if ((long)array.Length * Unsafe.SizeOf() is long bytes and >= LeastCounted) + if (!_forgotten && (long)array.Length * Unsafe.SizeOf() is long bytes and >= LeastCounted) { Unreserve(_memory!, bytes); } @@ -434,6 +435,17 @@ internal void LetGo() } } + /// + /// A lane's shelf whose tables are a result delivered, its query's memory given back whole: what + /// they give back goes to the process's shelf, counted nowhere. + /// + internal void Forget() + { + _out = 0; + _credit = 0; + _forgotten = true; + } + /// The bytes of an array a lane's shelf hands out, reserved from what it holds ahead, or else more. private void Reserve(QueryMemory memory, long bytes) { diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index cca934e6d..69fe1d6a8 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -57,9 +57,8 @@ public sealed partial class GroupAllocationTests /// process's shelf holds moves it by a hundred bytes or so; at four, the lanes' schedule moves /// the core's splits and batches by a few hundred kilobytes from one run to the next, and the /// process's shelf holds what the suite's other tests left, so a ceiling of the core is the highest - /// floor seen and 256 KiB, rounded up to 64 KiB (1 % for the one-lane axes whose floor moves too); - /// four lanes without the core move by a few kilobytes: the highest floor seen and 16 KiB, rounded - /// up to 4 KiB. + /// floor seen and 256 KiB, rounded up to 64 KiB; four lanes without the core move by a few + /// kilobytes: the highest floor seen and 16 KiB, rounded up to 4 KiB. /// /// /// Measured without the shuffle of every result's order the tests make: 3 to 4 MB less on the @@ -82,30 +81,35 @@ public sealed partial class GroupAllocationTests /// their first use or on the stack: a lane alone's small axes three to five times lower, the core's /// axes a sixth to a third. /// + /// + /// The table a lane alone holds as the result, or the largest a merge in series kept, goes back to + /// the process's shelf once delivered, and its order is lent by it: a lane alone no longer grows a + /// table of its own at every query, its axes 2 to 1 000 times lower (six keys 105 MB to 108 KB). + /// /// private static readonly (string Axis, int Lanes, GroupCoreReason Path, long Ceiling, Func> Query)[] Axes = [ // A thousand integers numbered by their value, a table a lane. - ("small integers", 1, GroupCoreReason.None, 76_800, SmallAsync), - ("small integers", 4, GroupCoreReason.None, 163_840, SmallAsync), + ("small integers", 1, GroupCoreReason.None, 35_840, SmallAsync), + ("small integers", 4, GroupCoreReason.None, 126_976, SmallAsync), // A thousand short texts, hashed as their words. - ("short texts", 1, GroupCoreReason.None, 155_136, NamesAsync), - ("short texts", 4, GroupCoreReason.None, 237_568, NamesAsync), + ("short texts", 1, GroupCoreReason.None, 37_376, NamesAsync), + ("short texts", 4, GroupCoreReason.None, 143_360, NamesAsync), // A long integer a row, hashed: at four lanes, a lane's first rows are all new groups. - ("hashed integers", 1, GroupCoreReason.None, 58_327_040, WideAsync), + ("hashed integers", 1, GroupCoreReason.None, 100_352, WideAsync), // // On this axis and the core's two others, each part is built in an order its shelf lends: 3 to // 4 MB less, four bytes a group. ("hashed integers", 4, GroupCoreReason.FirstRows, 1_245_184, WideAsync), // Integers in no order over a span of 2·10⁶, the one the statistics bound them to: the core by pages. - ("spread integers", 1, GroupCoreReason.None, 33_488_896, DenseAsync), + ("spread integers", 1, GroupCoreReason.None, 106_496, DenseAsync), ("spread integers", 4, GroupCoreReason.Spread, 1_245_184, DenseAsync), // Six keys, three texts and three integers, nearly a group a row: tuples of their values. - ("six keys", 1, GroupCoreReason.None, 106_561_536, SixAsync), + ("six keys", 1, GroupCoreReason.None, 109_056, SixAsync), ("six keys", 4, GroupCoreReason.FirstRows, 1_638_400, SixAsync), ]; From cd75bcbbe6b1be2582a7cfd4e007eb6f19fb6c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 02:37:08 +0200 Subject: [PATCH 41/56] perf(query): a delivered result takes the place of past queries' arrays only Given back with the shelf's usual eviction, a table larger than the process's budget let go of the arrays its own query had grown through, which the next query takes first; that query then let go of the table's arrays in turn: q10 at one lane made 722 MB a query, against 517 when the table went to the collector. Offered only where there was room, it found the shelf full of what the queries before it left (six keys at one lane, 64 MB a query). A pile now knows when it was last used, and a delivered result's arrays take the place of piles last used before its query began, never of those its own query used. q10 at one lane 496 MB a query, six keys at one lane 108 KB. Alternated runners against c7a5f1b6, warm, at one lane: s7 x0.928 (115 to 107 ms), k7 x0.959, q10 x1.006; at fourteen, s7 x0.986, q10 x0.996. --- src/Vorticity/Aggregation/ArrayShelf.cs | 64 ++++++++++++++++++------ src/Vorticity/Aggregation/QueryMemory.cs | 3 ++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index 6cdd09674..40766cc5e 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -309,7 +310,18 @@ internal void Give(T[] array) { if (Lane) { - if (!_forgotten && (long)array.Length * Unsafe.SizeOf() is long bytes and >= LeastCounted) + // A result delivered offers its arrays: kept in place of what queries before its own left, never + // of what its own used. A table larger than the process's budget, given back whole at the end, + // let go the arrays the next query's table takes first as it grows, and that query then let go + // of these (q10 at one lane, 722 MB a query against 517 when the table went to the collector); + // kept only where there was room, it found the shelf full of the queries' before it. + if (_forgotten) + { + Retained.Offer(array, _memory!.Started); + return; + } + + if ((long)array.Length * Unsafe.SizeOf() is long bytes and >= LeastCounted) { Unreserve(_memory!, bytes); } @@ -318,6 +330,28 @@ internal void Give(T[] array) return; } + Clean(array); + + // Let go past the shelf's budget, or while the shelf lets go of what it is given: it leaves the query's count. + if (Drops || !Give(typeof(T), array, Unsafe.SizeOf())) + { + Leave(array.Length * (long)Unsafe.SizeOf()); + } + } + + /// + /// An array put on the process's shelf where its budget has room, or in place of arrays of lengths + /// no one took or gave since , in ticks. + /// + private void Offer(T[] array, long since) + { + Clean(array); + Give(typeof(T), array, Unsafe.SizeOf(), since); + } + + /// An array given back made safe to keep: poisoned under the tests, its references cleared. + private static void Clean(T[] array) + { // An array of references kept would keep what it points to: the objects of the query that gave it. if (PoisonsGiven) { @@ -327,12 +361,6 @@ internal void Give(T[] array) { Array.Clear(array); } - - // Let go past the shelf's budget, or while the shelf lets go of what it is given: it leaves the query's count. - if (Drops || !Give(typeof(T), array, Unsafe.SizeOf())) - { - Leave(array.Length * (long)Unsafe.SizeOf()); - } } /// @@ -522,10 +550,11 @@ internal void Clear() /// /// An array of elements of , each, on the shelf, /// within its budget: whether it kept it. Past the budget, the arrays of the lengths taken or given - /// least recently go first: a shelf full of another query's lengths kept them while every query - /// went on taking from it, and the arrays of the queries running now could not come in. + /// least recently go first, those last used before only: a shelf full of + /// another query's lengths kept them while every query went on taking from it, and the arrays of the + /// queries running now could not come in. /// - private bool Give(Type type, Array array, int elementBytes) + private bool Give(Type type, Array array, int elementBytes, long before = long.MaxValue) { int length = array.Length; if (length == 0) @@ -544,7 +573,7 @@ private bool Give(Type type, Array array, int elementBytes) Touch(pile); long budget = pile.Small ? _smallBudget : _budget; long held = pile.Small ? _heldSmall : _held - _heldSmall; - if (held + pile.Bytes > budget && (pile.Bytes > budget || !Evict(held + pile.Bytes - budget, pile))) + if (held + pile.Bytes > budget && (pile.Bytes > budget || !Evict(held + pile.Bytes - budget, pile, before))) { return false; } @@ -567,15 +596,16 @@ private void Hold(Pile pile, long bytes) /// /// Lets go of at least of the arrays on the piles of 's - /// kind taken from or given to least recently, never 's, under the lock: - /// whether it could. Past , a pile emptied goes, so that the lengths a process + /// kind taken from or given to least recently, never 's nor one used since + /// , under the lock: whether it could. Past , a pile emptied goes, so that the lengths a process /// once met do not pile up; short of them it stays, its next array a push. /// - private bool Evict(long bytes, Pile kept) + private bool Evict(long bytes, Pile kept, long before) { + // The piles lie in the order of their last use: the first one used since the bound ends the walk. ref PileOrder order = ref OrderOf(kept); Pile? pile = order.Oldest; - while (bytes > 0 && pile is not null) + while (bytes > 0 && pile is not null && pile.Touched < before) { Pile? newer = pile.Newer; if (pile != kept) @@ -606,6 +636,7 @@ private bool Evict(long bytes, Pile kept) /// Makes the one of its kind used most recently, under the lock. private void Touch(Pile pile) { + pile.Touched = Stopwatch.GetTimestamp(); ref PileOrder order = ref OrderOf(pile); if (order.Newest == pile) { @@ -696,6 +727,9 @@ private sealed class Pile((Type Type, int Length) key, int elementBytes) /// Whether its arrays lie on the small object heap. internal bool Small => Bytes < LargeBytes; + /// When it was last taken from or given to, in ticks. + internal long Touched { get; set; } + /// The pile used next more recently, null for the newest. internal Pile? Newer { get; set; } diff --git a/src/Vorticity/Aggregation/QueryMemory.cs b/src/Vorticity/Aggregation/QueryMemory.cs index d81d83b4a..e07572ef8 100644 --- a/src/Vorticity/Aggregation/QueryMemory.cs +++ b/src/Vorticity/Aggregation/QueryMemory.cs @@ -88,6 +88,9 @@ internal QueryMemory(QueryMemoryBudget budget) budget.Enter(); } + /// When the query began, in ticks: what the process's shelf held from before it is another query's. + internal long Started { get; } = System.Diagnostics.Stopwatch.GetTimestamp(); + /// The bytes the query holds of its budget. internal long Held => Volatile.Read(ref _held); From d1a3f94c6d3721d78b1a29664b4bf35e3a237f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 02:56:52 +0200 Subject: [PATCH 42/56] perf(query): the process's shelf keeps what the running queries need, counted A query's working memory is zero allocation only if kept from one query to the next, and a shelf of a quarter of a gigabyte made most of it new at every query: q10 on 10^7 rows at fourteen lanes holds more than a gigabyte and a half, paid again in page faults and zeroed pages. The process's shelf now counts its large arrays in the process's budget, reserved and measured as the queries' tables are: kept within the ceiling, handed over to a query that counts it, and let go to the collector otherwise. Past its quarter of a gigabyte it keeps what the queries running lately need: the most a query held at once, grown by what it refused or let go for want of room since a query last ended, up to a quarter of the memory the process may use. A query the budget refuses first makes the shelf let its large arrays go, for the collection that follows to take them; two seconds with no query bring the shelf back to its quarter of a gigabyte, the collector asked to run when that let go much. Warm, per query: q10 on 4M rows at fourteen lanes 589 MB to 5.5 MB, at one lane 496 MB to 0.2 MB; s7 134 to 2.3 MB, k7 66 to 4.3 MB. Alternated runners against 14c8690c at fourteen lanes: q10 on 10M x0.877 (150 to 131 ms), 10^7 hashed keys on 40M x0.897, 10^7 keys x0.876, 10^6 hashed keys x0.893, q3 x0.946; q10 on 10M at one lane x0.868 (672 to 584 ms). --- src/Vorticity/Aggregation/ArrayShelf.cs | 270 ++++++++++++++++++--- src/Vorticity/Aggregation/QueryMemory.cs | 3 + src/Vorticity/Session/QueryMemoryBudget.cs | 52 +++- 3 files changed, 288 insertions(+), 37 deletions(-) diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index 40766cc5e..d8cfdf419 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -58,6 +58,19 @@ internal sealed class ArrayShelf : ISweptAfterCollections private long _heldSmall; private bool _used; + // The process's shelf counts its large arrays in the process's budget (Governed). Past its budget, it + // keeps what the queries running lately need (Expected), up to a quarter of the memory the process + // may use (Cap), back to its budget once idle: the bytes of large arrays it refused or let go for + // want of room since a query last ended (Short), whether it was used since the last tick of its + // timer, and the ticks it has been idle. + private readonly bool _governed; + private readonly long _cap; + private long _expected; + private long _short; + private bool _active; + private int _idle; + private Timer? _timer; + // The piles of each kind from the one taken from or given to most recently to the one least // recently, linked: past its budget, the shelf lets go of the oldest first. private PileOrder _large; @@ -103,30 +116,157 @@ internal ArrayShelf(QueryMemory memory) /// The query's memory the shelf counts its arrays under: its run's; null for a shelf that counts nothing. internal QueryMemory? Memory => _memory; - private ArrayShelf(ArrayShelf? parent, long budget, long smallBudget) + private ArrayShelf(ArrayShelf? parent, long budget, long smallBudget, bool governed = false) { _parent = parent; _budget = budget; _smallBudget = smallBudget; + _governed = governed; + _cap = governed ? Math.Max(budget, GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 4) : budget; } /// /// The process's shelf: what queries left, for the next query not to touch fresh memory again: of /// large arrays, up to a quarter of a gigabyte or a sixteenth of the memory the process may use, - /// whichever is less; of small ones, a quarter of that again, apart, so that a query whose large - /// arrays its budget does not hold still finds its small ones. Swept after collections: emptied when - /// no query took from it since the last, or when the machine's memory load is high. + /// whichever is less, and while queries run, up to the most one of them held lately + /// (); of small ones, a quarter of the first again, apart, so that a query whose + /// large arrays its budget does not hold still finds its small ones. Its large arrays count in the + /// process's budget as the queries' tables do, and go when a query needs their room + /// (). Back to its budget after two seconds with no query, and swept after + /// collections: emptied when no query took from it since the last, or when the machine's memory load + /// is high. /// + /// + /// A query's working memory is zero allocation only if kept from one query to the next: q10 on 10⁷ + /// rows at fourteen lanes holds more than a gigabyte and a half, which a shelf of a quarter of a + /// gigabyte made new at every query, in page faults and zeroed pages (×0.88 kept, ×0.90 for the keys + /// of 10⁷ values on 4·10⁷ rows). Kept only while queries come, counted, and given up for them. + /// internal static ArrayShelf Retained { get; } = NewRetained(); private static ArrayShelf NewRetained() { long budget = Math.Min(256L << 20, GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 16); - ArrayShelf shelf = new ArrayShelf(parent: null, budget, budget / 4); + ArrayShelf shelf = new ArrayShelf(parent: null, budget, budget / 4, governed: true); CollectionSweeper.Register(shelf); return shelf; } + /// How often the process's shelf, past its budget, looks whether queries still come. + private static readonly TimeSpan IdleTick = TimeSpan.FromSeconds(1); + + /// The ticks with no query after which the process's shelf goes back to its budget. + private const int IdleTicks = 2; + + /// + /// A query that held bytes of its budget at once ended: the process's shelf + /// keeps up to as many large arrays for the next, and more by what it refused or let go for want of + /// room since a query last ended, which the next of the kind would make again; up to + /// , until two seconds pass with no query. A query's peak alone fell short: what + /// its lanes gave back early and its core took late is never held at once (s7 on 4·10⁶ rows at + /// fourteen lanes, 134 MB a query made anew with the shelf at its peak). + /// + internal void Expect(long peak) + { + if (!_governed) + { + return; + } + + long shortBy = Interlocked.Exchange(ref _short, 0); + long want = Math.Min(_cap, Math.Max(peak, shortBy > 0 ? LargeBudget + shortBy : 0)); + if (want <= _budget) + { + return; + } + + long expected = Volatile.Read(ref _expected); + while (want > expected) + { + long seen = Interlocked.CompareExchange(ref _expected, want, expected); + if (seen == expected) + { + break; + } + + expected = seen; + } + + Watch(); + } + + /// + /// A query needs room the process's budget refused: the shelf's large arrays all go to the collector, + /// which the budget then asks for. A cache, not a holder: never in a query's way. + /// + internal void Relieve() + { + if (!_governed || Volatile.Read(ref _held) == Volatile.Read(ref _heldSmall)) + { + return; + } + + lock (_gate) + { + Volatile.Write(ref _expected, 0); + Evict(long.MaxValue, ref _large, kept: null, before: long.MaxValue); + } + } + + /// The timer that brings the process's shelf back to its budget once idle, started when it keeps past it. + private void Watch() + { + if (Volatile.Read(ref _timer) is null) + { + Timer timer = new Timer(static state => ((ArrayShelf)state!).Tick(), this, IdleTick, IdleTick); + if (Interlocked.CompareExchange(ref _timer, timer, null) is not null) + { + timer.Dispose(); + } + } + } + + /// + /// A tick of the timer: a shelf used since the last waits on; idle for , it lets + /// go of what it keeps past its budget, to the collector, which it asks to run when that is much. + /// + private void Tick() + { + long trimmed = 0; + lock (_gate) + { + if (_active) + { + _active = false; + _idle = 0; + return; + } + + if (++_idle < IdleTicks) + { + return; + } + + _idle = 0; + Volatile.Write(ref _expected, 0); + long over = _held - _heldSmall - _budget; + if (over > 0) + { + long before = _held; + Evict(over, ref _large, kept: null, before: long.MaxValue); + trimmed = before - _held; + } + + _timer?.Dispose(); + _timer = null; + } + + if (trimmed >= _budget) + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Optimized, blocking: false); + } + } + /// /// An array of elements at least, its power of two from sixteen, from the /// process's shelf, holding whatever it held: the shared array pool's rent, but found again on any @@ -167,7 +307,10 @@ internal long Held internal long CopiedBytes { get; private set; } /// The most bytes the shelf keeps on its piles, its large arrays' and its small ones'; past it, what it is given goes. - internal long Budget => _budget > long.MaxValue - _smallBudget ? long.MaxValue : _budget + _smallBudget; + internal long Budget => LargeBudget > long.MaxValue - _smallBudget ? long.MaxValue : LargeBudget + _smallBudget; + + /// The most bytes of large arrays the shelf keeps: its budget, or for the process's, what a query held lately past it. + private long LargeBudget => _governed ? Math.Max(_budget, Volatile.Read(ref _expected)) : _budget; /// /// Whether a lane's shelf takes an array its budget refuses all the same, counted past the ceiling: @@ -238,10 +381,11 @@ internal static void Resize(ArrayShelf? shelf, ref T[] array, int length) /// An array of elements, from the shelf, the process's, or new; zeroed /// when . Under a query's memory, reserved first, then new; past the budget /// when , for a lane emptying its table into the core, whose table is - /// given back right after. + /// given back right after. : whether the taker counts it in its query's + /// memory, so that the process's shelf hands its count over rather than leave it to the rest. /// /// The query's budget does not grant the array. - internal T[] Take(int length, bool zeroed, bool overdraw = false) + internal T[] Take(int length, bool zeroed, bool overdraw = false, bool counted = false) { if (Lane) { @@ -253,13 +397,14 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) Handed++; HandedBytes += bytes; - return Retained.Take(length, zeroed); + return Retained.Take(length, zeroed, counted: bytes >= LeastCounted); } Array? found = null; lock (_gate) { _used = true; + _active = true; if (_piles.TryGetValue((typeof(T), length), out Pile? pile)) { Touch(pile); @@ -267,6 +412,10 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) { found = pile.Arrays.Pop(); Hold(pile, -pile.Bytes); + if (_governed && !pile.Small) + { + QueryMemoryBudget.Process.Unkeep(pile.Bytes, counted); + } } } } @@ -282,22 +431,24 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) } // An array the query did not hold enters its count, exactly, before it comes. - if (_memory is { } memory && (long)length * Unsafe.SizeOf() is long counted and >= LeastCounted) + bool counts = false; + if (_memory is { } memory && (long)length * Unsafe.SizeOf() is long asked and >= LeastCounted) { - if (!memory.TryGrow(counted)) + if (!memory.TryGrow(asked)) { if (!overdraw && !Overdraws && Pressure is not { Owed: true }) { - throw memory.Exceeded("group by", -1, counted); + throw memory.Exceeded("group by", -1, asked); } - memory.Force(counted); + memory.Force(asked); } - memory.Measure(counted); + memory.Measure(asked); + counts = true; } - return _parent is not null ? _parent.Take(length, zeroed) + return _parent is not null ? _parent.Take(length, zeroed, counted: counts) : zeroed ? new T[length] : GC.AllocateUninitializedArray(length); } @@ -305,8 +456,10 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false) /// /// Puts an array nothing holds any more back on the shelf; past the shelf's budget, lets it go. /// A lane's shelf gives its bytes back to the query's memory and the array to the process's shelf. + /// : whether the giver counted it in its query's memory, which it gave back + /// as let go, so that the process's shelf counts it as kept instead. /// - internal void Give(T[] array) + internal void Give(T[] array, bool counted = false) { if (Lane) { @@ -321,19 +474,20 @@ internal void Give(T[] array) return; } - if ((long)array.Length * Unsafe.SizeOf() is long bytes and >= LeastCounted) + bool lent = (long)array.Length * Unsafe.SizeOf() >= LeastCounted; + if (lent) { - Unreserve(_memory!, bytes); + Unreserve(_memory!, (long)array.Length * Unsafe.SizeOf()); } - Retained.Give(array); + Retained.Give(array, lent); return; } Clean(array); // Let go past the shelf's budget, or while the shelf lets go of what it is given: it leaves the query's count. - if (Drops || !Give(typeof(T), array, Unsafe.SizeOf())) + if (Drops || !Give(typeof(T), array, Unsafe.SizeOf(), fromQuery: counted)) { Leave(array.Length * (long)Unsafe.SizeOf()); } @@ -346,7 +500,7 @@ internal void Give(T[] array) private void Offer(T[] array, long since) { Clean(array); - Give(typeof(T), array, Unsafe.SizeOf(), since); + Give(typeof(T), array, Unsafe.SizeOf(), since, fromQuery: true); } /// An array given back made safe to keep: poisoned under the tests, its references cleared. @@ -537,8 +691,9 @@ internal void Clear() while (pile.Arrays.Count > 0) { Array array = pile.Arrays.Pop(); - _parent.Give(entry.Key.Type, array, pile.ElementBytes); - Leave((long)array.Length * pile.ElementBytes); + long bytes = (long)array.Length * pile.ElementBytes; + _parent.Give(entry.Key.Type, array, pile.ElementBytes, fromQuery: _memory is not null && bytes >= LeastCounted); + Leave(bytes); } } } @@ -552,9 +707,11 @@ internal void Clear() /// within its budget: whether it kept it. Past the budget, the arrays of the lengths taken or given /// least recently go first, those last used before only: a shelf full of /// another query's lengths kept them while every query went on taking from it, and the arrays of the - /// queries running now could not come in. + /// queries running now could not come in. The process's shelf counts a large one in the process's + /// budget, and lets it go when the budget has no room for it; : whether + /// its giver had counted it, and given it back as let go. /// - private bool Give(Type type, Array array, int elementBytes, long before = long.MaxValue) + private bool Give(Type type, Array array, int elementBytes, long before = long.MaxValue, bool fromQuery = false) { int length = array.Length; if (length == 0) @@ -562,6 +719,7 @@ private bool Give(Type type, Array array, int elementBytes, long before = long.M return true; } + bool watch = false; lock (_gate) { if (!_piles.TryGetValue((type, length), out Pile? pile)) @@ -571,17 +729,47 @@ private bool Give(Type type, Array array, int elementBytes, long before = long.M } Touch(pile); - long budget = pile.Small ? _smallBudget : _budget; + _active = true; + long budget = pile.Small ? _smallBudget : LargeBudget; long held = pile.Small ? _heldSmall : _held - _heldSmall; - if (held + pile.Bytes > budget && (pile.Bytes > budget || !Evict(held + pile.Bytes - budget, pile, before))) + if (held + pile.Bytes > budget) + { + // Short of room: what goes for it, or it, the next query of its kind makes again. + long heldBefore = _held; + bool room = pile.Bytes <= budget && Evict(held + pile.Bytes - budget, ref OrderOf(pile), pile, before); + if (_governed && !pile.Small) + { + Interlocked.Add(ref _short, heldBefore - _held + (room ? 0 : pile.Bytes)); + } + + if (!room) + { + return false; + } + + held = pile.Small ? _heldSmall : _held - _heldSmall; + } + + if (_governed && !pile.Small) { - return false; + if (!QueryMemoryBudget.Process.TryKeep(pile.Bytes, fromQuery)) + { + return false; + } + + watch = held + pile.Bytes > _budget; } pile.Arrays.Push(array); Hold(pile, pile.Bytes); - return true; } + + if (watch) + { + Watch(); + } + + return true; } /// The bytes the shelf holds moved by , an array of 's taken or given. @@ -595,15 +783,16 @@ private void Hold(Pile pile, long bytes) } /// - /// Lets go of at least of the arrays on the piles of 's - /// kind taken from or given to least recently, never 's nor one used since - /// , under the lock: whether it could. Past , a pile emptied goes, so that the lengths a process - /// once met do not pile up; short of them it stays, its next array a push. + /// Lets go of at least of the arrays on the piles of + /// taken from or given to least recently, never 's nor one used since + /// , under the lock: whether it could. Past , a pile + /// emptied goes, so that the lengths a process once met do not pile up; short of them it stays, its + /// next array a push. The process's shelf takes a large array it lets go out of the process's budget, + /// to the collector. /// - private bool Evict(long bytes, Pile kept, long before) + private bool Evict(long bytes, ref PileOrder order, Pile? kept, long before) { // The piles lie in the order of their last use: the first one used since the bound ends the walk. - ref PileOrder order = ref OrderOf(kept); Pile? pile = order.Oldest; while (bytes > 0 && pile is not null && pile.Touched < before) { @@ -614,6 +803,11 @@ private bool Evict(long bytes, Pile kept, long before) { pile.Arrays.Pop(); Hold(pile, -pile.Bytes); + if (_governed && !pile.Small) + { + QueryMemoryBudget.Process.Unkeep(pile.Bytes, counted: false); + } + bytes -= pile.Bytes; } @@ -682,6 +876,12 @@ private static void Unlink(ref PileOrder order, Pile pile) /// Every pile gone, under the lock. private void Empty() { + if (_governed && _held > _heldSmall) + { + QueryMemoryBudget.Process.Unkeep(_held - _heldSmall, counted: false); + Volatile.Write(ref _expected, 0); + } + _piles.Clear(); _large = default; _small = default; diff --git a/src/Vorticity/Aggregation/QueryMemory.cs b/src/Vorticity/Aggregation/QueryMemory.cs index e07572ef8..f90c2654d 100644 --- a/src/Vorticity/Aggregation/QueryMemory.cs +++ b/src/Vorticity/Aggregation/QueryMemory.cs @@ -264,6 +264,9 @@ private void GiveBack() if (Interlocked.Exchange(ref _left, 1) == 0) { _budget.Leave(); + + // The next query of its kind will hold as much: the process's shelf keeps up to that, for it. + ArrayShelf.Retained.Expect(Peak); } } diff --git a/src/Vorticity/Session/QueryMemoryBudget.cs b/src/Vorticity/Session/QueryMemoryBudget.cs index a0854fef5..d0a6697d1 100644 --- a/src/Vorticity/Session/QueryMemoryBudget.cs +++ b/src/Vorticity/Session/QueryMemoryBudget.cs @@ -120,13 +120,23 @@ internal void Leave() _process?.Leave(); } - /// Reserves under this budget and the process's; false, and nothing reserved, when either would pass its ceiling. + /// + /// Reserves under this budget and the process's; false, and nothing reserved, + /// when either would pass its ceiling. Short of room, the process's shelf lets its large arrays go + /// first, for the collection that follows to take them. + /// internal bool TryReserve(long bytes) { if (_process is null) { Observe(); - return TryAdd(ref _reserved, bytes, Available()) || (Collect(bytes) is long after && TryAdd(ref _reserved, bytes, after)); + if (TryAdd(ref _reserved, bytes, Available())) + { + return true; + } + + Aggregating.ArrayShelf.Retained.Relieve(); + return Collect(bytes) is long after && TryAdd(ref _reserved, bytes, after); } if (!TryAdd(ref _reserved, bytes, _ceiling)) @@ -227,6 +237,44 @@ internal void Measure(long delta) /// Arrays of the queries' tables replaced with larger ones: in the heap until a collection takes them. internal void Discard(long bytes) => Interlocked.Add(ref (_process ?? this)._pending, bytes); + /// + /// The process's shelf keeps an array of for the next query: reserved and + /// measured as the queries' tables are, within the ceiling and with no collection asked for; false, + /// and nothing counted, past it. : whether its query had counted it and + /// given it back as let go, which it no longer is. The process's own budget. + /// + internal bool TryKeep(long bytes, bool fromQuery) + { + Observe(); + if (!TryAdd(ref _reserved, bytes, Available())) + { + return false; + } + + Interlocked.Add(ref _measured, bytes); + if (fromQuery) + { + Interlocked.Add(ref _pending, -bytes); + } + + return true; + } + + /// + /// The process's shelf lets an array of go: to a query that counts it, its + /// count handed over (); else to the collector or to code that counts + /// nothing, the rest of the process until the next full collection reads it again. + /// + internal void Unkeep(long bytes, bool counted) + { + Interlocked.Add(ref _reserved, -bytes); + Interlocked.Add(ref _measured, -bytes); + if (!counted) + { + Interlocked.Add(ref _pending, bytes); + } + } + /// /// What the process's limits leave the queries past the rest of its memory: the collector's, past /// the rest of its heap; in a container, the container's too, past the heap's rest and the engine's From cbe9411620834b170e906184afc0e422bbfb457e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 03:13:36 +0200 Subject: [PATCH 43/56] perf(query): a core's sub-table holds 65 536 groups before it splits A split moves half a sub-table, and each level of splits moves every group again, where a table held whole moves each about twice as it doubles. Sub-tables of half a megabyte split q10's parts on 10^7 rows three times over, a key of 8 bytes once, and made each part's objects again at every split. Bigger arrays were a cost while the process's shelf could not keep them; it now keeps what the running queries need. Out of pressure, a sub-table now splits past 65 536 groups, or past eight megabytes of wide groups; under pressure, the half megabyte as before. Alternated runners against 4d39474f at fourteen lanes, warm: q10 on 10M x0.760 (132 to 100 ms), 10^7 hashed keys on 40M x0.784 (139 to 106 ms), the rest flat (10^6 hashed keys x0.990, 10^7 keys x1.011, q3 x0.984, q2 x0.983). Swept first: 128 000 groups the same, 256 000 x1.10 on the hashed keys, and 40 000, just past q10's 39 000 a part, x1.20, a single split of a sub-table that size costing more than it saves. --- src/Vorticity/Aggregation/GroupCore.cs | 32 ++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index 89cc56ba4..e139067e1 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -69,15 +69,33 @@ internal sealed partial class GroupCore private const int DefaultFloor = 16_384; /// - /// The bytes of a sub-table past which it splits, S: within the private cache of any current core, - /// half a megabyte where the smallest hold one. Measured on the Mac on 2026-10-07 at fourteen - /// lanes on 20M rows, sub-tables of 16 000 groups against 7 900 (a count and a sum, 528 KB against - /// 256 KiB) took ×0.91 to ×0.95 off 10⁷, the stride, ten rows a key and a key a row, nothing below, - /// and ×0.83 off four aggregates at ≈ 900 KB; 32 000 groups more off 10⁷ (×0.81), but a megabyte - /// fills the whole private cache of a core of a current x64. + /// The bytes of a sub-table past which it splits under pressure, S: within the private cache of any + /// current core, half a megabyte where the smallest hold one. Measured on the Mac on 2026-10-07 at + /// fourteen lanes on 20M rows, sub-tables of 16 000 groups against 7 900 (a count and a sum, 528 KB + /// against 256 KiB) took ×0.91 to ×0.95 off 10⁷, the stride, ten rows a key and a key a row, nothing + /// below, and ×0.83 off four aggregates at ≈ 900 KB; 32 000 groups more off 10⁷ (×0.81), but a + /// megabyte fills the whole private cache of a core of a current x64. /// private const long TableBytes = 512 * 1024; + /// + /// The groups past which a sub-table splits, its memory not pressed: 65 536, or what holds in + /// of wide groups. A split moves half a sub-table, and each level of + /// splits moves every group again, where a table held whole moves each about twice as it doubles: + /// 10⁷ groups make 39 000 a part, which sub-tables of half a megabyte split three times over (q10) + /// or once (a key of 8 bytes). Measured on the Mac on 2026-10-10 at fourteen lanes, the process's + /// shelf keeping their arrays: against half a megabyte, 64 000 groups took ×0.754 off q10 on 10M + /// (132 to 100 ms) and ×0.787 off the keys of 10⁷ hashed on 40M; 128 000 the same, 256 000 ×1.10 on + /// the hashed keys (the first sub-table reserves what is pending, four times their groups there), + /// and 40 000, just past q10's parts, ×1.20: a single split of a sub-table that size costs more than + /// it saves. A burst's reads then spread over megabytes, a private cache on a current x64 and the + /// shared cache past it: to measure there. + /// + private const int WholeTableGroups = 65_536; + + /// The bytes past which a sub-table of wide groups splits before : eight megabytes. + private const long WideTableBytes = 8L << 20; + /// The most α: past it, reading the sub-tables back weighs a few percent of the batches' bytes, and only the memory grows. private const int MostAlpha = 8; @@ -221,7 +239,7 @@ private GroupCore( FlushAt = Math.Max(1, (int)(2L * Capacity / 3)); Bypass = plan.CoreBypass ?? DefaultBypass; BypassPeriod = plan.CoreBypassPeriod ?? DefaultBypassPeriod; - _tableGroups = plan.CoreTableGroups ?? Math.Max(64, tableBytes / groupBytes); + _tableGroups = plan.CoreTableGroups ?? (lean ? Math.Max(64, tableBytes / groupBytes) : Math.Clamp(WideTableBytes / groupBytes, 64, WholeTableGroups)); _tableBytes = _tableGroups * groupBytes; _batchEntries = plan.CoreBatchEntries ?? (int)Math.Max(1, batchBytes / entryBytes); _batchWords = _batchEntries * Shape.Words; From 5557a340c822105b5d324ca06dbff99c6a300e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 03:18:27 +0200 Subject: [PATCH 44/56] fix(query): the process's shelf takes back from the rest what it gave it An array the process's shelf handed to code that counts nothing went to the budget's rest, and came back from it untaken: every query added its scratch, its lent orders and its rows' groups to the rest again, until the budget refused a reservation, the shelf let everything go, and a query made 240 MB anew every few rounds (q10 on 4M rows at fourteen lanes). Every array given to the shelf was let go to the rest first, a query's when it gave it back, another's when the shelf handed it out or made it new for it: the shelf now always takes it back from there. Sixteen rounds of q10 on 4M rows, warm: 1.7 to 11 MB each from the third, none past. --- src/Vorticity/Aggregation/ArrayShelf.cs | 40 +++++++++++++--------- src/Vorticity/Session/QueryMemoryBudget.cs | 13 +++---- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index d8cfdf419..e3b9a45b0 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -448,18 +448,26 @@ internal T[] Take(int length, bool zeroed, bool overdraw = false, bool counte counts = true; } - return _parent is not null ? _parent.Take(length, zeroed, counted: counts) - : zeroed ? new T[length] - : GC.AllocateUninitializedArray(length); + if (_parent is not null) + { + return _parent.Take(length, zeroed, counted: counts); + } + + // Made new for code that counts nothing: the rest of the process, which the shelf takes it out of + // when it is given back. + if (_governed && !counted && (long)length * Unsafe.SizeOf() is long made and >= LargeBytes) + { + QueryMemoryBudget.Process.Discard(made); + } + + return zeroed ? new T[length] : GC.AllocateUninitializedArray(length); } /// /// Puts an array nothing holds any more back on the shelf; past the shelf's budget, lets it go. /// A lane's shelf gives its bytes back to the query's memory and the array to the process's shelf. - /// : whether the giver counted it in its query's memory, which it gave back - /// as let go, so that the process's shelf counts it as kept instead. /// - internal void Give(T[] array, bool counted = false) + internal void Give(T[] array) { if (Lane) { @@ -474,20 +482,19 @@ internal void Give(T[] array, bool counted = false) return; } - bool lent = (long)array.Length * Unsafe.SizeOf() >= LeastCounted; - if (lent) + if ((long)array.Length * Unsafe.SizeOf() is long bytes and >= LeastCounted) { - Unreserve(_memory!, (long)array.Length * Unsafe.SizeOf()); + Unreserve(_memory!, bytes); } - Retained.Give(array, lent); + Retained.Give(array); return; } Clean(array); // Let go past the shelf's budget, or while the shelf lets go of what it is given: it leaves the query's count. - if (Drops || !Give(typeof(T), array, Unsafe.SizeOf(), fromQuery: counted)) + if (Drops || !Give(typeof(T), array, Unsafe.SizeOf())) { Leave(array.Length * (long)Unsafe.SizeOf()); } @@ -500,7 +507,7 @@ internal void Give(T[] array, bool counted = false) private void Offer(T[] array, long since) { Clean(array); - Give(typeof(T), array, Unsafe.SizeOf(), since, fromQuery: true); + Give(typeof(T), array, Unsafe.SizeOf(), since); } /// An array given back made safe to keep: poisoned under the tests, its references cleared. @@ -692,8 +699,8 @@ internal void Clear() { Array array = pile.Arrays.Pop(); long bytes = (long)array.Length * pile.ElementBytes; - _parent.Give(entry.Key.Type, array, pile.ElementBytes, fromQuery: _memory is not null && bytes >= LeastCounted); Leave(bytes); + _parent.Give(entry.Key.Type, array, pile.ElementBytes); } } } @@ -708,10 +715,9 @@ internal void Clear() /// least recently go first, those last used before only: a shelf full of /// another query's lengths kept them while every query went on taking from it, and the arrays of the /// queries running now could not come in. The process's shelf counts a large one in the process's - /// budget, and lets it go when the budget has no room for it; : whether - /// its giver had counted it, and given it back as let go. + /// budget, and lets it go when the budget has no room for it. /// - private bool Give(Type type, Array array, int elementBytes, long before = long.MaxValue, bool fromQuery = false) + private bool Give(Type type, Array array, int elementBytes, long before = long.MaxValue) { int length = array.Length; if (length == 0) @@ -752,7 +758,7 @@ private bool Give(Type type, Array array, int elementBytes, long before = long.M if (_governed && !pile.Small) { - if (!QueryMemoryBudget.Process.TryKeep(pile.Bytes, fromQuery)) + if (!QueryMemoryBudget.Process.TryKeep(pile.Bytes)) { return false; } diff --git a/src/Vorticity/Session/QueryMemoryBudget.cs b/src/Vorticity/Session/QueryMemoryBudget.cs index d0a6697d1..98e7bd6f8 100644 --- a/src/Vorticity/Session/QueryMemoryBudget.cs +++ b/src/Vorticity/Session/QueryMemoryBudget.cs @@ -240,10 +240,11 @@ internal void Measure(long delta) /// /// The process's shelf keeps an array of for the next query: reserved and /// measured as the queries' tables are, within the ceiling and with no collection asked for; false, - /// and nothing counted, past it. : whether its query had counted it and - /// given it back as let go, which it no longer is. The process's own budget. + /// and nothing counted, past it. Every array given to it was let go to the rest first: a query's, + /// when it gave it back; another's, when the shelf handed it out () or made it + /// new for code that counts nothing. The process's own budget. /// - internal bool TryKeep(long bytes, bool fromQuery) + internal bool TryKeep(long bytes) { Observe(); if (!TryAdd(ref _reserved, bytes, Available())) @@ -252,11 +253,7 @@ internal bool TryKeep(long bytes, bool fromQuery) } Interlocked.Add(ref _measured, bytes); - if (fromQuery) - { - Interlocked.Add(ref _pending, -bytes); - } - + Interlocked.Add(ref _pending, -bytes); return true; } From 0960f79f9c9931724f3323c65813fbf5414b4802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 03:33:14 +0200 Subject: [PATCH 45/56] perf(query): a part's first sub-table at a quarter step, the shelf idle only with no query A part's first sub-table reserved exactly what its entries could hold: lengths no other part nor query asked again, which the process's shelf piled one by one and never found again (830 new piles a query on q10). It now rounds its room up to a quarter of a power of two, four lengths an octave, a quarter wasted at most. The process's shelf went back to its budget after two ticks without a use, which a query between two uses could take: under a debugger, q10's scan went seconds without one, and its shelf, let go mid-query, made 850 MB again a round. A query running now keeps it. Alternated runners against 9e8697c1 at fourteen lanes, warm: q10 on 10M x0.950 (101 to 95 ms), on 4M x0.960, 10^7 hashed keys on 40M x0.968. --- src/Vorticity/Aggregation/ArrayShelf.cs | 8 +++++--- src/Vorticity/Aggregation/GroupCore.cs | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/Vorticity/Aggregation/ArrayShelf.cs b/src/Vorticity/Aggregation/ArrayShelf.cs index e3b9a45b0..587a45757 100644 --- a/src/Vorticity/Aggregation/ArrayShelf.cs +++ b/src/Vorticity/Aggregation/ArrayShelf.cs @@ -227,15 +227,17 @@ private void Watch() } /// - /// A tick of the timer: a shelf used since the last waits on; idle for , it lets - /// go of what it keeps past its budget, to the collector, which it asks to run when that is much. + /// A tick of the timer: a shelf used since the last, or while a query runs, waits on; idle for + /// , it lets go of what it keeps past its budget, to the collector, which it + /// asks to run when that is much. A query between two uses of the shelf is not idle: under a + /// debugger, q10's scan went seconds without one, and its shelf, let go mid-query, made 850 MB again. /// private void Tick() { long trimmed = 0; lock (_gate) { - if (_active) + if (_active || QueryMemoryBudget.Process.ActiveQueries > 0) { _active = false; _idle = 0; diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index e139067e1..df053d548 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -314,7 +314,10 @@ private GroupCore( /// /// A part's first sub-table: its pages side by side, numbered whole, in a paged core; else one of /// hashes made at once for the groups entries may hold, which would - /// otherwise double some ten times, rehashing, through the part's first bursts. + /// otherwise double some ten times, rehashing, through the part's first bursts. Its room is rounded + /// up to a quarter of a power of two: a part's exact count made arrays of lengths no other part nor + /// query asked again, which the process's shelf piled one by one and found never (830 piles a query + /// on q10). /// internal SubTable FirstTable(CorePart part, int entries) { @@ -324,10 +327,22 @@ internal SubTable FirstTable(CorePart part, int entries) } SubTable first = NewTable(0); - first.Reserve((int)Math.Min(entries, _tableGroups)); + first.Reserve((int)Math.Min(QuarterStep(entries), _tableGroups)); return first; } + /// rounded up to a multiple of a quarter of the power of two at or below it: four lengths an octave, a quarter wasted at most. + internal static int QuarterStep(int count) + { + if (count <= 4) + { + return Math.Max(count, 1); + } + + int step = 1 << (BitOperations.Log2((uint)count) - 2); + return (int)Math.Min(int.MaxValue, ((long)count + step - 1) & -(long)step); + } + /// The sub-table of a part of a paged core: the part's pages side by side, numbered whole. private SubTable NewPagedTable(int part) { From 79599dc28dd98848b119880fd899c5933f6dc8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 04:10:05 +0200 Subject: [PATCH 46/56] docs: the comparison with DuckDB taken again, every row ahead Two sessions at 7445c2a5, one after the other, agreeing within 10 % on every row. Vorticity now leads every query on both bases at one and at fourteen threads: db-benchmark's q10 at fourteen threads goes from 0.51x to 1.05x over DuckDB's own table and from 0.65x to 1.26x over its reader, 10^7 hashed keys from 0.96x to 1.76x, the memory of the groups kept from one query to the next and the shared table's parts held whole. --- docs/guide/benchmarks-duckdb.md | 141 ++++++++++++++++---------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/docs/guide/benchmarks-duckdb.md b/docs/guide/benchmarks-duckdb.md index f4c497c91..a81f1b57c 100644 --- a/docs/guide/benchmarks-duckdb.md +++ b/docs/guide/benchmarks-duckdb.md @@ -53,37 +53,37 @@ Warm: our file kept mapped, DuckDB's own table. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 31.2 | 117 | 3.73x | 3.54–3.84 | 6 | -| count and sum by 10⁶ keys | 40M | 130.2 | 762 | 5.90x | 5.54–6.18 | 6 | -| count and sum by 10⁷ keys | 40M | 581.4 | 1 352 | 2.32x | 2.29–2.37 | 6 | -| count, least and largest by 10⁶ keys | 40M | 284.4 | 882 | 3.11x | 3.07–3.14 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 275.4 | 764 | 2.75x | 2.62–2.89 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 690.9 | 1 326 | 1.92x | 1.90–1.96 | 6 | -| db q1: sum v1 by id1 | 10M | 39.8 | 73.5 | 1.85x | 1.82–1.88 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 83.0 | 138 | 1.66x | 1.65–1.72 | 6 | -| db q3: sum v1, mean v3 by id3 | 10M | 102.7 | 179 | 1.73x | 1.69–1.76 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 23.7 | 44.0 | 1.86x | 1.79–1.88 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 48.3 | 117 | 2.42x | 2.37–2.47 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 86.2 | 174 | 2.02x | 2.01–2.05 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 682.9 | 826 | 1.21x | 1.18–1.29 | 9 | +| count and sum by 10³ keys | 40M | 31.1 | 119 | 3.81x | 3.60–3.83 | 6 | +| count and sum by 10⁶ keys | 40M | 114.0 | 717 | 6.28x | 5.95–6.53 | 9 | +| count and sum by 10⁷ keys | 40M | 554.3 | 1 328 | 2.40x | 2.39–2.41 | 6 | +| count, least and largest by 10⁶ keys | 40M | 274.8 | 866 | 3.15x | 3.10–3.18 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 254.7 | 741 | 2.90x | 2.75–3.04 | 9 | +| count and sum by 10⁷ hashed keys | 40M | 679.5 | 1 332 | 1.96x | 1.95–1.98 | 6 | +| db q1: sum v1 by id1 | 10M | 39.3 | 73.5 | 1.87x | 1.85–1.87 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 83.3 | 138 | 1.66x | 1.64–1.66 | 6 | +| db q3: sum v1, mean v3 by id3 | 10M | 102.5 | 177 | 1.73x | 1.59–1.73 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 23.3 | 43.5 | 1.86x | 1.80–1.89 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 47.4 | 118 | 2.48x | 2.45–2.51 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 86.0 | 176 | 2.04x | 2.02–2.06 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 586.4 | 814 | 1.39x | 1.38–1.40 | 6 | Cold: our file mapped anew at every run, DuckDB's Vortex reader. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 52.2 | 242 | 4.64x | 4.55–4.70 | 6 | -| count and sum by 10⁶ keys | 40M | 177.0 | 889 | 5.03x | 4.57–5.21 | 9 | -| count and sum by 10⁷ keys | 40M | 597.4 | 1 435 | 2.40x | 2.30–2.44 | 6 | -| count, least and largest by 10⁶ keys | 40M | 313.3 | 955 | 3.05x | 2.98–3.08 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 346.2 | 939 | 2.71x | 2.65–2.77 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 723.4 | 1 437 | 2.00x | 1.97–2.02 | 6 | -| db q1: sum v1 by id1 | 10M | 42.9 | 139 | 3.23x | 3.19–3.29 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 87.8 | 259 | 2.96x | 2.84–3.02 | 9 | -| db q3: sum v1, mean v3 by id3 | 10M | 108.6 | 202 | 1.86x | 1.73–1.90 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 31.7 | 63.5 | 2.02x | 1.95–2.05 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 55.0 | 124 | 2.25x | 2.21–2.28 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 90.0 | 192 | 2.14x | 2.05–2.22 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 694.5 | 914 | 1.32x | 1.28–1.41 | 12 | +| count and sum by 10³ keys | 40M | 50.6 | 246 | 4.84x | 4.72–4.96 | 6 | +| count and sum by 10⁶ keys | 40M | 148.0 | 812 | 5.54x | 5.12–5.71 | 10 | +| count and sum by 10⁷ keys | 40M | 580.1 | 1 422 | 2.44x | 2.42–2.45 | 6 | +| count, least and largest by 10⁶ keys | 40M | 304.1 | 943 | 3.10x | 3.01–3.15 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 294.6 | 889 | 3.00x | 2.93–3.06 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 718.2 | 1 451 | 2.02x | 1.99–2.03 | 6 | +| db q1: sum v1 by id1 | 10M | 41.3 | 136 | 3.30x | 3.24–3.37 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 86.2 | 262 | 3.05x | 2.80–3.08 | 6 | +| db q3: sum v1, mean v3 by id3 | 10M | 108.6 | 195 | 1.79x | 1.75–1.84 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 31.5 | 65.0 | 2.08x | 2.02–2.11 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 54.8 | 124 | 2.26x | 2.19–2.31 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 90.3 | 199 | 2.20x | 2.09–2.33 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 596.2 | 917 | 1.53x | 1.52–1.58 | 6 | ## Fourteen threads @@ -91,58 +91,59 @@ Warm: our file kept mapped, DuckDB's own table. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 3.6 | 13.2 | 3.60x | 3.54–3.68 | 6 | -| count and sum by 10⁶ keys | 40M | 65.2 | 145 | 2.23x | 2.15–2.41 | 9 | -| count and sum by 10⁷ keys | 40M | 73.0 | 178 | 2.47x | 2.34–2.56 | 9 | -| count, least and largest by 10⁶ keys | 40M | 97.5 | 175 | 1.79x | 1.67–1.91 | 9 | -| count and sum by 10⁶ hashed keys | 40M | 112.6 | 143 | 1.27x | 1.22–1.31 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 188.7 | 182 | 0.96x | 0.93–1.00 | 9 | -| db q1: sum v1 by id1 | 10M | 5.3 | 11.3 | 2.12x | 1.94–2.20 | 9 | -| db q2: sum v1 by id1, id2 | 10M | 13.4 | 16.9 | 1.25x | 1.16–1.28 | 9 | -| db q3: sum v1, mean v3 by id3 | 10M | 39.8 | 46.0 | 1.16x | 1.14–1.19 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 3.1 | 5.3 | 1.74x | 1.69–1.80 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 20.8 | 45.5 | 2.20x | 2.12–2.30 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 27.5 | 49.0 | 1.78x | 1.70–1.87 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 211.0 | 107 | 0.51x | 0.50–0.52 | 6 | +| count and sum by 10³ keys | 40M | 3.6 | 13.2 | 3.66x | 3.47–3.77 | 9 | +| count and sum by 10⁶ keys | 40M | 61.9 | 146 | 2.36x | 2.10–2.48 | 11 | +| count and sum by 10⁷ keys | 40M | 65.3 | 179 | 2.73x | 2.65–2.88 | 6 | +| count, least and largest by 10⁶ keys | 40M | 84.8 | 171 | 2.04x | 1.95–2.16 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 76.6 | 144 | 1.88x | 1.84–1.91 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 102.3 | 181 | 1.76x | 1.74–1.80 | 6 | +| db q1: sum v1 by id1 | 10M | 5.2 | 11.1 | 2.09x | 2.03–2.16 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 12.4 | 16.6 | 1.35x | 1.26–1.37 | 9 | +| db q3: sum v1, mean v3 by id3 | 10M | 35.4 | 45.5 | 1.29x | 1.24–1.30 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 3.0 | 5.2 | 1.72x | 1.67–1.80 | 10 | +| db q5: sum v1:v3 by id6 | 10M | 17.2 | 44.5 | 2.60x | 2.50–2.65 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 22.9 | 48.0 | 2.11x | 1.95–2.13 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 97.4 | 103 | 1.05x | 1.02–1.07 | 6 | Cold: our file mapped anew at every run, DuckDB's Vortex reader. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 12.4 | 52.8 | 4.23x | 3.50–4.62 | 16 | -| count and sum by 10⁶ keys | 40M | 72.7 | 183 | 2.51x | 2.46–2.66 | 6 | -| count and sum by 10⁷ keys | 40M | 79.9 | 228 | 2.87x | 2.80–3.02 | 6 | -| count, least and largest by 10⁶ keys | 40M | 103.8 | 200 | 1.95x | 1.89–2.00 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 123.2 | 183 | 1.49x | 1.45–1.56 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 192.3 | 226 | 1.18x | 1.13–1.20 | 6 | -| db q1: sum v1 by id1 | 10M | 6.9 | 21.5 | 3.13x | 2.87–3.20 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 15.7 | 35.5 | 2.23x | 2.18–2.31 | 6 | -| db q3: sum v1, mean v3 by id3 | 10M | 44.6 | 60.5 | 1.36x | 1.34–1.40 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 6.4 | 17.9 | 2.79x | 2.73–2.98 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 23.6 | 57.2 | 2.42x | 2.27–2.58 | 9 | -| db q7: max v1 − min v2 by id3 | 10M | 31.6 | 56.5 | 1.81x | 1.74–1.94 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 205.6 | 133 | 0.65x | 0.61–0.67 | 9 | +| count and sum by 10³ keys | 40M | 12.0 | 51.5 | 4.29x | 3.39–4.75 | 16 | +| count and sum by 10⁶ keys | 40M | 66.4 | 187 | 2.81x | 2.75–2.89 | 6 | +| count and sum by 10⁷ keys | 40M | 71.4 | 231 | 3.19x | 2.99–3.33 | 10 | +| count, least and largest by 10⁶ keys | 40M | 90.6 | 201 | 2.24x | 2.11–2.31 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 84.2 | 189 | 2.23x | 2.09–2.26 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 111.3 | 229 | 2.04x | 1.94–2.14 | 10 | +| db q1: sum v1 by id1 | 10M | 6.7 | 21.0 | 3.15x | 3.04–3.31 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 14.4 | 34.5 | 2.39x | 2.30–2.48 | 6 | +| db q3: sum v1, mean v3 by id3 | 10M | 38.1 | 58.5 | 1.55x | 1.48–1.58 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 6.1 | 17.6 | 2.90x | 2.82–2.97 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 19.5 | 55.0 | 2.82x | 2.70–2.86 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 26.0 | 60.2 | 2.28x | 2.00–2.42 | 14 | +| db q10: sum v3, count by id1:id6 | 10M | 101.8 | 128 | 1.26x | 1.21–1.29 | 6 | ## Where Vorticity leads, and where it trails -* **On one thread it leads every query on both bases**: 1.21x to 5.90x over DuckDB's own table, 1.32x - to 5.03x over its reader. db-benchmark's q10, six keys of which three are texts and a group for - nearly every one of 10⁷ rows, leads by the least: 1.21x and 1.32x. -* **On fourteen threads it leads every query but two**: 1.16x to 3.60x over DuckDB's own table and - 1.18x to 4.23x over its reader. Integer keys of 10³ to 10⁷ values lead by 1.79x to 3.60x warm, and - db-benchmark's q1 to q7 by 1.16x to 2.20x warm, 1.36x to 3.13x cold. -* **db-benchmark's q10 trails on fourteen threads**: 0.51x over DuckDB's own table, 0.65x over its - reader. Its six keys are held as one tuple of their values, which the lanes hand to a shared table - of groups by parts; DuckDB writes rows nearly all unique straight to its partitions. -* **10⁷ hashed keys are level with DuckDB's own table on fourteen threads**: 0.96x, its interval - reaching 1.00x; 1.18x over its reader. - -Measured on 2026-10-09 on an Apple M4 Pro (14 cores: 10 performance, 4 efficiency) under macOS, at -commit `41467eaf`: two sessions one after the other, from 21:24 to 21:35 and from 21:35 to 21:53, and a -third for the two rows they did not agree on (10⁶ keys and 10⁶ hashed keys on one thread, cold, which -took 4.95x and 2.70x there). The machine was not at rest: in the second session another process held -a query back 356 s in all, and its canaries strayed up to 30 % from their first values, which the -pairs share; the first strayed up to 16 %. +* **On one thread it leads every query on both bases**: 1.39x to 6.28x over DuckDB's own table, 1.53x + to 5.54x over its reader. db-benchmark's q10, six keys of which three are texts and a group for + nearly every one of 10⁷ rows, leads by the least: 1.39x and 1.53x. +* **On fourteen threads it leads every query on both bases**: 1.05x to 3.66x over DuckDB's own table + and 1.26x to 4.29x over its reader. Integer keys of 10³ to 10⁷ values lead by 2.04x to 3.66x warm, + hashed ones by 1.76x and 1.88x, and db-benchmark's q1 to q7 by 1.29x to 2.60x warm, 1.55x to 3.15x + cold. +* **db-benchmark's q10 leads by the least on fourteen threads**: 1.05x over DuckDB's own table, its + interval 1.02x to 1.07x, and 1.26x over its reader. Its six keys are held as one tuple of their + values, which the lanes hand to a shared table of groups by parts; DuckDB writes rows nearly all + unique straight to its partitions. It trailed at 0.51x on 2026-10-09, as 10⁷ hashed keys did at + 0.96x (1.76x now): the memory of the groups is now kept from one query to the next rather than made + anew, and each part of the shared table is held whole rather than split as it grows. + +Measured on 2026-10-10 on an Apple M4 Pro (14 cores: 10 performance, 4 efficiency) under macOS, at +commit `7445c2a5`: two sessions one after the other, from 03:45 to 03:56 and from 03:56 to 04:08, which +agreed within 10 % on every row. The machine was near rest: the canaries strayed up to 7.6 % and 8.0 % +from their first values, another process held a query back 24 s and 6 s in all, and two rows of the +second session ran again. Until 2026-10-09 this page timed every DuckDB query, then every one of ours: a perturbation of the machine weighed on one side unseen, and DuckDB's q10 on fourteen threads read 257 ms where it took 102 From bfd524c360109ad4a1c1634d0d5bc5361436abc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 04:26:04 +0200 Subject: [PATCH 47/56] docs: benchmarks.md measured again, section by section Every section of the page rewritten by its own instrument at 32b3a82b: the report's scenarios and decoders, the in-process ratios, taking and writing per encoding, the encodings' trade-offs and the advice, and the four kernel classes. Every ratio-check axis inside its ceiling; Vorticity decodes 51 of 57 files faster than Rust (median 1.27x), takes rows faster on 56 of 57 (1.15x) and writes faster on 56 of 56 (3.76x). --- docs/guide/benchmarks.md | 980 +++++++++++++++++++-------------------- 1 file changed, 490 insertions(+), 490 deletions(-) diff --git a/docs/guide/benchmarks.md b/docs/guide/benchmarks.md index 9cb9ff677..12b7105ed 100644 --- a/docs/guide/benchmarks.md +++ b/docs/guide/benchmarks.md @@ -88,41 +88,41 @@ AVX-512) under Windows 11, every section measured there, against a Rust built on | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.1 (0.1-0.2) | 0.5 (0.5-0.6) | 3.65x | — | — | 5.4 KiB | 10 / 11 MiB | -| `open` | Vortex Rust's | 0.1 (0.1-0.2) | 0.5 (0.4-0.6) | 3.60x | — | — | 5.0 KiB | 10 / 11 MiB | -| `scan` | Vorticity's | 4.3 (4.1-4.5) | 8.5 (8.4-8.7) | 1.98x | 6.2 | 3.1 | 14.7 KiB | 15 / 18 MiB | -| `scan` | Vortex Rust's | 1.9 (1.8-2.3) | 2.6 (2.5-2.7) | 1.38x | 14.1 | 10.2 | 11.1 KiB | 23 / 25 MiB | -| `project` | Vorticity's | 0.4 (0.4-0.4) | 0.9 (0.8-1.0) | 2.23x | 21.7 | 9.7 | 17.0 KiB | 11 / 13 MiB | -| `project` | Vortex Rust's | 0.4 (0.4-0.4) | 0.9 (0.8-0.9) | 2.12x | 20.9 | 9.8 | 13.3 KiB | 11 / 14 MiB | -| `filter-narrow` | Vorticity's | 0.9 (0.8-0.9) | 1.2 (1.1-1.2) | 1.35x | 0.3 | 0.2 | 41.4 KiB | 14 / 15 MiB | -| `filter-narrow` | Vortex Rust's | 0.8 (0.7-0.8) | 1.1 (1.1-1.2) | 1.43x | 0.3 | 0.2 | 21.2 KiB | 15 / 15 MiB | -| `filter-wide` | Vorticity's | 2.4 (2.4-2.7) | 5.0 (4.9-5.4) | 2.05x | 5.5 | 2.7 | 23.9 KiB | 14 / 17 MiB | -| `filter-wide` | Vortex Rust's | 1.3 (1.3-1.5) | 2.2 (2.2-2.3) | 1.66x | 10.0 | 6.0 | 21.2 KiB | 20 / 20 MiB | -| `take` | Vorticity's | 4.0 (3.9-4.2) | 6.4 (6.3-6.8) | 1.59x | — | — | 38.4 KiB | 13 / 17 MiB | -| `take` | Vortex Rust's | 0.7 (0.6-0.8) | 1.4 (1.4-1.6) | 2.08x | — | — | 34.8 KiB | 18 / 21 MiB | -| `write` | Vorticity's | 24.0 (23.7-24.4) | 57.5 (56.7-60.2) | 2.39x | 1.1 | 0.5 | 30.9 KiB | 20 / 76 MiB | -| `write` | Vortex Rust's | 21.4 (21.4-21.9) | 51.3 (50.8-52.9) | 2.39x | 1.2 | 0.5 | 27.2 KiB | 28 / 81 MiB | -| `append` | Vorticity's | 11.3 (11.0-11.4) | not asked | n/a | — | — | 213.5 KiB | 18 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.5 (0.5-0.5) | 3.88x | — | — | 5.4 KiB | 11 / 11 MiB | +| `open` | Vortex Rust's | 0.1 (0.1-0.1) | 0.5 (0.5-0.7) | 3.96x | — | — | 5.0 KiB | 11 / 11 MiB | +| `scan` | Vorticity's | 4.3 (4.2-4.4) | 8.6 (8.4-8.7) | 1.97x | 6.1 | 3.1 | 14.7 KiB | 17 / 18 MiB | +| `scan` | Vortex Rust's | 1.8 (1.8-1.8) | 2.6 (2.5-2.7) | 1.46x | 14.7 | 10.1 | 11.1 KiB | 25 / 25 MiB | +| `project` | Vorticity's | 0.4 (0.3-0.4) | 0.9 (0.8-1.0) | 2.41x | 23.4 | 9.7 | 17.0 KiB | 13 / 13 MiB | +| `project` | Vortex Rust's | 0.4 (0.4-0.4) | 0.9 (0.8-1.6) | 2.27x | 21.8 | 9.6 | 13.3 KiB | 13 / 14 MiB | +| `filter-narrow` | Vorticity's | 0.8 (0.8-0.8) | 1.1 (1.1-1.2) | 1.45x | 0.3 | 0.2 | 41.4 KiB | 16 / 15 MiB | +| `filter-narrow` | Vortex Rust's | 0.7 (0.6-0.8) | 1.1 (0.9-1.1) | 1.47x | 0.4 | 0.3 | 21.2 KiB | 18 / 15 MiB | +| `filter-wide` | Vorticity's | 2.5 (2.4-2.6) | 5.0 (4.9-5.2) | 2.04x | 5.4 | 2.7 | 23.9 KiB | 16 / 17 MiB | +| `filter-wide` | Vortex Rust's | 1.3 (1.2-1.3) | 2.1 (2.1-2.2) | 1.63x | 10.5 | 6.4 | 21.2 KiB | 22 / 22 MiB | +| `take` | Vorticity's | 3.8 (3.8-4.1) | 6.4 (6.3-6.5) | 1.68x | — | — | 38.4 KiB | 16 / 17 MiB | +| `take` | Vortex Rust's | 0.6 (0.6-0.7) | 1.3 (1.1-1.4) | 2.06x | — | — | 34.8 KiB | 20 / 21 MiB | +| `write` | Vorticity's | 23.9 (23.7-23.9) | 55.7 (55.4-58.1) | 2.33x | 1.1 | 0.5 | 30.9 KiB | 22 / 75 MiB | +| `write` | Vortex Rust's | 21.5 (21.1-22.2) | 49.7 (49.4-50.7) | 2.31x | 1.2 | 0.5 | 27.2 KiB | 31 / 81 MiB | +| `append` | Vorticity's | 6.4 (5.4-7.3) | not asked | n/a | — | — | 2.5 MiB | 21 / — MiB | #### All 14 cores | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.1 (0.1-0.2) | 0.7 (0.6-0.8) | 4.82x | — | — | 5.4 KiB | 10 / 12 MiB | -| `open` | Vortex Rust's | 0.2 (0.2-0.2) | 0.7 (0.7-0.7) | 4.45x | — | — | 5.0 KiB | 10 / 12 MiB | -| `scan` | Vorticity's | 1.7 (1.7-1.8) | 3.9 (3.7-4.2) | 2.31x | 15.7 | 6.8 | 39.1 KiB | 43 / 49 MiB | -| `scan` | Vortex Rust's | 1.7 (1.6-1.7) | 4.0 (3.6-4.0) | 2.34x | 15.8 | 6.7 | 23.7 KiB | 62 / 55 MiB | -| `project` | Vorticity's | 0.6 (0.6-0.7) | 1.4 (1.4-1.5) | 2.24x | 13.0 | 5.8 | 31.6 KiB | 18 / 22 MiB | -| `project` | Vortex Rust's | 0.7 (0.6-0.7) | 1.5 (1.4-1.7) | 2.17x | 12.3 | 5.7 | 22.2 KiB | 19 / 22 MiB | -| `filter-narrow` | Vorticity's | 0.6 (0.5-0.6) | 1.6 (1.5-1.7) | 2.89x | 0.5 | 0.2 | 43.2 KiB | 12 / 18 MiB | -| `filter-narrow` | Vortex Rust's | 0.5 (0.4-0.5) | 1.6 (1.5-1.7) | 3.00x | 0.5 | 0.2 | 24.2 KiB | 12 / 19 MiB | -| `filter-wide` | Vorticity's | 1.2 (1.1-1.3) | 2.7 (2.6-2.8) | 2.31x | 11.3 | 4.9 | 35.1 KiB | 29 / 36 MiB | -| `filter-wide` | Vortex Rust's | 1.2 (1.1-1.3) | 2.6 (2.5-2.7) | 2.27x | 11.5 | 5.1 | 29.2 KiB | 37 / 39 MiB | -| `take` | Vorticity's | 1.9 (1.8-2.0) | 2.2 (2.1-2.2) | 1.17x | — | — | 81.2 KiB | 24 / 28 MiB | -| `take` | Vortex Rust's | 0.9 (0.8-0.9) | 1.6 (1.5-1.6) | 1.75x | — | — | 77.8 KiB | 20 / 26 MiB | -| `write` | Vorticity's | 10.8 (10.7-11.0) | 14.8 (14.8-15.4) | 1.37x | 2.5 | 1.8 | 2.5 MiB | 54 / 109 MiB | -| `write` | Vortex Rust's | 10.5 (10.5-10.7) | 15.0 (14.4-15.3) | 1.43x | 2.5 | 1.8 | 275.1 KiB | 73 / 113 MiB | -| `append` | Vorticity's | 10.9 (10.5-13.2) | not asked | n/a | — | — | 211.2 KiB | 19 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.7 (0.6-0.8) | 5.20x | — | — | 5.4 KiB | 11 / 12 MiB | +| `open` | Vortex Rust's | 0.1 (0.1-0.1) | 0.8 (0.7-0.8) | 5.29x | — | — | 5.0 KiB | 11 / 12 MiB | +| `scan` | Vorticity's | 1.6 (1.6-1.7) | 3.8 (3.6-3.9) | 2.30x | 16.2 | 7.0 | 36.1 KiB | 45 / 49 MiB | +| `scan` | Vortex Rust's | 1.6 (1.5-1.6) | 3.6 (3.4-3.8) | 2.28x | 17.1 | 7.5 | 24.5 KiB | 64 / 54 MiB | +| `project` | Vorticity's | 0.6 (0.6-0.6) | 1.4 (1.2-1.5) | 2.35x | 13.7 | 5.8 | 31.2 KiB | 20 / 22 MiB | +| `project` | Vortex Rust's | 0.6 (0.5-0.6) | 1.4 (1.1-1.6) | 2.53x | 14.8 | 5.8 | 21.6 KiB | 21 / 23 MiB | +| `filter-narrow` | Vorticity's | 0.5 (0.5-0.6) | 1.5 (1.3-1.7) | 2.97x | 0.5 | 0.2 | 42.7 KiB | 14 / 18 MiB | +| `filter-narrow` | Vortex Rust's | 0.5 (0.5-0.6) | 1.4 (1.3-1.5) | 2.95x | 0.6 | 0.2 | 42.0 KiB | 14 / 18 MiB | +| `filter-wide` | Vorticity's | 1.3 (1.2-1.3) | 2.8 (2.6-3.3) | 2.12x | 10.3 | 4.8 | 37.2 KiB | 31 / 34 MiB | +| `filter-wide` | Vortex Rust's | 1.1 (1.0-1.3) | 2.5 (2.3-2.7) | 2.31x | 12.3 | 5.3 | 30.2 KiB | 39 / 39 MiB | +| `take` | Vorticity's | 2.1 (1.9-2.3) | 2.2 (2.2-2.4) | 1.02x | — | — | 82.1 KiB | 26 / 28 MiB | +| `take` | Vortex Rust's | 0.8 (0.8-0.8) | 1.5 (1.5-1.7) | 1.88x | — | — | 76.4 KiB | 22 / 26 MiB | +| `write` | Vorticity's | 10.5 (10.5-10.7) | 14.4 (14.2-15.1) | 1.37x | 2.5 | 1.8 | 2.3 MiB | 57 / 105 MiB | +| `write` | Vortex Rust's | 10.2 (10.0-10.5) | 14.8 (14.3-15.0) | 1.44x | 2.6 | 1.8 | 1.3 MiB | 76 / 110 MiB | +| `append` | Vorticity's | 5.9 (5.5-6.9) | not asked | n/a | — | — | 1.3 MiB | 22 / — MiB | ### 10,485,760 rows @@ -137,56 +137,56 @@ AVX-512) under Windows 11, every section measured there, against a Rust built on | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.2 (0.1-0.2) | 0.5 (0.4-0.6) | 3.41x | — | — | 21.1 KiB | 10 / 11 MiB | -| `open` | Vortex Rust's | 0.2 (0.1-0.2) | 0.5 (0.5-0.6) | 3.19x | — | — | 9.5 KiB | 10 / 11 MiB | -| `scan` | Vorticity's | 39.6 (39.1-39.9) | 76.7 (76.2-76.8) | 1.94x | 6.7 | 3.5 | 74.4 KiB | 36 / 40 MiB | -| `scan` | Vortex Rust's | 13.9 (13.8-17.6) | 17.4 (17.2-17.8) | 1.25x | 19.2 | 15.3 | 28.5 KiB | 89 / 91 MiB | -| `project` | Vorticity's | 1.4 (1.3-1.5) | 2.0 (1.9-2.2) | 1.41x | 59.6 | 42.1 | 76.6 KiB | 13 / 16 MiB | -| `project` | Vortex Rust's | 1.1 (1.1-2.0) | 1.6 (1.5-1.8) | 1.44x | 75.3 | 52.1 | 30.6 KiB | 11 / 14 MiB | -| `filter-narrow` | Vorticity's | 1.3 (1.3-2.8) | 2.4 (2.4-2.5) | 1.82x | 2.0 | 1.1 | 107.1 KiB | 15 / 18 MiB | -| `filter-narrow` | Vortex Rust's | 1.4 (1.3-4.2) | 1.8 (1.8-3.6) | 1.34x | 2.0 | 1.5 | 62.2 KiB | 17 / 20 MiB | -| `filter-wide` | Vorticity's | 20.0 (19.9-20.6) | 40.6 (40.4-41.3) | 2.03x | 6.7 | 3.3 | 107.1 KiB | 25 / 29 MiB | -| `filter-wide` | Vortex Rust's | 10.3 (10.1-12.4) | 11.5 (10.9-12.8) | 1.11x | 12.9 | 11.6 | 62.2 KiB | 53 / 56 MiB | -| `take` | Vorticity's | 29.3 (28.9-30.0) | 55.0 (54.8-55.2) | 1.88x | — | — | 98.0 KiB | 34 / 40 MiB | -| `take` | Vortex Rust's | 1.9 (1.8-4.6) | 3.8 (3.2-5.1) | 1.97x | — | — | 52.1 KiB | 37 / 42 MiB | -| `write` | Vorticity's | 218.2 (216.3-218.3) | 526.7 (522.7-533.8) | 2.41x | 1.2 | 0.5 | 95.5 KiB | 42 / 265 MiB | -| `write` | Vortex Rust's | 191.6 (190.5-194.3) | 474.9 (464.8-480.6) | 2.48x | 1.4 | 0.6 | 49.6 KiB | 97 / 396 MiB | -| `append` | Vorticity's | 85.4 (85.0-85.6) | not asked | n/a | — | — | 1.4 MiB | 23 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.5 (0.5-0.6) | 3.97x | — | — | 21.1 KiB | 11 / 11 MiB | +| `open` | Vortex Rust's | 0.2 (0.1-0.2) | 0.6 (0.5-0.8) | 3.50x | — | — | 9.5 KiB | 11 / 11 MiB | +| `scan` | Vorticity's | 39.6 (39.0-40.0) | 76.7 (76.4-77.9) | 1.93x | 6.7 | 3.5 | 74.4 KiB | 38 / 40 MiB | +| `scan` | Vortex Rust's | 13.2 (13.1-13.5) | 17.3 (17.0-17.9) | 1.31x | 20.2 | 15.4 | 28.5 KiB | 91 / 91 MiB | +| `project` | Vorticity's | 1.4 (1.3-1.4) | 2.0 (1.9-2.3) | 1.45x | 61.0 | 42.0 | 76.6 KiB | 15 / 16 MiB | +| `project` | Vortex Rust's | 1.1 (1.1-3.4) | 1.6 (1.6-2.4) | 1.44x | 73.4 | 50.8 | 30.6 KiB | 13 / 14 MiB | +| `filter-narrow` | Vorticity's | 1.2 (1.2-1.3) | 2.4 (2.4-2.9) | 1.94x | 2.2 | 1.1 | 107.1 KiB | 17 / 18 MiB | +| `filter-narrow` | Vortex Rust's | 1.3 (1.3-4.5) | 1.7 (1.7-1.9) | 1.34x | 2.1 | 1.5 | 62.2 KiB | 19 / 19 MiB | +| `filter-wide` | Vorticity's | 20.1 (20.0-21.5) | 40.7 (40.5-40.9) | 2.02x | 6.6 | 3.3 | 107.1 KiB | 27 / 29 MiB | +| `filter-wide` | Vortex Rust's | 10.3 (9.8-12.9) | 11.0 (10.6-12.8) | 1.06x | 12.9 | 12.2 | 62.2 KiB | 56 / 56 MiB | +| `take` | Vorticity's | 16.8 (16.8-17.2) | 55.1 (54.7-58.9) | 3.28x | — | — | 98.0 KiB | 36 / 40 MiB | +| `take` | Vortex Rust's | 1.5 (1.5-4.7) | 3.3 (3.2-3.5) | 2.12x | — | — | 52.1 KiB | 39 / 42 MiB | +| `write` | Vorticity's | 217.8 (216.1-224.7) | 530.6 (525.0-536.2) | 2.44x | 1.2 | 0.5 | 95.5 KiB | 44 / 260 MiB | +| `write` | Vortex Rust's | 191.5 (190.4-201.8) | 469.7 (464.6-480.9) | 2.45x | 1.4 | 0.6 | 49.7 KiB | 100 / 398 MiB | +| `append` | Vorticity's | 49.6 (49.3-53.9) | not asked | n/a | — | — | 2.4 MiB | 28 / — MiB | #### All 14 cores | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.2 (0.1-0.2) | 0.8 (0.6-0.8) | 4.39x | — | — | 21.1 KiB | 10 / 12 MiB | -| `open` | Vortex Rust's | 0.2 (0.1-0.2) | 0.7 (0.7-3.9) | 4.48x | — | — | 9.5 KiB | 10 / 12 MiB | -| `scan` | Vorticity's | 6.0 (5.6-6.6) | 11.0 (10.6-14.0) | 1.83x | 44.2 | 24.2 | 109.0 KiB | 64 / 79 MiB | -| `scan` | Vortex Rust's | 6.9 (5.2-8.7) | 7.5 (6.9-9.7) | 1.07x | 38.4 | 35.8 | 59.4 KiB | 161 / 131 MiB | -| `project` | Vorticity's | 1.2 (1.1-1.3) | 2.1 (1.9-3.1) | 1.77x | 72.3 | 40.9 | 92.6 KiB | 20 / 26 MiB | -| `project` | Vortex Rust's | 1.1 (1.0-4.4) | 2.0 (2.0-2.1) | 1.88x | 77.4 | 41.1 | 47.8 KiB | 25 / 23 MiB | -| `filter-narrow` | Vorticity's | 1.0 (0.9-1.1) | 2.6 (2.5-3.0) | 2.60x | 2.6 | 1.0 | 109.2 KiB | 16 / 23 MiB | -| `filter-narrow` | Vortex Rust's | 1.4 (1.3-4.8) | 2.4 (2.2-3.3) | 1.70x | 1.9 | 1.1 | 65.0 KiB | 17 / 23 MiB | -| `filter-wide` | Vorticity's | 3.9 (3.7-3.9) | 7.6 (7.6-7.8) | 1.98x | 34.6 | 17.5 | 145.9 KiB | 53 / 67 MiB | -| `filter-wide` | Vortex Rust's | 5.5 (5.1-8.3) | 6.3 (5.4-6.7) | 1.15x | 24.2 | 21.1 | 93.0 KiB | 131 / 92 MiB | -| `take` | Vorticity's | 4.9 (4.5-6.6) | 7.4 (7.2-8.4) | 1.52x | — | — | 175.2 KiB | 56 / 54 MiB | -| `take` | Vortex Rust's | 1.9 (1.8-3.2) | 3.0 (2.9-3.4) | 1.59x | — | — | 103.5 KiB | 40 / 47 MiB | -| `write` | Vorticity's | 67.6 (67.2-68.4) | 91.7 (90.9-103.7) | 1.36x | 3.9 | 2.9 | 2.6 MiB | 77 / 343 MiB | -| `write` | Vortex Rust's | 68.3 (67.4-68.6) | 92.7 (92.0-94.3) | 1.36x | 3.9 | 2.9 | 2.6 MiB | 177 / 426 MiB | -| `append` | Vorticity's | 82.9 (82.2-83.7) | not asked | n/a | — | — | 3.1 MiB | 53 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.2) | 0.8 (0.7-0.9) | 5.69x | — | — | 21.1 KiB | 11 / 12 MiB | +| `open` | Vortex Rust's | 0.1 (0.1-2.6) | 0.8 (0.7-0.9) | 5.57x | — | — | 9.5 KiB | 11 / 12 MiB | +| `scan` | Vorticity's | 5.9 (5.6-6.2) | 10.3 (10.1-10.9) | 1.74x | 45.1 | 25.9 | 111.4 KiB | 66 / 79 MiB | +| `scan` | Vortex Rust's | 4.9 (4.7-5.7) | 6.8 (6.7-9.3) | 1.39x | 54.9 | 39.5 | 62.5 KiB | 164 / 126 MiB | +| `project` | Vorticity's | 1.2 (1.1-1.2) | 1.9 (1.9-2.1) | 1.66x | 72.8 | 43.8 | 95.0 KiB | 23 / 26 MiB | +| `project` | Vortex Rust's | 1.2 (1.1-4.0) | 1.9 (1.8-2.2) | 1.59x | 70.0 | 43.9 | 46.8 KiB | 27 / 23 MiB | +| `filter-narrow` | Vorticity's | 0.9 (0.9-2.2) | 2.6 (2.6-3.0) | 2.83x | 2.8 | 1.0 | 109.2 KiB | 18 / 23 MiB | +| `filter-narrow` | Vortex Rust's | 1.3 (1.1-4.1) | 2.3 (2.2-5.3) | 1.71x | 2.0 | 1.2 | 65.0 KiB | 19 / 23 MiB | +| `filter-wide` | Vorticity's | 3.6 (3.3-4.4) | 7.5 (7.2-8.1) | 2.07x | 36.6 | 17.7 | 144.9 KiB | 55 / 70 MiB | +| `filter-wide` | Vortex Rust's | 4.5 (4.5-5.9) | 5.6 (5.3-6.4) | 1.24x | 29.5 | 23.8 | 87.6 KiB | 133 / 90 MiB | +| `take` | Vorticity's | 3.7 (3.2-4.1) | 7.4 (7.3-7.9) | 1.98x | — | — | 169.2 KiB | 54 / 54 MiB | +| `take` | Vortex Rust's | 1.8 (1.8-2.9) | 2.9 (2.8-3.5) | 1.56x | — | — | 107.0 KiB | 42 / 47 MiB | +| `write` | Vorticity's | 68.0 (67.1-70.5) | 90.9 (90.4-110.8) | 1.34x | 3.9 | 2.9 | 2.6 MiB | 79 / 345 MiB | +| `write` | Vortex Rust's | 66.8 (66.4-67.3) | 91.8 (89.7-118.3) | 1.37x | 4.0 | 2.9 | 2.5 MiB | 179 / 426 MiB | +| `append` | Vorticity's | 47.5 (47.1-48.3) | not asked | n/a | — | — | 2.4 MiB | 60 / — MiB | `open`: open the file and read no rows. `scan`: read every column of every row. `project`: read one column of four. `filter-narrow`: read the rows of a band holding about one in a hundred. `filter-wide`: read the rows of a band holding about half. `take`: take a thousand rows spread across the file. `write`: read the file and encode it back out. `append`: append a tenth of the rows to a copy of the file. **On one core.** Vorticity took less time than Rust on 28 of 28 compared rows. -The highest speedup is `open` at 1,048,576 rows on Vorticity's file (3.65x), the lowest -`filter-wide` at 10,485,760 rows on Vortex Rust's file (1.11x). +The highest speedup is `open` at 10,485,760 rows on Vorticity's file (3.97x), the lowest +`filter-wide` at 10,485,760 rows on Vortex Rust's file (1.06x). **On all 14 cores.** Vorticity took less time than Rust on 28 of 28 compared rows. -The highest speedup is `open` at 1,048,576 rows on Vorticity's file (4.82x), the lowest -`scan` at 10,485,760 rows on Vortex Rust's file (1.07x). +The highest speedup is `open` at 10,485,760 rows on Vorticity's file (5.69x), the lowest +`take` at 1,048,576 rows on Vorticity's file (1.02x). **The process start** is not in the figures: 15 ms for Vorticity's native binary and 14 ms for Rust's, most of it the operating system starting a binary, and 42 ms for the managed runtime on the JIT. -**Rust's splits.** On one core, one split per chunk was the faster of its two on 12 of 14 rows on Vorticity's file and 11 of 14 rows on Vortex Rust's file; +**Rust's splits.** On one core, one split per chunk was the faster of its two on 8 of 14 rows on Vorticity's file and 11 of 14 rows on Vortex Rust's file; the two differ only where a chunk holds more than 100,000 rows. **Not asked of Rust**: `append`, which its harness has no entry point for; those figures @@ -200,16 +200,16 @@ Milliseconds, the median of the same runs. | scenario | 1,048,576 rows, Vorticity's file | 1,048,576 rows, Vortex Rust's file | 10,485,760 rows, Vorticity's file | 10,485,760 rows, Vortex Rust's file | |---|---:|---:|---:|---:| -| `open` | 12.3 | 11.5 | 11.6 | 11.8 | -| `scan` | 76.2 | 65.2 | 165.4 | 116.6 | -| `project` | 44.4 | 43.4 | 44.1 | 46.6 | -| `filter-narrow` | 103.5 | 84.6 | 98.0 | 93.4 | -| `filter-wide` | 86.2 | 78.7 | 128.9 | 113.2 | -| `take` | 85.8 | 60.9 | 144.6 | 66.6 | -| `write` | 210.8 | 200.6 | 583.0 | 555.8 | -| `append` | 204.3 | — | 315.1 | — | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 15dd2240 with uncommitted changes, 2026-10-03 21:40 UTC.* +| `open` | 12.6 | 11.6 | 12.3 | 12.7 | +| `scan` | 78.3 | 65.3 | 163.8 | 119.9 | +| `project` | 46.0 | 46.2 | 45.4 | 46.9 | +| `filter-narrow` | 104.1 | 84.3 | 103.4 | 97.2 | +| `filter-wide` | 88.3 | 79.9 | 132.9 | 117.8 | +| `take` | 85.7 | 61.8 | 126.0 | 66.5 | +| `write` | 208.9 | 206.3 | 591.2 | 551.6 | +| `append` | 199.7 | — | 268.5 | — | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 32b3a82b with uncommitted changes, 2026-10-10 02:12 UTC.* @@ -226,73 +226,73 @@ layout. Throughput is over the plain size, as above. | encoding | rows | Vorticity, ns/row | Vortex Rust, ns/row | speedup | Vorticity, GB/s | Vortex Rust, GB/s | |---|---:|---:|---:|---:|---:|---:| -| `alp` | 1,000,000 | 0.64 | 0.82 | 1.27x | 12.4 | 9.8 | -| `alp_no_patches` | 1,000,000 | 0.39 | 0.59 | 1.50x | 20.5 | 13.6 | -| `alp_patched_no_chunk_offsets` | 1,000,000 | 0.40 | 0.60 | 1.51x | 20.2 | 13.3 | -| `alprd` | 1,000,000 | 0.76 | 0.78 | 1.02x | 10.6 | 10.3 | -| `bool` | 1,000,000 | 0.03 | 0.04 | 1.06x | 3.7 | 3.5 | -| `bool_bit_offset3` | 1,000,000 | 0.04 | 0.04 | 1.04x | 3.5 | 3.4 | -| `bool_bit_offset7` | 1,000,000 | 0.04 | 0.04 | 0.99x | 3.4 | 3.5 | -| `bool_bit_offset_straddle` | 1,000,000 | 0.04 | 0.04 | 0.80x | 2.8 | 3.5 | -| `bytebool` | 1,000,000 | 0.07 | 0.08 | 1.14x | 1.8 | 1.6 | -| `chunked` | 1,000,000 | 0.05 | 0.14 | 2.93x | 173.9 | 59.3 | -| `chunked_bool` | 1,000,000 | 0.04 | 0.05 | 1.17x | 2.8 | 2.4 | -| `chunked_decimal` | 1,000,000 | 0.05 | 0.16 | 3.32x | 161.6 | 48.6 | -| `chunked_empty_chunks` | 1,000,000 | 0.04 | 0.12 | 2.67x | 177.8 | 66.7 | -| `chunked_mixed_validity` | 1,000,000 | 0.13 | 0.16 | 1.26x | 63.7 | 50.7 | -| `chunked_one_chunk` | 1,000,000 | 0.05 | 0.06 | 1.41x | 175.8 | 125.0 | -| `chunked_varbinview` | 1,000,000 | 1.71 | 6.25 | 3.65x | 7.9 | 2.2 | -| `constant` | 1,000,000 | 0.04 | 0.04 | 1.06x | 205.1 | 192.8 | -| `datetimeparts` | 1,000,000 | 0.98 | 1.15 | 1.18x | 8.2 | 6.9 | -| `decimal` | 1,000,000 | 0.04 | 0.04 | 1.00x | 190.5 | 190.5 | -| `decimal_byte_parts` | 1,000,000 | 0.04 | 0.05 | 1.21x | 197.5 | 163.3 | -| `dict` | 1,000,000 | 0.33 | 0.42 | 1.26x | 18.0 | 14.3 | -| `dict_nullable_codes` | 1,000,000 | 0.60 | 1.94 | 3.23x | 9.2 | 2.9 | -| `dict_nullable_values_nonnull_codes` | 1,000,000 | 0.41 | 0.64 | 1.57x | 14.1 | 9.0 | -| `dict_u64_codes` | 1,000,000 | 0.53 | 0.58 | 1.10x | 11.3 | 10.3 | -| `dict_u8_codes` | 1,000,000 | 0.25 | 0.35 | 1.37x | 31.6 | 23.1 | -| `ext` | 1,000,000 | 0.04 | 0.04 | 1.01x | 188.2 | 186.0 | -| `fastlanes_bitpacked` | 1,000,000 | 0.13 | 0.14 | 1.05x | 30.8 | 29.3 | -| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,000,000 | 0.14 | 0.14 | 1.06x | 29.6 | 27.9 | -| `fastlanes_delta` | 1,000,000 | 0.47 | 0.88 | 1.89x | 17.1 | 9.0 | -| `fastlanes_for` | 1,000,000 | 0.37 | 0.37 | 1.01x | 21.9 | 21.7 | -| `fastlanes_rle` | 1,000,000 | 0.29 | 0.39 | 1.35x | 13.7 | 10.2 | -| `fixed_size_list` | 1,000,000 | 0.04 | 0.05 | 1.02x | 269.7 | 263.7 | -| `fsst` | 1,000,000 | 3.85 | 4.80 | 1.25x | 14.6 | 11.7 | -| `list` | 1,000,000 | 0.29 | 0.76 | 2.59x | 34.1 | 13.2 | -| `listview` | 1,000,000 | 0.66 | 1.07 | 1.62x | 15.1 | 9.3 | -| `map` | 1,000,000 | 1.78 | 5.53 | 3.11x | 11.2 | 3.6 | -| `masked` | 1,000,000 | 0.05 | 0.05 | 0.99x | 89.7 | 90.7 | -| `masked_all_invalid` | 1,000,000 | 0.06 | 0.06 | 0.96x | 63.5 | 66.1 | -| `masked_all_valid` | 1,000,000 | 0.05 | 0.04 | 0.93x | 85.1 | 92.0 | -| `null` | 1,000,000 | 0.03 | 0.04 | 1.29x | — | — | -| `onpair` | 1,000,000 | 1.98 | 1.92 | 0.97x | 11.0 | 11.3 | -| `parquet_variant` | 1,000,000 | 1.85 | 4.36 | 2.35x | 6.5 | 2.8 | -| `pco` | 1,000,000 | 0.41 | 1.20 | 2.96x | 19.7 | 6.6 | -| `primitive` | 1,000,000 | 0.04 | 0.05 | 1.14x | 181.8 | 160.0 | -| `runend` | 1,000,000 | 0.08 | 0.11 | 1.34x | 49.7 | 37.0 | -| `sequence` | 1,000,000 | 0.10 | 0.10 | 1.04x | 83.8 | 80.8 | -| `sparse` | 1,000,000 | 0.10 | 0.10 | 1.01x | 38.8 | 38.5 | -| `struct` | 1,000,000 | 0.88 | 4.47 | 5.10x | 17.0 | 3.3 | -| `table_mixed` | 1,000,000 | 4.77 | 11.0 | 2.31x | 14.7 | 6.4 | -| `table_wide` | 50,000 | 1.11 | 1.49 | 1.34x | 360.4 | 268.5 | -| `varbin` | 1,000,000 | 1.56 | 8.90 | 5.70x | 10.2 | 1.8 | -| `varbinview` | 1,000,000 | 1.56 | 5.82 | 3.74x | 8.7 | 2.3 | -| `variant` | 1,000,000 | 0.04 | 0.05 | 1.16x | 390.2 | 336.8 | -| `zigzag` | 1,000,000 | 0.21 | 0.28 | 1.33x | 19.0 | 14.3 | -| `zstd` | 1,000,000 | 3.26 | 6.07 | 1.86x | 9.8 | 5.2 | -| `zstd_buffers` | 1,000,000 | 0.19 | 0.29 | 1.57x | 43.0 | 27.4 | -| `zstd_nullable` | 1,000,000 | 1.46 | 2.77 | 1.89x | 5.6 | 2.9 | +| `alp` | 1,000,000 | 0.65 | 0.82 | 1.25x | 12.3 | 9.8 | +| `alp_no_patches` | 1,000,000 | 0.40 | 0.61 | 1.53x | 20.3 | 13.2 | +| `alp_patched_no_chunk_offsets` | 1,000,000 | 0.43 | 0.61 | 1.40x | 18.4 | 13.1 | +| `alprd` | 1,000,000 | 0.77 | 0.77 | 1.00x | 10.4 | 10.4 | +| `bool` | 1,000,000 | 0.04 | 0.04 | 0.91x | 3.2 | 3.5 | +| `bool_bit_offset3` | 1,000,000 | 0.03 | 0.04 | 1.09x | 3.6 | 3.3 | +| `bool_bit_offset7` | 1,000,000 | 0.04 | 0.04 | 1.03x | 3.2 | 3.1 | +| `bool_bit_offset_straddle` | 1,000,000 | 0.04 | 0.04 | 1.11x | 3.4 | 3.0 | +| `bytebool` | 1,000,000 | 0.07 | 0.08 | 1.12x | 1.8 | 1.6 | +| `chunked` | 1,000,000 | 0.04 | 0.14 | 3.49x | 192.8 | 55.2 | +| `chunked_bool` | 1,000,000 | 0.04 | 0.05 | 1.07x | 2.9 | 2.7 | +| `chunked_decimal` | 1,000,000 | 0.05 | 0.15 | 2.98x | 163.3 | 54.8 | +| `chunked_empty_chunks` | 1,000,000 | 0.04 | 0.12 | 2.70x | 179.8 | 66.7 | +| `chunked_mixed_validity` | 1,000,000 | 0.13 | 0.17 | 1.27x | 61.1 | 48.0 | +| `chunked_one_chunk` | 1,000,000 | 0.04 | 0.07 | 1.64x | 197.5 | 120.3 | +| `chunked_varbinview` | 1,000,000 | 1.80 | 6.29 | 3.49x | 7.5 | 2.1 | +| `constant` | 1,000,000 | 0.03 | 0.04 | 1.26x | 235.3 | 186.0 | +| `datetimeparts` | 1,000,000 | 0.89 | 1.07 | 1.20x | 9.0 | 7.4 | +| `decimal` | 1,000,000 | 0.04 | 0.04 | 1.07x | 210.5 | 197.5 | +| `decimal_byte_parts` | 1,000,000 | 0.04 | 0.04 | 1.13x | 205.1 | 181.8 | +| `dict` | 1,000,000 | 0.33 | 0.43 | 1.30x | 18.4 | 14.1 | +| `dict_nullable_codes` | 1,000,000 | 0.60 | 1.98 | 3.28x | 9.2 | 2.8 | +| `dict_nullable_values_nonnull_codes` | 1,000,000 | 0.41 | 0.66 | 1.63x | 14.1 | 8.7 | +| `dict_u64_codes` | 1,000,000 | 0.54 | 0.59 | 1.08x | 11.0 | 10.2 | +| `dict_u8_codes` | 1,000,000 | 0.26 | 0.35 | 1.34x | 30.9 | 23.1 | +| `ext` | 1,000,000 | 0.04 | 0.04 | 0.98x | 190.5 | 195.1 | +| `fastlanes_bitpacked` | 1,000,000 | 0.13 | 0.14 | 1.07x | 30.8 | 28.9 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,000,000 | 0.14 | 0.15 | 1.06x | 28.6 | 26.8 | +| `fastlanes_delta` | 1,000,000 | 0.47 | 0.89 | 1.88x | 16.9 | 9.0 | +| `fastlanes_for` | 1,000,000 | 0.37 | 0.41 | 1.10x | 21.4 | 19.4 | +| `fastlanes_rle` | 1,000,000 | 0.30 | 0.41 | 1.35x | 13.2 | 9.8 | +| `fixed_size_list` | 1,000,000 | 0.05 | 0.04 | 0.90x | 260.9 | 289.2 | +| `fsst` | 1,000,000 | 3.86 | 4.83 | 1.25x | 14.5 | 11.6 | +| `list` | 1,000,000 | 0.29 | 0.70 | 2.41x | 34.4 | 14.2 | +| `listview` | 1,000,000 | 0.65 | 1.04 | 1.61x | 15.4 | 9.6 | +| `map` | 1,000,000 | 1.81 | 5.03 | 2.78x | 11.1 | 4.0 | +| `masked` | 1,000,000 | 0.04 | 0.05 | 1.10x | 91.7 | 83.3 | +| `masked_all_invalid` | 1,000,000 | 0.05 | 0.05 | 0.98x | 81.6 | 83.3 | +| `masked_all_valid` | 1,000,000 | 0.04 | 0.05 | 1.06x | 93.0 | 87.9 | +| `null` | 1,000,000 | 0.03 | 0.04 | 1.19x | — | — | +| `onpair` | 1,000,000 | 1.94 | 1.92 | 0.99x | 11.2 | 11.3 | +| `parquet_variant` | 1,000,000 | 1.82 | 4.39 | 2.42x | 6.6 | 2.7 | +| `pco` | 1,000,000 | 0.40 | 1.22 | 3.03x | 19.9 | 6.6 | +| `primitive` | 1,000,000 | 0.04 | 0.04 | 1.09x | 195.1 | 179.8 | +| `runend` | 1,000,000 | 0.08 | 0.11 | 1.44x | 52.3 | 36.4 | +| `sequence` | 1,000,000 | 0.10 | 0.10 | 1.01x | 80.0 | 78.8 | +| `sparse` | 1,000,000 | 0.09 | 0.09 | 0.99x | 45.7 | 46.2 | +| `struct` | 1,000,000 | 0.66 | 4.58 | 6.89x | 22.4 | 3.3 | +| `table_mixed` | 1,000,000 | 4.15 | 11.4 | 2.75x | 16.9 | 6.1 | +| `table_wide` | 50,000 | 1.13 | 1.54 | 1.36x | 354.0 | 259.7 | +| `varbin` | 1,000,000 | 1.53 | 8.94 | 5.85x | 10.4 | 1.8 | +| `varbinview` | 1,000,000 | 1.45 | 5.97 | 4.13x | 9.3 | 2.3 | +| `variant` | 1,000,000 | 0.04 | 0.05 | 1.18x | 415.6 | 351.6 | +| `zigzag` | 1,000,000 | 0.21 | 0.31 | 1.47x | 19.2 | 13.1 | +| `zstd` | 1,000,000 | 3.30 | 6.18 | 1.87x | 9.6 | 5.2 | +| `zstd_buffers` | 1,000,000 | 0.18 | 0.28 | 1.54x | 43.4 | 28.2 | +| `zstd_nullable` | 1,000,000 | 1.47 | 2.85 | 1.94x | 5.5 | 2.9 | Rust is timed under each of its two ways of splitting a scan, three processes each, and its figure is the faster: its default cuts a chunk of more than 100,000 rows into splits of 100,000 and decodes the whole chunk again for each, which on these files of one chunk of a million rows is ten times; one split per chunk decodes it once, but builds a chunk made of smaller arrays in one -piece. One split per chunk was the faster on 44 of 57 files, its default on `alprd`, `chunked`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `datetimeparts`, `fastlanes_for`, `fsst`, `masked_all_invalid`, `onpair`, `table_wide`, `zigzag`. +piece. One split per chunk was the faster on 45 of 57 files, its default on `alprd`, `chunked`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `datetimeparts`, `fastlanes_delta`, `fsst`, `parquet_variant`, `table_wide`, `zigzag`. -Vorticity decoded 50 of 57 files in less time than Rust; the median speedup is 1.27x. At 1.00x or below: `bool_bit_offset_straddle` 0.80x, `masked_all_valid` 0.93x, `masked_all_invalid` 0.96x, `onpair` 0.97x, `bool_bit_offset7` 0.99x, `masked` 0.99x, `decimal` 1.00x. +Vorticity decoded 51 of 57 files in less time than Rust; the median speedup is 1.27x. At 1.00x or below: `fixed_size_list` 0.90x, `bool` 0.91x, `ext` 0.98x, `masked_all_invalid` 0.98x, `sparse` 0.99x, `onpair` 0.99x. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 15dd2240 with uncommitted changes, 2026-10-03 21:40 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 32b3a82b with uncommitted changes, 2026-10-10 02:12 UTC.* @@ -307,30 +307,30 @@ one the `--ratio-check` gate holds to a ceiling; what each one reads and asks is | axis | Vorticity, µs | Vortex Rust, µs | speedup | 95 % interval | |---|---:|---:|---:|---| -| full scan | 495.9 | 1,586.2 | 3.20x | 3.19 to 3.21 | -| projected scan, 1 of 5 columns | 108.6 | 217.1 | 2.02x | 1.98 to 2.04 | -| open to first batch | 94.8 | 1,420.2 | 15.1x | 14.2 to 15.3 | -| open, footer only | 30.9 | 52.6 | 1.69x | 1.65 to 1.75 | -| read and write back | 4,326.4 | 13,434.3 | 3.04x | 3.02 to 3.19 | -| filtered scan, 1% band | 89.5 | 293.9 | 3.26x | 3.23 to 3.31 | -| filtered scan, half the rows | 171.4 | 585.4 | 3.42x | 3.38 to 3.44 | -| scattered take, 64 of 64 splits | 323.4 | 1,455.2 | 4.51x | 4.44 to 4.55 | -| key order, sorted column, 1% band | 76.9 | 99.8 | 1.30x | 1.25 to 1.34 | -| key order, uncorrelated, 64 rows | 242.8 | 265.6 | 1.11x | 1.07 to 1.19 | -| count, exact cover, 1% band | 134.1 | 141.4 | 1.14x | 1.06 to 1.17 | -| filtered scan, string equality, fsst | 203.7 | 374.2 | 1.85x | 1.79 to 1.89 | -| filtered scan, string prefix, fsst | 767.7 | 716.1 | 0.93x | 0.92 to 0.94 | -| filtered scan, string equality, dict | 93.8 | 116.6 | 1.23x | 1.17 to 1.27 | -| filtered scan, string prefix, dict | 103.5 | 129.0 | 1.24x | 1.24 to 1.29 | -| filtered scan, band, runend | 70.9 | 83.7 | 1.21x | 1.15 to 1.23 | -| filtered scan, band, bitpacked | 82.7 | 81.6 | 0.98x | 0.94 to 0.99 | -| full scan, 1M table | 4,259.9 | 10,957.8 | 2.64x | 2.59 to 2.72 | -| projected scan, 1 of 50 columns | 46.3 | 66.4 | 1.41x | 1.40 to 1.46 | - -Vorticity took less time on 17 of 19 axes. At 1.00x or below: filtered scan, string prefix, fsst (0.93x), filtered scan, band, bitpacked (0.98x). -Rust is timed under the faster of its two splits, axis by axis: one split per chunk on 12 of 19, its default on open, footer only; read and write back; key order, sorted column, 1% band; filtered scan, string prefix, fsst; filtered scan, string equality, dict; filtered scan, string prefix, dict; filtered scan, band, runend. - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 49dacfd9920c; commit 15dd2240 with uncommitted changes, 2026-10-03 21:41 UTC.* +| full scan | 504.5 | 1,609.7 | 3.18x | 3.16 to 3.21 | +| projected scan, 1 of 5 columns | 115.6 | 229.9 | 1.99x | 1.97 to 2.01 | +| open to first batch | 98.8 | 1,439.0 | 14.7x | 14.0 to 15.2 | +| open, footer only | 29.3 | 51.5 | 1.75x | 1.75 to 1.77 | +| read and write back | 4,384.6 | 13,744.0 | 3.24x | 3.04 to 3.27 | +| filtered scan, 1% band | 91.8 | 302.6 | 3.25x | 3.22 to 3.33 | +| filtered scan, half the rows | 164.0 | 581.1 | 3.55x | 3.51 to 3.58 | +| scattered take, 64 of 64 splits | 332.4 | 1,491.0 | 4.49x | 4.43 to 4.52 | +| key order, sorted column, 1% band | 78.6 | 105.8 | 1.35x | 1.29 to 1.35 | +| key order, uncorrelated, 64 rows | 237.6 | 266.5 | 1.13x | 1.06 to 1.16 | +| count, exact cover, 1% band | 141.7 | 154.0 | 1.17x | 1.11 to 1.21 | +| filtered scan, string equality, fsst | 308.5 | 428.5 | 1.38x | 1.32 to 1.45 | +| filtered scan, string prefix, fsst | 772.9 | 711.8 | 0.93x | 0.92 to 0.94 | +| filtered scan, string equality, dict | 85.8 | 104.1 | 1.21x | 1.17 to 1.23 | +| filtered scan, string prefix, dict | 109.7 | 136.1 | 1.24x | 1.23 to 1.26 | +| filtered scan, band, runend | 64.6 | 81.5 | 1.25x | 1.22 to 1.28 | +| filtered scan, band, bitpacked | 76.8 | 79.6 | 1.03x | 1.02 to 1.05 | +| full scan, 1M table | 4,184.0 | 11,042.7 | 2.65x | 2.52 to 2.74 | +| projected scan, 1 of 50 columns | 46.6 | 65.5 | 1.44x | 1.37 to 1.46 | + +Vorticity took less time on 18 of 19 axes. At 1.00x or below: filtered scan, string prefix, fsst (0.93x). +Rust is timed under the faster of its two splits, axis by axis: one split per chunk on 10 of 19, its default on full scan; open, footer only; filtered scan, half the rows; key order, sorted column, 1% band; key order, uncorrelated, 64 rows; filtered scan, string equality, fsst; filtered scan, string prefix, fsst; filtered scan, string prefix, dict; filtered scan, band, bitpacked. + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 32b3a82b with uncommitted changes, 2026-10-10 02:13 UTC.* @@ -344,68 +344,68 @@ at least 9 rounds after a 1-second warm-up per file, our side on the JIT, warmed | encoding | Vorticity, µs | Vortex Rust, µs | speedup | |---|---:|---:|---:| -| `alp` | 179 | 190 | 1.09x | -| `alp_no_patches` | 63 | 69 | 1.12x | -| `alp_patched_no_chunk_offsets` | 66 | 72 | 1.10x | -| `alprd` | 67 | 71 | 1.07x | -| `bool` | 35 | 41 | 1.16x | -| `bool_bit_offset3` | 36 | 41 | 1.14x | -| `bool_bit_offset7` | 36 | 41 | 1.12x | -| `bool_bit_offset_straddle` | 36 | 41 | 1.13x | -| `bytebool` | 56 | 85 | 1.52x | -| `chunked` | 64 | 72 | 1.12x | -| `chunked_bool` | 38 | 46 | 1.22x | -| `chunked_decimal` | 68 | 77 | 1.13x | -| `chunked_empty_chunks` | 83 | 92 | 1.10x | -| `chunked_mixed_validity` | 74 | 89 | 1.20x | -| `chunked_one_chunk` | 71 | 85 | 1.14x | -| `chunked_varbinview` | 206 | 6,418 | 31.4x | -| `constant` | 34 | 45 | 1.33x | -| `datetimeparts` | 114 | 117 | 1.04x | -| `decimal` | 67 | 72 | 1.06x | -| `decimal_byte_parts` | 68 | 74 | 1.07x | -| `dict` | 70 | 72 | 1.04x | -| `dict_nullable_codes` | 66 | 76 | 1.14x | -| `dict_nullable_values_nonnull_codes` | 59 | 64 | 1.08x | -| `dict_u64_codes` | 67 | 73 | 1.07x | -| `dict_u8_codes` | 56 | 61 | 1.10x | -| `ext` | 63 | 68 | 1.06x | -| `fastlanes_bitpacked` | 58 | 62 | 1.06x | -| `fastlanes_bitpacked_patched_no_chunk_offsets` | 67 | 68 | 1.08x | -| `fastlanes_delta` | 146 | 955 | 6.64x | -| `fastlanes_for` | 64 | 69 | 1.10x | -| `fastlanes_rle` | 329 | 431 | 1.32x | -| `fixed_size_list` | 67 | 89 | 1.37x | -| `fsst` | 121 | 123 | 1.00x | -| `list` | 84 | 238 | 2.81x | -| `listview` | 645 | 959 | 1.52x | -| `map` | 1,500 | 4,655 | 3.16x | -| `masked` | 38 | 44 | 1.14x | -| `masked_all_invalid` | 39 | 44 | 1.13x | -| `masked_all_valid` | 123 | 134 | 1.07x | -| `null` | 28 | 37 | 1.35x | -| `onpair` | 114 | 135 | 1.20x | -| `parquet_variant` | 129 | 138 | 1.06x | -| `pco` | 376 | 1,236 | 3.27x | -| `primitive` | 62 | 66 | 1.06x | -| `runend` | 41 | 46 | 1.12x | -| `sequence` | 29 | 38 | 1.32x | -| `sparse` | 43 | 47 | 1.12x | -| `struct` | 727 | 4,682 | 6.58x | -| `table_mixed` | 3,835 | 11,060 | 2.89x | -| `table_wide` | 105 | 152 | 1.41x | -| `varbin` | 142 | 6,713 | 47.6x | -| `varbinview` | 1,280 | 5,940 | 4.64x | -| `variant` | 30 | 39 | 1.32x | -| `zigzag` | 58 | 62 | 1.08x | -| `zstd` | 382 | 6,102 | 16.0x | -| `zstd_buffers` | 202 | 312 | 1.54x | -| `zstd_nullable` | 235 | 2,808 | 11.9x | - -Vorticity took less time on 57 of 57 files; the median speedup is 1.14x. +| `alp` | 174 | 195 | 1.10x | +| `alp_no_patches` | 60 | 66 | 1.09x | +| `alp_patched_no_chunk_offsets` | 67 | 69 | 1.04x | +| `alprd` | 63 | 69 | 1.11x | +| `bool` | 35 | 41 | 1.15x | +| `bool_bit_offset3` | 36 | 41 | 1.12x | +| `bool_bit_offset7` | 35 | 40 | 1.13x | +| `bool_bit_offset_straddle` | 36 | 41 | 1.16x | +| `bytebool` | 55 | 84 | 1.52x | +| `chunked` | 60 | 70 | 1.16x | +| `chunked_bool` | 38 | 46 | 1.21x | +| `chunked_decimal` | 61 | 71 | 1.16x | +| `chunked_empty_chunks` | 63 | 71 | 1.14x | +| `chunked_mixed_validity` | 65 | 80 | 1.21x | +| `chunked_one_chunk` | 63 | 70 | 1.11x | +| `chunked_varbinview` | 156 | 6,027 | 40.2x | +| `constant` | 28 | 38 | 1.37x | +| `datetimeparts` | 116 | 119 | 1.02x | +| `decimal` | 59 | 62 | 1.06x | +| `decimal_byte_parts` | 58 | 63 | 1.10x | +| `dict` | 57 | 62 | 1.08x | +| `dict_nullable_codes` | 61 | 73 | 1.17x | +| `dict_nullable_values_nonnull_codes` | 60 | 64 | 1.09x | +| `dict_u64_codes` | 59 | 65 | 1.09x | +| `dict_u8_codes` | 56 | 61 | 1.09x | +| `ext` | 61 | 66 | 1.08x | +| `fastlanes_bitpacked` | 57 | 62 | 1.09x | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 60 | 65 | 1.08x | +| `fastlanes_delta` | 145 | 939 | 6.49x | +| `fastlanes_for` | 60 | 64 | 1.07x | +| `fastlanes_rle` | 333 | 443 | 1.33x | +| `fixed_size_list` | 63 | 90 | 1.41x | +| `fsst` | 121 | 119 | 1.00x | +| `list` | 84 | 232 | 2.72x | +| `listview` | 665 | 976 | 1.47x | +| `map` | 1,652 | 5,284 | 3.18x | +| `masked` | 39 | 44 | 1.15x | +| `masked_all_invalid` | 39 | 44 | 1.14x | +| `masked_all_valid` | 61 | 65 | 1.07x | +| `null` | 27 | 37 | 1.35x | +| `onpair` | 114 | 139 | 1.22x | +| `parquet_variant` | 133 | 139 | 1.06x | +| `pco` | 381 | 1,224 | 3.23x | +| `primitive` | 58 | 63 | 1.08x | +| `runend` | 40 | 45 | 1.14x | +| `sequence` | 29 | 38 | 1.34x | +| `sparse` | 43 | 49 | 1.13x | +| `struct` | 737 | 4,679 | 6.62x | +| `table_mixed` | 3,861 | 11,057 | 2.93x | +| `table_wide` | 102 | 147 | 1.44x | +| `varbin` | 140 | 6,987 | 49.0x | +| `varbinview` | 1,332 | 6,156 | 4.60x | +| `variant` | 29 | 39 | 1.33x | +| `zigzag` | 56 | 61 | 1.08x | +| `zstd` | 394 | 6,116 | 15.6x | +| `zstd_buffers` | 209 | 318 | 1.51x | +| `zstd_nullable` | 220 | 2,835 | 12.9x | + +Vorticity took less time on 56 of 57 files; the median speedup is 1.15x. At 1.00x or below: `fsst` 1.00x. Rust is timed under the faster of its two splits, file by file: one split per chunk on 56 of 57 files, its default on `table_wide`. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 49dacfd9920c; commit 15dd2240 with uncommitted changes, 2026-10-03 21:42 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 32b3a82b with uncommitted changes, 2026-10-10 02:15 UTC.* @@ -421,69 +421,69 @@ A writer can be fast by compressing less, so each one's bytes are beside its tim | encoding | Vorticity, µs | Vortex Rust, µs | speedup | Vorticity, bytes | Vortex Rust, bytes | |---|---:|---:|---:|---:|---:| -| `alp` | 5,550 | 22,320 | 4.03x | 2,740,820 | 2,976,924 | -| `alp_no_patches` | 5,018 | 15,898 | 3.16x | 1,955,668 | 1,956,060 | -| `alp_patched_no_chunk_offsets` | 4,643 | 17,550 | 3.79x | 2,155,956 | 2,141,772 | -| `alprd` | 5,996 | 17,073 | 2.86x | 6,508,892 | 6,509,316 | -| `bool` | 137 | 408 | 2.99x | 127,108 | 127,108 | -| `bool_bit_offset3` | 153 | 550 | 3.48x | 127,108 | 127,108 | -| `bool_bit_offset7` | 178 | 548 | 3.10x | 127,108 | 127,108 | -| `bool_bit_offset_straddle` | 180 | 548 | 3.05x | 127,108 | 127,108 | -| `bytebool` | 172 | 448 | 2.60x | 127,108 | 127,108 | -| `chunked` | 1,433 | 10,723 | 7.86x | 890,228 | 890,736 | -| `chunked_bool` | 151 | 421 | 2.79x | 127,236 | 127,248 | -| `chunked_decimal` | 3,059 | 13,182 | 4.31x | 1,822,388 | 1,825,056 | -| `chunked_empty_chunks` | 1,034 | 10,174 | 10.00x | 316,532 | 317,080 | -| `chunked_mixed_validity` | 2,856 | 10,266 | 3.61x | 1,431,516 | 1,374,016 | -| `chunked_one_chunk` | 918 | 9,901 | 11.1x | 5,044 | 4,640 | -| `chunked_varbinview` | 13,621 | 26,430 | 1.95x | 643,644 | 628,628 | -| `constant` | 127 | 2,829 | 22.7x | 4,212 | 3,300 | -| `datetimeparts` | 3,799 | 23,943 | 6.56x | 3,380,476 | 3,383,628 | -| `decimal` | 1,790 | 12,736 | 7.10x | 3,372 | 5,116 | -| `decimal_byte_parts` | 1,682 | 12,188 | 7.46x | 3,372 | 5,116 | -| `dict` | 5,905 | 18,805 | 3.16x | 383,892 | 378,288 | -| `dict_nullable_codes` | 9,405 | 17,475 | 1.85x | 384,628 | 378,772 | -| `dict_nullable_values_nonnull_codes` | 8,782 | 16,052 | 1.84x | 384,628 | 378,764 | -| `dict_u64_codes` | 6,150 | 18,968 | 3.15x | 383,892 | 378,288 | -| `dict_u8_codes` | 9,435 | 18,052 | 1.91x | 38,740 | 1,004,160 | -| `ext` | 954 | 18,885 | 19.7x | 3,420 | 4,988 | -| `fastlanes_bitpacked` | 1,565 | 4,592 | 2.89x | 1,254,340 | 1,253,668 | -| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,275 | 5,241 | 4.10x | 1,270,452 | 1,272,740 | -| `fastlanes_delta` | 1,321 | 10,948 | 8.29x | 4,908 | 4,532 | -| `fastlanes_for` | 2,443 | 5,460 | 2.23x | 881,364 | 879,732 | -| `fastlanes_rle` | 1,287 | 5,005 | 3.86x | 3,964 | 3,812 | -| `fixed_size_list` | 1,190 | 22,015 | 18.6x | 4,268 | 4,724 | -| `fsst` | 30,120 | 248,871 | 8.27x | 933,668 | 8,574,500 | -| `list` | 6,880 | 20,777 | 3.02x | 4,782,492 | 5,122,712 | -| `listview` | 7,562 | 32,596 | 4.39x | 4,745,468 | 4,502,136 | -| `map` | 14,326 | 56,126 | 3.85x | 4,426,852 | 4,811,136 | -| `masked` | 3,673 | 12,373 | 3.37x | 2,380,060 | 2,381,156 | -| `masked_all_invalid` | 302 | 724 | 2.41x | 2,636 | 2,700 | -| `masked_all_valid` | 503 | 8,320 | 16.7x | 3,484 | 3,324 | -| `null` | 65 | 267 | 4.28x | 2,052 | 2,164 | -| `onpair` | 17,720 | 24,455 | 1.39x | 60,588 | 1,005,840 | -| `pco` | 851 | 11,093 | 13.2x | 4,908 | 4,508 | -| `primitive` | 848 | 9,393 | 11.1x | 5,356 | 4,572 | -| `runend` | 838 | 5,640 | 7.02x | 4,028 | 3,500 | -| `sequence` | 625 | 9,870 | 15.9x | 4,908 | 4,516 | -| `sparse` | 732 | 2,656 | 3.63x | 131,172 | 3,884 | -| `struct` | 26,846 | 88,810 | 3.31x | 638,412 | 5,274,480 | -| `table_mixed` | 50,215 | 278,227 | 5.57x | 7,865,164 | 14,856,384 | -| `table_wide` | 6,078 | 32,219 | 5.31x | 5,163,108 | 5,168,320 | -| `varbin` | 31,210 | 185,834 | 5.95x | 637,012 | 5,066,580 | -| `varbinview` | 13,071 | 25,643 | 1.95x | 643,516 | 628,504 | -| `variant` | 90 | 530 | 5.94x | 2,220 | 52,520 | -| `zigzag` | 1,741 | 10,348 | 5.85x | 2,531,652 | 2,380,300 | -| `zstd` | 22,382 | 28,860 | 1.29x | 786,948 | 754,336 | -| `zstd_buffers` | 2,268 | 4,781 | 2.11x | 630,732 | 629,028 | -| `zstd_nullable` | 5,273 | 10,336 | 1.94x | 756,500 | 628,740 | - -Vorticity took less time on 56 of 56 files; the median speedup is 3.82x. -Rust is timed under the faster of its two splits, file by file: one split per chunk on 46 of 56 files, its default on `alp_patched_no_chunk_offsets`, `alprd`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `decimal`, `fsst`, `list`, `onpair`, `zigzag`. -Vorticity's bytes were within 5 % of Rust's or fewer on 45 of 56 files; more than 5 % above them on `sparse` (33.8 times Rust's), `constant` (1.28 times), `zstd_nullable` (1.20 times), `primitive` (1.17 times), `runend` (1.15 times), `pco` (1.09 times), `chunked_one_chunk` (1.09 times), `sequence` (1.09 times), `fastlanes_delta` (1.08 times), `zigzag` (1.06 times), `listview` (1.05 times). +| `alp` | 5,432 | 22,809 | 4.23x | 2,740,820 | 2,976,924 | +| `alp_no_patches` | 5,064 | 16,103 | 3.21x | 1,955,668 | 1,956,060 | +| `alp_patched_no_chunk_offsets` | 4,707 | 17,734 | 3.75x | 2,155,956 | 2,141,772 | +| `alprd` | 6,111 | 17,386 | 2.84x | 6,508,892 | 6,509,316 | +| `bool` | 143 | 409 | 2.87x | 127,108 | 127,108 | +| `bool_bit_offset3` | 173 | 543 | 3.14x | 127,108 | 127,108 | +| `bool_bit_offset7` | 168 | 540 | 3.19x | 127,108 | 127,108 | +| `bool_bit_offset_straddle` | 172 | 541 | 3.16x | 127,108 | 127,108 | +| `bytebool` | 177 | 449 | 2.55x | 127,108 | 127,108 | +| `chunked` | 1,405 | 10,760 | 7.65x | 890,228 | 890,736 | +| `chunked_bool` | 150 | 419 | 2.81x | 127,236 | 127,248 | +| `chunked_decimal` | 2,964 | 13,001 | 4.37x | 1,822,388 | 1,825,056 | +| `chunked_empty_chunks` | 1,113 | 10,411 | 9.68x | 316,532 | 317,080 | +| `chunked_mixed_validity` | 2,923 | 10,027 | 3.43x | 1,431,516 | 1,374,016 | +| `chunked_one_chunk` | 841 | 9,803 | 11.6x | 5,044 | 4,640 | +| `chunked_varbinview` | 13,666 | 26,292 | 1.94x | 643,644 | 628,628 | +| `constant` | 89 | 2,707 | 30.5x | 4,212 | 3,300 | +| `datetimeparts` | 3,487 | 23,519 | 6.75x | 3,382,604 | 3,383,628 | +| `decimal` | 1,579 | 11,992 | 7.60x | 3,372 | 5,116 | +| `decimal_byte_parts` | 1,635 | 12,337 | 7.62x | 3,372 | 5,116 | +| `dict` | 5,779 | 18,454 | 3.22x | 383,892 | 378,288 | +| `dict_nullable_codes` | 9,440 | 17,501 | 1.85x | 384,628 | 378,772 | +| `dict_nullable_values_nonnull_codes` | 8,784 | 16,352 | 1.86x | 384,628 | 378,764 | +| `dict_u64_codes` | 6,088 | 19,178 | 3.15x | 383,892 | 378,288 | +| `dict_u8_codes` | 9,476 | 18,196 | 1.93x | 38,740 | 1,004,160 | +| `ext` | 858 | 18,873 | 21.9x | 5,548 | 4,988 | +| `fastlanes_bitpacked` | 1,552 | 4,658 | 3.00x | 1,254,340 | 1,253,668 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,330 | 5,444 | 4.08x | 1,270,452 | 1,272,740 | +| `fastlanes_delta` | 1,260 | 10,873 | 8.63x | 4,908 | 4,532 | +| `fastlanes_for` | 2,418 | 5,453 | 2.24x | 881,364 | 879,732 | +| `fastlanes_rle` | 1,216 | 4,826 | 3.97x | 3,964 | 3,812 | +| `fixed_size_list` | 1,103 | 22,142 | 20.2x | 4,268 | 4,724 | +| `fsst` | 30,374 | 252,930 | 8.32x | 933,668 | 8,574,500 | +| `list` | 6,662 | 20,680 | 3.09x | 4,782,492 | 5,122,712 | +| `listview` | 7,170 | 32,519 | 4.64x | 4,745,468 | 4,502,136 | +| `map` | 14,359 | 53,516 | 3.76x | 4,426,852 | 4,811,136 | +| `masked` | 3,720 | 12,425 | 3.30x | 2,380,060 | 2,381,156 | +| `masked_all_invalid` | 324 | 727 | 2.26x | 2,636 | 2,700 | +| `masked_all_valid` | 495 | 8,374 | 17.1x | 3,484 | 3,324 | +| `null` | 61 | 265 | 4.35x | 2,052 | 2,164 | +| `onpair` | 17,348 | 24,544 | 1.42x | 60,588 | 1,005,840 | +| `pco` | 878 | 11,101 | 12.7x | 4,908 | 4,508 | +| `primitive` | 865 | 9,487 | 11.2x | 5,356 | 4,572 | +| `runend` | 811 | 5,572 | 6.94x | 4,028 | 3,500 | +| `sequence` | 637 | 9,956 | 15.7x | 4,908 | 4,516 | +| `sparse` | 733 | 2,655 | 3.61x | 131,172 | 3,884 | +| `struct` | 26,670 | 89,248 | 3.34x | 638,412 | 5,274,480 | +| `table_mixed` | 50,023 | 283,674 | 5.64x | 7,867,284 | 14,856,384 | +| `table_wide` | 5,987 | 31,005 | 5.23x | 5,163,108 | 5,168,320 | +| `varbin` | 30,788 | 185,512 | 6.00x | 637,012 | 5,066,580 | +| `varbinview` | 13,200 | 26,138 | 1.98x | 643,516 | 628,504 | +| `variant` | 95 | 535 | 5.63x | 2,220 | 52,520 | +| `zigzag` | 1,713 | 9,932 | 5.84x | 2,531,652 | 2,380,300 | +| `zstd` | 22,478 | 28,500 | 1.25x | 786,948 | 754,336 | +| `zstd_buffers` | 2,299 | 4,768 | 2.09x | 630,732 | 629,028 | +| `zstd_nullable` | 5,080 | 10,126 | 1.98x | 756,500 | 628,740 | + +Vorticity took less time on 56 of 56 files; the median speedup is 3.76x. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 48 of 56 files, its default on `alprd`, `chunked`, `chunked_decimal`, `chunked_one_chunk`, `dict`, `dict_u8_codes`, `fsst`, `list`. +Vorticity's bytes were within 5 % of Rust's or fewer on 44 of 56 files; more than 5 % above them on `sparse` (33.8 times Rust's), `constant` (1.28 times), `zstd_nullable` (1.20 times), `primitive` (1.17 times), `runend` (1.15 times), `ext` (1.11 times), `pco` (1.09 times), `chunked_one_chunk` (1.09 times), `sequence` (1.09 times), `fastlanes_delta` (1.08 times), `zigzag` (1.06 times), `listview` (1.05 times). Rust's writer declines `parquet_variant`, which is left out. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 49dacfd9920c; commit 15dd2240 with uncommitted changes, 2026-10-03 21:44 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 32b3a82b with uncommitted changes, 2026-10-10 02:16 UTC.* @@ -508,27 +508,27 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| a sequence | Sequence x77 | 0.00 | 5 | 3.3 | 0.37 | | -| sorted runs of 1 000 | RunEnd x77 | 0.01 | 15 | 3.5 | 0.48 | Smallest: RunEnd x76, Pco x1, 0.01 B/value, scan 3.7 ms, take 0.86 ms; reads faster below 184 MB/s | -| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 32 | 6.4 | 0.80 | Smallest: Pco x77, 1.35 B/value, scan 70.2 ms, take 67.29 ms; reads faster below 319 MB/s | -| random in 0..999 | BitPacked x77 | 1.25 | 25 | 4.0 | 0.63 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 47 | 5.2 | 0.55 | | -| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 36 | 6.2 | 1.24 | Smallest: Pco x77, 0.06 B/value, scan 69.1 ms, take 66.63 ms; reads faster below 727 MB/s | -| uniform 64-bit | Canonical x77 | 8.00 | 35 | 5.3 | 0.88 | | -| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 48 | 5.5 | 0.71 | Smallest: Pco x82, 1.26 B/value, scan 31.2 ms, take 29.54 ms; reads faster below 48 MB/s | +| a sequence | Sequence x77 | 0.00 | 6 | 3.5 | 0.20 | | +| sorted runs of 1 000 | RunEnd x77 | 0.01 | 15 | 3.3 | 0.26 | Smallest: RunEnd x76, Pco x1, 0.01 B/value, scan 3.8 ms, take 0.65 ms; reads faster below 71 MB/s | +| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 33 | 6.4 | 0.63 | Smallest: Pco x77, 1.35 B/value, scan 70.2 ms, take 67.22 ms; reads faster below 319 MB/s | +| random in 0..999 | BitPacked x77 | 1.25 | 25 | 4.2 | 0.50 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 48 | 4.7 | 0.37 | | +| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 33 | 5.4 | 0.61 | Smallest: Pco x77, 0.06 B/value, scan 69.3 ms, take 67.81 ms; reads faster below 715 MB/s | +| uniform 64-bit | Canonical x77 | 8.00 | 20 | 5.6 | 0.76 | | +| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 47 | 5.6 | 0.56 | Smallest: Pco x82, 1.26 B/value, scan 30.7 ms, take 27.95 ms; reads faster below 49 MB/s |
a sequence: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.37 | | -| Fastest | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.36 | same | -| Smallest | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.36 | same | -| None | Canonical x77 | 80,043,724 | 8.00 | 26 | 5.4 | 0.88 | reads slower at any throughput | -| hint Dictionary | Sequence x77 | 38,844 | 0.00 | 75 | 3.2 | 0.35 | same | -| hint BitPacked | Sequence x77 | 38,844 | 0.00 | 5 | 3.2 | 0.36 | same | -| hint RunEnd | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.37 | same | -| hint Zstd | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.39 | same | +| Auto | Sequence x77 | 38,844 | 0.00 | 6 | 3.5 | 0.20 | | +| Fastest | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.20 | same | +| Smallest | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.20 | same | +| None | Canonical x77 | 80,043,724 | 8.00 | 18 | 5.6 | 0.80 | reads slower at any throughput | +| hint Dictionary | Sequence x77 | 38,844 | 0.00 | 76 | 3.3 | 0.22 | same | +| hint BitPacked | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.20 | same | +| hint RunEnd | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.22 | same | +| hint Zstd | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.20 | same |
@@ -536,14 +536,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x77 | 88,004 | 0.01 | 15 | 3.5 | 0.48 | | -| Fastest | RunEnd x77 | 88,004 | 0.01 | 14 | 3.5 | 0.49 | same | -| Smallest | RunEnd x76, Pco x1 | 53,748 | 0.01 | 89 | 3.7 | 0.86 | reads faster below 184 MB/s | -| None | Canonical x77 | 80,043,716 | 8.00 | 33 | 5.4 | 0.87 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 1,449,892 | 0.14 | 23 | 6.6 | 2.00 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 10,034,460 | 1.00 | 26 | 5.2 | 0.57 | reads slower at any throughput | -| hint RunEnd | RunEnd x77 | 88,004 | 0.01 | 15 | 3.5 | 0.48 | same | -| hint Zstd | Zstd x77 | 136,628 | 0.01 | 15 | 20.2 | 15.61 | reads slower at any throughput | +| Auto | RunEnd x77 | 88,004 | 0.01 | 15 | 3.3 | 0.26 | | +| Fastest | RunEnd x77 | 88,004 | 0.01 | 15 | 3.4 | 0.27 | same | +| Smallest | RunEnd x76, Pco x1 | 53,748 | 0.01 | 90 | 3.8 | 0.65 | reads faster below 71 MB/s | +| None | Canonical x77 | 80,043,716 | 8.00 | 23 | 5.2 | 0.70 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 1,449,892 | 0.14 | 23 | 4.9 | 0.63 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 10,034,460 | 1.00 | 25 | 5.4 | 0.44 | reads slower at any throughput | +| hint RunEnd | RunEnd x77 | 88,004 | 0.01 | 15 | 3.4 | 0.26 | same | +| hint Zstd | Zstd x77 | 136,628 | 0.01 | 15 | 4.5 | 1.17 | reads slower at any throughput | @@ -551,14 +551,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 33,795,172 | 3.38 | 32 | 6.4 | 0.80 | | -| Fastest | BitPacked x77 | 33,795,172 | 3.38 | 30 | 6.4 | 0.83 | same | -| Smallest | Pco x77 | 13,472,444 | 1.35 | 361 | 70.2 | 67.29 | reads faster below 319 MB/s | -| None | Canonical x77 | 80,043,724 | 8.00 | 35 | 5.9 | 0.98 | reads faster above 91,436 MB/s | -| hint Dictionary | BitPacked x77 | 33,795,172 | 3.38 | 110 | 6.3 | 0.76 | same | -| hint BitPacked | BitPacked x77 | 33,795,172 | 3.38 | 31 | 6.4 | 0.78 | same | -| hint RunEnd | BitPacked x77 | 33,795,172 | 3.38 | 40 | 6.4 | 0.82 | same | -| hint Zstd | Zstd x77 | 20,700,796 | 2.07 | 136 | 58.3 | 46.65 | reads faster below 252 MB/s | +| Auto | BitPacked x77 | 33,795,172 | 3.38 | 33 | 6.4 | 0.63 | | +| Fastest | BitPacked x77 | 33,795,172 | 3.38 | 33 | 6.5 | 0.72 | same | +| Smallest | Pco x77 | 13,472,444 | 1.35 | 343 | 70.2 | 67.22 | reads faster below 319 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 24 | 5.3 | 0.68 | reads faster above 38,890 MB/s | +| hint Dictionary | BitPacked x77 | 33,795,172 | 3.38 | 98 | 6.3 | 0.64 | same | +| hint BitPacked | BitPacked x77 | 33,795,172 | 3.38 | 28 | 6.3 | 0.60 | same | +| hint RunEnd | BitPacked x77 | 33,795,172 | 3.38 | 39 | 6.5 | 0.72 | same | +| hint Zstd | Zstd x77 | 20,700,796 | 2.07 | 118 | 42.1 | 17.50 | reads faster below 367 MB/s | @@ -566,14 +566,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.0 | 0.63 | | -| Fastest | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.1 | 0.62 | same | -| Smallest | BitPacked x76, Pco x1 | 12,543,924 | 1.25 | 272 | 4.0 | 0.67 | same | -| None | Canonical x77 | 80,043,716 | 8.00 | 30 | 5.4 | 0.93 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 12,652,724 | 1.27 | 58 | 5.8 | 0.81 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 12,544,284 | 1.25 | 26 | 4.1 | 0.62 | same | -| hint RunEnd | BitPacked x77 | 12,544,284 | 1.25 | 24 | 3.9 | 0.59 | same | -| hint Zstd | Zstd x77 | 20,141,172 | 2.01 | 167 | 82.3 | 66.88 | reads slower at any throughput | +| Auto | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.2 | 0.50 | | +| Fastest | BitPacked x77 | 12,544,284 | 1.25 | 26 | 4.1 | 0.49 | same | +| Smallest | BitPacked x76, Pco x1 | 12,543,924 | 1.25 | 257 | 4.2 | 0.62 | same | +| None | Canonical x77 | 80,043,716 | 8.00 | 25 | 5.4 | 0.70 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,652,724 | 1.27 | 57 | 5.2 | 0.60 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 12,544,284 | 1.25 | 26 | 4.1 | 0.48 | same | +| hint RunEnd | BitPacked x77 | 12,544,284 | 1.25 | 22 | 4.1 | 0.49 | same | +| hint Zstd | Zstd x77 | 20,141,172 | 2.01 | 159 | 48.0 | 22.40 | reads slower at any throughput | @@ -581,15 +581,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,748 | 0.51 | 47 | 5.2 | 0.55 | | -| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 48 | 5.3 | 0.55 | same | -| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 491 | 5.2 | 0.72 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 24 | 5.5 | 0.87 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 47 | 5.4 | 0.57 | same | -| hint BitPacked | Dictionary x77 | 5,063,748 | 0.51 | 83 | 5.6 | 0.69 | reads slower at any throughput | -| hint RunEnd | Dictionary x77 | 5,063,748 | 0.51 | 79 | 5.2 | 0.54 | same | -| hint Zstd | Zstd x77 | 10,539,396 | 1.05 | 114 | 66.3 | 53.99 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 63 | 5.8 | 0.63 | reads slower at any throughput | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.7 | 0.37 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.7 | 0.36 | same | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 488 | 4.9 | 0.56 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.7 | 0.75 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.7 | 0.36 | same | +| hint BitPacked | Dictionary x77 | 5,063,748 | 0.51 | 83 | 4.8 | 0.37 | same | +| hint RunEnd | Dictionary x77 | 5,063,748 | 0.51 | 76 | 4.7 | 0.38 | same | +| hint Zstd | Zstd x77 | 10,539,396 | 1.05 | 116 | 37.5 | 15.70 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 54 | 4.7 | 0.33 | same | @@ -597,15 +597,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 46,295,652 | 4.63 | 36 | 6.2 | 1.24 | | -| Fastest | BitPacked x77 | 46,295,652 | 4.63 | 35 | 5.6 | 0.87 | reads faster at any throughput | -| Smallest | Pco x77 | 580,924 | 0.06 | 343 | 69.1 | 66.63 | reads faster below 727 MB/s | -| None | Canonical x77 | 80,043,724 | 8.00 | 32 | 5.3 | 0.92 | reads faster above 38,472 MB/s | -| hint Dictionary | BitPacked x77 | 46,295,652 | 4.63 | 95 | 5.7 | 0.90 | reads faster at any throughput | -| hint BitPacked | BitPacked x77 | 46,295,652 | 4.63 | 30 | 5.7 | 0.86 | reads faster at any throughput | -| hint RunEnd | BitPacked x77 | 46,295,652 | 4.63 | 49 | 5.5 | 0.86 | reads faster at any throughput | -| hint Zstd | Zstd x77 | 51,815,868 | 5.18 | 127 | 66.5 | 52.46 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 23,603,548 | 2.36 | 81 | 6.3 | 0.88 | reads faster at any throughput | +| Auto | BitPacked x77 | 46,295,652 | 4.63 | 33 | 5.4 | 0.61 | | +| Fastest | BitPacked x77 | 46,295,652 | 4.63 | 33 | 5.6 | 0.68 | reads slower at any throughput | +| Smallest | Pco x77 | 580,924 | 0.06 | 339 | 69.3 | 67.81 | reads faster below 715 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 20 | 5.3 | 0.75 | reads slower at any throughput | +| hint Dictionary | BitPacked x77 | 46,295,652 | 4.63 | 94 | 5.8 | 0.71 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 46,295,652 | 4.63 | 32 | 5.7 | 0.71 | reads slower at any throughput | +| hint RunEnd | BitPacked x77 | 46,295,652 | 4.63 | 49 | 5.7 | 0.70 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 51,815,868 | 5.18 | 95 | 55.0 | 37.55 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 23,603,548 | 2.36 | 81 | 6.0 | 0.65 | reads faster below 35,974 MB/s | @@ -613,14 +613,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x77 | 80,043,732 | 8.00 | 35 | 5.3 | 0.88 | | -| Fastest | Canonical x77 | 80,043,732 | 8.00 | 31 | 5.6 | 0.90 | reads slower at any throughput | -| Smallest | Canonical x77 | 80,043,732 | 8.00 | 243 | 5.5 | 0.92 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 41 | 5.8 | 0.97 | reads slower at any throughput | -| hint Dictionary | Canonical x77 | 80,043,732 | 8.00 | 113 | 6.3 | 1.03 | reads slower at any throughput | -| hint BitPacked | Canonical x77 | 80,043,732 | 8.00 | 85 | 5.5 | 0.92 | same | -| hint RunEnd | Canonical x77 | 80,043,732 | 8.00 | 90 | 5.4 | 0.86 | same | -| hint Zstd | Canonical x77 | 80,043,732 | 8.00 | 91 | 5.4 | 0.89 | same | +| Auto | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.6 | 0.76 | | +| Fastest | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.3 | 0.72 | reads faster at any throughput | +| Smallest | Canonical x77 | 80,043,732 | 8.00 | 242 | 5.4 | 0.75 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.3 | 0.69 | reads faster at any throughput | +| hint Dictionary | Canonical x77 | 80,043,732 | 8.00 | 109 | 5.3 | 0.74 | same | +| hint BitPacked | Canonical x77 | 80,043,732 | 8.00 | 85 | 5.4 | 0.71 | same | +| hint RunEnd | Canonical x77 | 80,043,732 | 8.00 | 78 | 5.3 | 0.76 | same | +| hint Zstd | Canonical x77 | 80,043,732 | 8.00 | 88 | 5.3 | 0.70 | reads faster at any throughput | @@ -628,14 +628,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x82 | 13,800,460 | 1.38 | 48 | 5.5 | 0.71 | | -| Fastest | BitPacked x82 | 13,800,460 | 1.38 | 48 | 5.5 | 0.69 | same | -| Smallest | Pco x82 | 12,567,268 | 1.26 | 285 | 31.2 | 29.54 | reads faster below 48 MB/s | -| None | Canonical x82 | 81,294,668 | 8.13 | 42 | 5.5 | 0.96 | reads slower at any throughput | -| hint Dictionary | Dictionary x82 | 12,676,396 | 1.27 | 79 | 7.0 | 0.85 | reads faster below 759 MB/s | -| hint BitPacked | BitPacked x82 | 13,800,460 | 1.38 | 45 | 5.5 | 0.73 | same | -| hint RunEnd | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.8 | 0.85 | same | -| hint Zstd | Zstd x82 | 19,283,940 | 1.93 | 173 | 87.4 | 69.82 | reads slower at any throughput | +| Auto | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.6 | 0.56 | | +| Fastest | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.8 | 0.59 | same | +| Smallest | Pco x82 | 12,567,268 | 1.26 | 277 | 30.7 | 27.95 | reads faster below 49 MB/s | +| None | Canonical x82 | 81,294,668 | 8.13 | 40 | 5.3 | 0.74 | reads faster above 209,674 MB/s | +| hint Dictionary | Dictionary x82 | 12,676,396 | 1.27 | 80 | 7.1 | 0.60 | reads faster below 766 MB/s | +| hint BitPacked | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.5 | 0.54 | same | +| hint RunEnd | BitPacked x82 | 13,800,460 | 1.38 | 48 | 5.5 | 0.54 | same | +| hint Zstd | Zstd x82 | 19,283,940 | 1.93 | 168 | 49.4 | 21.00 | reads slower at any throughput | @@ -643,23 +643,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| prices (2 decimals) | Alp x77 | 2.50 | 48 | 7.2 | 0.84 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 45 | 5.2 | 0.54 | | -| 1 000 distinct prices | Dictionary x77 | 1.28 | 60 | 5.9 | 0.84 | | -| 100 003 distinct, repeating | Zstd x77 | 1.69 | 136 | 60.3 | 47.97 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 9.3 ms, take 3.77 ms; reads faster above 102 MB/s; None: Canonical x77, 8.00 B/value, scan 5.7 ms, take 0.89 ms; reads faster above 1,156 MB/s | -| uniform in [0, 1) | AlpRd x77 | 6.91 | 41 | 11.0 | 1.28 | None: Canonical x77, 8.00 B/value, scan 5.4 ms, take 0.95 ms; reads faster above 1,955 MB/s | +| prices (2 decimals) | Alp x77 | 2.50 | 40 | 7.2 | 0.66 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 46 | 4.8 | 0.38 | | +| 1 000 distinct prices | Dictionary x77 | 1.28 | 60 | 5.2 | 0.58 | | +| 100 003 distinct, repeating | Zstd x77 | 1.69 | 119 | 35.0 | 14.98 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 7.5 ms, take 2.30 ms; reads faster above 190 MB/s; None: Canonical x77, 8.00 B/value, scan 5.4 ms, take 0.75 ms; reads faster above 2,134 MB/s | +| uniform in [0, 1) | AlpRd x77 | 6.91 | 39 | 11.1 | 1.00 | None: Canonical x77, 8.00 B/value, scan 5.3 ms, take 0.70 ms; reads faster above 1,902 MB/s |
prices (2 decimals): every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Alp x77 | 25,049,732 | 2.50 | 48 | 7.2 | 0.84 | | -| Fastest | Alp x77 | 25,049,732 | 2.50 | 51 | 7.1 | 0.85 | same | -| Smallest | Alp x77 | 25,048,868 | 2.50 | 719 | 7.2 | 0.89 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 24 | 5.4 | 0.85 | reads faster above 29,831 MB/s | -| hint Dictionary | Alp x77 | 25,049,732 | 2.50 | 133 | 7.0 | 0.82 | same | -| hint Alp | Alp x77 | 25,049,732 | 2.50 | 39 | 7.1 | 0.86 | same | -| hint Zstd | Zstd x77 | 41,240,260 | 4.12 | 246 | 83.3 | 67.23 | reads slower at any throughput | +| Auto | Alp x77 | 25,049,732 | 2.50 | 40 | 7.2 | 0.66 | | +| Fastest | Alp x77 | 25,049,732 | 2.50 | 40 | 7.2 | 0.64 | same | +| Smallest | Alp x77 | 25,048,868 | 2.50 | 709 | 7.4 | 0.80 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.4 | 0.77 | reads faster above 31,123 MB/s | +| hint Dictionary | Alp x77 | 25,049,732 | 2.50 | 136 | 7.3 | 0.66 | same | +| hint Alp | Alp x77 | 25,049,732 | 2.50 | 40 | 7.2 | 0.64 | same | +| hint Zstd | Zstd x77 | 41,240,260 | 4.12 | 246 | 51.4 | 28.31 | reads slower at any throughput |
@@ -667,14 +667,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,748 | 0.51 | 45 | 5.2 | 0.54 | | -| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 45 | 5.2 | 0.54 | same | -| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 487 | 5.5 | 0.74 | reads slower at any throughput | -| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.2 | 0.84 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 44 | 5.5 | 0.58 | reads slower at any throughput | -| hint Alp | Dictionary x77 | 5,063,748 | 0.51 | 93 | 5.2 | 0.54 | same | -| hint Zstd | Zstd x77 | 10,545,668 | 1.05 | 118 | 64.8 | 51.36 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 52 | 5.3 | 0.50 | same | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 46 | 4.8 | 0.38 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 46 | 4.8 | 0.37 | same | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 491 | 5.0 | 0.55 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.3 | 0.68 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 47 | 4.8 | 0.37 | same | +| hint Alp | Dictionary x77 | 5,063,748 | 0.51 | 95 | 4.7 | 0.36 | same | +| hint Zstd | Zstd x77 | 10,545,668 | 1.05 | 119 | 37.3 | 15.72 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 51 | 4.7 | 0.34 | same | @@ -682,14 +682,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.9 | 0.84 | | -| Fastest | Dictionary x77 | 12,756,260 | 1.28 | 59 | 5.8 | 0.85 | same | -| Smallest | Dictionary x77 | 12,751,356 | 1.28 | 412 | 6.0 | 1.57 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 27 | 5.4 | 0.85 | reads faster above 130,529 MB/s | -| hint Dictionary | Dictionary x77 | 12,756,260 | 1.28 | 59 | 5.8 | 0.82 | same | -| hint Alp | Alp x77 | 12,756,260 | 1.28 | 95 | 7.1 | 0.91 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 26,073,412 | 2.61 | 205 | 69.4 | 56.27 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 12,545,732 | 1.25 | 59 | 6.9 | 1.11 | reads faster below 210 MB/s | +| Auto | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.2 | 0.58 | | +| Fastest | Dictionary x77 | 12,756,260 | 1.28 | 59 | 5.3 | 0.60 | same | +| Smallest | Dictionary x77 | 12,751,356 | 1.28 | 401 | 5.4 | 0.78 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.5 | 0.82 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.3 | 0.61 | same | +| hint Alp | Alp x77 | 12,756,260 | 1.28 | 94 | 6.6 | 0.59 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 26,073,412 | 2.61 | 192 | 39.3 | 18.93 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 12,545,732 | 1.25 | 58 | 5.2 | 0.53 | reads faster at any throughput | @@ -697,14 +697,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x77 | 16,909,892 | 1.69 | 136 | 60.3 | 47.97 | | -| Fastest | Zstd x77 | 16,909,892 | 1.69 | 135 | 60.9 | 47.96 | same | -| Smallest | Zstd x77 | 16,909,892 | 1.69 | 245 | 60.6 | 48.80 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.7 | 0.89 | reads faster above 1,156 MB/s | -| hint Dictionary | Zstd x77 | 16,909,892 | 1.69 | 223 | 60.7 | 48.34 | same | -| hint Alp | Zstd x77 | 16,909,892 | 1.69 | 208 | 60.5 | 48.15 | same | -| hint Zstd | Zstd x77 | 16,909,892 | 1.69 | 137 | 59.3 | 47.44 | same | -| Auto, 16 MiB chunks | Dictionary x5 | 22,129,988 | 2.21 | 87 | 9.3 | 3.77 | reads faster above 102 MB/s | +| Auto | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.0 | 14.98 | | +| Fastest | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.1 | 14.89 | same | +| Smallest | Zstd x77 | 16,909,892 | 1.69 | 229 | 35.0 | 14.81 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.4 | 0.75 | reads faster above 2,134 MB/s | +| hint Dictionary | Zstd x77 | 16,909,892 | 1.69 | 205 | 35.3 | 15.24 | same | +| hint Alp | Zstd x77 | 16,909,892 | 1.69 | 191 | 34.8 | 14.88 | same | +| hint Zstd | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.1 | 14.94 | same | +| Auto, 16 MiB chunks | Dictionary x5 | 22,129,988 | 2.21 | 87 | 7.5 | 2.30 | reads faster above 190 MB/s | @@ -712,13 +712,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | AlpRd x77 | 69,054,380 | 6.91 | 41 | 11.0 | 1.28 | | -| Fastest | AlpRd x77 | 69,054,380 | 6.91 | 51 | 11.0 | 1.35 | same | -| Smallest | Pco x77 | 67,609,668 | 6.76 | 415 | 70.5 | 68.03 | reads faster below 24 MB/s | -| None | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.4 | 0.95 | reads faster above 1,955 MB/s | -| hint Dictionary | AlpRd x77 | 69,054,380 | 6.91 | 192 | 11.1 | 1.33 | same | -| hint Alp | AlpRd x77 | 69,054,380 | 6.91 | 197 | 11.0 | 1.28 | same | -| hint Zstd | AlpRd x77 | 69,054,380 | 6.91 | 213 | 11.9 | 1.47 | reads slower at any throughput | +| Auto | AlpRd x77 | 69,054,380 | 6.91 | 39 | 11.1 | 1.00 | | +| Fastest | AlpRd x77 | 69,054,380 | 6.91 | 40 | 11.1 | 1.04 | same | +| Smallest | Pco x77 | 67,609,668 | 6.76 | 414 | 67.9 | 66.15 | reads faster below 25 MB/s | +| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.3 | 0.70 | reads faster above 1,902 MB/s | +| hint Dictionary | AlpRd x77 | 69,054,380 | 6.91 | 185 | 11.0 | 0.98 | same | +| hint Alp | AlpRd x77 | 69,054,380 | 6.91 | 164 | 11.1 | 1.03 | same | +| hint Zstd | AlpRd x77 | 69,054,380 | 6.91 | 207 | 11.2 | 1.01 | same | @@ -726,23 +726,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| 16 cities, random order | Dictionary x175 | 0.51 | 102 | 8.3 | 0.79 | | -| 10 000 distinct ids | Dictionary x306 | 2.54 | 578 | 37.6 | 30.00 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 12.6 ms, take 3.75 ms; reads faster at any throughput | -| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 592 | 39.4 | 28.57 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 13.2 ms, take 3.45 ms; reads faster at any throughput | -| unique UUIDs | Zstd x611 | 20.61 | 1301 | 316.3 | 311.29 | hint Fsst: Fsst x611, 22.44 B/value, scan 82.8 ms, take 5.09 ms; reads faster above 78 MB/s | -| log lines (~100 B) | Zstd x1221 | 13.71 | 1227 | 414.7 | 341.97 | hint Fsst: Fsst x1221, 22.01 B/value, scan 100.5 ms, take 6.36 ms; reads faster above 264 MB/s | +| 16 cities, random order | Dictionary x175 | 0.51 | 108 | 7.7 | 0.58 | | +| 10 000 distinct ids | Dictionary x306 | 2.54 | 566 | 29.1 | 20.92 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 11.0 ms, take 2.08 ms; reads faster at any throughput | +| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 581 | 31.4 | 21.05 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 11.7 ms, take 2.07 ms; reads faster at any throughput | +| unique UUIDs | Zstd x611 | 20.61 | 1283 | 240.1 | 235.37 | hint Fsst: Fsst x611, 22.44 B/value, scan 80.8 ms, take 4.84 ms; reads faster above 115 MB/s | +| log lines (~100 B) | Zstd x1221 | 13.71 | 1244 | 309.7 | 250.45 | hint Fsst: Fsst x1221, 22.01 B/value, scan 100.9 ms, take 6.26 ms; reads faster above 397 MB/s |
16 cities, random order: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x175 | 5,117,284 | 0.51 | 102 | 8.3 | 0.79 | | -| Fastest | Dictionary x175 | 5,117,284 | 0.51 | 103 | 8.5 | 0.81 | same | -| Smallest | Dictionary x175 | 5,117,284 | 0.51 | 266 | 8.5 | 0.81 | same | -| None | Canonical x175 | 96,746,756 | 9.67 | 122 | 26.5 | 8.12 | reads slower at any throughput | -| hint Dictionary | Dictionary x175 | 5,117,284 | 0.51 | 104 | 8.5 | 0.84 | same | -| hint Fsst | Fsst x175 | 41,142,948 | 4.11 | 256 | 27.8 | 4.11 | reads slower at any throughput | -| hint Zstd | Zstd x175 | 17,427,100 | 1.74 | 210 | 92.1 | 67.05 | reads slower at any throughput | +| Auto | Dictionary x175 | 5,117,284 | 0.51 | 108 | 7.7 | 0.58 | | +| Fastest | Dictionary x175 | 5,117,284 | 0.51 | 108 | 7.6 | 0.56 | same | +| Smallest | Dictionary x175 | 5,117,284 | 0.51 | 269 | 7.6 | 0.57 | same | +| None | Canonical x175 | 96,746,756 | 9.67 | 123 | 26.5 | 5.64 | reads slower at any throughput | +| hint Dictionary | Dictionary x175 | 5,117,284 | 0.51 | 108 | 7.7 | 0.57 | same | +| hint Fsst | Fsst x175 | 41,142,948 | 4.11 | 255 | 26.8 | 2.67 | reads slower at any throughput | +| hint Zstd | Zstd x175 | 17,427,100 | 1.74 | 214 | 71.1 | 48.75 | reads slower at any throughput |
@@ -750,14 +750,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 25,433,788 | 2.54 | 578 | 37.6 | 30.00 | | -| Fastest | Dictionary x306 | 25,433,788 | 2.54 | 582 | 37.5 | 29.08 | same | -| Smallest | Dictionary x305, Zstd x1 | 23,452,500 | 2.35 | 1578 | 106.2 | 125.85 | reads faster below 29 MB/s | -| None | Canonical x306 | 140,098,172 | 14.01 | 133 | 24.1 | 11.69 | reads faster above 8,489 MB/s | -| hint Dictionary | Dictionary x306 | 25,433,788 | 2.54 | 569 | 37.9 | 29.75 | same | -| hint Fsst | Fsst x306 | 62,863,260 | 6.29 | 344 | 29.6 | 3.81 | reads faster above 4,661 MB/s | -| hint Zstd | Zstd x306 | 27,097,628 | 2.71 | 330 | 91.6 | 65.95 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 18,014,012 | 1.80 | 308 | 12.6 | 3.75 | reads faster at any throughput | +| Auto | Dictionary x306 | 25,433,788 | 2.54 | 566 | 29.1 | 20.92 | | +| Fastest | Dictionary x306 | 25,433,788 | 2.54 | 566 | 29.0 | 20.59 | same | +| Smallest | Dictionary x305, Zstd x1 | 23,452,500 | 2.35 | 1560 | 96.9 | 90.28 | reads faster below 29 MB/s | +| None | Canonical x306 | 140,098,172 | 14.01 | 124 | 24.2 | 9.90 | reads faster above 23,317 MB/s | +| hint Dictionary | Dictionary x306 | 25,433,788 | 2.54 | 565 | 28.9 | 20.65 | same | +| hint Fsst | Fsst x306 | 62,863,260 | 6.29 | 339 | 29.5 | 3.40 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 27,097,628 | 2.71 | 315 | 64.7 | 44.04 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 18,014,012 | 1.80 | 306 | 11.0 | 2.08 | reads faster at any throughput | @@ -765,14 +765,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 24,539,284 | 2.45 | 592 | 39.4 | 28.57 | | -| Fastest | Dictionary x306 | 24,539,284 | 2.45 | 570 | 40.1 | 29.35 | same | -| Smallest | Dictionary x305, Zstd x1 | 22,854,628 | 2.29 | 1569 | 108.0 | 123.97 | reads faster below 25 MB/s | -| None | Canonical x306 | 151,883,732 | 15.19 | 182 | 35.5 | 11.54 | reads faster above 31,978 MB/s | -| hint Dictionary | Dictionary x306 | 24,539,284 | 2.45 | 576 | 40.5 | 29.06 | same | -| hint Fsst | Fsst x306 | 64,408,284 | 6.44 | 392 | 46.8 | 8.47 | reads slower at any throughput | -| hint Zstd | Zstd x306 | 25,595,636 | 2.56 | 339 | 102.8 | 60.21 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 17,982,604 | 1.80 | 358 | 13.2 | 3.45 | reads faster at any throughput | +| Auto | Dictionary x306 | 24,539,284 | 2.45 | 581 | 31.4 | 21.05 | | +| Fastest | Dictionary x306 | 24,539,284 | 2.45 | 578 | 31.4 | 21.23 | same | +| Smallest | Dictionary x305, Zstd x1 | 22,854,628 | 2.29 | 1572 | 99.2 | 89.20 | reads faster below 25 MB/s | +| None | Canonical x306 | 151,883,732 | 15.19 | 179 | 35.5 | 9.91 | reads slower at any throughput | +| hint Dictionary | Dictionary x306 | 24,539,284 | 2.45 | 583 | 31.4 | 21.13 | same | +| hint Fsst | Fsst x306 | 64,408,284 | 6.44 | 403 | 39.4 | 3.96 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 25,595,636 | 2.56 | 323 | 78.5 | 39.35 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 17,982,604 | 1.80 | 351 | 11.7 | 2.07 | reads faster at any throughput | @@ -780,13 +780,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x611 | 206,058,588 | 20.61 | 1301 | 316.3 | 311.29 | | -| Fastest | Zstd x611 | 206,058,588 | 20.61 | 1343 | 312.6 | 312.17 | same | -| Smallest | Zstd x611 | 206,058,588 | 20.61 | 2840 | 315.1 | 310.42 | same | -| None | Canonical x611 | 360,155,516 | 36.02 | 132 | 45.4 | 26.35 | reads faster above 569 MB/s | -| hint Dictionary | Zstd x610, Fsst x1 | 206,059,580 | 20.61 | 5809 | 314.6 | 312.02 | same | -| hint Fsst | Fsst x611 | 224,384,604 | 22.44 | 1439 | 82.8 | 5.09 | reads faster above 78 MB/s | -| hint Zstd | Zstd x611 | 206,058,588 | 20.61 | 1297 | 315.5 | 312.45 | same | +| Auto | Zstd x611 | 206,058,588 | 20.61 | 1283 | 240.1 | 235.37 | | +| Fastest | Zstd x611 | 206,058,588 | 20.61 | 1273 | 238.9 | 235.45 | same | +| Smallest | Zstd x611 | 206,058,588 | 20.61 | 2794 | 239.8 | 234.84 | same | +| None | Canonical x611 | 360,155,516 | 36.02 | 118 | 45.1 | 25.77 | reads faster above 790 MB/s | +| hint Dictionary | Zstd x610, Fsst x1 | 206,059,580 | 20.61 | 5780 | 238.7 | 235.21 | same | +| hint Fsst | Fsst x611 | 224,384,604 | 22.44 | 1423 | 80.8 | 4.84 | reads faster above 115 MB/s | +| hint Zstd | Zstd x611 | 206,058,588 | 20.61 | 1267 | 240.6 | 235.82 | same | @@ -794,13 +794,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x1221 | 137,107,188 | 13.71 | 1227 | 414.7 | 341.97 | | -| Fastest | Zstd x1221 | 137,107,188 | 13.71 | 1222 | 412.5 | 334.07 | same | -| Smallest | Zstd x1221 | 137,107,188 | 13.71 | 2623 | 419.8 | 338.92 | same | -| None | Canonical x1221 | 725,468,444 | 72.55 | 404 | 79.7 | 42.77 | reads faster above 1,757 MB/s | -| hint Dictionary | Zstd x1221 | 137,107,188 | 13.71 | 5601 | 414.9 | 340.21 | same | -| hint Fsst | Fsst x1221 | 220,058,684 | 22.01 | 1676 | 100.5 | 6.36 | reads faster above 264 MB/s | -| hint Zstd | Zstd x1221 | 137,107,188 | 13.71 | 1230 | 412.0 | 333.23 | same | +| Auto | Zstd x1221 | 137,107,188 | 13.71 | 1244 | 309.7 | 250.45 | | +| Fastest | Zstd x1221 | 137,107,188 | 13.71 | 1212 | 308.8 | 249.97 | same | +| Smallest | Zstd x1221 | 137,107,188 | 13.71 | 2666 | 309.7 | 249.00 | same | +| None | Canonical x1221 | 725,468,444 | 72.55 | 431 | 79.0 | 41.25 | reads faster above 2,550 MB/s | +| hint Dictionary | Zstd x1221 | 137,107,188 | 13.71 | 5652 | 309.3 | 250.08 | same | +| hint Fsst | Fsst x1221 | 220,058,684 | 22.01 | 1737 | 100.9 | 6.26 | reads faster above 397 MB/s | +| hint Zstd | Zstd x1221 | 137,107,188 | 13.71 | 1212 | 310.7 | 251.04 | same | @@ -808,18 +808,18 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| half true | Canonical x10 | 0.13 | 2 | 0.2 | 0.34 | | -| 1 % true | RunEnd x10 | 0.05 | 3 | 2.2 | 1.76 | None: Canonical x10, 0.13 B/value, scan 0.2 ms, take 0.28 ms; reads faster above 358 MB/s | +| half true | Canonical x10 | 0.13 | 3 | 0.2 | 0.15 | | +| 1 % true | RunEnd x10 | 0.05 | 3 | 1.7 | 0.30 | None: Canonical x10, 0.13 B/value, scan 0.2 ms, take 0.18 ms; reads faster above 488 MB/s |
half true: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.34 | | -| Fastest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.29 | same | -| Smallest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.29 | same | -| None | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.30 | same | -| hint RunEnd | Canonical x10 | 1,262,556 | 0.13 | 5 | 0.2 | 0.29 | same | +| Auto | Canonical x10 | 1,262,556 | 0.13 | 3 | 0.2 | 0.15 | | +| Fastest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.1 | 0.13 | reads faster at any throughput | +| Smallest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.22 | reads slower at any throughput | +| None | Canonical x10 | 1,262,556 | 0.13 | 1 | 0.2 | 0.19 | reads slower at any throughput | +| hint RunEnd | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.21 | reads slower at any throughput |
@@ -827,20 +827,20 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x10 | 548,092 | 0.05 | 3 | 2.2 | 1.76 | | -| Fastest | RunEnd x10 | 548,092 | 0.05 | 3 | 2.2 | 1.77 | same | -| Smallest | RunEnd x10 | 164,692 | 0.02 | 10 | 3.3 | 2.94 | reads faster below 329 MB/s | -| None | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.28 | reads faster above 358 MB/s | -| hint RunEnd | RunEnd x10 | 548,092 | 0.05 | 4 | 2.1 | 1.71 | same | +| Auto | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.30 | | +| Fastest | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.31 | same | +| Smallest | RunEnd x10 | 164,692 | 0.02 | 10 | 2.9 | 1.46 | reads faster below 314 MB/s | +| None | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.18 | reads faster above 488 MB/s | +| hint RunEnd | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.28 | same | ### The profiles side by side -`Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *i64, random in 0..999*, *i64, 16 distinct, random order*, *i64, 100 003 distinct, repeating*, *i64, random in 0..999, 10 % null*, *f64, 16 distinct, random order*, *f64, uniform in [0, 1)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 15.1 times `Auto`'s write, 719 ms against 48 ms on *f64, prices (2 decimals)*, since it tries every scheme on every chunk. +`Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *i64, random in 0..999*, *i64, 16 distinct, random order*, *i64, 100 003 distinct, repeating*, *i64, random in 0..999, 10 % null*, *f64, 16 distinct, random order*, *f64, uniform in [0, 1)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 17.6 times `Auto`'s write, 709 ms against 40 ms on *f64, prices (2 decimals)*, since it tries every scheme on every chunk. `None`, the plain form, made the largest file on 20 of 20 columns. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 671b43b6 with uncommitted changes, 2026-10-02 00:42 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 32b3a82b with uncommitted changes, 2026-10-10 02:22 UTC.* @@ -856,9 +856,9 @@ does not; a column it never departs on is left out. | i64, timestamps (ms, increasing, jittered) | | `Pco` | | `Zstd` | | | i64, 100 003 distinct, repeating | 16 MiB chunks | `Pco` | 16 MiB chunks | | `Dictionary`, 16 MiB chunks | | i64, random in 0..999, 10 % null | | `Dictionary` | | `Dictionary` | `Dictionary` | -| f64, 100 003 distinct, repeating | 16 MiB chunks | | `Canonical` | | `Dictionary`, 16 MiB chunks | +| f64, 100 003 distinct, repeating | 16 MiB chunks | | 16 MiB chunks | | `Dictionary`, 16 MiB chunks | | f64, uniform in [0, 1) | | | `Canonical` | | | -| utf8, 10 000 distinct ids | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `OnPair` | 16 MiB chunks | +| utf8, 10 000 distinct ids | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `Zstd` | 16 MiB chunks | | utf8, 10 000 distinct ids, 10 % null | 16 MiB chunks | 16 MiB chunks | 16 MiB chunks | `Zstd` | 16 MiB chunks | | utf8, unique UUIDs | `Fsst` | | `Canonical` | `Fsst` | | | utf8, log lines (~100 B) | `OnPair` | | `Fsst` | `OnPair` | | @@ -866,7 +866,7 @@ does not; a column it never departs on is left out. It keeps the writer's choice under every goal on the 10 other columns. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 671b43b6 with uncommitted changes, 2026-10-02 00:45 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 32b3a82b with uncommitted changes, 2026-10-10 02:24 UTC.* ## Kernels, against their baselines @@ -883,32 +883,32 @@ reference profile on that class, the one to quote a small difference from. | Method | BitWidth | Mean | Error | StdDev | Speedup | MannWhitney(5%) | ns/row | GB/s | Allocated | Alloc Ratio | |------------- |--------- |----------:|----------:|----------:|--------:|---------------- |-------:|------:|----------:|------------:| -| 'i64 scalar' | 17 | 77.379 μs | 5.5814 μs | 1.4495 μs | 1.00x | Baseline | 1.18 | 6.78 | - | NA | -| 'i64 vector' | 17 | 8.458 μs | 0.2551 μs | 0.0663 μs | 9.15x | Faster | 0.13 | 61.99 | - | NA | -| 'i32 scalar' | 17 | 87.267 μs | 1.8560 μs | 0.4820 μs | 0.89x | Same | 1.33 | 3.00 | - | NA | -| 'i64 pack' | 17 | 22.376 μs | 0.2826 μs | 0.0437 μs | 3.46x | Faster | 0.34 | 23.43 | - | NA | -| 'i32 pack' | 17 | 11.464 μs | 0.3956 μs | 0.1027 μs | 6.75x | Faster | 0.17 | 22.87 | - | NA | -| 'i32 vector' | 17 | 5.012 μs | 0.9929 μs | 0.2579 μs | 15.4x | Faster | 0.08 | 52.30 | - | NA | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 9e94f340 with uncommitted changes, 2026-10-02 06:49 UTC.* +| 'i64 scalar' | 17 | 76.387 μs | 1.8615 μs | 0.4834 μs | 1.00x | Baseline | 1.17 | 6.86 | - | NA | +| 'i64 vector' | 17 | 8.280 μs | 0.1418 μs | 0.0368 μs | 9.23x | Faster | 0.13 | 63.32 | - | NA | +| 'i32 scalar' | 17 | 86.997 μs | 1.7466 μs | 0.4536 μs | 0.88x | Same | 1.33 | 3.01 | - | NA | +| 'i64 pack' | 17 | 22.053 μs | 0.4604 μs | 0.1196 μs | 3.46x | Faster | 0.34 | 23.77 | - | NA | +| 'i32 pack' | 17 | 11.711 μs | 0.3273 μs | 0.0507 μs | 6.52x | Faster | 0.18 | 22.38 | - | NA | +| 'i32 vector' | 17 | 4.745 μs | 0.0168 μs | 0.0043 μs | 16.1x | Faster | 0.07 | 55.25 | - | NA | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* ### `RowEncodingBenchmarks` -| Method | Entry | Mean | Error | StdDev | ns/row | GB/s | Allocated | -|----------------------------------------- |--------------------- |-----------:|-----------:|-----------:|-------:|-----:|----------:| -| **'encode a batch to row keys'** | **conta(...)nical [33]** | **48.506 μs** | **2.4528 μs** | **0.6370 μs** | **11.84** | **3.63** | **48 B** | -| 'encode a batch to row keys, descending' | conta(...)nical [33] | 47.236 μs | 3.3545 μs | 0.8712 μs | 11.53 | 3.73 | 48 B | -| 'encode, then sort rows by key' | conta(...)nical [33] | 359.607 μs | 41.8745 μs | 10.8747 μs | 87.79 | 0.49 | 48 B | -| **'encode a batch to row keys'** | **conta(...)nulls [33]** | **14.697 μs** | **0.4671 μs** | **0.0723 μs** | **14.35** | **4.60** | **48 B** | -| 'encode a batch to row keys, descending' | conta(...)nulls [33] | 14.971 μs | 0.3792 μs | 0.0985 μs | 14.62 | 4.51 | 48 B | -| 'encode, then sort rows by key' | conta(...)nulls [33] | 46.457 μs | 2.5397 μs | 0.3930 μs | 45.37 | 1.45 | 48 B | -| **'encode a batch to row keys'** | **encod(...)r1025 [31]** | **6.427 μs** | **0.1677 μs** | **0.0436 μs** | **6.27** | **1.44** | **48 B** | -| 'encode a batch to row keys, descending' | encod(...)r1025 [31] | 6.450 μs | 0.0778 μs | 0.0120 μs | 6.29 | 1.43 | 48 B | -| 'encode, then sort rows by key' | encod(...)r1025 [31] | 37.092 μs | 1.5607 μs | 0.4053 μs | 36.19 | 0.25 | 48 B | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 9e94f340 with uncommitted changes, 2026-10-02 06:49 UTC.* +| Method | Entry | Mean | Error | StdDev | ns/row | GB/s | Allocated | +|----------------------------------------- |--------------------- |-----------:|-----------:|----------:|-------:|-----:|----------:| +| **'encode a batch to row keys'** | **conta(...)nical [33]** | **48.246 μs** | **0.4884 μs** | **0.1268 μs** | **11.78** | **3.65** | **48 B** | +| 'encode a batch to row keys, descending' | conta(...)nical [33] | 45.855 μs | 0.2506 μs | 0.0388 μs | 11.19 | 3.84 | 48 B | +| 'encode, then sort rows by key' | conta(...)nical [33] | 344.474 μs | 27.3616 μs | 4.2342 μs | 84.10 | 0.51 | 48 B | +| **'encode a batch to row keys'** | **conta(...)nulls [33]** | **14.714 μs** | **0.0875 μs** | **0.0227 μs** | **14.37** | **4.59** | **48 B** | +| 'encode a batch to row keys, descending' | conta(...)nulls [33] | 14.614 μs | 0.0758 μs | 0.0117 μs | 14.27 | 4.62 | 48 B | +| 'encode, then sort rows by key' | conta(...)nulls [33] | 47.087 μs | 3.1135 μs | 0.8086 μs | 45.98 | 1.44 | 48 B | +| **'encode a batch to row keys'** | **encod(...)r1025 [31]** | **6.381 μs** | **0.1115 μs** | **0.0290 μs** | **6.22** | **1.45** | **48 B** | +| 'encode a batch to row keys, descending' | encod(...)r1025 [31] | 6.400 μs | 0.1260 μs | 0.0327 μs | 6.24 | 1.44 | 48 B | +| 'encode, then sort rows by key' | encod(...)r1025 [31] | 36.293 μs | 2.3583 μs | 0.6125 μs | 35.41 | 0.25 | 48 B | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* @@ -916,25 +916,25 @@ reference profile on that class, the one to quote a small difference from. | Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | |----------------------------- |------- |--------------:|------------:|------------:|--------:|---------------- |----------:|------------:| -| 'Choose, the whole decision' | i64 | 45,172.787 ns | 474.2953 ns | 123.1730 ns | 1.00x | Baseline | 72 B | 1.00 | -| 'candidate: sequence' | i64 | 8.113 ns | 0.2090 ns | 0.0323 ns | 5,568x | Faster | - | 0.00 | -| 'candidate: bit packing' | i64 | 6,696.626 ns | 144.5110 ns | 37.5290 ns | 6.75x | Faster | - | 0.00 | -| 'candidate: FSST' | i64 | 1.535 ns | 0.0219 ns | 0.0057 ns | 29,430x | Faster | - | 0.00 | -| 'candidate: zstd' | i64 | 9,739.484 ns | 266.9906 ns | 41.3171 ns | 4.64x | Faster | 72 B | 1.00 | -| 'candidate: ALP' | i64 | 1.723 ns | 0.0458 ns | 0.0071 ns | 26,211x | Faster | - | 0.00 | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 9e94f340 with uncommitted changes, 2026-10-02 06:49 UTC.* +| 'Choose, the whole decision' | i64 | 43,085.478 ns | 792.0855 ns | 205.7021 ns | 1.00x | Baseline | - | NA | +| 'candidate: sequence' | i64 | 7.948 ns | 0.0725 ns | 0.0188 ns | 5,421x | Faster | - | NA | +| 'candidate: bit packing' | i64 | 6,606.360 ns | 310.5553 ns | 80.6502 ns | 6.52x | Faster | - | NA | +| 'candidate: FSST' | i64 | 1.550 ns | 0.0173 ns | 0.0045 ns | 27,793x | Faster | - | NA | +| 'candidate: zstd' | i64 | 8,612.868 ns | 124.5499 ns | 19.2742 ns | 5.00x | Faster | - | NA | +| 'candidate: ALP' | i64 | 1.760 ns | 0.0394 ns | 0.0102 ns | 24,480x | Faster | - | NA | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* ### `LanesBenchmarks` -| Method | Lanes | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Completed Work Items | Lock Contentions | Gen0 | Allocated | Alloc Ratio | -|----------------------- |------ |---------:|----------:|---------:|--------:|---------------- |---------------------:|-----------------:|-------:|----------:|------------:| -| 'ours, n lanes' | 4 | 388.7 μs | 114.32 μs | 29.69 μs | 1.00x | Baseline | 98.8160 | 0.0035 | 3.4722 | 46957 B | 1.00 | -| 'reference, n workers' | 4 | 944.6 μs | 35.39 μs | 9.19 μs | 0.41x | Slower | - | - | - | - | 0.00 | +| Method | Lanes | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Gen0 | Completed Work Items | Lock Contentions | Allocated | Alloc Ratio | +|----------------------- |------ |---------:|---------:|--------:|--------:|---------------- |-------:|---------------------:|-----------------:|----------:|------------:| +| 'ours, n lanes' | 4 | 344.5 μs | 9.00 μs | 2.34 μs | 1.00x | Baseline | 3.4722 | 103.5903 | - | 46720 B | 1.00 | +| 'reference, n workers' | 4 | 787.7 μs | 30.70 μs | 7.97 μs | 0.44x | Slower | - | - | - | - | 0.00 | -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 9e94f340 with uncommitted changes, 2026-10-02 06:49 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* ## What this does not measure From f464aac3db64400a47175d0433915602e7c2f095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 04:26:04 +0200 Subject: [PATCH 48/56] bench: profile.sh launches the runner with VORTICITY_RUNNER_WARM Instruments launches the runner without the caller's environment, so a profile of the warm base mapped its file anew at every round. The variable, when set, now goes to the launched runner through --env. --- bench/profile.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bench/profile.sh b/bench/profile.sh index 5d1e7499c..05fa3f571 100755 --- a/bench/profile.sh +++ b/bench/profile.sh @@ -28,6 +28,10 @@ # millisecond, and what the caller's thread and the pool's workers spent their cycles on. # # The reports land in the output directory, by default under $TMPDIR/vorticity-profile. +# +# VORTICITY_RUNNER_WARM=1 in the environment keeps the runner's file mapped from one round to the next, +# as the warm base of the comparison with DuckDB does: Instruments launches the runner with it, which +# it would not pass on by itself. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -50,6 +54,8 @@ while [ $# -gt 0 ]; do done lanes=() [ -z "$threads" ] || lanes=(--threads "$threads") +envs=() +[ -z "${VORTICITY_RUNNER_WARM:-}" ] || envs=(--env "VORTICITY_RUNNER_WARM=$VORTICITY_RUNNER_WARM") case "$mode" in cycles|allocations|trace) ;; *) usage ;; esac [ -f "$file" ] || { echo "no such file: $file" >&2; exit 2; } file="$(cd "$(dirname "$file")" && pwd)/$(basename "$file")" @@ -90,7 +96,7 @@ case "$mode" in "$xctrace" record --template 'CPU Profiler' --show-recording-options 2> /dev/null | sed 's/"highFrequency" : false/"highFrequency" : true/' > "$out/options.json" "$xctrace" record --template 'CPU Profiler' --recording-options "$out/options.json" --no-prompt \ - --output "$out/cycles.trace" --target-stdout "$out/runner.txt" \ + --output "$out/cycles.trace" --target-stdout "$out/runner.txt" ${envs[@]+"${envs[@]}"} \ --launch -- "$runner" --scenario "$scenario" "$file" "$rows" --repeat "$rounds" ${lanes[@]+"${lanes[@]}"} > /dev/null "$xctrace" export --input "$out/cycles.trace" \ --xpath '/trace-toc/run[@number="1"]/data/table[@schema="cpu-profile"]' \ @@ -113,7 +119,7 @@ case "$mode" in ;; trace) "$xctrace" record --template 'Processor Trace' --no-prompt \ - --output "$out/processor.trace" --target-stdout "$out/runner.txt" \ + --output "$out/processor.trace" --target-stdout "$out/runner.txt" ${envs[@]+"${envs[@]}"} \ --launch -- "$runner" --scenario "$scenario" "$file" "$rows" --repeat "$rounds" ${lanes[@]+"${lanes[@]}"} echo "open it with: open '$out/processor.trace'" ;; From 1aff1ddf88d6be1e5f9aec0456fbb94be911f9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 04:51:39 +0200 Subject: [PATCH 49/56] perf(query): a key hashed once by the core, a lane's table read ahead past its cache The core found a part's new keys at their home first, then hashed each missed key again to add it, and a third time when its home was taken: three hashes of a 64-byte tuple where one does. The home pass now keeps each hash, and a key it missed is added under it while the table keeps its seed; a table condensed during the batch, or reseeded, hashes again. Alternated runners against 7445c2a5 at fourteen lanes: q10 on 10M x0.985 (cycles x0.983), 10^7 hashed keys x0.977. A lane's table of hashed keys now reads each slot sixteen rows ahead once it outgrows its lane's share of a cache, 768 KB at several lanes and 8 MB alone, unless the plan sets a distance: a table of 10^5 short texts per lane missed at every probe. Tuples keep their two passes as they were, their 64 bytes still compared once the slot is read (q10 at one lane gave back 1.5 %). Against the commit before, at fourteen lanes: q3 x0.963, q7 x0.947, q2 and q10 flat; at one lane flat. --- .../Aggregation/AggregationEngine.cs | 58 +++++++++++++------ src/Vorticity/Aggregation/GroupCore.cs | 4 +- src/Vorticity/Aggregation/GroupKeys.cs | 39 +++++++++++-- src/Vorticity/Aggregation/ShortTextKeys.cs | 8 ++- src/Vorticity/Aggregation/WideKeyTable.cs | 50 +++++++++++++--- 5 files changed, 122 insertions(+), 37 deletions(-) diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index 3fd898820..d5d3e96ad 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -223,6 +223,25 @@ internal AggregationPlan(ReadOnlySpan results, ColumnShape[] keys) /// internal const int DefaultProbeAhead = 0; + /// + /// The bytes of a lane's table of hashed keys past which, left at its + /// default, its first pass reads each slot rows ahead: the share of a private + /// cache a lane's cache gets, past which every probe misses. Each run sets it from its lanes; none + /// reads ahead before it does. + /// + internal long ReadAheadBytes { get; set; } = long.MaxValue; + + /// + /// The rows ahead a lane's table read its slots once it outgrew . Measured + /// on 2026-10-10 on the Mac at fourteen lanes, every table read ahead: 16 rows took ×0.929 off q3 + /// (10⁵ short texts, a lane's table of 2.4 MB) and nothing off q2 nor 10⁶ hashed keys, 32 rows + /// ×0.939; keys in a row, at a stride or hot gave back 2 to 8 % at one lane on 2026-10-06, their + /// tables in cache, which the size spares. Read ahead past the size alone: q3 ×0.963 and q7 ×0.947 + /// at fourteen lanes, the rest flat; never the tuples of several columns, whose slot read ahead + /// still leaves their 64 bytes to compare: q10 ×1.015 at one lane. + /// + internal const int ReadAhead = 16; + /// The most groups the plan's last run held at once (). internal long PeakGroups { get; set; } @@ -473,7 +492,7 @@ private static (IAggregateNode Row, int[] Columns)[] Rows(IChosenColumn[] chosen /// internal GroupKeys CreateKeys(bool sorted, KeyFacts? facts = null, ArrayShelf? shelf = null, QueryMemory? memory = null) => Keys.Length switch { - 1 => Single(Keys[0], sorted, sorted ? null : facts?.Bounds[0], ProbeAhead, facts?.Rows ?? -1, shelf), + 1 => Single(Keys[0], sorted, sorted ? null : facts?.Bounds[0], ProbeAhead, facts?.Rows ?? -1, shelf, ReadAheadBytes), 2 or 3 or 4 when Raw(facts) is { } layout => layout.Bits <= 64 ? new RawKeys(layout, shelf: shelf) : new RawKeys(layout, shelf: shelf), 2 => new PackedKeys(Keys, facts, shelf: shelf), 3 or 4 => new PackedKeys(Keys, facts, shelf: shelf), @@ -548,28 +567,29 @@ private static (IAggregateNode Row, int[] Columns)[] Rows(IChosenColumn[] chosen } /// The index of a key of one column. - internal static GroupKeys Single(ColumnShape key, bool sorted, KeyBounds? bounds = null, int probeAhead = DefaultProbeAhead, long rows = -1, ArrayShelf? shelf = null) => + internal static GroupKeys Single( + ColumnShape key, bool sorted, KeyBounds? bounds = null, int probeAhead = DefaultProbeAhead, long rows = -1, ArrayShelf? shelf = null, long aheadBytes = long.MaxValue) => key.Kind switch { StorageKind.Primitive => key.PType switch { - PType.I8 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.I16 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.I32 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.I64 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.U8 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.U16 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.U32 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.U64 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf), - PType.F16 => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf), - PType.F32 => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf), - _ => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf), + PType.I8 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.I16 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.I32 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.I64 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.U8 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.U16 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.U32 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.U64 => new FixedKeys(key, sorted, bounds, probeAhead, rows, shelf: shelf, aheadBytes: aheadBytes), + PType.F16 => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes), + PType.F32 => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes), + _ => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes), }, - StorageKind.Decimal => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf), - StorageKind.Decimal256 => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf), - StorageKind.Uuid => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf), + StorageKind.Decimal => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes), + StorageKind.Decimal256 => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes), + StorageKind.Uuid => new FixedKeys(key, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes), StorageKind.Bool => new BoolKeys(key), - StorageKind.Bytes => new ShortTextKeys(key, sorted, probeAhead, shelf), + StorageKind.Bytes => new ShortTextKeys(key, sorted, probeAhead, shelf, aheadBytes), _ => throw key.Unsupported("a group key"), }; } @@ -3328,6 +3348,10 @@ internal static async ValueTask RunAsync( long splitBytes = ranges is null ? 0 : source.ReadAheadBytes(pass); int lanes = Admit(memory, asked, pass.Options.BatchRows, spilledRow, splitBytes, out int ahead); + // A lane's table reads its slots ahead once past its share of a cache: one lane alone has its + // cluster's to itself. + plan.ReadAheadBytes = lanes > 1 ? GroupCore.LaneCacheBytes : GroupCore.AloneCacheBytes; + // The core holds the groups once a lane's cache fills: each lane's // partition is then its cache, which no table of groups sized on the source's rows fills. core = GroupCore.Of(plan, settled, columns, inputs, source, facts, sorted, top, lanes, memory); diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index df053d548..ca8b3922a 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -117,10 +117,10 @@ internal sealed partial class GroupCore /// smallest core it may run on, a megabyte on current ones, less its open batches' lines and a /// window of rows. A fixed default until the topology is read. /// - private const long LaneCacheBytes = 768 * 1024; + internal const long LaneCacheBytes = 768 * 1024; /// The bytes of the cache of a lane alone, which has the level of cache its cluster shares to itself. - private const long AloneCacheBytes = 8L * 1024 * 1024; + internal const long AloneCacheBytes = 8L * 1024 * 1024; /// /// The share of its rows a cache finds below which its lane bypasses it, ε: a row the cache misses diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index 0d4a60a30..9bccbdeb8 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -624,8 +624,12 @@ internal sealed class FixedKeys : GroupKeys private ValuesCache _entries; // The hashed path in two passes (AggregationPlan.ProbeAhead): each row's home slot, then the rows - // the first pass left; what its reads ahead found, written so that they stay. + // the first pass left; what its reads ahead found, written so that they stay. Left at its default, + // the slots read ahead once the table outgrew its lane's share of a cache (AggregationPlan.ReadAheadBytes): + // its bytes, and its groups by a key and a slot each. private readonly int _probeAhead; + private readonly long _aheadBytes; + private readonly int _aheadFrom; private uint[] _homes = []; private int[] _left = []; private int _sink; @@ -672,12 +676,14 @@ internal sealed class FixedKeys : GroupKeys internal FixedKeys( ColumnShape shape, bool sorted, KeyBounds? bounds = null, int probeAhead = AggregationPlan.DefaultProbeAhead, long rows = -1, bool appending = false, - ArrayShelf? shelf = null) + ArrayShelf? shelf = null, long aheadBytes = long.MaxValue) { _shape = shape; _sorted = sorted; _bounds = bounds; _probeAhead = probeAhead; + _aheadBytes = aheadBytes; + _aheadFrom = aheadBytes == long.MaxValue ? int.MaxValue : (int)Math.Clamp(aheadBytes / (Unsafe.SizeOf() + sizeof(ulong)), 1, int.MaxValue); _rows = rows; _appending = appending; _shelf = shelf; @@ -1236,13 +1242,14 @@ internal void AssignChunk(ReadOnlySpan values, int start, ReadOnlySpan values, ReadOnlySpan validity, int start, int[] rowGroups) { int rows = values.Length; - if (_probeAhead > 0) + int ahead = _probeAhead > 0 ? _probeAhead : Count >= _aheadFrom ? AggregationPlan.ReadAhead : 0; + if (ahead > 0) { Scratch.Grow(ref _homes, rows); } Span groups = rowGroups.AsSpan(start, rows); - _sink ^= FindAtHome(values, groups, _homes, _probeAhead, out bool missed); + _sink ^= FindAtHome(values, groups, _homes, ahead, out bool missed); // Every row found its group, none null: the usual batch once the keys are known. if (!missed && validity.IsEmpty) @@ -1719,7 +1726,7 @@ private bool DirectSlot(TValue value, out ulong number) return number < _span; } - internal override GroupKeys Fresh() => new FixedKeys(_shape, _sorted, _bounds, _probeAhead, _rows); + internal override GroupKeys Fresh() => new FixedKeys(_shape, _sorted, _bounds, _probeAhead, _rows, aheadBytes: _aheadBytes); /// Whether the groups are numbered by value, in the pages of a table of groups, rather than hashed. internal bool ByValue => _pages is not null; @@ -2151,21 +2158,41 @@ internal override void GroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan< } groups = groups[..count]; + bool hashed = Compact; + ulong seed = hashed ? _wide.Seed : 0; _sink ^= FindAtHome(keys, groups, homes, 0, out bool missed); if (!missed) { return; } + // A key missed added under the hash its home was found by, while the table's seed holds: only a + // table of a hash and a group keeps them, and only from before the batch (one condensed during + // it was not the one the homes were taken in). for (int i = 0; i < count; i++) { if (groups[i] < 0) { - groups[i] = Lookup(keys[i]); + groups[i] = hashed && _wide.Seed == seed ? LookupHashed(keys[i], homes[i]) : Lookup(keys[i]); } } } + /// + /// of a key hashed in a table of a hash and a group, under its seed now: a wide + /// key hashed once, where a new one was hashed again at its home, then at its line. + /// + private int LookupHashed(TValue value, uint hash) + { + int group = _wide.GetOrAdd(value, hash, Count, _keys); + if (group == Count) + { + Add(value); + } + + return group; + } + /// /// for a paged core's part, numbered whole: an entry's group is its number, its /// page's place among the part's then its place in the page, read off its value with no call, its bit diff --git a/src/Vorticity/Aggregation/ShortTextKeys.cs b/src/Vorticity/Aggregation/ShortTextKeys.cs index 79993dc13..5881c2f10 100644 --- a/src/Vorticity/Aggregation/ShortTextKeys.cs +++ b/src/Vorticity/Aggregation/ShortTextKeys.cs @@ -145,6 +145,7 @@ internal sealed class ShortTextKeys : GroupKeys private readonly ColumnShape _shape; private readonly bool _sorted; private readonly int _probeAhead; + private readonly long _aheadBytes; private readonly ArrayShelf? _shelf; // The words while every value met is short, then the bytes for good: one of them, never both. @@ -152,13 +153,14 @@ internal sealed class ShortTextKeys : GroupKeys private BytesKeys? _bytes; private TextWord[] _block = []; - internal ShortTextKeys(ColumnShape shape, bool sorted, int probeAhead = AggregationPlan.DefaultProbeAhead, ArrayShelf? shelf = null) + internal ShortTextKeys(ColumnShape shape, bool sorted, int probeAhead = AggregationPlan.DefaultProbeAhead, ArrayShelf? shelf = null, long aheadBytes = long.MaxValue) { _shape = shape; _sorted = sorted; _probeAhead = probeAhead; + _aheadBytes = aheadBytes; _shelf = shelf; - _words = new FixedKeys(shape, sorted, probeAhead: probeAhead, shelf: shelf); + _words = new FixedKeys(shape, sorted, probeAhead: probeAhead, shelf: shelf, aheadBytes: aheadBytes); } /// Whether the keys are still words: every value met so far 12 bytes or less. @@ -454,7 +456,7 @@ internal static bool TryWord(ReadOnlySpan value, out TextWord word) private ReadOnlySpan KeyOf(int group, Span buffer) => _words is { } words ? BytesOf(words.KeyAt(group), buffer) : _bytes!.KeyOf(group); - internal override GroupKeys Fresh() => new ShortTextKeys(_shape, _sorted, _probeAhead); + internal override GroupKeys Fresh() => new ShortTextKeys(_shape, _sorted, _probeAhead, aheadBytes: _aheadBytes); internal override GroupKeys ForTable(ArrayShelf shelf) => new ShortTextKeys(_shape, sorted: false, _probeAhead, shelf); diff --git a/src/Vorticity/Aggregation/WideKeyTable.cs b/src/Vorticity/Aggregation/WideKeyTable.cs index 5dae5e0b1..f64744331 100644 --- a/src/Vorticity/Aggregation/WideKeyTable.cs +++ b/src/Vorticity/Aggregation/WideKeyTable.cs @@ -161,12 +161,22 @@ internal bool Squeeze(int more) /// , which it then holds, and which the caller's keys hold from then on. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal int GetOrAdd(TValue key, int next, ReadOnlySpan keys) + internal int GetOrAdd(TValue key, int next, ReadOnlySpan keys) => GetOrAdd(key, HashOf(key, _seed), next, keys); + + /// The seed the keys are hashed under: a hash taken under another is no longer theirs. + internal readonly ulong Seed => _seed; + + /// + /// of a key whose + /// took under the table's : a wide key hashed once where a + /// new one was hashed three times, its home looked at, then its slot, then its line (q10's tuples). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal int GetOrAdd(TValue key, uint hash, int next, ReadOnlySpan keys) { Span slots = Slots; if (slots.Length != 0) { - uint hash = HashOf(key, _seed); ref Slot slot = ref slots[(int)KeyTable.SlotOf(hash, (uint)slots.Length, _multiplier)]; if (slot.Group != 0) { @@ -184,7 +194,7 @@ internal int GetOrAdd(TValue key, int next, ReadOnlySpan keys) } } - return Search(key, next, keys); + return Search(key, hash, next, keys); } /// @@ -194,7 +204,7 @@ internal int GetOrAdd(TValue key, int next, ReadOnlySpan keys) /// /// The keys of the batch. /// Each key's group, or -1. - /// Room for each key's hash, as many as the keys, when reads ahead. + /// Room for each key's hash, as many as the keys: filled, for the keys missed to be added under it while holds. /// How many rows on a slot is read before its row compares; 0 for none. /// The keys of the groups. /// Whether a key found no group: the rows left to the lookup, none when false. @@ -205,6 +215,14 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp { groups[..keys.Length].Fill(-1); missed = keys.Length > 0; + if (hashes.Length >= keys.Length) + { + for (int i = 0; i < keys.Length; i++) + { + hashes[i] = HashOf(keys[i], _seed); + } + } + return 0; } @@ -221,10 +239,17 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp // Every row's slot first, its group where the hash agrees, then every candidate's key: two // loops, no read of a key waiting on its row's read of a slot. In one loop, a count by a // million UUIDs at one lane took 1.06 times as long as slots that hold their keys; in two, - // 0.86 of one loop's time, measured on 2026-10-08. + // 0.86 of one loop's time, measured on 2026-10-08. Each hash kept for the keys missed, where + // the caller has room for them. + bool keep = hashes.Length >= keys.Length; for (int i = 0; i < keys.Length; i++) { uint hash = HashOf(keys[i], seed, prepared); + if (keep) + { + hashes[i] = hash; + } + ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.SlotOf(hash, length, multiplier)); groups[i] = (home.Group & -Unsafe.BitCast(home.Hash == hash)) - 1; } @@ -341,7 +366,14 @@ internal void Release() /// A key whose home is taken: the rest of the home's line, then the line's chain. [MethodImpl(MethodImplOptions.NoInlining)] - private int Search(TValue key, int next, ReadOnlySpan keys) + private int Search(TValue key, int next, ReadOnlySpan keys) => Search(key, HashOf(key, _seed), next, keys); + + /// + /// of a key whose was + /// taken under the table's seed, which a resize keeps and a new seed does not. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private int Search(TValue key, uint hash, int next, ReadOnlySpan keys) { if (_length == 0) { @@ -349,7 +381,6 @@ private int Search(TValue key, int next, ReadOnlySpan keys) } Span slots = Slots; - uint hash = HashOf(key, _seed); uint at = KeyTable.SlotOf(hash, (uint)slots.Length, _multiplier); uint first = at & ~(uint)(Width - 1); uint end = Math.Min(first + (uint)Width, (uint)slots.Length); @@ -362,7 +393,7 @@ private int Search(TValue key, int next, ReadOnlySpan keys) if (_count >= _growAt) { Resize(KeyTable.PrimeAtLeast(GroupKeys.Doubled(slots.Length))); - return Search(key, next, keys); + return Search(key, hash, next, keys); } slot.Hash = hash; @@ -397,9 +428,10 @@ private int Search(TValue key, int next, ReadOnlySpan keys) if (_count >= _growAt || (!_squeezed && 8L * _overflowed > _count && 10L * _count > 3L * slots.Length)) { Resize(KeyTable.PrimeAtLeast(GroupKeys.Doubled(slots.Length))); - return Search(key, next, keys); + return Search(key, hash, next, keys); } + // A new seed: the key hashed again under it. if (links >= MaxChain && _seed == 0) { Reseed((ulong)Random.Shared.NextInt64(1, long.MaxValue) | 1, keys); From da606212d7958af73341a158cd59e2ea13eebb57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 05:05:20 +0200 Subject: [PATCH 50/56] perf(query): an entry carries its wide key's hash to the core A lane scattering a key wider than two words folded its words for the key's part, and the core folded them again to find the key's group in its sub-table: four 128-bit products a tuple, twice. An entry of such a key now carries the four bytes of its sub-table hash under no seed, before the key, in the record's padding when there is room (q10's entry keeps its 104 bytes); the lane writes it as it scatters, a spill as it copies, and the core reads it while the sub-table has no seed, folding the words again otherwise. Alternated runners against e89d9a6c at fourteen lanes, warm: q10 on 10M x0.960 (94.5 to 90.7 ms, cycles x0.961), on 4M x0.999 (cycles x0.982). --- src/Vorticity/Aggregation/GroupCore.cs | 90 +++++++++++++++++++++-- src/Vorticity/Aggregation/GroupKeys.cs | 17 +++-- src/Vorticity/Aggregation/WideKeyTable.cs | 37 +++++++--- 3 files changed, 124 insertions(+), 20 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupCore.cs b/src/Vorticity/Aggregation/GroupCore.cs index ca8b3922a..9f0f69bdf 100644 --- a/src/Vorticity/Aggregation/GroupCore.cs +++ b/src/Vorticity/Aggregation/GroupCore.cs @@ -1486,16 +1486,33 @@ internal enum CoreReason /// The words of a record, copied whole from a lane's cache. /// The words of an entry. /// The byte of an entry its key starts at: in the record's padding when it fits there, past the record otherwise. -internal readonly record struct EntryShape(int RecordWords, int Words, int KeyOffset) +/// +/// The byte of an entry its key's hash lies at, four bytes, as a sub-table under no seed hashes it; -1 when +/// entries carry none. +/// +internal readonly record struct EntryShape(int RecordWords, int Words, int KeyOffset, int HashOffset = -1) { - /// The entries of records laid out as , or of none, and keys of , a power of two. + /// + /// The entries of records laid out as , or of none, and keys of , a power of two. + /// A key wider than two words carries its hash, before the key, in the record's padding when there is + /// room: the lane that scatters it has its words folded already, where the core folded them again, + /// four products for a tuple (q10). + /// internal static EntryShape Of(RecordLayout? layout, int keyBytes) { int recordWords = layout?.Stride ?? 0; int alignment = Math.Min(sizeof(ulong), keyBytes); - int keyOffset = ((layout?.Width ?? 0) + alignment - 1) & -alignment; + int width = layout?.Width ?? 0; + int hashOffset = -1; + if (keyBytes > 2 * sizeof(ulong)) + { + hashOffset = (width + sizeof(uint) - 1) & -sizeof(uint); + width = hashOffset + sizeof(uint); + } + + int keyOffset = (width + alignment - 1) & -alignment; int bytes = Math.Max(recordWords * sizeof(ulong), keyOffset + keyBytes); - return new EntryShape(recordWords, (bytes + sizeof(ulong) - 1) / sizeof(ulong), keyOffset); + return new EntryShape(recordWords, (bytes + sizeof(ulong) - 1) / sizeof(ulong), keyOffset, hashOffset); } } @@ -2713,6 +2730,36 @@ internal static void Scatter(ReadOnlySpan keys, int skip, ReadOnlySp } ref ulong first = ref MemoryMarshal.GetReference(records); + if (shape.HashOffset >= 0) + { + // A wide key's words folded once: its part from them, and its hash in the sub-table, which the + // core then reads where it folded the words again. + nint hashOffset = shape.HashOffset; + for (int g = 0; g < keys.Length; g++) + { + if (g == skip) + { + continue; + } + + TKey key = keys[g]; + (ulong low, ulong high) = KeyWords.Of(key); + ref ulong entry = ref lane.Entry((int)(MergeHash.Of(low, high, MergeHash.Seed) >> GroupCore.PartShift)); + ref ulong record = ref Unsafe.Add(ref first, (nint)g * stride); + for (int w = 0; w < stride; w++) + { + Unsafe.Add(ref entry, w) = Unsafe.Add(ref record, w); + } + + // The hash and the key last: they may lie in the record's padding. + ref byte bytes = ref Unsafe.As(ref entry); + Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref bytes, hashOffset), TableHash(low, high)); + Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref bytes, keyOffset), key); + } + + return; + } + for (int g = 0; g < keys.Length; g++) { if (g == skip) @@ -2759,8 +2806,15 @@ internal static int Copy(ReadOnlySpan keys, int skip, ReadOnlySpan(ref MemoryMarshal.GetReference(entry)), keyOffset), keys[g]); + // The hash and the key last: they may lie in the record's padding. + ref byte bytes = ref Unsafe.As(ref MemoryMarshal.GetReference(entry)); + if (shape.HashOffset >= 0) + { + (ulong low, ulong high) = KeyWords.Of(keys[g]); + Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref bytes, shape.HashOffset), TableHash(low, high)); + } + + Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref bytes, keyOffset), keys[g]); n++; } @@ -2820,4 +2874,28 @@ internal static TKey KeyAt(PartBatch batch, EntryShape shape, int entry) ref byte first = ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(batch.Words), (nint)batch.Start)); return Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref first, ((nint)entry * shape.Words * sizeof(ulong)) + shape.KeyOffset)); } + + /// The hash entry of a batch carries (), as took it. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint HashAt(PartBatch batch, EntryShape shape, int entry) + { + if ((uint)entry >= (uint)batch.Count || shape.HashOffset < 0) + { + throw new ArgumentOutOfRangeException(nameof(entry)); + } + + ref byte first = ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(batch.Words), (nint)batch.Start)); + return Unsafe.ReadUnaligned(ref Unsafe.AddByteOffset(ref first, ((nint)entry * shape.Words * sizeof(ulong)) + shape.HashOffset)); + } + + /// + /// A key's hash in a sub-table under no seed, its words folded () then their + /// halves: what takes for any key but a short text's. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint TableHash(ulong low, ulong high) + { + ulong folded = low ^ high; + return (uint)(folded ^ (folded >> 32)); + } } diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index 9bccbdeb8..e61c9321d 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -811,8 +811,8 @@ private static bool CompactAlways /// The first pass of the index the groups are in (). [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int FindAtHome(ReadOnlySpan keys, Span groups, Span homes, int ahead, out bool missed) => - Compact ? _wide.FindAtHome(keys, groups, homes, ahead, _keys.AsSpan(0, Count), out missed) : _index.FindAtHome(keys, groups, homes, ahead, out missed); + private int FindAtHome(ReadOnlySpan keys, Span groups, Span homes, int ahead, out bool missed, bool given = false) => + Compact ? _wide.FindAtHome(keys, groups, homes, ahead, given, _keys.AsSpan(0, Count), out missed) : _index.FindAtHome(keys, groups, homes, ahead, out missed); /// The group of in the index the groups are in, when it is new. [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -2152,15 +2152,22 @@ internal override void GroupsOf(PartBatch batch, EntryShape shape, ReadOnlySpan< int keyWords = (Unsafe.SizeOf() + sizeof(ulong) - 1) / sizeof(ulong); Span keys = MemoryMarshal.Cast(scratch[..(count * keyWords)])[..count]; Span homes = MemoryMarshal.Cast(scratch.Slice(count * keyWords, count))[..count]; + bool hashed = Compact; + ulong seed = hashed ? _wide.Seed : 0; + + // The hashes the entries carry are a sub-table's under no seed (EntryKeys.TableHash). + bool given = hashed && seed == 0 && shape.HashOffset >= 0; for (int i = 0; i < count; i++) { keys[i] = EntryKeys.KeyAt(batch, shape, entries[i]); + if (given) + { + homes[i] = EntryKeys.HashAt(batch, shape, entries[i]); + } } groups = groups[..count]; - bool hashed = Compact; - ulong seed = hashed ? _wide.Seed : 0; - _sink ^= FindAtHome(keys, groups, homes, 0, out bool missed); + _sink ^= FindAtHome(keys, groups, homes, 0, out bool missed, given); if (!missed) { return; diff --git a/src/Vorticity/Aggregation/WideKeyTable.cs b/src/Vorticity/Aggregation/WideKeyTable.cs index f64744331..08c081685 100644 --- a/src/Vorticity/Aggregation/WideKeyTable.cs +++ b/src/Vorticity/Aggregation/WideKeyTable.cs @@ -168,8 +168,8 @@ internal bool Squeeze(int more) /// /// of a key whose - /// took under the table's : a wide key hashed once where a - /// new one was hashed three times, its home looked at, then its slot, then its line (q10's tuples). + /// the home pass took under the table's : a wide key hashed once where a new one was + /// hashed three times, its home looked at, then its slot, then its line (q10's tuples). /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal int GetOrAdd(TValue key, uint hash, int next, ReadOnlySpan keys) @@ -208,14 +208,22 @@ internal int GetOrAdd(TValue key, uint hash, int next, ReadOnlySpan keys /// How many rows on a slot is read before its row compares; 0 for none. /// The keys of the groups. /// Whether a key found no group: the rows left to the lookup, none when false. - internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Span hashes, int ahead, ReadOnlySpan held, out bool missed) + internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Span hashes, int ahead, ReadOnlySpan held, out bool missed) => + FindAtHome(keys, groups, hashes, ahead, given: false, held, out missed); + + /// + /// , + /// the keys' hashes under the table's seed already in when + /// : the hashes a core's entries carry, read where the words were folded again. + /// + internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Span hashes, int ahead, bool given, ReadOnlySpan held, out bool missed) { Span slots = Slots; if (slots.Length == 0 || held.IsEmpty) { groups[..keys.Length].Fill(-1); missed = keys.Length > 0; - if (hashes.Length >= keys.Length) + if (!given && hashes.Length >= keys.Length) { for (int i = 0; i < keys.Length; i++) { @@ -244,10 +252,18 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp bool keep = hashes.Length >= keys.Length; for (int i = 0; i < keys.Length; i++) { - uint hash = HashOf(keys[i], seed, prepared); - if (keep) + uint hash; + if (given) + { + hash = hashes[i]; + } + else { - hashes[i] = hash; + hash = HashOf(keys[i], seed, prepared); + if (keep) + { + hashes[i] = hash; + } } ref Slot home = ref Unsafe.Add(ref first, (nint)KeyTable.SlotOf(hash, length, multiplier)); @@ -268,9 +284,12 @@ internal readonly int FindAtHome(ReadOnlySpan keys, Span groups, Sp } hashes = hashes[..keys.Length]; - for (int i = 0; i < keys.Length; i++) + if (!given) { - hashes[i] = HashOf(keys[i], seed, prepared); + for (int i = 0; i < keys.Length; i++) + { + hashes[i] = HashOf(keys[i], seed, prepared); + } } int sink = 0; From 3ea30a52a71c99e1d423e4d190fb2056d8e715fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 05:14:41 +0200 Subject: [PATCH 51/56] perf(query): a group by writes its keys into the result at once A key's values went into the result's column one call at a time, each checking the column's room and moving its count: nine cycles a text and six an integer of q10's ten million tuples, in a sixth of its cycles. A part's keys are now written in place, the column's room made once: up to the first null or long text, the rest one at a time as before. Texts as views through the new VarBinStore.InlineViews and AdvanceViews, integers through FixedStore.GetSpan and Advance. Alternated runners against 9b4d1cb5, warm: q10 on 10M x0.956 at fourteen lanes (91.2 to 88.0 ms) and x0.948 at one (609 to 577 ms); the keys of one column flat at fourteen (q3 x0.991, 10^7 keys x0.990). --- src/Vorticity/Aggregation/GroupKeys.cs | 18 ++++++- src/Vorticity/Aggregation/ShortTextKeys.cs | 21 +++++++- src/Vorticity/Aggregation/TupleKeys.cs | 50 ++++++++++++++++++-- src/Vorticity/Writing/Builders/LeafStores.cs | 27 +++++++++++ 4 files changed, 111 insertions(+), 5 deletions(-) diff --git a/src/Vorticity/Aggregation/GroupKeys.cs b/src/Vorticity/Aggregation/GroupKeys.cs index e61c9321d..5c8bd2747 100644 --- a/src/Vorticity/Aggregation/GroupKeys.cs +++ b/src/Vorticity/Aggregation/GroupKeys.cs @@ -2300,7 +2300,23 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpan()) { - foreach (int group in groups) + // Into the values at once up to the null group, then one at a time from it: a call a value + // took a group by's output six cycles a key (q10's tuples, 2026-10-10). + Span into = leaf.GetSpan(groups.Length); + int done = 0; + for (; done < groups.Length; done++) + { + int group = groups[done]; + if (group == nullGroup) + { + break; + } + + into[done] = keys[group]; + } + + leaf.Advance(done); + foreach (int group in groups[done..]) { if (group == nullGroup) { diff --git a/src/Vorticity/Aggregation/ShortTextKeys.cs b/src/Vorticity/Aggregation/ShortTextKeys.cs index 5881c2f10..216e1f2ae 100644 --- a/src/Vorticity/Aggregation/ShortTextKeys.cs +++ b/src/Vorticity/Aggregation/ShortTextKeys.cs @@ -697,7 +697,26 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpan views = leaf.InlineViews(groups.Length); + ref byte view = ref MemoryMarshal.GetReference(views); + int done = 0; + for (; done < groups.Length; done++) + { + int group = groups[done]; + if (group == nullGroup) + { + break; + } + + ref readonly TextWord word = ref words.KeyRef(group); + ref byte at = ref Unsafe.Add(ref view, (nint)done * (2 * sizeof(ulong))); + Unsafe.WriteUnaligned(ref at, word.Low); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref at, sizeof(ulong)), word.High); + } + + leaf.AdvanceViews(done); + foreach (int group in groups[done..]) { if (group == nullGroup) { diff --git a/src/Vorticity/Aggregation/TupleKeys.cs b/src/Vorticity/Aggregation/TupleKeys.cs index 8050a36ef..c1b28350f 100644 --- a/src/Vorticity/Aggregation/TupleKeys.cs +++ b/src/Vorticity/Aggregation/TupleKeys.cs @@ -609,9 +609,33 @@ internal override void Append(int component, ColumnStore store, ReadOnlySpan views = leaf.InlineViews(groups.Length); + ref byte view = ref MemoryMarshal.GetReference(views); + for (; done < groups.Length; done++) + { + ref byte tuple = ref TupleOf(groups[done]); + if (IsNull(ref tuple, component)) + { + break; + } + + TextWord word = Read(ref tuple, offset); + if (TupleLayout.IsLong(word)) + { + break; + } + + ref byte at = ref Unsafe.Add(ref view, (nint)done * (2 * sizeof(ulong))); + Unsafe.WriteUnaligned(ref at, word.Low); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref at, sizeof(ulong)), word.High); + } + + leaf.AdvanceViews(done); + foreach (int group in groups[done..]) { ref byte tuple = ref TupleOf(group); if (IsNull(ref tuple, component)) @@ -668,7 +692,27 @@ private void AppendIntegers(int component, int offset, ColumnStore store where TValue : unmanaged { FixedStore? leaf = store.Leaf is FixedStore fixedStore && fixedStore.Width == Unsafe.SizeOf() ? fixedStore : null; - foreach (int group in groups) + + // Into the values at once up to the first null, then one at a time from it. + int done = 0; + if (leaf is not null) + { + Span into = leaf.GetSpan(groups.Length); + for (; done < groups.Length; done++) + { + ref byte tuple = ref TupleOf(groups[done]); + if (IsNull(ref tuple, component)) + { + break; + } + + into[done] = Read(ref tuple, offset); + } + + leaf.Advance(done); + } + + foreach (int group in groups[done..]) { ref byte tuple = ref TupleOf(group); if (IsNull(ref tuple, component)) diff --git a/src/Vorticity/Writing/Builders/LeafStores.cs b/src/Vorticity/Writing/Builders/LeafStores.cs index 76ae8f460..7b116fcfc 100644 --- a/src/Vorticity/Writing/Builders/LeafStores.cs +++ b/src/Vorticity/Writing/Builders/LeafStores.cs @@ -420,6 +420,33 @@ internal void Append(ReadOnlySpan value) /// which its decoder validated as it read it, kept as a group's key or extreme. Checking them again /// took a tenth of a group by of a million texts. /// + /// + /// Room for views past the last, each to be written in place as + /// writes one, then held by : a part's key texts + /// written at once, where a call a value took nine cycles of a group by's output (q10). + /// + internal Span InlineViews(int count) + { + if (_capacity - Count < count) + { + Grow(Pool, ref _viewsOwner, ref _views, ref _viewBytes, (long)Count * ViewSize, ((long)Count + count) * ViewSize); + _capacity = _viewBytes / ViewSize; + } + + return new Span(_views + ((long)Count * ViewSize), count * ViewSize); + } + + /// The first views of the last written, now the store's. + internal void AdvanceViews(int count) + { + if ((uint)count > (uint)(_capacity - Count)) + { + throw new ArgumentOutOfRangeException(nameof(count), count, $"The last views hold {_capacity - Count} values."); + } + + Count += count; + } + internal void AppendValidated(ReadOnlySpan value) { EnsureViews(); From 426873139cb2a8354c27d3ab6c713e39125e17d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 05:36:40 +0200 Subject: [PATCH 52/56] perf(query): a short text key reads the views this library cut as its words A key of short texts copied each view of a block into a word before looking it up, though the views the varbin decoder cuts already are those words: every inline value zero past its bytes, as ViewKernels' cuts and CanonicalSupport.WriteView write it. A canonical record now says whether this library wrote its views (PaddedViews, set by the varbin decoder for every row cut, false for views read as they came), and a block without nulls whose values are all twelve bytes or fewer hands its views to the key as words where they lie, one pass over the lengths judging them. Alternated runners against d72a693f, warm: q2 at one lane x0.927 (84 to 78 ms) and at fourteen cycles x0.928 (its median noisy), q1 x0.952 at fourteen and x0.941 at one, q3 x0.975 at fourteen. --- src/Vorticity/Aggregation/ColumnAccess.cs | 40 ++++++++++++++++++- src/Vorticity/Aggregation/ShortTextKeys.cs | 8 ++++ src/Vorticity/Arrays/CanonicalArena.cs | 24 ++++++++++- .../Decoders/Canonical/VarBinDecoder.cs | 7 +++- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/Vorticity/Aggregation/ColumnAccess.cs b/src/Vorticity/Aggregation/ColumnAccess.cs index c6788b7ff..48756e8ee 100644 --- a/src/Vorticity/Aggregation/ColumnAccess.cs +++ b/src/Vorticity/Aggregation/ColumnAccess.cs @@ -357,25 +357,61 @@ internal readonly ref struct BytesBlock // The first data buffer, resolved once: the one a decoded block's values lie in, as a rule. private readonly ReadOnlySpan _first; - private BytesBlock(CanonicalArena arena, ReadOnlySpan views, int dataStart, int dataCount, int length) + private BytesBlock(CanonicalArena arena, ReadOnlySpan views, int dataStart, int dataCount, int length, bool padded = false) { _arena = arena; _views = views; _dataStart = dataStart; _dataCount = dataCount; _first = dataCount > 0 ? arena.DataBufferAt(dataStart).Span : default; + _padded = padded; Length = length; } internal int Length { get; } + // Whether the views are this library's, zero past each inline value (CanonicalRecord.PaddedViews). + private readonly bool _padded; + /// The block of in canonical form, decoding it when it is encoded, and its validity. internal static BytesBlock Canonical(CanonicalArena arena, int node, out ReadOnlySpan validity) { int views = EncodedForms.Canonical(arena, node); validity = ArenaWords.Validity(arena, views); ref readonly CanonicalRecord record = ref arena.RecordRef(views); - return new BytesBlock(arena, record.BufferA.Span, record.DataBufferStart, record.DataBufferCount, record.Length); + return new BytesBlock(arena, record.BufferA.Span, record.DataBufferStart, record.DataBufferCount, record.Length, record.PaddedViews); + } + + /// + /// The words of rows from where their views lie, as + /// would copy them: when this library's kernels wrote the views, each inline value + /// zero past its bytes, and every value is twelve bytes or fewer, which a pass over the lengths alone + /// tells. Copying views that already are words took a tenth of the cycles of a sum by two columns of a + /// hundred short texts (q2 at fourteen lanes, 2026-10-10). + /// + internal bool TryWordsInPlace(int start, int count, out ReadOnlySpan words) + { + words = default; + if (!_padded) + { + return false; + } + + ReadOnlySpan views = _views.Slice(start * ViewSize, count * ViewSize); + ref byte view = ref MemoryMarshal.GetReference(views); + uint longest = 0; + for (int i = 0; i < count; i++) + { + longest = Math.Max(longest, Unsafe.ReadUnaligned(ref Unsafe.Add(ref view, i * ViewSize))); + } + + if (longest > MaxInline) + { + return false; + } + + words = MemoryMarshal.Cast(views); + return true; } /// A canonical block over its views and data buffers, resolved once by a caller that compares many of its values. diff --git a/src/Vorticity/Aggregation/ShortTextKeys.cs b/src/Vorticity/Aggregation/ShortTextKeys.cs index 216e1f2ae..78e0d3ffd 100644 --- a/src/Vorticity/Aggregation/ShortTextKeys.cs +++ b/src/Vorticity/Aggregation/ShortTextKeys.cs @@ -222,6 +222,14 @@ internal override bool Assign(CanonicalArena arena, ReadOnlySpan nodes, int for (int start = 0; start < rows; start += Chunk) { int end = Math.Min(rows, start + Chunk); + + // Views this library cut are the words already: read where they lie. + if (validity.IsEmpty && canonical.TryWordsInPlace(start, end - start, out ReadOnlySpan inPlace)) + { + words.TwoPasses(inPlace, validity, start, rowGroups); + continue; + } + if (!FillWords(canonical, validity, start, end)) { Demote(); diff --git a/src/Vorticity/Arrays/CanonicalArena.cs b/src/Vorticity/Arrays/CanonicalArena.cs index 1a26c9491..df05ea120 100644 --- a/src/Vorticity/Arrays/CanonicalArena.cs +++ b/src/Vorticity/Arrays/CanonicalArena.cs @@ -1172,11 +1172,26 @@ public int AddVarBinView( int length, Validity validity, VortexBuffer views, - ReadOnlySpan dataBuffers) + ReadOnlySpan dataBuffers) => + AddVarBinView(dtype, length, validity, views, dataBuffers, padded: false); + + /// + /// , saying + /// whether this library's kernels wrote the views, every inline value zero past its bytes + /// (). + /// + internal int AddVarBinView( + DType dtype, + int length, + Validity validity, + VortexBuffer views, + ReadOnlySpan dataBuffers, + bool padded) { RequireExactLength(views.Length, length, 16, "VarBinView"); CanonicalRecord r = New(CanonicalKind.VarBinView, dtype, length, validity); r.BufferA = views; + r.PaddedViews = padded; r.DataBufferStart = _dataBufferCount; r.DataBufferCount = dataBuffers.Length; for (int i = 0; i < dataBuffers.Length; i++) @@ -2147,6 +2162,13 @@ internal struct CanonicalRecord internal int DataBufferCount; internal uint FixedSize; + /// + /// For a : whether this library's kernels wrote its views, every + /// inline value zero past its bytes, so that a view of twelve bytes or fewer is a short text key's word + /// as it lies; false for views read as they came, whose padding nothing promises. + /// + internal bool PaddedViews; + /// /// For a , or /// node: the materialized twin, once someone has asked for a diff --git a/src/Vorticity/Arrays/Decoders/Canonical/VarBinDecoder.cs b/src/Vorticity/Arrays/Decoders/Canonical/VarBinDecoder.cs index 63980532d..91dfa61d8 100644 --- a/src/Vorticity/Arrays/Decoders/Canonical/VarBinDecoder.cs +++ b/src/Vorticity/Arrays/Decoders/Canonical/VarBinDecoder.cs @@ -150,16 +150,19 @@ private static int Core( context.MarkNodeChecked(in node); } + // The cut of every row writes each inline value zero past its bytes (ViewKernels.Place, the + // vector cuts by their masks), as the emptied views of the null rows are. + bool padded = !selective; if (bytes.Length == 0) { // Every value is empty, so no view can reference a buffer; matching upstream, which // returns `Vec::new()` for an empty heap. - return context.Canonical.AddVarBinView(dtype, produced, validity, views, default); + return context.Canonical.AddVarBinView(dtype, produced, validity, views, default, padded); } Span single = stackalloc VortexBuffer[1]; single[0] = bytes; - return context.Canonical.AddVarBinView(dtype, produced, validity, views, single); + return context.Canonical.AddVarBinView(dtype, produced, validity, views, single, padded); } /// From 00ca71febe3bc3126240bbde910ebca707290326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 05:41:05 +0200 Subject: [PATCH 53/56] perf(query): a tuple takes its short texts from the views where they lie The tuples copied a block's text views into words, then the words into the tuples. Views this library cut already are the words: they now go into the tuples from where they lie, the copy kept for any other views. Alternated runners against ddc0a4d5, warm: q10 on 10M x0.983 at one lane (575 to 565 ms), flat at fourteen. --- src/Vorticity/Aggregation/TupleKeys.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Vorticity/Aggregation/TupleKeys.cs b/src/Vorticity/Aggregation/TupleKeys.cs index c1b28350f..5f9c72f30 100644 --- a/src/Vorticity/Aggregation/TupleKeys.cs +++ b/src/Vorticity/Aggregation/TupleKeys.cs @@ -367,6 +367,18 @@ private void Texts(CanonicalArena arena, int node, int start, int end, ref byte { BytesBlock values = BytesBlock.Canonical(arena, node, out ReadOnlySpan validity); int rows = end - start; + + // Views this library cut are the words already: into the tuples from where they lie. + if (validity.IsEmpty && values.TryWordsInPlace(start, rows, out ReadOnlySpan inPlace)) + { + for (int i = 0; i < rows; i++) + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref first, ((nint)i * TupleLayout.Bytes) + offset), inPlace[i]); + } + + return; + } + Scratch.Grow(ref _words, rows); Span words = _words.AsSpan(0, rows); if (validity.IsEmpty && values.TryWords(start, words)) From 9df22c4f9df2896468d2c91a90e1a5abf092bb06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 06:09:14 +0200 Subject: [PATCH 54/56] docs: the comparison with DuckDB taken again after the profiling pass Two sessions at 7734c472, one after the other, and a third for the one row they did not agree on. Every row still ahead, the thinnest leads wider: db-benchmark's q10 at fourteen threads 1.05x to 1.18x over DuckDB's own table and 1.26x to 1.39x over its reader, at one thread 1.39x to 1.43x; q3 at fourteen 1.29x to 1.35x, q2 1.35x to 1.44x. --- docs/guide/benchmarks-duckdb.md | 139 ++++++++++++++++---------------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/docs/guide/benchmarks-duckdb.md b/docs/guide/benchmarks-duckdb.md index a81f1b57c..8c29f3cd5 100644 --- a/docs/guide/benchmarks-duckdb.md +++ b/docs/guide/benchmarks-duckdb.md @@ -53,37 +53,37 @@ Warm: our file kept mapped, DuckDB's own table. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 31.1 | 119 | 3.81x | 3.60–3.83 | 6 | -| count and sum by 10⁶ keys | 40M | 114.0 | 717 | 6.28x | 5.95–6.53 | 9 | -| count and sum by 10⁷ keys | 40M | 554.3 | 1 328 | 2.40x | 2.39–2.41 | 6 | -| count, least and largest by 10⁶ keys | 40M | 274.8 | 866 | 3.15x | 3.10–3.18 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 254.7 | 741 | 2.90x | 2.75–3.04 | 9 | -| count and sum by 10⁷ hashed keys | 40M | 679.5 | 1 332 | 1.96x | 1.95–1.98 | 6 | -| db q1: sum v1 by id1 | 10M | 39.3 | 73.5 | 1.87x | 1.85–1.87 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 83.3 | 138 | 1.66x | 1.64–1.66 | 6 | -| db q3: sum v1, mean v3 by id3 | 10M | 102.5 | 177 | 1.73x | 1.59–1.73 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 23.3 | 43.5 | 1.86x | 1.80–1.89 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 47.4 | 118 | 2.48x | 2.45–2.51 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 86.0 | 176 | 2.04x | 2.02–2.06 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 586.4 | 814 | 1.39x | 1.38–1.40 | 6 | +| count and sum by 10³ keys | 40M | 31.2 | 115 | 3.69x | 3.56–3.76 | 6 | +| count and sum by 10⁶ keys | 40M | 117.0 | 723 | 6.15x | 6.02–6.36 | 6 | +| count and sum by 10⁷ keys | 40M | 553.2 | 1 333 | 2.41x | 2.39–2.42 | 6 | +| count, least and largest by 10⁶ keys | 40M | 277.1 | 866 | 3.12x | 3.09–3.24 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 292.2 | 737 | 2.52x | 2.45–2.66 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 598.8 | 1 334 | 2.23x | 2.17–2.25 | 6 | +| db q1: sum v1 by id1 | 10M | 35.5 | 72.5 | 2.05x | 2.02–2.08 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 76.1 | 138 | 1.81x | 1.78–1.83 | 6 | +| db q3: sum v1, mean v3 by id3 | 10M | 102.6 | 177 | 1.72x | 1.69–1.75 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 23.3 | 43.0 | 1.85x | 1.79–1.90 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 47.2 | 117 | 2.48x | 2.47–2.49 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 86.2 | 175 | 2.02x | 2.02–2.06 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 569.0 | 813 | 1.43x | 1.42–1.45 | 6 | Cold: our file mapped anew at every run, DuckDB's Vortex reader. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 50.6 | 246 | 4.84x | 4.72–4.96 | 6 | -| count and sum by 10⁶ keys | 40M | 148.0 | 812 | 5.54x | 5.12–5.71 | 10 | -| count and sum by 10⁷ keys | 40M | 580.1 | 1 422 | 2.44x | 2.42–2.45 | 6 | -| count, least and largest by 10⁶ keys | 40M | 304.1 | 943 | 3.10x | 3.01–3.15 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 294.6 | 889 | 3.00x | 2.93–3.06 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 718.2 | 1 451 | 2.02x | 1.99–2.03 | 6 | -| db q1: sum v1 by id1 | 10M | 41.3 | 136 | 3.30x | 3.24–3.37 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 86.2 | 262 | 3.05x | 2.80–3.08 | 6 | -| db q3: sum v1, mean v3 by id3 | 10M | 108.6 | 195 | 1.79x | 1.75–1.84 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 31.5 | 65.0 | 2.08x | 2.02–2.11 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 54.8 | 124 | 2.26x | 2.19–2.31 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 90.3 | 199 | 2.20x | 2.09–2.33 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 596.2 | 917 | 1.53x | 1.52–1.58 | 6 | +| count and sum by 10³ keys | 40M | 49.7 | 241 | 4.84x | 4.79–4.91 | 6 | +| count and sum by 10⁶ keys | 40M | 145.3 | 819 | 5.65x | 5.29–5.86 | 9 | +| count and sum by 10⁷ keys | 40M | 575.6 | 1 428 | 2.48x | 2.47–2.51 | 6 | +| count, least and largest by 10⁶ keys | 40M | 303.6 | 943 | 3.10x | 3.07–3.16 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 321.5 | 887 | 2.76x | 2.63–2.81 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 634.0 | 1 461 | 2.30x | 2.28–2.32 | 6 | +| db q1: sum v1 by id1 | 10M | 37.5 | 135 | 3.60x | 3.56–3.64 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 78.1 | 261 | 3.34x | 3.30–3.36 | 6 | +| db q3: sum v1, mean v3 by id3 | 10M | 108.9 | 197 | 1.77x | 1.71–1.87 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 30.6 | 64.5 | 2.12x | 2.09–2.14 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 53.8 | 124 | 2.30x | 2.18–2.35 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 89.8 | 193 | 2.14x | 2.06–2.16 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 580.4 | 915 | 1.58x | 1.57–1.60 | 6 | ## Fourteen threads @@ -91,59 +91,62 @@ Warm: our file kept mapped, DuckDB's own table. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 3.6 | 13.2 | 3.66x | 3.47–3.77 | 9 | -| count and sum by 10⁶ keys | 40M | 61.9 | 146 | 2.36x | 2.10–2.48 | 11 | -| count and sum by 10⁷ keys | 40M | 65.3 | 179 | 2.73x | 2.65–2.88 | 6 | -| count, least and largest by 10⁶ keys | 40M | 84.8 | 171 | 2.04x | 1.95–2.16 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 76.6 | 144 | 1.88x | 1.84–1.91 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 102.3 | 181 | 1.76x | 1.74–1.80 | 6 | -| db q1: sum v1 by id1 | 10M | 5.2 | 11.1 | 2.09x | 2.03–2.16 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 12.4 | 16.6 | 1.35x | 1.26–1.37 | 9 | -| db q3: sum v1, mean v3 by id3 | 10M | 35.4 | 45.5 | 1.29x | 1.24–1.30 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 3.0 | 5.2 | 1.72x | 1.67–1.80 | 10 | -| db q5: sum v1:v3 by id6 | 10M | 17.2 | 44.5 | 2.60x | 2.50–2.65 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 22.9 | 48.0 | 2.11x | 1.95–2.13 | 6 | -| db q10: sum v3, count by id1:id6 | 10M | 97.4 | 103 | 1.05x | 1.02–1.07 | 6 | +| count and sum by 10³ keys | 40M | 3.7 | 12.8 | 3.50x | 3.26–3.60 | 7 | +| count and sum by 10⁶ keys | 40M | 58.9 | 145 | 2.46x | 2.35–2.58 | 9 | +| count and sum by 10⁷ keys | 40M | 65.9 | 176 | 2.66x | 2.50–2.90 | 9 | +| count, least and largest by 10⁶ keys | 40M | 84.3 | 173 | 2.06x | 1.88–2.09 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 77.6 | 146 | 1.87x | 1.82–1.92 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 104.0 | 180 | 1.75x | 1.66–1.78 | 6 | +| db q1: sum v1 by id1 | 10M | 4.9 | 11.3 | 2.31x | 2.22–2.43 | 6 | +| db q2: sum v1 by id1, id2 | 10M | 11.6 | 16.6 | 1.44x | 1.31–1.51 | 15 | +| db q3: sum v1, mean v3 by id3 | 10M | 33.6 | 46.0 | 1.35x | 1.32–1.39 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 3.0 | 5.1 | 1.66x | 1.59–1.75 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 17.2 | 44.0 | 2.56x | 2.40–2.60 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 21.7 | 48.5 | 2.21x | 2.13–2.24 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 87.2 | 103 | 1.18x | 1.15–1.20 | 6 | Cold: our file mapped anew at every run, DuckDB's Vortex reader. | query | rows | ours (ms) | DuckDB (ms) | speedup | 95 % | pairs | |---|---:|---:|---:|---:|---:|---:| -| count and sum by 10³ keys | 40M | 12.0 | 51.5 | 4.29x | 3.39–4.75 | 16 | -| count and sum by 10⁶ keys | 40M | 66.4 | 187 | 2.81x | 2.75–2.89 | 6 | -| count and sum by 10⁷ keys | 40M | 71.4 | 231 | 3.19x | 2.99–3.33 | 10 | -| count, least and largest by 10⁶ keys | 40M | 90.6 | 201 | 2.24x | 2.11–2.31 | 6 | -| count and sum by 10⁶ hashed keys | 40M | 84.2 | 189 | 2.23x | 2.09–2.26 | 6 | -| count and sum by 10⁷ hashed keys | 40M | 111.3 | 229 | 2.04x | 1.94–2.14 | 10 | -| db q1: sum v1 by id1 | 10M | 6.7 | 21.0 | 3.15x | 3.04–3.31 | 6 | -| db q2: sum v1 by id1, id2 | 10M | 14.4 | 34.5 | 2.39x | 2.30–2.48 | 6 | -| db q3: sum v1, mean v3 by id3 | 10M | 38.1 | 58.5 | 1.55x | 1.48–1.58 | 6 | -| db q4: mean v1:v3 by id4 | 10M | 6.1 | 17.6 | 2.90x | 2.82–2.97 | 6 | -| db q5: sum v1:v3 by id6 | 10M | 19.5 | 55.0 | 2.82x | 2.70–2.86 | 6 | -| db q7: max v1 − min v2 by id3 | 10M | 26.0 | 60.2 | 2.28x | 2.00–2.42 | 14 | -| db q10: sum v3, count by id1:id6 | 10M | 101.8 | 128 | 1.26x | 1.21–1.29 | 6 | +| count and sum by 10³ keys | 40M | 11.9 | 53.0 | 4.47x | 4.26–5.41 | 11 | +| count and sum by 10⁶ keys | 40M | 64.9 | 184 | 2.79x | 2.68–2.92 | 6 | +| count and sum by 10⁷ keys | 40M | 71.4 | 227 | 3.16x | 2.82–3.35 | 16 | +| count, least and largest by 10⁶ keys | 40M | 89.8 | 202 | 2.24x | 2.18–2.34 | 6 | +| count and sum by 10⁶ hashed keys | 40M | 83.8 | 184 | 2.19x | 2.14–2.26 | 6 | +| count and sum by 10⁷ hashed keys | 40M | 110.8 | 226 | 2.03x | 1.95–2.18 | 15 | +| db q1: sum v1 by id1 | 10M | 6.1 | 21.0 | 3.44x | 3.26–3.49 | 9 | +| db q2: sum v1 by id1, id2 | 10M | 13.8 | 34.0 | 2.46x | 2.27–2.53 | 6 | +| db q3: sum v1, mean v3 by id3 | 10M | 36.7 | 58.0 | 1.60x | 1.56–1.62 | 6 | +| db q4: mean v1:v3 by id4 | 10M | 5.9 | 17.7 | 3.00x | 2.93–3.06 | 6 | +| db q5: sum v1:v3 by id6 | 10M | 19.4 | 56.0 | 2.89x | 2.80–2.97 | 6 | +| db q7: max v1 − min v2 by id3 | 10M | 24.4 | 60.0 | 2.47x | 2.36–2.52 | 6 | +| db q10: sum v3, count by id1:id6 | 10M | 91.5 | 129 | 1.39x | 1.31–1.42 | 6 | ## Where Vorticity leads, and where it trails -* **On one thread it leads every query on both bases**: 1.39x to 6.28x over DuckDB's own table, 1.53x - to 5.54x over its reader. db-benchmark's q10, six keys of which three are texts and a group for - nearly every one of 10⁷ rows, leads by the least: 1.39x and 1.53x. -* **On fourteen threads it leads every query on both bases**: 1.05x to 3.66x over DuckDB's own table - and 1.26x to 4.29x over its reader. Integer keys of 10³ to 10⁷ values lead by 2.04x to 3.66x warm, - hashed ones by 1.76x and 1.88x, and db-benchmark's q1 to q7 by 1.29x to 2.60x warm, 1.55x to 3.15x +* **On one thread it leads every query on both bases**: 1.43x to 6.15x over DuckDB's own table, 1.58x + to 5.65x over its reader. db-benchmark's q10, six keys of which three are texts and a group for + nearly every one of 10⁷ rows, leads by the least: 1.43x and 1.58x. +* **On fourteen threads it leads every query on both bases**: 1.18x to 3.50x over DuckDB's own table + and 1.39x to 4.47x over its reader. Integer keys of 10³ to 10⁷ values lead by 2.06x to 3.50x warm, + hashed ones by 1.75x and 1.87x, and db-benchmark's q1 to q7 by 1.35x to 2.56x warm, 1.60x to 3.44x cold. -* **db-benchmark's q10 leads by the least on fourteen threads**: 1.05x over DuckDB's own table, its - interval 1.02x to 1.07x, and 1.26x over its reader. Its six keys are held as one tuple of their - values, which the lanes hand to a shared table of groups by parts; DuckDB writes rows nearly all - unique straight to its partitions. It trailed at 0.51x on 2026-10-09, as 10⁷ hashed keys did at - 0.96x (1.76x now): the memory of the groups is now kept from one query to the next rather than made - anew, and each part of the shared table is held whole rather than split as it grows. +* **db-benchmark's q10 leads by the least on fourteen threads**: 1.18x over DuckDB's own table, its + interval 1.15x to 1.20x, and 1.39x over its reader. Its six keys are held as one tuple of their + values, 64 bytes, which the lanes hand to a shared table of groups by parts; DuckDB packs the same + keys into 38 bytes before it groups them, and writes rows nearly all unique straight to its + partitions. It trailed at 0.51x on 2026-10-09, as 10⁷ hashed keys did at 0.96x (1.75x now): the + memory of the groups is now kept from one query to the next rather than made anew, each part of the + shared table is held whole rather than split as it grows, a key is hashed once on its way, and the + keys are written into the result a part at a time. Measured on 2026-10-10 on an Apple M4 Pro (14 cores: 10 performance, 4 efficiency) under macOS, at -commit `7445c2a5`: two sessions one after the other, from 03:45 to 03:56 and from 03:56 to 04:08, which -agreed within 10 % on every row. The machine was near rest: the canaries strayed up to 7.6 % and 8.0 % -from their first values, another process held a query back 24 s and 6 s in all, and two rows of the -second session ran again. +commit `7734c472`: two sessions one after the other, from 05:45 to 05:57 and from 05:57 to 06:07, and a +third for the one row they did not agree on (10³ keys on fourteen threads, cold, which took 3.85x and +4.64x, then 4.31x). The machine was near rest: the canaries strayed up to 9.8 % and 10.3 % from their +first values, another process held a query back 8 s and 6 s in all, and one row of the first session +ran again. Until 2026-10-09 this page timed every DuckDB query, then every one of ours: a perturbation of the machine weighed on one side unseen, and DuckDB's q10 on fourteen threads read 257 ms where it took 102 From a2986a10ca847dde5a305d4a5ae9351f3e1a0d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 06:24:46 +0200 Subject: [PATCH 55/56] docs: benchmarks.md measured again after the profiling pass Every section rewritten by its instrument at 6403ae1a. Every ratio-check axis inside its ceiling; Vorticity decodes 47 of 57 files faster than Rust (median 1.28x, the files near 1.00x moving from run to run) and takes rows faster on 56 of 57 (median 1.14x). --- docs/guide/benchmarks.md | 952 +++++++++++++++++++-------------------- 1 file changed, 476 insertions(+), 476 deletions(-) diff --git a/docs/guide/benchmarks.md b/docs/guide/benchmarks.md index 12b7105ed..ef4148fd3 100644 --- a/docs/guide/benchmarks.md +++ b/docs/guide/benchmarks.md @@ -88,41 +88,41 @@ AVX-512) under Windows 11, every section measured there, against a Rust built on | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.5 (0.5-0.5) | 3.88x | — | — | 5.4 KiB | 11 / 11 MiB | -| `open` | Vortex Rust's | 0.1 (0.1-0.1) | 0.5 (0.5-0.7) | 3.96x | — | — | 5.0 KiB | 11 / 11 MiB | -| `scan` | Vorticity's | 4.3 (4.2-4.4) | 8.6 (8.4-8.7) | 1.97x | 6.1 | 3.1 | 14.7 KiB | 17 / 18 MiB | -| `scan` | Vortex Rust's | 1.8 (1.8-1.8) | 2.6 (2.5-2.7) | 1.46x | 14.7 | 10.1 | 11.1 KiB | 25 / 25 MiB | -| `project` | Vorticity's | 0.4 (0.3-0.4) | 0.9 (0.8-1.0) | 2.41x | 23.4 | 9.7 | 17.0 KiB | 13 / 13 MiB | -| `project` | Vortex Rust's | 0.4 (0.4-0.4) | 0.9 (0.8-1.6) | 2.27x | 21.8 | 9.6 | 13.3 KiB | 13 / 14 MiB | -| `filter-narrow` | Vorticity's | 0.8 (0.8-0.8) | 1.1 (1.1-1.2) | 1.45x | 0.3 | 0.2 | 41.4 KiB | 16 / 15 MiB | -| `filter-narrow` | Vortex Rust's | 0.7 (0.6-0.8) | 1.1 (0.9-1.1) | 1.47x | 0.4 | 0.3 | 21.2 KiB | 18 / 15 MiB | -| `filter-wide` | Vorticity's | 2.5 (2.4-2.6) | 5.0 (4.9-5.2) | 2.04x | 5.4 | 2.7 | 23.9 KiB | 16 / 17 MiB | -| `filter-wide` | Vortex Rust's | 1.3 (1.2-1.3) | 2.1 (2.1-2.2) | 1.63x | 10.5 | 6.4 | 21.2 KiB | 22 / 22 MiB | -| `take` | Vorticity's | 3.8 (3.8-4.1) | 6.4 (6.3-6.5) | 1.68x | — | — | 38.4 KiB | 16 / 17 MiB | -| `take` | Vortex Rust's | 0.6 (0.6-0.7) | 1.3 (1.1-1.4) | 2.06x | — | — | 34.8 KiB | 20 / 21 MiB | -| `write` | Vorticity's | 23.9 (23.7-23.9) | 55.7 (55.4-58.1) | 2.33x | 1.1 | 0.5 | 30.9 KiB | 22 / 75 MiB | -| `write` | Vortex Rust's | 21.5 (21.1-22.2) | 49.7 (49.4-50.7) | 2.31x | 1.2 | 0.5 | 27.2 KiB | 31 / 81 MiB | -| `append` | Vorticity's | 6.4 (5.4-7.3) | not asked | n/a | — | — | 2.5 MiB | 21 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.5 (0.5-0.5) | 3.52x | — | — | 5.4 KiB | 11 / 11 MiB | +| `open` | Vortex Rust's | 0.1 (0.1-0.1) | 0.4 (0.4-0.5) | 3.56x | — | — | 5.0 KiB | 11 / 11 MiB | +| `scan` | Vorticity's | 4.3 (4.2-4.6) | 8.5 (8.5-8.7) | 1.99x | 6.2 | 3.1 | 14.7 KiB | 17 / 18 MiB | +| `scan` | Vortex Rust's | 1.8 (1.7-1.9) | 2.5 (2.5-2.7) | 1.39x | 14.9 | 10.7 | 11.1 KiB | 25 / 25 MiB | +| `project` | Vorticity's | 0.4 (0.3-0.5) | 0.8 (0.8-0.8) | 2.31x | 23.5 | 10.2 | 17.0 KiB | 13 / 13 MiB | +| `project` | Vortex Rust's | 0.4 (0.3-0.4) | 0.7 (0.7-0.8) | 2.04x | 23.4 | 11.5 | 13.3 KiB | 13 / 14 MiB | +| `filter-narrow` | Vorticity's | 0.8 (0.8-0.9) | 1.2 (1.1-1.2) | 1.45x | 0.3 | 0.2 | 41.4 KiB | 16 / 15 MiB | +| `filter-narrow` | Vortex Rust's | 0.7 (0.7-0.8) | 1.1 (1.0-1.2) | 1.60x | 0.4 | 0.2 | 21.2 KiB | 18 / 15 MiB | +| `filter-wide` | Vorticity's | 2.4 (2.4-2.4) | 5.0 (4.9-5.1) | 2.08x | 5.6 | 2.7 | 23.9 KiB | 16 / 17 MiB | +| `filter-wide` | Vortex Rust's | 1.3 (1.2-1.3) | 2.0 (1.9-2.0) | 1.59x | 10.6 | 6.7 | 21.2 KiB | 22 / 22 MiB | +| `take` | Vorticity's | 3.9 (3.8-4.0) | 6.4 (6.3-6.4) | 1.64x | — | — | 38.4 KiB | 16 / 17 MiB | +| `take` | Vortex Rust's | 0.6 (0.6-0.7) | 1.2 (1.1-1.3) | 2.01x | — | — | 34.8 KiB | 20 / 21 MiB | +| `write` | Vorticity's | 23.8 (23.7-24.4) | 56.3 (55.4-58.9) | 2.37x | 1.1 | 0.5 | 30.9 KiB | 22 / 76 MiB | +| `write` | Vortex Rust's | 21.2 (21.0-21.5) | 50.2 (50.1-51.7) | 2.37x | 1.3 | 0.5 | 27.2 KiB | 31 / 81 MiB | +| `append` | Vorticity's | 6.2 (6.1-6.3) | not asked | n/a | — | — | 1.3 MiB | 21 / — MiB | #### All 14 cores | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.7 (0.6-0.8) | 5.20x | — | — | 5.4 KiB | 11 / 12 MiB | -| `open` | Vortex Rust's | 0.1 (0.1-0.1) | 0.8 (0.7-0.8) | 5.29x | — | — | 5.0 KiB | 11 / 12 MiB | -| `scan` | Vorticity's | 1.6 (1.6-1.7) | 3.8 (3.6-3.9) | 2.30x | 16.2 | 7.0 | 36.1 KiB | 45 / 49 MiB | -| `scan` | Vortex Rust's | 1.6 (1.5-1.6) | 3.6 (3.4-3.8) | 2.28x | 17.1 | 7.5 | 24.5 KiB | 64 / 54 MiB | -| `project` | Vorticity's | 0.6 (0.6-0.6) | 1.4 (1.2-1.5) | 2.35x | 13.7 | 5.8 | 31.2 KiB | 20 / 22 MiB | -| `project` | Vortex Rust's | 0.6 (0.5-0.6) | 1.4 (1.1-1.6) | 2.53x | 14.8 | 5.8 | 21.6 KiB | 21 / 23 MiB | -| `filter-narrow` | Vorticity's | 0.5 (0.5-0.6) | 1.5 (1.3-1.7) | 2.97x | 0.5 | 0.2 | 42.7 KiB | 14 / 18 MiB | -| `filter-narrow` | Vortex Rust's | 0.5 (0.5-0.6) | 1.4 (1.3-1.5) | 2.95x | 0.6 | 0.2 | 42.0 KiB | 14 / 18 MiB | -| `filter-wide` | Vorticity's | 1.3 (1.2-1.3) | 2.8 (2.6-3.3) | 2.12x | 10.3 | 4.8 | 37.2 KiB | 31 / 34 MiB | -| `filter-wide` | Vortex Rust's | 1.1 (1.0-1.3) | 2.5 (2.3-2.7) | 2.31x | 12.3 | 5.3 | 30.2 KiB | 39 / 39 MiB | -| `take` | Vorticity's | 2.1 (1.9-2.3) | 2.2 (2.2-2.4) | 1.02x | — | — | 82.1 KiB | 26 / 28 MiB | -| `take` | Vortex Rust's | 0.8 (0.8-0.8) | 1.5 (1.5-1.7) | 1.88x | — | — | 76.4 KiB | 22 / 26 MiB | -| `write` | Vorticity's | 10.5 (10.5-10.7) | 14.4 (14.2-15.1) | 1.37x | 2.5 | 1.8 | 2.3 MiB | 57 / 105 MiB | -| `write` | Vortex Rust's | 10.2 (10.0-10.5) | 14.8 (14.3-15.0) | 1.44x | 2.6 | 1.8 | 1.3 MiB | 76 / 110 MiB | -| `append` | Vorticity's | 5.9 (5.5-6.9) | not asked | n/a | — | — | 1.3 MiB | 22 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.7 (0.7-0.8) | 5.75x | — | — | 5.4 KiB | 11 / 12 MiB | +| `open` | Vortex Rust's | 0.1 (0.1-0.2) | 0.7 (0.7-0.8) | 5.45x | — | — | 5.0 KiB | 11 / 12 MiB | +| `scan` | Vorticity's | 1.5 (1.5-1.6) | 3.7 (3.6-3.7) | 2.48x | 17.8 | 7.2 | 36.7 KiB | 45 / 49 MiB | +| `scan` | Vortex Rust's | 1.5 (1.5-1.6) | 3.7 (2.9-4.0) | 2.45x | 17.7 | 7.2 | 26.7 KiB | 64 / 54 MiB | +| `project` | Vorticity's | 0.6 (0.6-0.6) | 1.5 (1.2-1.5) | 2.51x | 14.3 | 5.7 | 31.6 KiB | 20 / 22 MiB | +| `project` | Vortex Rust's | 0.5 (0.5-0.7) | 1.5 (1.3-1.5) | 2.69x | 15.3 | 5.7 | 21.6 KiB | 21 / 23 MiB | +| `filter-narrow` | Vorticity's | 0.5 (0.5-0.5) | 1.6 (1.5-1.7) | 3.01x | 0.5 | 0.2 | 42.7 KiB | 14 / 18 MiB | +| `filter-narrow` | Vortex Rust's | 0.5 (0.5-0.6) | 1.5 (1.4-1.6) | 3.11x | 0.6 | 0.2 | 24.0 KiB | 14 / 18 MiB | +| `filter-wide` | Vorticity's | 1.1 (1.0-1.3) | 2.7 (2.4-3.2) | 2.37x | 11.7 | 4.9 | 37.9 KiB | 32 / 34 MiB | +| `filter-wide` | Vortex Rust's | 1.1 (1.0-1.1) | 2.6 (2.3-2.7) | 2.44x | 12.6 | 5.2 | 28.4 KiB | 39 / 38 MiB | +| `take` | Vorticity's | 2.2 (2.0-2.2) | 2.4 (2.2-2.5) | 1.11x | — | — | 98.5 KiB | 26 / 26 MiB | +| `take` | Vortex Rust's | 0.9 (0.8-0.9) | 1.5 (1.4-1.6) | 1.69x | — | — | 76.5 KiB | 22 / 26 MiB | +| `write` | Vorticity's | 10.6 (10.3-11.3) | 14.2 (14.0-14.3) | 1.35x | 2.5 | 1.9 | 2.5 MiB | 57 / 109 MiB | +| `write` | Vortex Rust's | 10.0 (10.0-10.3) | 14.6 (14.3-14.7) | 1.45x | 2.7 | 1.8 | 263.9 KiB | 76 / 111 MiB | +| `append` | Vorticity's | 5.7 (5.6-5.8) | not asked | n/a | — | — | 255.5 KiB | 22 / — MiB | ### 10,485,760 rows @@ -137,56 +137,56 @@ AVX-512) under Windows 11, every section measured there, against a Rust built on | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.5 (0.5-0.6) | 3.97x | — | — | 21.1 KiB | 11 / 11 MiB | -| `open` | Vortex Rust's | 0.2 (0.1-0.2) | 0.6 (0.5-0.8) | 3.50x | — | — | 9.5 KiB | 11 / 11 MiB | -| `scan` | Vorticity's | 39.6 (39.0-40.0) | 76.7 (76.4-77.9) | 1.93x | 6.7 | 3.5 | 74.4 KiB | 38 / 40 MiB | -| `scan` | Vortex Rust's | 13.2 (13.1-13.5) | 17.3 (17.0-17.9) | 1.31x | 20.2 | 15.4 | 28.5 KiB | 91 / 91 MiB | -| `project` | Vorticity's | 1.4 (1.3-1.4) | 2.0 (1.9-2.3) | 1.45x | 61.0 | 42.0 | 76.6 KiB | 15 / 16 MiB | -| `project` | Vortex Rust's | 1.1 (1.1-3.4) | 1.6 (1.6-2.4) | 1.44x | 73.4 | 50.8 | 30.6 KiB | 13 / 14 MiB | -| `filter-narrow` | Vorticity's | 1.2 (1.2-1.3) | 2.4 (2.4-2.9) | 1.94x | 2.2 | 1.1 | 107.1 KiB | 17 / 18 MiB | -| `filter-narrow` | Vortex Rust's | 1.3 (1.3-4.5) | 1.7 (1.7-1.9) | 1.34x | 2.1 | 1.5 | 62.2 KiB | 19 / 19 MiB | -| `filter-wide` | Vorticity's | 20.1 (20.0-21.5) | 40.7 (40.5-40.9) | 2.02x | 6.6 | 3.3 | 107.1 KiB | 27 / 29 MiB | -| `filter-wide` | Vortex Rust's | 10.3 (9.8-12.9) | 11.0 (10.6-12.8) | 1.06x | 12.9 | 12.2 | 62.2 KiB | 56 / 56 MiB | -| `take` | Vorticity's | 16.8 (16.8-17.2) | 55.1 (54.7-58.9) | 3.28x | — | — | 98.0 KiB | 36 / 40 MiB | -| `take` | Vortex Rust's | 1.5 (1.5-4.7) | 3.3 (3.2-3.5) | 2.12x | — | — | 52.1 KiB | 39 / 42 MiB | -| `write` | Vorticity's | 217.8 (216.1-224.7) | 530.6 (525.0-536.2) | 2.44x | 1.2 | 0.5 | 95.5 KiB | 44 / 260 MiB | -| `write` | Vortex Rust's | 191.5 (190.4-201.8) | 469.7 (464.6-480.9) | 2.45x | 1.4 | 0.6 | 49.7 KiB | 100 / 398 MiB | -| `append` | Vorticity's | 49.6 (49.3-53.9) | not asked | n/a | — | — | 2.4 MiB | 28 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.1) | 0.5 (0.5-0.6) | 3.90x | — | — | 21.1 KiB | 11 / 11 MiB | +| `open` | Vortex Rust's | 0.1 (0.1-0.1) | 0.5 (0.4-0.6) | 4.16x | — | — | 9.5 KiB | 11 / 11 MiB | +| `scan` | Vorticity's | 39.3 (39.1-39.5) | 76.3 (75.8-76.5) | 1.94x | 6.8 | 3.5 | 74.4 KiB | 38 / 40 MiB | +| `scan` | Vortex Rust's | 13.3 (13.2-16.5) | 17.0 (16.9-17.3) | 1.28x | 20.1 | 15.7 | 28.5 KiB | 91 / 91 MiB | +| `project` | Vorticity's | 1.3 (1.3-1.4) | 1.9 (1.9-2.2) | 1.46x | 63.1 | 43.2 | 76.6 KiB | 15 / 16 MiB | +| `project` | Vortex Rust's | 1.1 (1.1-1.2) | 1.7 (1.6-2.2) | 1.48x | 73.3 | 49.5 | 30.6 KiB | 13 / 14 MiB | +| `filter-narrow` | Vorticity's | 1.2 (1.2-1.3) | 2.3 (2.2-2.4) | 1.85x | 2.1 | 1.2 | 107.1 KiB | 17 / 18 MiB | +| `filter-narrow` | Vortex Rust's | 1.3 (1.2-1.3) | 1.6 (1.6-1.8) | 1.26x | 2.1 | 1.7 | 62.2 KiB | 19 / 20 MiB | +| `filter-wide` | Vorticity's | 20.3 (19.9-20.3) | 40.6 (40.2-40.9) | 2.00x | 6.6 | 3.3 | 107.1 KiB | 27 / 29 MiB | +| `filter-wide` | Vortex Rust's | 9.9 (9.8-10.1) | 10.7 (10.6-11.8) | 1.09x | 13.5 | 12.4 | 62.2 KiB | 56 / 56 MiB | +| `take` | Vorticity's | 16.8 (16.7-17.0) | 55.0 (54.5-55.1) | 3.27x | — | — | 98.0 KiB | 36 / 40 MiB | +| `take` | Vortex Rust's | 1.7 (1.5-3.7) | 3.3 (3.1-3.5) | 1.95x | — | — | 52.1 KiB | 39 / 42 MiB | +| `write` | Vorticity's | 217.6 (215.4-220.4) | 526.7 (525.0-533.7) | 2.42x | 1.2 | 0.5 | 95.5 KiB | 44 / 264 MiB | +| `write` | Vortex Rust's | 190.6 (189.8-191.0) | 467.6 (464.8-475.8) | 2.45x | 1.4 | 0.6 | 49.7 KiB | 100 / 396 MiB | +| `append` | Vorticity's | 56.1 (55.8-59.7) | not asked | n/a | — | — | 2.4 MiB | 28 / — MiB | #### All 14 cores | scenario | file | Vorticity, ms | Vortex Rust, ms | speedup | Vorticity, GB/s | Vortex Rust, GB/s | Vorticity, allocated | peak, Vorticity / Rust | |---|---|---:|---:|---:|---:|---:|---:|---:| -| `open` | Vorticity's | 0.1 (0.1-0.2) | 0.8 (0.7-0.9) | 5.69x | — | — | 21.1 KiB | 11 / 12 MiB | -| `open` | Vortex Rust's | 0.1 (0.1-2.6) | 0.8 (0.7-0.9) | 5.57x | — | — | 9.5 KiB | 11 / 12 MiB | -| `scan` | Vorticity's | 5.9 (5.6-6.2) | 10.3 (10.1-10.9) | 1.74x | 45.1 | 25.9 | 111.4 KiB | 66 / 79 MiB | -| `scan` | Vortex Rust's | 4.9 (4.7-5.7) | 6.8 (6.7-9.3) | 1.39x | 54.9 | 39.5 | 62.5 KiB | 164 / 126 MiB | -| `project` | Vorticity's | 1.2 (1.1-1.2) | 1.9 (1.9-2.1) | 1.66x | 72.8 | 43.8 | 95.0 KiB | 23 / 26 MiB | -| `project` | Vortex Rust's | 1.2 (1.1-4.0) | 1.9 (1.8-2.2) | 1.59x | 70.0 | 43.9 | 46.8 KiB | 27 / 23 MiB | -| `filter-narrow` | Vorticity's | 0.9 (0.9-2.2) | 2.6 (2.6-3.0) | 2.83x | 2.8 | 1.0 | 109.2 KiB | 18 / 23 MiB | -| `filter-narrow` | Vortex Rust's | 1.3 (1.1-4.1) | 2.3 (2.2-5.3) | 1.71x | 2.0 | 1.2 | 65.0 KiB | 19 / 23 MiB | -| `filter-wide` | Vorticity's | 3.6 (3.3-4.4) | 7.5 (7.2-8.1) | 2.07x | 36.6 | 17.7 | 144.9 KiB | 55 / 70 MiB | -| `filter-wide` | Vortex Rust's | 4.5 (4.5-5.9) | 5.6 (5.3-6.4) | 1.24x | 29.5 | 23.8 | 87.6 KiB | 133 / 90 MiB | -| `take` | Vorticity's | 3.7 (3.2-4.1) | 7.4 (7.3-7.9) | 1.98x | — | — | 169.2 KiB | 54 / 54 MiB | -| `take` | Vortex Rust's | 1.8 (1.8-2.9) | 2.9 (2.8-3.5) | 1.56x | — | — | 107.0 KiB | 42 / 47 MiB | -| `write` | Vorticity's | 68.0 (67.1-70.5) | 90.9 (90.4-110.8) | 1.34x | 3.9 | 2.9 | 2.6 MiB | 79 / 345 MiB | -| `write` | Vortex Rust's | 66.8 (66.4-67.3) | 91.8 (89.7-118.3) | 1.37x | 4.0 | 2.9 | 2.5 MiB | 179 / 426 MiB | -| `append` | Vorticity's | 47.5 (47.1-48.3) | not asked | n/a | — | — | 2.4 MiB | 60 / — MiB | +| `open` | Vorticity's | 0.1 (0.1-0.2) | 0.8 (0.8-0.8) | 5.39x | — | — | 21.1 KiB | 11 / 12 MiB | +| `open` | Vortex Rust's | 0.2 (0.1-1.8) | 0.8 (0.6-0.8) | 5.18x | — | — | 9.5 KiB | 11 / 12 MiB | +| `scan` | Vorticity's | 5.7 (5.6-6.6) | 10.2 (10.0-11.8) | 1.79x | 46.7 | 26.1 | 112.4 KiB | 66 / 79 MiB | +| `scan` | Vortex Rust's | 5.2 (4.7-6.4) | 6.8 (6.1-7.0) | 1.29x | 51.1 | 39.5 | 60.8 KiB | 164 / 127 MiB | +| `project` | Vorticity's | 1.1 (1.0-1.1) | 2.0 (1.9-2.8) | 1.83x | 77.1 | 42.0 | 94.9 KiB | 23 / 25 MiB | +| `project` | Vortex Rust's | 1.0 (1.0-4.2) | 1.8 (1.8-1.9) | 1.76x | 80.1 | 45.6 | 47.3 KiB | 27 / 23 MiB | +| `filter-narrow` | Vorticity's | 0.9 (0.8-1.1) | 2.8 (2.6-2.9) | 3.07x | 3.0 | 1.0 | 109.5 KiB | 18 / 23 MiB | +| `filter-narrow` | Vortex Rust's | 1.2 (1.2-2.8) | 2.2 (2.1-2.3) | 1.78x | 2.2 | 1.2 | 65.0 KiB | 19 / 24 MiB | +| `filter-wide` | Vorticity's | 3.6 (3.5-5.2) | 7.4 (7.2-7.6) | 2.03x | 36.8 | 18.1 | 143.8 KiB | 55 / 67 MiB | +| `filter-wide` | Vortex Rust's | 4.6 (4.6-6.5) | 5.8 (5.0-6.8) | 1.26x | 29.0 | 23.1 | 92.0 KiB | 133 / 90 MiB | +| `take` | Vorticity's | 3.5 (3.3-3.9) | 7.4 (7.3-7.9) | 2.11x | — | — | 175.3 KiB | 54 / 54 MiB | +| `take` | Vortex Rust's | 1.9 (1.7-3.2) | 2.8 (2.7-5.7) | 1.50x | — | — | 104.8 KiB | 42 / 47 MiB | +| `write` | Vorticity's | 66.9 (66.4-68.0) | 89.9 (89.6-92.2) | 1.34x | 4.0 | 3.0 | 2.6 MiB | 79 / 348 MiB | +| `write` | Vortex Rust's | 66.5 (65.6-67.0) | 91.2 (90.9-91.5) | 1.37x | 4.0 | 2.9 | 2.6 MiB | 179 / 413 MiB | +| `append` | Vorticity's | 53.6 (53.1-54.9) | not asked | n/a | — | — | 3.2 MiB | 60 / — MiB | `open`: open the file and read no rows. `scan`: read every column of every row. `project`: read one column of four. `filter-narrow`: read the rows of a band holding about one in a hundred. `filter-wide`: read the rows of a band holding about half. `take`: take a thousand rows spread across the file. `write`: read the file and encode it back out. `append`: append a tenth of the rows to a copy of the file. **On one core.** Vorticity took less time than Rust on 28 of 28 compared rows. -The highest speedup is `open` at 10,485,760 rows on Vorticity's file (3.97x), the lowest -`filter-wide` at 10,485,760 rows on Vortex Rust's file (1.06x). +The highest speedup is `open` at 10,485,760 rows on Vortex Rust's file (4.16x), the lowest +`filter-wide` at 10,485,760 rows on Vortex Rust's file (1.09x). **On all 14 cores.** Vorticity took less time than Rust on 28 of 28 compared rows. -The highest speedup is `open` at 10,485,760 rows on Vorticity's file (5.69x), the lowest -`take` at 1,048,576 rows on Vorticity's file (1.02x). +The highest speedup is `open` at 1,048,576 rows on Vorticity's file (5.75x), the lowest +`take` at 1,048,576 rows on Vorticity's file (1.11x). -**The process start** is not in the figures: 15 ms for Vorticity's native binary and 14 ms for Rust's, -most of it the operating system starting a binary, and 42 ms for the managed runtime on the JIT. +**The process start** is not in the figures: 16 ms for Vorticity's native binary and 14 ms for Rust's, +most of it the operating system starting a binary, and 43 ms for the managed runtime on the JIT. -**Rust's splits.** On one core, one split per chunk was the faster of its two on 8 of 14 rows on Vorticity's file and 11 of 14 rows on Vortex Rust's file; +**Rust's splits.** On one core, one split per chunk was the faster of its two on 7 of 14 rows on Vorticity's file and 14 of 14 rows on Vortex Rust's file; the two differ only where a chunk holds more than 100,000 rows. **Not asked of Rust**: `append`, which its harness has no entry point for; those figures @@ -200,16 +200,16 @@ Milliseconds, the median of the same runs. | scenario | 1,048,576 rows, Vorticity's file | 1,048,576 rows, Vortex Rust's file | 10,485,760 rows, Vorticity's file | 10,485,760 rows, Vortex Rust's file | |---|---:|---:|---:|---:| -| `open` | 12.6 | 11.6 | 12.3 | 12.7 | -| `scan` | 78.3 | 65.3 | 163.8 | 119.9 | -| `project` | 46.0 | 46.2 | 45.4 | 46.9 | -| `filter-narrow` | 104.1 | 84.3 | 103.4 | 97.2 | -| `filter-wide` | 88.3 | 79.9 | 132.9 | 117.8 | -| `take` | 85.7 | 61.8 | 126.0 | 66.5 | -| `write` | 208.9 | 206.3 | 591.2 | 551.6 | -| `append` | 199.7 | — | 268.5 | — | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 32b3a82b with uncommitted changes, 2026-10-10 02:12 UTC.* +| `open` | 12.7 | 11.5 | 12.5 | 11.7 | +| `scan` | 77.4 | 66.0 | 162.7 | 118.1 | +| `project` | 46.8 | 43.5 | 45.8 | 47.0 | +| `filter-narrow` | 104.2 | 85.5 | 101.1 | 95.3 | +| `filter-wide` | 88.0 | 81.0 | 131.8 | 114.9 | +| `take` | 86.4 | 60.7 | 126.1 | 64.7 | +| `write` | 207.7 | 203.2 | 586.4 | 550.9 | +| `append` | 199.7 | — | 273.8 | — | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 6403ae1a with uncommitted changes, 2026-10-10 04:11 UTC.* @@ -226,73 +226,73 @@ layout. Throughput is over the plain size, as above. | encoding | rows | Vorticity, ns/row | Vortex Rust, ns/row | speedup | Vorticity, GB/s | Vortex Rust, GB/s | |---|---:|---:|---:|---:|---:|---:| -| `alp` | 1,000,000 | 0.65 | 0.82 | 1.25x | 12.3 | 9.8 | -| `alp_no_patches` | 1,000,000 | 0.40 | 0.61 | 1.53x | 20.3 | 13.2 | -| `alp_patched_no_chunk_offsets` | 1,000,000 | 0.43 | 0.61 | 1.40x | 18.4 | 13.1 | +| `alp` | 1,000,000 | 0.66 | 0.82 | 1.25x | 12.2 | 9.8 | +| `alp_no_patches` | 1,000,000 | 0.39 | 0.59 | 1.52x | 20.7 | 13.6 | +| `alp_patched_no_chunk_offsets` | 1,000,000 | 0.40 | 0.61 | 1.51x | 19.8 | 13.1 | | `alprd` | 1,000,000 | 0.77 | 0.77 | 1.00x | 10.4 | 10.4 | -| `bool` | 1,000,000 | 0.04 | 0.04 | 0.91x | 3.2 | 3.5 | -| `bool_bit_offset3` | 1,000,000 | 0.03 | 0.04 | 1.09x | 3.6 | 3.3 | -| `bool_bit_offset7` | 1,000,000 | 0.04 | 0.04 | 1.03x | 3.2 | 3.1 | -| `bool_bit_offset_straddle` | 1,000,000 | 0.04 | 0.04 | 1.11x | 3.4 | 3.0 | +| `bool` | 1,000,000 | 0.04 | 0.04 | 0.90x | 3.1 | 3.4 | +| `bool_bit_offset3` | 1,000,000 | 0.04 | 0.04 | 1.03x | 3.5 | 3.4 | +| `bool_bit_offset7` | 1,000,000 | 0.04 | 0.04 | 1.03x | 3.3 | 3.2 | +| `bool_bit_offset_straddle` | 1,000,000 | 0.03 | 0.04 | 1.25x | 3.6 | 2.9 | | `bytebool` | 1,000,000 | 0.07 | 0.08 | 1.12x | 1.8 | 1.6 | -| `chunked` | 1,000,000 | 0.04 | 0.14 | 3.49x | 192.8 | 55.2 | -| `chunked_bool` | 1,000,000 | 0.04 | 0.05 | 1.07x | 2.9 | 2.7 | -| `chunked_decimal` | 1,000,000 | 0.05 | 0.15 | 2.98x | 163.3 | 54.8 | -| `chunked_empty_chunks` | 1,000,000 | 0.04 | 0.12 | 2.70x | 179.8 | 66.7 | -| `chunked_mixed_validity` | 1,000,000 | 0.13 | 0.17 | 1.27x | 61.1 | 48.0 | -| `chunked_one_chunk` | 1,000,000 | 0.04 | 0.07 | 1.64x | 197.5 | 120.3 | -| `chunked_varbinview` | 1,000,000 | 1.80 | 6.29 | 3.49x | 7.5 | 2.1 | -| `constant` | 1,000,000 | 0.03 | 0.04 | 1.26x | 235.3 | 186.0 | -| `datetimeparts` | 1,000,000 | 0.89 | 1.07 | 1.20x | 9.0 | 7.4 | -| `decimal` | 1,000,000 | 0.04 | 0.04 | 1.07x | 210.5 | 197.5 | -| `decimal_byte_parts` | 1,000,000 | 0.04 | 0.04 | 1.13x | 205.1 | 181.8 | -| `dict` | 1,000,000 | 0.33 | 0.43 | 1.30x | 18.4 | 14.1 | -| `dict_nullable_codes` | 1,000,000 | 0.60 | 1.98 | 3.28x | 9.2 | 2.8 | -| `dict_nullable_values_nonnull_codes` | 1,000,000 | 0.41 | 0.66 | 1.63x | 14.1 | 8.7 | -| `dict_u64_codes` | 1,000,000 | 0.54 | 0.59 | 1.08x | 11.0 | 10.2 | -| `dict_u8_codes` | 1,000,000 | 0.26 | 0.35 | 1.34x | 30.9 | 23.1 | -| `ext` | 1,000,000 | 0.04 | 0.04 | 0.98x | 190.5 | 195.1 | -| `fastlanes_bitpacked` | 1,000,000 | 0.13 | 0.14 | 1.07x | 30.8 | 28.9 | -| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,000,000 | 0.14 | 0.15 | 1.06x | 28.6 | 26.8 | -| `fastlanes_delta` | 1,000,000 | 0.47 | 0.89 | 1.88x | 16.9 | 9.0 | -| `fastlanes_for` | 1,000,000 | 0.37 | 0.41 | 1.10x | 21.4 | 19.4 | -| `fastlanes_rle` | 1,000,000 | 0.30 | 0.41 | 1.35x | 13.2 | 9.8 | -| `fixed_size_list` | 1,000,000 | 0.05 | 0.04 | 0.90x | 260.9 | 289.2 | -| `fsst` | 1,000,000 | 3.86 | 4.83 | 1.25x | 14.5 | 11.6 | -| `list` | 1,000,000 | 0.29 | 0.70 | 2.41x | 34.4 | 14.2 | -| `listview` | 1,000,000 | 0.65 | 1.04 | 1.61x | 15.4 | 9.6 | -| `map` | 1,000,000 | 1.81 | 5.03 | 2.78x | 11.1 | 4.0 | -| `masked` | 1,000,000 | 0.04 | 0.05 | 1.10x | 91.7 | 83.3 | -| `masked_all_invalid` | 1,000,000 | 0.05 | 0.05 | 0.98x | 81.6 | 83.3 | -| `masked_all_valid` | 1,000,000 | 0.04 | 0.05 | 1.06x | 93.0 | 87.9 | -| `null` | 1,000,000 | 0.03 | 0.04 | 1.19x | — | — | -| `onpair` | 1,000,000 | 1.94 | 1.92 | 0.99x | 11.2 | 11.3 | -| `parquet_variant` | 1,000,000 | 1.82 | 4.39 | 2.42x | 6.6 | 2.7 | -| `pco` | 1,000,000 | 0.40 | 1.22 | 3.03x | 19.9 | 6.6 | -| `primitive` | 1,000,000 | 0.04 | 0.04 | 1.09x | 195.1 | 179.8 | -| `runend` | 1,000,000 | 0.08 | 0.11 | 1.44x | 52.3 | 36.4 | -| `sequence` | 1,000,000 | 0.10 | 0.10 | 1.01x | 80.0 | 78.8 | -| `sparse` | 1,000,000 | 0.09 | 0.09 | 0.99x | 45.7 | 46.2 | -| `struct` | 1,000,000 | 0.66 | 4.58 | 6.89x | 22.4 | 3.3 | -| `table_mixed` | 1,000,000 | 4.15 | 11.4 | 2.75x | 16.9 | 6.1 | -| `table_wide` | 50,000 | 1.13 | 1.54 | 1.36x | 354.0 | 259.7 | -| `varbin` | 1,000,000 | 1.53 | 8.94 | 5.85x | 10.4 | 1.8 | -| `varbinview` | 1,000,000 | 1.45 | 5.97 | 4.13x | 9.3 | 2.3 | -| `variant` | 1,000,000 | 0.04 | 0.05 | 1.18x | 415.6 | 351.6 | -| `zigzag` | 1,000,000 | 0.21 | 0.31 | 1.47x | 19.2 | 13.1 | -| `zstd` | 1,000,000 | 3.30 | 6.18 | 1.87x | 9.6 | 5.2 | -| `zstd_buffers` | 1,000,000 | 0.18 | 0.28 | 1.54x | 43.4 | 28.2 | -| `zstd_nullable` | 1,000,000 | 1.47 | 2.85 | 1.94x | 5.5 | 2.9 | +| `chunked` | 1,000,000 | 0.04 | 0.14 | 3.49x | 195.1 | 55.9 | +| `chunked_bool` | 1,000,000 | 0.04 | 0.05 | 1.08x | 2.9 | 2.7 | +| `chunked_decimal` | 1,000,000 | 0.04 | 0.15 | 3.55x | 190.5 | 53.7 | +| `chunked_empty_chunks` | 1,000,000 | 0.05 | 0.12 | 2.45x | 164.9 | 67.2 | +| `chunked_mixed_validity` | 1,000,000 | 0.13 | 0.16 | 1.22x | 60.9 | 49.8 | +| `chunked_one_chunk` | 1,000,000 | 0.04 | 0.08 | 1.88x | 195.1 | 103.9 | +| `chunked_varbinview` | 1,000,000 | 1.71 | 6.35 | 3.71x | 7.9 | 2.1 | +| `constant` | 1,000,000 | 0.03 | 0.04 | 1.28x | 231.9 | 181.8 | +| `datetimeparts` | 1,000,000 | 0.91 | 1.14 | 1.26x | 8.8 | 7.0 | +| `decimal` | 1,000,000 | 0.04 | 0.04 | 0.96x | 200.0 | 207.8 | +| `decimal_byte_parts` | 1,000,000 | 0.04 | 0.04 | 0.96x | 200.0 | 207.8 | +| `dict` | 1,000,000 | 0.33 | 0.42 | 1.28x | 18.3 | 14.3 | +| `dict_nullable_codes` | 1,000,000 | 0.60 | 1.97 | 3.28x | 9.2 | 2.8 | +| `dict_nullable_values_nonnull_codes` | 1,000,000 | 0.40 | 0.64 | 1.61x | 14.3 | 8.9 | +| `dict_u64_codes` | 1,000,000 | 0.57 | 0.60 | 1.05x | 10.5 | 10.0 | +| `dict_u8_codes` | 1,000,000 | 0.27 | 0.36 | 1.34x | 30.2 | 22.5 | +| `ext` | 1,000,000 | 0.04 | 0.05 | 1.10x | 177.8 | 161.6 | +| `fastlanes_bitpacked` | 1,000,000 | 0.14 | 0.15 | 1.12x | 29.4 | 26.2 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,000,000 | 0.15 | 0.14 | 0.98x | 27.1 | 27.6 | +| `fastlanes_delta` | 1,000,000 | 0.47 | 0.89 | 1.89x | 17.0 | 9.0 | +| `fastlanes_for` | 1,000,000 | 0.36 | 0.39 | 1.07x | 22.1 | 20.7 | +| `fastlanes_rle` | 1,000,000 | 0.30 | 0.40 | 1.35x | 13.4 | 9.9 | +| `fixed_size_list` | 1,000,000 | 0.04 | 0.05 | 1.11x | 282.4 | 255.3 | +| `fsst` | 1,000,000 | 3.95 | 4.84 | 1.23x | 14.2 | 11.6 | +| `list` | 1,000,000 | 0.29 | 0.67 | 2.34x | 35.0 | 15.0 | +| `listview` | 1,000,000 | 0.64 | 0.97 | 1.50x | 15.5 | 10.4 | +| `map` | 1,000,000 | 1.63 | 5.36 | 3.28x | 12.2 | 3.7 | +| `masked` | 1,000,000 | 0.05 | 0.05 | 0.98x | 87.8 | 89.7 | +| `masked_all_invalid` | 1,000,000 | 0.04 | 0.04 | 1.00x | 92.0 | 92.0 | +| `masked_all_valid` | 1,000,000 | 0.04 | 0.04 | 0.96x | 98.8 | 102.6 | +| `null` | 1,000,000 | 0.03 | 0.04 | 1.17x | — | — | +| `onpair` | 1,000,000 | 2.01 | 1.92 | 0.95x | 10.8 | 11.3 | +| `parquet_variant` | 1,000,000 | 1.83 | 4.38 | 2.39x | 6.6 | 2.7 | +| `pco` | 1,000,000 | 0.40 | 1.21 | 3.06x | 20.2 | 6.6 | +| `primitive` | 1,000,000 | 0.04 | 0.04 | 1.01x | 190.5 | 188.2 | +| `runend` | 1,000,000 | 0.08 | 0.11 | 1.40x | 51.0 | 36.4 | +| `sequence` | 1,000,000 | 0.10 | 0.10 | 1.00x | 78.4 | 78.4 | +| `sparse` | 1,000,000 | 0.09 | 0.09 | 1.04x | 46.0 | 44.2 | +| `struct` | 1,000,000 | 0.66 | 4.59 | 6.92x | 22.4 | 3.2 | +| `table_mixed` | 1,000,000 | 4.33 | 11.6 | 2.67x | 16.2 | 6.1 | +| `table_wide` | 50,000 | 1.12 | 1.53 | 1.37x | 357.1 | 261.4 | +| `varbin` | 1,000,000 | 1.58 | 9.58 | 6.08x | 10.1 | 1.7 | +| `varbinview` | 1,000,000 | 1.42 | 6.02 | 4.24x | 9.5 | 2.2 | +| `variant` | 1,000,000 | 0.04 | 0.05 | 1.23x | 426.7 | 347.8 | +| `zigzag` | 1,000,000 | 0.22 | 0.30 | 1.34x | 18.1 | 13.5 | +| `zstd` | 1,000,000 | 3.29 | 6.22 | 1.89x | 9.7 | 5.1 | +| `zstd_buffers` | 1,000,000 | 0.18 | 0.28 | 1.54x | 44.0 | 28.6 | +| `zstd_nullable` | 1,000,000 | 1.47 | 2.81 | 1.91x | 5.5 | 2.9 | Rust is timed under each of its two ways of splitting a scan, three processes each, and its figure is the faster: its default cuts a chunk of more than 100,000 rows into splits of 100,000 and decodes the whole chunk again for each, which on these files of one chunk of a million rows is ten times; one split per chunk decodes it once, but builds a chunk made of smaller arrays in one -piece. One split per chunk was the faster on 45 of 57 files, its default on `alprd`, `chunked`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `datetimeparts`, `fastlanes_delta`, `fsst`, `parquet_variant`, `table_wide`, `zigzag`. +piece. One split per chunk was the faster on 44 of 57 files, its default on `alp_no_patches`, `alp_patched_no_chunk_offsets`, `alprd`, `chunked`, `chunked_decimal`, `chunked_empty_chunks`, `chunked_mixed_validity`, `chunked_one_chunk`, `dict_u64_codes`, `fastlanes_for`, `fsst`, `table_wide`, `zigzag`. -Vorticity decoded 51 of 57 files in less time than Rust; the median speedup is 1.27x. At 1.00x or below: `fixed_size_list` 0.90x, `bool` 0.91x, `ext` 0.98x, `masked_all_invalid` 0.98x, `sparse` 0.99x, `onpair` 0.99x. +Vorticity decoded 47 of 57 files in less time than Rust; the median speedup is 1.28x. At 1.00x or below: `bool` 0.90x, `onpair` 0.95x, `decimal` 0.96x, `decimal_byte_parts` 0.96x, `masked_all_valid` 0.96x, `masked` 0.98x, `fastlanes_bitpacked_patched_no_chunk_offsets` 0.98x, `alprd` 1.00x, `masked_all_invalid` 1.00x, `sequence` 1.00x. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 32b3a82b with uncommitted changes, 2026-10-10 02:12 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1, rustc 1.98.1 (48a229cea 2026-09-01) (Homebrew); commit 6403ae1a with uncommitted changes, 2026-10-10 04:11 UTC.* @@ -307,30 +307,30 @@ one the `--ratio-check` gate holds to a ceiling; what each one reads and asks is | axis | Vorticity, µs | Vortex Rust, µs | speedup | 95 % interval | |---|---:|---:|---:|---| -| full scan | 504.5 | 1,609.7 | 3.18x | 3.16 to 3.21 | -| projected scan, 1 of 5 columns | 115.6 | 229.9 | 1.99x | 1.97 to 2.01 | -| open to first batch | 98.8 | 1,439.0 | 14.7x | 14.0 to 15.2 | -| open, footer only | 29.3 | 51.5 | 1.75x | 1.75 to 1.77 | -| read and write back | 4,384.6 | 13,744.0 | 3.24x | 3.04 to 3.27 | -| filtered scan, 1% band | 91.8 | 302.6 | 3.25x | 3.22 to 3.33 | -| filtered scan, half the rows | 164.0 | 581.1 | 3.55x | 3.51 to 3.58 | -| scattered take, 64 of 64 splits | 332.4 | 1,491.0 | 4.49x | 4.43 to 4.52 | -| key order, sorted column, 1% band | 78.6 | 105.8 | 1.35x | 1.29 to 1.35 | -| key order, uncorrelated, 64 rows | 237.6 | 266.5 | 1.13x | 1.06 to 1.16 | -| count, exact cover, 1% band | 141.7 | 154.0 | 1.17x | 1.11 to 1.21 | -| filtered scan, string equality, fsst | 308.5 | 428.5 | 1.38x | 1.32 to 1.45 | -| filtered scan, string prefix, fsst | 772.9 | 711.8 | 0.93x | 0.92 to 0.94 | -| filtered scan, string equality, dict | 85.8 | 104.1 | 1.21x | 1.17 to 1.23 | -| filtered scan, string prefix, dict | 109.7 | 136.1 | 1.24x | 1.23 to 1.26 | -| filtered scan, band, runend | 64.6 | 81.5 | 1.25x | 1.22 to 1.28 | -| filtered scan, band, bitpacked | 76.8 | 79.6 | 1.03x | 1.02 to 1.05 | -| full scan, 1M table | 4,184.0 | 11,042.7 | 2.65x | 2.52 to 2.74 | -| projected scan, 1 of 50 columns | 46.6 | 65.5 | 1.44x | 1.37 to 1.46 | +| full scan | 505.8 | 1,618.0 | 3.20x | 3.19 to 3.24 | +| projected scan, 1 of 5 columns | 116.4 | 228.1 | 1.97x | 1.94 to 1.99 | +| open to first batch | 99.1 | 1,441.3 | 14.4x | 14.1 to 15.0 | +| open, footer only | 29.1 | 51.1 | 1.75x | 1.73 to 1.78 | +| read and write back | 4,364.7 | 13,638.5 | 3.11x | 3.05 to 3.22 | +| filtered scan, 1% band | 92.9 | 301.6 | 3.27x | 3.19 to 3.29 | +| filtered scan, half the rows | 175.7 | 600.0 | 3.36x | 3.33 to 3.50 | +| scattered take, 64 of 64 splits | 333.8 | 1,484.0 | 4.47x | 4.43 to 4.50 | +| key order, sorted column, 1% band | 77.6 | 103.1 | 1.35x | 1.31 to 1.37 | +| key order, uncorrelated, 64 rows | 245.3 | 269.4 | 1.09x | 1.05 to 1.16 | +| count, exact cover, 1% band | 143.4 | 148.6 | 1.14x | 1.09 to 1.18 | +| filtered scan, string equality, fsst | 206.5 | 354.4 | 1.72x | 1.71 to 1.73 | +| filtered scan, string prefix, fsst | 781.6 | 722.2 | 0.93x | 0.92 to 0.93 | +| filtered scan, string equality, dict | 87.6 | 100.2 | 1.16x | 1.13 to 1.21 | +| filtered scan, string prefix, dict | 109.5 | 134.8 | 1.24x | 1.20 to 1.26 | +| filtered scan, band, runend | 70.4 | 85.7 | 1.24x | 1.16 to 1.28 | +| filtered scan, band, bitpacked | 78.3 | 79.5 | 1.02x | 0.99 to 1.04 | +| full scan, 1M table | 4,084.7 | 11,273.5 | 2.74x | 2.70 to 2.77 | +| projected scan, 1 of 50 columns | 46.5 | 67.3 | 1.46x | 1.37 to 1.51 | Vorticity took less time on 18 of 19 axes. At 1.00x or below: filtered scan, string prefix, fsst (0.93x). -Rust is timed under the faster of its two splits, axis by axis: one split per chunk on 10 of 19, its default on full scan; open, footer only; filtered scan, half the rows; key order, sorted column, 1% band; key order, uncorrelated, 64 rows; filtered scan, string equality, fsst; filtered scan, string prefix, fsst; filtered scan, string prefix, dict; filtered scan, band, bitpacked. +Rust is timed under the faster of its two splits, axis by axis: one split per chunk on 7 of 19, its default on the other 12, one split per chunk on projected scan, 1 of 5 columns; open to first batch; filtered scan, half the rows; key order, sorted column, 1% band; count, exact cover, 1% band; full scan, 1M table; projected scan, 1 of 50 columns. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 32b3a82b with uncommitted changes, 2026-10-10 02:13 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 6403ae1a with uncommitted changes, 2026-10-10 04:12 UTC.* @@ -344,68 +344,68 @@ at least 9 rounds after a 1-second warm-up per file, our side on the JIT, warmed | encoding | Vorticity, µs | Vortex Rust, µs | speedup | |---|---:|---:|---:| -| `alp` | 174 | 195 | 1.10x | -| `alp_no_patches` | 60 | 66 | 1.09x | -| `alp_patched_no_chunk_offsets` | 67 | 69 | 1.04x | -| `alprd` | 63 | 69 | 1.11x | -| `bool` | 35 | 41 | 1.15x | -| `bool_bit_offset3` | 36 | 41 | 1.12x | -| `bool_bit_offset7` | 35 | 40 | 1.13x | -| `bool_bit_offset_straddle` | 36 | 41 | 1.16x | -| `bytebool` | 55 | 84 | 1.52x | -| `chunked` | 60 | 70 | 1.16x | -| `chunked_bool` | 38 | 46 | 1.21x | -| `chunked_decimal` | 61 | 71 | 1.16x | -| `chunked_empty_chunks` | 63 | 71 | 1.14x | -| `chunked_mixed_validity` | 65 | 80 | 1.21x | -| `chunked_one_chunk` | 63 | 70 | 1.11x | -| `chunked_varbinview` | 156 | 6,027 | 40.2x | -| `constant` | 28 | 38 | 1.37x | -| `datetimeparts` | 116 | 119 | 1.02x | -| `decimal` | 59 | 62 | 1.06x | -| `decimal_byte_parts` | 58 | 63 | 1.10x | -| `dict` | 57 | 62 | 1.08x | -| `dict_nullable_codes` | 61 | 73 | 1.17x | -| `dict_nullable_values_nonnull_codes` | 60 | 64 | 1.09x | -| `dict_u64_codes` | 59 | 65 | 1.09x | -| `dict_u8_codes` | 56 | 61 | 1.09x | -| `ext` | 61 | 66 | 1.08x | -| `fastlanes_bitpacked` | 57 | 62 | 1.09x | -| `fastlanes_bitpacked_patched_no_chunk_offsets` | 60 | 65 | 1.08x | -| `fastlanes_delta` | 145 | 939 | 6.49x | -| `fastlanes_for` | 60 | 64 | 1.07x | -| `fastlanes_rle` | 333 | 443 | 1.33x | -| `fixed_size_list` | 63 | 90 | 1.41x | -| `fsst` | 121 | 119 | 1.00x | -| `list` | 84 | 232 | 2.72x | -| `listview` | 665 | 976 | 1.47x | -| `map` | 1,652 | 5,284 | 3.18x | -| `masked` | 39 | 44 | 1.15x | -| `masked_all_invalid` | 39 | 44 | 1.14x | -| `masked_all_valid` | 61 | 65 | 1.07x | -| `null` | 27 | 37 | 1.35x | -| `onpair` | 114 | 139 | 1.22x | -| `parquet_variant` | 133 | 139 | 1.06x | -| `pco` | 381 | 1,224 | 3.23x | -| `primitive` | 58 | 63 | 1.08x | -| `runend` | 40 | 45 | 1.14x | -| `sequence` | 29 | 38 | 1.34x | -| `sparse` | 43 | 49 | 1.13x | -| `struct` | 737 | 4,679 | 6.62x | -| `table_mixed` | 3,861 | 11,057 | 2.93x | -| `table_wide` | 102 | 147 | 1.44x | -| `varbin` | 140 | 6,987 | 49.0x | -| `varbinview` | 1,332 | 6,156 | 4.60x | -| `variant` | 29 | 39 | 1.33x | -| `zigzag` | 56 | 61 | 1.08x | -| `zstd` | 394 | 6,116 | 15.6x | -| `zstd_buffers` | 209 | 318 | 1.51x | -| `zstd_nullable` | 220 | 2,835 | 12.9x | - -Vorticity took less time on 56 of 57 files; the median speedup is 1.15x. At 1.00x or below: `fsst` 1.00x. -Rust is timed under the faster of its two splits, file by file: one split per chunk on 56 of 57 files, its default on `table_wide`. - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 32b3a82b with uncommitted changes, 2026-10-10 02:15 UTC.* +| `alp` | 176 | 197 | 1.10x | +| `alp_no_patches` | 60 | 67 | 1.10x | +| `alp_patched_no_chunk_offsets` | 63 | 68 | 1.09x | +| `alprd` | 63 | 69 | 1.10x | +| `bool` | 35 | 40 | 1.15x | +| `bool_bit_offset3` | 35 | 40 | 1.15x | +| `bool_bit_offset7` | 36 | 41 | 1.14x | +| `bool_bit_offset_straddle` | 36 | 41 | 1.13x | +| `bytebool` | 57 | 88 | 1.52x | +| `chunked` | 62 | 73 | 1.17x | +| `chunked_bool` | 39 | 49 | 1.28x | +| `chunked_decimal` | 61 | 70 | 1.17x | +| `chunked_empty_chunks` | 64 | 72 | 1.14x | +| `chunked_mixed_validity` | 65 | 80 | 1.23x | +| `chunked_one_chunk` | 60 | 66 | 1.08x | +| `chunked_varbinview` | 160 | 6,096 | 38.6x | +| `constant` | 28 | 39 | 1.35x | +| `datetimeparts` | 107 | 116 | 1.07x | +| `decimal` | 58 | 63 | 1.07x | +| `decimal_byte_parts` | 60 | 64 | 1.09x | +| `dict` | 59 | 64 | 1.08x | +| `dict_nullable_codes` | 62 | 71 | 1.14x | +| `dict_nullable_values_nonnull_codes` | 59 | 64 | 1.09x | +| `dict_u64_codes` | 60 | 66 | 1.11x | +| `dict_u8_codes` | 57 | 64 | 1.11x | +| `ext` | 59 | 64 | 1.06x | +| `fastlanes_bitpacked` | 56 | 61 | 1.10x | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 60 | 64 | 1.06x | +| `fastlanes_delta` | 142 | 943 | 6.58x | +| `fastlanes_for` | 59 | 63 | 1.08x | +| `fastlanes_rle` | 334 | 438 | 1.31x | +| `fixed_size_list` | 61 | 88 | 1.43x | +| `fsst` | 115 | 115 | 0.98x | +| `list` | 86 | 237 | 2.79x | +| `listview` | 652 | 987 | 1.50x | +| `map` | 1,547 | 4,849 | 3.31x | +| `masked` | 39 | 43 | 1.10x | +| `masked_all_invalid` | 39 | 46 | 1.14x | +| `masked_all_valid` | 60 | 65 | 1.08x | +| `null` | 28 | 37 | 1.34x | +| `onpair` | 108 | 130 | 1.18x | +| `parquet_variant` | 131 | 136 | 1.03x | +| `pco` | 379 | 1,240 | 3.26x | +| `primitive` | 58 | 63 | 1.07x | +| `runend` | 39 | 44 | 1.12x | +| `sequence` | 31 | 40 | 1.31x | +| `sparse` | 43 | 49 | 1.14x | +| `struct` | 687 | 4,748 | 7.01x | +| `table_mixed` | 3,915 | 11,151 | 2.83x | +| `table_wide` | 100 | 147 | 1.44x | +| `varbin` | 134 | 7,471 | 55.6x | +| `varbinview` | 1,310 | 6,112 | 4.67x | +| `variant` | 30 | 39 | 1.33x | +| `zigzag` | 57 | 62 | 1.06x | +| `zstd` | 385 | 6,214 | 16.1x | +| `zstd_buffers` | 206 | 312 | 1.52x | +| `zstd_nullable` | 222 | 2,835 | 12.9x | + +Vorticity took less time on 56 of 57 files; the median speedup is 1.14x. At 1.00x or below: `fsst` 0.98x. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 57 of 57 files, its default on none. + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 6403ae1a with uncommitted changes, 2026-10-10 04:13 UTC.* @@ -421,69 +421,69 @@ A writer can be fast by compressing less, so each one's bytes are beside its tim | encoding | Vorticity, µs | Vortex Rust, µs | speedup | Vorticity, bytes | Vortex Rust, bytes | |---|---:|---:|---:|---:|---:| -| `alp` | 5,432 | 22,809 | 4.23x | 2,740,820 | 2,976,924 | -| `alp_no_patches` | 5,064 | 16,103 | 3.21x | 1,955,668 | 1,956,060 | -| `alp_patched_no_chunk_offsets` | 4,707 | 17,734 | 3.75x | 2,155,956 | 2,141,772 | -| `alprd` | 6,111 | 17,386 | 2.84x | 6,508,892 | 6,509,316 | -| `bool` | 143 | 409 | 2.87x | 127,108 | 127,108 | -| `bool_bit_offset3` | 173 | 543 | 3.14x | 127,108 | 127,108 | -| `bool_bit_offset7` | 168 | 540 | 3.19x | 127,108 | 127,108 | -| `bool_bit_offset_straddle` | 172 | 541 | 3.16x | 127,108 | 127,108 | -| `bytebool` | 177 | 449 | 2.55x | 127,108 | 127,108 | -| `chunked` | 1,405 | 10,760 | 7.65x | 890,228 | 890,736 | -| `chunked_bool` | 150 | 419 | 2.81x | 127,236 | 127,248 | -| `chunked_decimal` | 2,964 | 13,001 | 4.37x | 1,822,388 | 1,825,056 | -| `chunked_empty_chunks` | 1,113 | 10,411 | 9.68x | 316,532 | 317,080 | -| `chunked_mixed_validity` | 2,923 | 10,027 | 3.43x | 1,431,516 | 1,374,016 | -| `chunked_one_chunk` | 841 | 9,803 | 11.6x | 5,044 | 4,640 | -| `chunked_varbinview` | 13,666 | 26,292 | 1.94x | 643,644 | 628,628 | -| `constant` | 89 | 2,707 | 30.5x | 4,212 | 3,300 | -| `datetimeparts` | 3,487 | 23,519 | 6.75x | 3,382,604 | 3,383,628 | -| `decimal` | 1,579 | 11,992 | 7.60x | 3,372 | 5,116 | -| `decimal_byte_parts` | 1,635 | 12,337 | 7.62x | 3,372 | 5,116 | -| `dict` | 5,779 | 18,454 | 3.22x | 383,892 | 378,288 | -| `dict_nullable_codes` | 9,440 | 17,501 | 1.85x | 384,628 | 378,772 | -| `dict_nullable_values_nonnull_codes` | 8,784 | 16,352 | 1.86x | 384,628 | 378,764 | -| `dict_u64_codes` | 6,088 | 19,178 | 3.15x | 383,892 | 378,288 | -| `dict_u8_codes` | 9,476 | 18,196 | 1.93x | 38,740 | 1,004,160 | -| `ext` | 858 | 18,873 | 21.9x | 5,548 | 4,988 | -| `fastlanes_bitpacked` | 1,552 | 4,658 | 3.00x | 1,254,340 | 1,253,668 | -| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,330 | 5,444 | 4.08x | 1,270,452 | 1,272,740 | -| `fastlanes_delta` | 1,260 | 10,873 | 8.63x | 4,908 | 4,532 | -| `fastlanes_for` | 2,418 | 5,453 | 2.24x | 881,364 | 879,732 | -| `fastlanes_rle` | 1,216 | 4,826 | 3.97x | 3,964 | 3,812 | -| `fixed_size_list` | 1,103 | 22,142 | 20.2x | 4,268 | 4,724 | -| `fsst` | 30,374 | 252,930 | 8.32x | 933,668 | 8,574,500 | -| `list` | 6,662 | 20,680 | 3.09x | 4,782,492 | 5,122,712 | -| `listview` | 7,170 | 32,519 | 4.64x | 4,745,468 | 4,502,136 | -| `map` | 14,359 | 53,516 | 3.76x | 4,426,852 | 4,811,136 | -| `masked` | 3,720 | 12,425 | 3.30x | 2,380,060 | 2,381,156 | -| `masked_all_invalid` | 324 | 727 | 2.26x | 2,636 | 2,700 | -| `masked_all_valid` | 495 | 8,374 | 17.1x | 3,484 | 3,324 | -| `null` | 61 | 265 | 4.35x | 2,052 | 2,164 | -| `onpair` | 17,348 | 24,544 | 1.42x | 60,588 | 1,005,840 | -| `pco` | 878 | 11,101 | 12.7x | 4,908 | 4,508 | -| `primitive` | 865 | 9,487 | 11.2x | 5,356 | 4,572 | -| `runend` | 811 | 5,572 | 6.94x | 4,028 | 3,500 | -| `sequence` | 637 | 9,956 | 15.7x | 4,908 | 4,516 | -| `sparse` | 733 | 2,655 | 3.61x | 131,172 | 3,884 | -| `struct` | 26,670 | 89,248 | 3.34x | 638,412 | 5,274,480 | -| `table_mixed` | 50,023 | 283,674 | 5.64x | 7,867,284 | 14,856,384 | -| `table_wide` | 5,987 | 31,005 | 5.23x | 5,163,108 | 5,168,320 | -| `varbin` | 30,788 | 185,512 | 6.00x | 637,012 | 5,066,580 | -| `varbinview` | 13,200 | 26,138 | 1.98x | 643,516 | 628,504 | -| `variant` | 95 | 535 | 5.63x | 2,220 | 52,520 | -| `zigzag` | 1,713 | 9,932 | 5.84x | 2,531,652 | 2,380,300 | -| `zstd` | 22,478 | 28,500 | 1.25x | 786,948 | 754,336 | -| `zstd_buffers` | 2,299 | 4,768 | 2.09x | 630,732 | 629,028 | -| `zstd_nullable` | 5,080 | 10,126 | 1.98x | 756,500 | 628,740 | +| `alp` | 5,512 | 22,660 | 4.10x | 2,740,820 | 2,976,924 | +| `alp_no_patches` | 5,148 | 16,205 | 3.17x | 1,955,668 | 1,956,060 | +| `alp_patched_no_chunk_offsets` | 4,600 | 17,569 | 3.82x | 2,155,956 | 2,141,772 | +| `alprd` | 6,046 | 17,204 | 2.84x | 6,508,892 | 6,509,316 | +| `bool` | 144 | 411 | 2.86x | 127,108 | 127,108 | +| `bool_bit_offset3` | 170 | 541 | 3.19x | 127,108 | 127,108 | +| `bool_bit_offset7` | 171 | 539 | 3.15x | 127,108 | 127,108 | +| `bool_bit_offset_straddle` | 171 | 543 | 3.16x | 127,108 | 127,108 | +| `bytebool` | 179 | 450 | 2.50x | 127,108 | 127,108 | +| `chunked` | 1,487 | 10,872 | 7.46x | 890,228 | 890,736 | +| `chunked_bool` | 150 | 418 | 2.78x | 127,236 | 127,248 | +| `chunked_decimal` | 3,030 | 13,352 | 4.36x | 1,822,388 | 1,825,056 | +| `chunked_empty_chunks` | 1,036 | 10,284 | 10.0x | 316,532 | 317,080 | +| `chunked_mixed_validity` | 2,882 | 9,869 | 3.41x | 1,431,516 | 1,374,016 | +| `chunked_one_chunk` | 860 | 9,814 | 11.5x | 5,044 | 4,640 | +| `chunked_varbinview` | 13,437 | 26,318 | 1.96x | 643,644 | 628,628 | +| `constant` | 94 | 2,730 | 28.9x | 4,212 | 3,300 | +| `datetimeparts` | 3,498 | 23,670 | 6.76x | 3,382,604 | 3,383,628 | +| `decimal` | 1,535 | 12,040 | 7.84x | 3,372 | 5,116 | +| `decimal_byte_parts` | 1,641 | 12,194 | 7.54x | 3,372 | 5,116 | +| `dict` | 5,815 | 18,324 | 3.14x | 383,892 | 378,288 | +| `dict_nullable_codes` | 9,289 | 17,471 | 1.87x | 384,628 | 378,772 | +| `dict_nullable_values_nonnull_codes` | 8,598 | 16,489 | 1.92x | 384,628 | 378,764 | +| `dict_u64_codes` | 6,046 | 18,625 | 3.06x | 383,892 | 378,288 | +| `dict_u8_codes` | 9,385 | 17,892 | 1.91x | 38,740 | 1,004,160 | +| `ext` | 859 | 18,841 | 21.8x | 5,548 | 4,988 | +| `fastlanes_bitpacked` | 1,542 | 4,658 | 3.01x | 1,254,340 | 1,253,668 | +| `fastlanes_bitpacked_patched_no_chunk_offsets` | 1,337 | 5,520 | 4.11x | 1,270,452 | 1,272,740 | +| `fastlanes_delta` | 1,282 | 10,861 | 8.49x | 4,908 | 4,532 | +| `fastlanes_for` | 2,359 | 5,469 | 2.29x | 881,364 | 879,732 | +| `fastlanes_rle` | 1,221 | 4,805 | 3.92x | 3,964 | 3,812 | +| `fixed_size_list` | 1,128 | 22,111 | 19.4x | 4,268 | 4,724 | +| `fsst` | 30,190 | 252,888 | 8.35x | 933,668 | 8,574,500 | +| `list` | 6,677 | 20,787 | 3.11x | 4,782,492 | 5,122,712 | +| `listview` | 7,320 | 33,529 | 4.57x | 4,745,468 | 4,502,136 | +| `map` | 14,399 | 53,606 | 3.71x | 4,426,852 | 4,811,136 | +| `masked` | 3,668 | 12,499 | 3.38x | 2,380,060 | 2,381,156 | +| `masked_all_invalid` | 317 | 727 | 2.29x | 2,636 | 2,700 | +| `masked_all_valid` | 496 | 8,370 | 17.0x | 3,484 | 3,324 | +| `null` | 62 | 264 | 4.28x | 2,052 | 2,164 | +| `onpair` | 17,657 | 24,726 | 1.40x | 60,588 | 1,005,840 | +| `pco` | 860 | 11,027 | 12.8x | 4,908 | 4,508 | +| `primitive` | 856 | 9,535 | 11.1x | 5,356 | 4,572 | +| `runend` | 796 | 5,563 | 6.96x | 4,028 | 3,500 | +| `sequence` | 633 | 10,112 | 16.0x | 4,908 | 4,516 | +| `sparse` | 734 | 2,668 | 3.56x | 131,172 | 3,884 | +| `struct` | 26,807 | 89,502 | 3.33x | 638,412 | 5,274,480 | +| `table_mixed` | 49,629 | 282,572 | 5.70x | 7,867,284 | 14,856,384 | +| `table_wide` | 5,825 | 31,087 | 5.33x | 5,163,108 | 5,168,320 | +| `varbin` | 31,121 | 184,599 | 5.95x | 637,012 | 5,066,580 | +| `varbinview` | 13,090 | 26,184 | 2.01x | 643,516 | 628,504 | +| `variant` | 93 | 537 | 5.82x | 2,220 | 52,520 | +| `zigzag` | 1,772 | 10,174 | 5.83x | 2,531,652 | 2,380,300 | +| `zstd` | 22,196 | 28,486 | 1.28x | 786,948 | 754,336 | +| `zstd_buffers` | 2,310 | 4,912 | 2.08x | 630,732 | 629,028 | +| `zstd_nullable` | 5,214 | 9,997 | 1.92x | 756,500 | 628,740 | Vorticity took less time on 56 of 56 files; the median speedup is 3.76x. -Rust is timed under the faster of its two splits, file by file: one split per chunk on 48 of 56 files, its default on `alprd`, `chunked`, `chunked_decimal`, `chunked_one_chunk`, `dict`, `dict_u8_codes`, `fsst`, `list`. +Rust is timed under the faster of its two splits, file by file: one split per chunk on 49 of 56 files, its default on `chunked_empty_chunks`, `chunked_one_chunk`, `dict_u64_codes`, `dict_u8_codes`, `fastlanes_for`, `list`, `table_wide`. Vorticity's bytes were within 5 % of Rust's or fewer on 44 of 56 files; more than 5 % above them on `sparse` (33.8 times Rust's), `constant` (1.28 times), `zstd_nullable` (1.20 times), `primitive` (1.17 times), `runend` (1.15 times), `ext` (1.11 times), `pco` (1.09 times), `chunked_one_chunk` (1.09 times), `sequence` (1.09 times), `fastlanes_delta` (1.08 times), `zigzag` (1.06 times), `listview` (1.05 times). Rust's writer declines `parquet_variant`, which is left out. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 32b3a82b with uncommitted changes, 2026-10-10 02:16 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; Vortex 0.86.1 through the C ABI of `tools/vxbench-rs`, binary 1be9345d4816; commit 6403ae1a with uncommitted changes, 2026-10-10 04:15 UTC.* @@ -508,27 +508,27 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| a sequence | Sequence x77 | 0.00 | 6 | 3.5 | 0.20 | | -| sorted runs of 1 000 | RunEnd x77 | 0.01 | 15 | 3.3 | 0.26 | Smallest: RunEnd x76, Pco x1, 0.01 B/value, scan 3.8 ms, take 0.65 ms; reads faster below 71 MB/s | -| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 33 | 6.4 | 0.63 | Smallest: Pco x77, 1.35 B/value, scan 70.2 ms, take 67.22 ms; reads faster below 319 MB/s | -| random in 0..999 | BitPacked x77 | 1.25 | 25 | 4.2 | 0.50 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 48 | 4.7 | 0.37 | | -| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 33 | 5.4 | 0.61 | Smallest: Pco x77, 0.06 B/value, scan 69.3 ms, take 67.81 ms; reads faster below 715 MB/s | -| uniform 64-bit | Canonical x77 | 8.00 | 20 | 5.6 | 0.76 | | -| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 47 | 5.6 | 0.56 | Smallest: Pco x82, 1.26 B/value, scan 30.7 ms, take 27.95 ms; reads faster below 49 MB/s | +| a sequence | Sequence x77 | 0.00 | 7 | 3.4 | 0.20 | | +| sorted runs of 1 000 | RunEnd x77 | 0.01 | 15 | 3.4 | 0.26 | Smallest: RunEnd x76, Pco x1, 0.01 B/value, scan 3.6 ms, take 0.58 ms; reads faster below 148 MB/s | +| timestamps (ms, increasing, jittered) | BitPacked x77 | 3.38 | 29 | 6.3 | 0.57 | Smallest: Pco x77, 1.35 B/value, scan 71.1 ms, take 70.68 ms; reads faster below 314 MB/s | +| random in 0..999 | BitPacked x77 | 1.25 | 25 | 4.1 | 0.50 | | +| 16 distinct, random order | Dictionary x77 | 0.51 | 48 | 4.9 | 0.35 | | +| 100 003 distinct, repeating | BitPacked x77 | 4.63 | 30 | 5.7 | 0.70 | Smallest: Pco x77, 0.06 B/value, scan 70.6 ms, take 68.60 ms; reads faster below 703 MB/s | +| uniform 64-bit | Canonical x77 | 8.00 | 21 | 5.3 | 0.71 | | +| random in 0..999, 10 % null | BitPacked x82 | 1.38 | 47 | 5.6 | 0.59 | Smallest: Pco x82, 1.26 B/value, scan 30.5 ms, take 27.74 ms; reads faster below 49 MB/s |
a sequence: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Sequence x77 | 38,844 | 0.00 | 6 | 3.5 | 0.20 | | +| Auto | Sequence x77 | 38,844 | 0.00 | 7 | 3.4 | 0.20 | | | Fastest | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.20 | same | -| Smallest | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.20 | same | -| None | Canonical x77 | 80,043,724 | 8.00 | 18 | 5.6 | 0.80 | reads slower at any throughput | -| hint Dictionary | Sequence x77 | 38,844 | 0.00 | 76 | 3.3 | 0.22 | same | +| Smallest | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.22 | same | +| None | Canonical x77 | 80,043,724 | 8.00 | 17 | 6.0 | 0.97 | reads slower at any throughput | +| hint Dictionary | Sequence x77 | 38,844 | 0.00 | 77 | 3.3 | 0.22 | same | | hint BitPacked | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.20 | same | -| hint RunEnd | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.22 | same | -| hint Zstd | Sequence x77 | 38,844 | 0.00 | 5 | 3.4 | 0.20 | same | +| hint RunEnd | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.21 | same | +| hint Zstd | Sequence x77 | 38,844 | 0.00 | 5 | 3.3 | 0.21 | same |
@@ -536,13 +536,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x77 | 88,004 | 0.01 | 15 | 3.3 | 0.26 | | -| Fastest | RunEnd x77 | 88,004 | 0.01 | 15 | 3.4 | 0.27 | same | -| Smallest | RunEnd x76, Pco x1 | 53,748 | 0.01 | 90 | 3.8 | 0.65 | reads faster below 71 MB/s | -| None | Canonical x77 | 80,043,716 | 8.00 | 23 | 5.2 | 0.70 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 1,449,892 | 0.14 | 23 | 4.9 | 0.63 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 10,034,460 | 1.00 | 25 | 5.4 | 0.44 | reads slower at any throughput | -| hint RunEnd | RunEnd x77 | 88,004 | 0.01 | 15 | 3.4 | 0.26 | same | +| Auto | RunEnd x77 | 88,004 | 0.01 | 15 | 3.4 | 0.26 | | +| Fastest | RunEnd x77 | 88,004 | 0.01 | 15 | 3.3 | 0.27 | same | +| Smallest | RunEnd x76, Pco x1 | 53,748 | 0.01 | 91 | 3.6 | 0.58 | reads faster below 148 MB/s | +| None | Canonical x77 | 80,043,716 | 8.00 | 23 | 5.4 | 0.72 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 1,449,892 | 0.14 | 23 | 4.9 | 0.62 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 10,034,460 | 1.00 | 25 | 5.4 | 0.45 | reads slower at any throughput | +| hint RunEnd | RunEnd x77 | 88,004 | 0.01 | 14 | 3.4 | 0.25 | same | | hint Zstd | Zstd x77 | 136,628 | 0.01 | 15 | 4.5 | 1.17 | reads slower at any throughput | @@ -551,14 +551,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 33,795,172 | 3.38 | 33 | 6.4 | 0.63 | | -| Fastest | BitPacked x77 | 33,795,172 | 3.38 | 33 | 6.5 | 0.72 | same | -| Smallest | Pco x77 | 13,472,444 | 1.35 | 343 | 70.2 | 67.22 | reads faster below 319 MB/s | -| None | Canonical x77 | 80,043,724 | 8.00 | 24 | 5.3 | 0.68 | reads faster above 38,890 MB/s | -| hint Dictionary | BitPacked x77 | 33,795,172 | 3.38 | 98 | 6.3 | 0.64 | same | -| hint BitPacked | BitPacked x77 | 33,795,172 | 3.38 | 28 | 6.3 | 0.60 | same | -| hint RunEnd | BitPacked x77 | 33,795,172 | 3.38 | 39 | 6.5 | 0.72 | same | -| hint Zstd | Zstd x77 | 20,700,796 | 2.07 | 118 | 42.1 | 17.50 | reads faster below 367 MB/s | +| Auto | BitPacked x77 | 33,795,172 | 3.38 | 29 | 6.3 | 0.57 | | +| Fastest | BitPacked x77 | 33,795,172 | 3.38 | 29 | 6.4 | 0.66 | same | +| Smallest | Pco x77 | 13,472,444 | 1.35 | 340 | 71.1 | 70.68 | reads faster below 314 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 39 | 5.2 | 0.71 | reads faster above 42,418 MB/s | +| hint Dictionary | BitPacked x77 | 33,795,172 | 3.38 | 100 | 6.4 | 0.64 | same | +| hint BitPacked | BitPacked x77 | 33,795,172 | 3.38 | 27 | 6.4 | 0.66 | same | +| hint RunEnd | BitPacked x77 | 33,795,172 | 3.38 | 37 | 6.3 | 0.63 | same | +| hint Zstd | Zstd x77 | 20,700,796 | 2.07 | 118 | 42.8 | 17.90 | reads faster below 359 MB/s | @@ -566,14 +566,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.2 | 0.50 | | -| Fastest | BitPacked x77 | 12,544,284 | 1.25 | 26 | 4.1 | 0.49 | same | -| Smallest | BitPacked x76, Pco x1 | 12,543,924 | 1.25 | 257 | 4.2 | 0.62 | same | -| None | Canonical x77 | 80,043,716 | 8.00 | 25 | 5.4 | 0.70 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 12,652,724 | 1.27 | 57 | 5.2 | 0.60 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 12,544,284 | 1.25 | 26 | 4.1 | 0.48 | same | -| hint RunEnd | BitPacked x77 | 12,544,284 | 1.25 | 22 | 4.1 | 0.49 | same | -| hint Zstd | Zstd x77 | 20,141,172 | 2.01 | 159 | 48.0 | 22.40 | reads slower at any throughput | +| Auto | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.1 | 0.50 | | +| Fastest | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.1 | 0.51 | same | +| Smallest | BitPacked x76, Pco x1 | 12,543,924 | 1.25 | 255 | 4.2 | 0.58 | same | +| None | Canonical x77 | 80,043,716 | 8.00 | 21 | 7.9 | 0.97 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,652,724 | 1.27 | 57 | 5.4 | 0.65 | reads slower at any throughput | +| hint BitPacked | BitPacked x77 | 12,544,284 | 1.25 | 25 | 4.1 | 0.56 | same | +| hint RunEnd | BitPacked x77 | 12,544,284 | 1.25 | 22 | 4.1 | 0.48 | same | +| hint Zstd | Zstd x77 | 20,141,172 | 2.01 | 162 | 47.8 | 22.27 | reads slower at any throughput | @@ -581,15 +581,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.7 | 0.37 | | -| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.7 | 0.36 | same | -| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 488 | 4.9 | 0.56 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.7 | 0.75 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.7 | 0.36 | same | -| hint BitPacked | Dictionary x77 | 5,063,748 | 0.51 | 83 | 4.8 | 0.37 | same | -| hint RunEnd | Dictionary x77 | 5,063,748 | 0.51 | 76 | 4.7 | 0.38 | same | -| hint Zstd | Zstd x77 | 10,539,396 | 1.05 | 116 | 37.5 | 15.70 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 54 | 4.7 | 0.33 | same | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.9 | 0.35 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.8 | 0.37 | same | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 497 | 5.0 | 0.57 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.9 | 0.82 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 47 | 4.8 | 0.40 | same | +| hint BitPacked | Dictionary x77 | 5,063,748 | 0.51 | 83 | 4.7 | 0.36 | same | +| hint RunEnd | Dictionary x77 | 5,063,748 | 0.51 | 78 | 4.7 | 0.38 | same | +| hint Zstd | Zstd x77 | 10,539,396 | 1.05 | 119 | 37.9 | 16.18 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 54 | 4.8 | 0.33 | same | @@ -597,15 +597,15 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x77 | 46,295,652 | 4.63 | 33 | 5.4 | 0.61 | | -| Fastest | BitPacked x77 | 46,295,652 | 4.63 | 33 | 5.6 | 0.68 | reads slower at any throughput | -| Smallest | Pco x77 | 580,924 | 0.06 | 339 | 69.3 | 67.81 | reads faster below 715 MB/s | -| None | Canonical x77 | 80,043,724 | 8.00 | 20 | 5.3 | 0.75 | reads slower at any throughput | -| hint Dictionary | BitPacked x77 | 46,295,652 | 4.63 | 94 | 5.8 | 0.71 | reads slower at any throughput | -| hint BitPacked | BitPacked x77 | 46,295,652 | 4.63 | 32 | 5.7 | 0.71 | reads slower at any throughput | -| hint RunEnd | BitPacked x77 | 46,295,652 | 4.63 | 49 | 5.7 | 0.70 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 51,815,868 | 5.18 | 95 | 55.0 | 37.55 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 23,603,548 | 2.36 | 81 | 6.0 | 0.65 | reads faster below 35,974 MB/s | +| Auto | BitPacked x77 | 46,295,652 | 4.63 | 30 | 5.7 | 0.70 | | +| Fastest | BitPacked x77 | 46,295,652 | 4.63 | 45 | 5.9 | 0.71 | same | +| Smallest | Pco x77 | 580,924 | 0.06 | 340 | 70.6 | 68.60 | reads faster below 703 MB/s | +| None | Canonical x77 | 80,043,724 | 8.00 | 19 | 5.1 | 0.61 | reads faster above 57,094 MB/s | +| hint Dictionary | BitPacked x77 | 46,295,652 | 4.63 | 97 | 5.6 | 0.69 | same | +| hint BitPacked | BitPacked x77 | 46,295,652 | 4.63 | 36 | 5.7 | 0.68 | same | +| hint RunEnd | BitPacked x77 | 46,295,652 | 4.63 | 52 | 5.7 | 0.68 | same | +| hint Zstd | Zstd x77 | 51,815,868 | 5.18 | 107 | 54.3 | 36.65 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 23,603,548 | 2.36 | 81 | 5.9 | 0.68 | reads faster at any throughput | @@ -613,14 +613,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.6 | 0.76 | | -| Fastest | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.3 | 0.72 | reads faster at any throughput | -| Smallest | Canonical x77 | 80,043,732 | 8.00 | 242 | 5.4 | 0.75 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.3 | 0.69 | reads faster at any throughput | -| hint Dictionary | Canonical x77 | 80,043,732 | 8.00 | 109 | 5.3 | 0.74 | same | -| hint BitPacked | Canonical x77 | 80,043,732 | 8.00 | 85 | 5.4 | 0.71 | same | -| hint RunEnd | Canonical x77 | 80,043,732 | 8.00 | 78 | 5.3 | 0.76 | same | -| hint Zstd | Canonical x77 | 80,043,732 | 8.00 | 88 | 5.3 | 0.70 | reads faster at any throughput | +| Auto | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.3 | 0.71 | | +| Fastest | Canonical x77 | 80,043,732 | 8.00 | 21 | 5.3 | 0.69 | same | +| Smallest | Canonical x77 | 80,043,732 | 8.00 | 241 | 6.5 | 0.82 | reads slower at any throughput | +| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.4 | 0.67 | same | +| hint Dictionary | Canonical x77 | 80,043,732 | 8.00 | 109 | 5.4 | 0.74 | same | +| hint BitPacked | Canonical x77 | 80,043,732 | 8.00 | 84 | 5.3 | 0.70 | same | +| hint RunEnd | Canonical x77 | 80,043,732 | 8.00 | 103 | 5.5 | 0.82 | same | +| hint Zstd | Canonical x77 | 80,043,732 | 8.00 | 88 | 5.3 | 0.70 | same | @@ -628,14 +628,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.6 | 0.56 | | -| Fastest | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.8 | 0.59 | same | -| Smallest | Pco x82 | 12,567,268 | 1.26 | 277 | 30.7 | 27.95 | reads faster below 49 MB/s | -| None | Canonical x82 | 81,294,668 | 8.13 | 40 | 5.3 | 0.74 | reads faster above 209,674 MB/s | -| hint Dictionary | Dictionary x82 | 12,676,396 | 1.27 | 80 | 7.1 | 0.60 | reads faster below 766 MB/s | -| hint BitPacked | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.5 | 0.54 | same | -| hint RunEnd | BitPacked x82 | 13,800,460 | 1.38 | 48 | 5.5 | 0.54 | same | -| hint Zstd | Zstd x82 | 19,283,940 | 1.93 | 168 | 49.4 | 21.00 | reads slower at any throughput | +| Auto | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.6 | 0.59 | | +| Fastest | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.6 | 0.53 | same | +| Smallest | Pco x82 | 12,567,268 | 1.26 | 278 | 30.5 | 27.74 | reads faster below 49 MB/s | +| None | Canonical x82 | 81,294,668 | 8.13 | 39 | 5.4 | 0.78 | reads slower at any throughput | +| hint Dictionary | Dictionary x82 | 12,676,396 | 1.27 | 80 | 7.1 | 0.61 | reads faster below 762 MB/s | +| hint BitPacked | BitPacked x82 | 13,800,460 | 1.38 | 47 | 5.5 | 0.53 | same | +| hint RunEnd | BitPacked x82 | 13,800,460 | 1.38 | 48 | 5.6 | 0.56 | same | +| hint Zstd | Zstd x82 | 19,283,940 | 1.93 | 170 | 49.1 | 20.84 | reads slower at any throughput | @@ -644,22 +644,22 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| | prices (2 decimals) | Alp x77 | 2.50 | 40 | 7.2 | 0.66 | | -| 16 distinct, random order | Dictionary x77 | 0.51 | 46 | 4.8 | 0.38 | | -| 1 000 distinct prices | Dictionary x77 | 1.28 | 60 | 5.2 | 0.58 | | -| 100 003 distinct, repeating | Zstd x77 | 1.69 | 119 | 35.0 | 14.98 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 7.5 ms, take 2.30 ms; reads faster above 190 MB/s; None: Canonical x77, 8.00 B/value, scan 5.4 ms, take 0.75 ms; reads faster above 2,134 MB/s | -| uniform in [0, 1) | AlpRd x77 | 6.91 | 39 | 11.1 | 1.00 | None: Canonical x77, 8.00 B/value, scan 5.3 ms, take 0.70 ms; reads faster above 1,902 MB/s | +| 16 distinct, random order | Dictionary x77 | 0.51 | 46 | 4.8 | 0.37 | | +| 1 000 distinct prices | Dictionary x77 | 1.28 | 60 | 5.3 | 0.60 | | +| 100 003 distinct, repeating | Zstd x77 | 1.69 | 119 | 35.0 | 14.82 | Auto, 16 MiB chunks: Dictionary x5, 2.21 B/value, scan 7.4 ms, take 2.23 ms; reads faster above 189 MB/s; None: Canonical x77, 8.00 B/value, scan 5.6 ms, take 0.78 ms; reads faster above 2,147 MB/s | +| uniform in [0, 1) | AlpRd x77 | 6.91 | 41 | 10.9 | 0.92 | None: Canonical x77, 8.00 B/value, scan 5.3 ms, take 0.72 ms; reads faster above 1,959 MB/s |
prices (2 decimals): every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| | Auto | Alp x77 | 25,049,732 | 2.50 | 40 | 7.2 | 0.66 | | -| Fastest | Alp x77 | 25,049,732 | 2.50 | 40 | 7.2 | 0.64 | same | -| Smallest | Alp x77 | 25,048,868 | 2.50 | 709 | 7.4 | 0.80 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.4 | 0.77 | reads faster above 31,123 MB/s | -| hint Dictionary | Alp x77 | 25,049,732 | 2.50 | 136 | 7.3 | 0.66 | same | -| hint Alp | Alp x77 | 25,049,732 | 2.50 | 40 | 7.2 | 0.64 | same | -| hint Zstd | Zstd x77 | 41,240,260 | 4.12 | 246 | 51.4 | 28.31 | reads slower at any throughput | +| Fastest | Alp x77 | 25,049,732 | 2.50 | 40 | 7.7 | 0.67 | reads slower at any throughput | +| Smallest | Alp x77 | 25,048,868 | 2.50 | 709 | 7.3 | 0.75 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.4 | 0.67 | reads faster above 29,330 MB/s | +| hint Dictionary | Alp x77 | 25,049,732 | 2.50 | 136 | 7.2 | 0.65 | same | +| hint Alp | Alp x77 | 25,049,732 | 2.50 | 39 | 7.3 | 0.71 | same | +| hint Zstd | Zstd x77 | 41,240,260 | 4.12 | 248 | 51.0 | 27.80 | reads slower at any throughput |
@@ -667,13 +667,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 5,063,748 | 0.51 | 46 | 4.8 | 0.38 | | -| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 46 | 4.8 | 0.37 | same | -| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 491 | 5.0 | 0.55 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.3 | 0.68 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 47 | 4.8 | 0.37 | same | -| hint Alp | Dictionary x77 | 5,063,748 | 0.51 | 95 | 4.7 | 0.36 | same | -| hint Zstd | Zstd x77 | 10,545,668 | 1.05 | 119 | 37.3 | 15.72 | reads slower at any throughput | +| Auto | Dictionary x77 | 5,063,748 | 0.51 | 46 | 4.8 | 0.37 | | +| Fastest | Dictionary x77 | 5,063,748 | 0.51 | 47 | 4.8 | 0.36 | same | +| Smallest | Dictionary x76, Pco x1 | 5,063,588 | 0.51 | 497 | 4.9 | 0.55 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.3 | 0.66 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 5,063,748 | 0.51 | 48 | 4.9 | 0.40 | same | +| hint Alp | Dictionary x77 | 5,063,748 | 0.51 | 97 | 4.8 | 0.36 | same | +| hint Zstd | Zstd x77 | 10,545,668 | 1.05 | 120 | 37.3 | 15.73 | reads slower at any throughput | | Auto, 16 MiB chunks | Dictionary x5 | 5,032,932 | 0.50 | 51 | 4.7 | 0.34 | same | @@ -682,14 +682,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.2 | 0.58 | | -| Fastest | Dictionary x77 | 12,756,260 | 1.28 | 59 | 5.3 | 0.60 | same | -| Smallest | Dictionary x77 | 12,751,356 | 1.28 | 401 | 5.4 | 0.78 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.5 | 0.82 | reads slower at any throughput | -| hint Dictionary | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.3 | 0.61 | same | -| hint Alp | Alp x77 | 12,756,260 | 1.28 | 94 | 6.6 | 0.59 | reads slower at any throughput | -| hint Zstd | Zstd x77 | 26,073,412 | 2.61 | 192 | 39.3 | 18.93 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x5 | 12,545,732 | 1.25 | 58 | 5.2 | 0.53 | reads faster at any throughput | +| Auto | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.3 | 0.60 | | +| Fastest | Dictionary x77 | 12,756,260 | 1.28 | 61 | 5.2 | 0.60 | same | +| Smallest | Dictionary x77 | 12,751,356 | 1.28 | 401 | 5.5 | 0.78 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.3 | 0.77 | reads slower at any throughput | +| hint Dictionary | Dictionary x77 | 12,756,260 | 1.28 | 60 | 5.3 | 0.60 | same | +| hint Alp | Alp x77 | 12,756,260 | 1.28 | 94 | 6.7 | 0.60 | reads slower at any throughput | +| hint Zstd | Zstd x77 | 26,073,412 | 2.61 | 194 | 39.6 | 19.30 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x5 | 12,545,732 | 1.25 | 59 | 5.1 | 0.54 | reads faster at any throughput | @@ -697,14 +697,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.0 | 14.98 | | -| Fastest | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.1 | 14.89 | same | -| Smallest | Zstd x77 | 16,909,892 | 1.69 | 229 | 35.0 | 14.81 | same | -| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.4 | 0.75 | reads faster above 2,134 MB/s | -| hint Dictionary | Zstd x77 | 16,909,892 | 1.69 | 205 | 35.3 | 15.24 | same | -| hint Alp | Zstd x77 | 16,909,892 | 1.69 | 191 | 34.8 | 14.88 | same | -| hint Zstd | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.1 | 14.94 | same | -| Auto, 16 MiB chunks | Dictionary x5 | 22,129,988 | 2.21 | 87 | 7.5 | 2.30 | reads faster above 190 MB/s | +| Auto | Zstd x77 | 16,909,892 | 1.69 | 119 | 35.0 | 14.82 | | +| Fastest | Zstd x77 | 16,909,892 | 1.69 | 119 | 34.9 | 15.22 | same | +| Smallest | Zstd x77 | 16,909,892 | 1.69 | 228 | 35.4 | 15.30 | same | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.6 | 0.78 | reads faster above 2,147 MB/s | +| hint Dictionary | Zstd x77 | 16,909,892 | 1.69 | 203 | 34.9 | 14.83 | same | +| hint Alp | Zstd x77 | 16,909,892 | 1.69 | 191 | 35.1 | 14.96 | same | +| hint Zstd | Zstd x77 | 16,909,892 | 1.69 | 118 | 35.3 | 15.39 | same | +| Auto, 16 MiB chunks | Dictionary x5 | 22,129,988 | 2.21 | 86 | 7.4 | 2.23 | reads faster above 189 MB/s | @@ -712,13 +712,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | AlpRd x77 | 69,054,380 | 6.91 | 39 | 11.1 | 1.00 | | -| Fastest | AlpRd x77 | 69,054,380 | 6.91 | 40 | 11.1 | 1.04 | same | -| Smallest | Pco x77 | 67,609,668 | 6.76 | 414 | 67.9 | 66.15 | reads faster below 25 MB/s | -| None | Canonical x77 | 80,043,732 | 8.00 | 19 | 5.3 | 0.70 | reads faster above 1,902 MB/s | -| hint Dictionary | AlpRd x77 | 69,054,380 | 6.91 | 185 | 11.0 | 0.98 | same | -| hint Alp | AlpRd x77 | 69,054,380 | 6.91 | 164 | 11.1 | 1.03 | same | -| hint Zstd | AlpRd x77 | 69,054,380 | 6.91 | 207 | 11.2 | 1.01 | same | +| Auto | AlpRd x77 | 69,054,380 | 6.91 | 41 | 10.9 | 0.92 | | +| Fastest | AlpRd x77 | 69,054,380 | 6.91 | 56 | 11.3 | 1.07 | same | +| Smallest | Pco x77 | 67,609,668 | 6.76 | 416 | 70.2 | 67.21 | reads faster below 24 MB/s | +| None | Canonical x77 | 80,043,732 | 8.00 | 20 | 5.3 | 0.72 | reads faster above 1,959 MB/s | +| hint Dictionary | AlpRd x77 | 69,054,380 | 6.91 | 184 | 11.0 | 1.03 | same | +| hint Alp | AlpRd x77 | 69,054,380 | 6.91 | 166 | 12.1 | 1.10 | reads slower at any throughput | +| hint Zstd | AlpRd x77 | 69,054,380 | 6.91 | 208 | 11.2 | 1.02 | same | @@ -726,23 +726,23 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| 16 cities, random order | Dictionary x175 | 0.51 | 108 | 7.7 | 0.58 | | -| 10 000 distinct ids | Dictionary x306 | 2.54 | 566 | 29.1 | 20.92 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 11.0 ms, take 2.08 ms; reads faster at any throughput | -| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 581 | 31.4 | 21.05 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 11.7 ms, take 2.07 ms; reads faster at any throughput | -| unique UUIDs | Zstd x611 | 20.61 | 1283 | 240.1 | 235.37 | hint Fsst: Fsst x611, 22.44 B/value, scan 80.8 ms, take 4.84 ms; reads faster above 115 MB/s | -| log lines (~100 B) | Zstd x1221 | 13.71 | 1244 | 309.7 | 250.45 | hint Fsst: Fsst x1221, 22.01 B/value, scan 100.9 ms, take 6.26 ms; reads faster above 397 MB/s | +| 16 cities, random order | Dictionary x175 | 0.51 | 107 | 7.4 | 0.56 | | +| 10 000 distinct ids | Dictionary x306 | 2.54 | 568 | 28.7 | 20.28 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 11.0 ms, take 2.07 ms; reads faster at any throughput | +| 10 000 distinct ids, 10 % null | Dictionary x306 | 2.45 | 581 | 30.9 | 20.88 | Auto, 16 MiB chunks: Dictionary x18, 1.80 B/value, scan 11.9 ms, take 2.09 ms; reads faster at any throughput | +| unique UUIDs | Zstd x611 | 20.61 | 1278 | 239.9 | 237.15 | hint Fsst: Fsst x611, 22.44 B/value, scan 81.1 ms, take 4.90 ms; reads faster above 115 MB/s | +| log lines (~100 B) | Zstd x1221 | 13.71 | 1225 | 309.9 | 250.07 | hint Fsst: Fsst x1221, 22.01 B/value, scan 112.8 ms, take 7.06 ms; reads faster above 421 MB/s |
16 cities, random order: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x175 | 5,117,284 | 0.51 | 108 | 7.7 | 0.58 | | -| Fastest | Dictionary x175 | 5,117,284 | 0.51 | 108 | 7.6 | 0.56 | same | -| Smallest | Dictionary x175 | 5,117,284 | 0.51 | 269 | 7.6 | 0.57 | same | -| None | Canonical x175 | 96,746,756 | 9.67 | 123 | 26.5 | 5.64 | reads slower at any throughput | -| hint Dictionary | Dictionary x175 | 5,117,284 | 0.51 | 108 | 7.7 | 0.57 | same | -| hint Fsst | Fsst x175 | 41,142,948 | 4.11 | 255 | 26.8 | 2.67 | reads slower at any throughput | -| hint Zstd | Zstd x175 | 17,427,100 | 1.74 | 214 | 71.1 | 48.75 | reads slower at any throughput | +| Auto | Dictionary x175 | 5,117,284 | 0.51 | 107 | 7.4 | 0.56 | | +| Fastest | Dictionary x175 | 5,117,284 | 0.51 | 106 | 7.6 | 0.55 | same | +| Smallest | Dictionary x175 | 5,117,284 | 0.51 | 268 | 7.5 | 0.56 | same | +| None | Canonical x175 | 96,746,756 | 9.67 | 122 | 26.7 | 5.76 | reads slower at any throughput | +| hint Dictionary | Dictionary x175 | 5,117,284 | 0.51 | 106 | 7.6 | 0.57 | same | +| hint Fsst | Fsst x175 | 41,142,948 | 4.11 | 256 | 26.7 | 2.58 | reads slower at any throughput | +| hint Zstd | Zstd x175 | 17,427,100 | 1.74 | 213 | 70.7 | 49.07 | reads slower at any throughput |
@@ -750,14 +750,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 25,433,788 | 2.54 | 566 | 29.1 | 20.92 | | -| Fastest | Dictionary x306 | 25,433,788 | 2.54 | 566 | 29.0 | 20.59 | same | -| Smallest | Dictionary x305, Zstd x1 | 23,452,500 | 2.35 | 1560 | 96.9 | 90.28 | reads faster below 29 MB/s | -| None | Canonical x306 | 140,098,172 | 14.01 | 124 | 24.2 | 9.90 | reads faster above 23,317 MB/s | -| hint Dictionary | Dictionary x306 | 25,433,788 | 2.54 | 565 | 28.9 | 20.65 | same | -| hint Fsst | Fsst x306 | 62,863,260 | 6.29 | 339 | 29.5 | 3.40 | reads slower at any throughput | -| hint Zstd | Zstd x306 | 27,097,628 | 2.71 | 315 | 64.7 | 44.04 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 18,014,012 | 1.80 | 306 | 11.0 | 2.08 | reads faster at any throughput | +| Auto | Dictionary x306 | 25,433,788 | 2.54 | 568 | 28.7 | 20.28 | | +| Fastest | Dictionary x306 | 25,433,788 | 2.54 | 566 | 29.0 | 20.48 | same | +| Smallest | Dictionary x305, Zstd x1 | 23,452,500 | 2.35 | 1552 | 98.4 | 91.02 | reads faster below 28 MB/s | +| None | Canonical x306 | 140,098,172 | 14.01 | 132 | 24.3 | 10.31 | reads faster above 26,217 MB/s | +| hint Dictionary | Dictionary x306 | 25,433,788 | 2.54 | 567 | 28.8 | 20.28 | same | +| hint Fsst | Fsst x306 | 62,863,260 | 6.29 | 341 | 28.7 | 3.09 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 27,097,628 | 2.71 | 318 | 64.9 | 43.54 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 18,014,012 | 1.80 | 305 | 11.0 | 2.07 | reads faster at any throughput | @@ -765,14 +765,14 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Dictionary x306 | 24,539,284 | 2.45 | 581 | 31.4 | 21.05 | | -| Fastest | Dictionary x306 | 24,539,284 | 2.45 | 578 | 31.4 | 21.23 | same | -| Smallest | Dictionary x305, Zstd x1 | 22,854,628 | 2.29 | 1572 | 99.2 | 89.20 | reads faster below 25 MB/s | -| None | Canonical x306 | 151,883,732 | 15.19 | 179 | 35.5 | 9.91 | reads slower at any throughput | -| hint Dictionary | Dictionary x306 | 24,539,284 | 2.45 | 583 | 31.4 | 21.13 | same | -| hint Fsst | Fsst x306 | 64,408,284 | 6.44 | 403 | 39.4 | 3.96 | reads slower at any throughput | -| hint Zstd | Zstd x306 | 25,595,636 | 2.56 | 323 | 78.5 | 39.35 | reads slower at any throughput | -| Auto, 16 MiB chunks | Dictionary x18 | 17,982,604 | 1.80 | 351 | 11.7 | 2.07 | reads faster at any throughput | +| Auto | Dictionary x306 | 24,539,284 | 2.45 | 581 | 30.9 | 20.88 | | +| Fastest | Dictionary x306 | 24,539,284 | 2.45 | 579 | 31.0 | 21.03 | same | +| Smallest | Dictionary x305, Zstd x1 | 22,854,628 | 2.29 | 1576 | 98.9 | 89.37 | reads faster below 25 MB/s | +| None | Canonical x306 | 151,883,732 | 15.19 | 184 | 36.0 | 9.80 | reads slower at any throughput | +| hint Dictionary | Dictionary x306 | 24,539,284 | 2.45 | 579 | 30.9 | 20.82 | same | +| hint Fsst | Fsst x306 | 64,408,284 | 6.44 | 409 | 39.2 | 3.85 | reads slower at any throughput | +| hint Zstd | Zstd x306 | 25,595,636 | 2.56 | 333 | 78.9 | 39.63 | reads slower at any throughput | +| Auto, 16 MiB chunks | Dictionary x18 | 17,982,604 | 1.80 | 356 | 11.9 | 2.09 | reads faster at any throughput | @@ -780,13 +780,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x611 | 206,058,588 | 20.61 | 1283 | 240.1 | 235.37 | | -| Fastest | Zstd x611 | 206,058,588 | 20.61 | 1273 | 238.9 | 235.45 | same | -| Smallest | Zstd x611 | 206,058,588 | 20.61 | 2794 | 239.8 | 234.84 | same | -| None | Canonical x611 | 360,155,516 | 36.02 | 118 | 45.1 | 25.77 | reads faster above 790 MB/s | -| hint Dictionary | Zstd x610, Fsst x1 | 206,059,580 | 20.61 | 5780 | 238.7 | 235.21 | same | -| hint Fsst | Fsst x611 | 224,384,604 | 22.44 | 1423 | 80.8 | 4.84 | reads faster above 115 MB/s | -| hint Zstd | Zstd x611 | 206,058,588 | 20.61 | 1267 | 240.6 | 235.82 | same | +| Auto | Zstd x611 | 206,058,588 | 20.61 | 1278 | 239.9 | 237.15 | | +| Fastest | Zstd x611 | 206,058,588 | 20.61 | 1300 | 240.7 | 235.42 | same | +| Smallest | Zstd x611 | 206,058,588 | 20.61 | 2766 | 239.4 | 235.93 | same | +| None | Canonical x611 | 360,155,516 | 36.02 | 132 | 45.3 | 25.52 | reads faster above 792 MB/s | +| hint Dictionary | Zstd x610, Fsst x1 | 206,059,580 | 20.61 | 5797 | 240.6 | 234.63 | same | +| hint Fsst | Fsst x611 | 224,384,604 | 22.44 | 1420 | 81.1 | 4.90 | reads faster above 115 MB/s | +| hint Zstd | Zstd x611 | 206,058,588 | 20.61 | 1270 | 239.6 | 234.74 | same | @@ -794,13 +794,13 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Zstd x1221 | 137,107,188 | 13.71 | 1244 | 309.7 | 250.45 | | -| Fastest | Zstd x1221 | 137,107,188 | 13.71 | 1212 | 308.8 | 249.97 | same | -| Smallest | Zstd x1221 | 137,107,188 | 13.71 | 2666 | 309.7 | 249.00 | same | -| None | Canonical x1221 | 725,468,444 | 72.55 | 431 | 79.0 | 41.25 | reads faster above 2,550 MB/s | -| hint Dictionary | Zstd x1221 | 137,107,188 | 13.71 | 5652 | 309.3 | 250.08 | same | -| hint Fsst | Fsst x1221 | 220,058,684 | 22.01 | 1737 | 100.9 | 6.26 | reads faster above 397 MB/s | -| hint Zstd | Zstd x1221 | 137,107,188 | 13.71 | 1212 | 310.7 | 251.04 | same | +| Auto | Zstd x1221 | 137,107,188 | 13.71 | 1225 | 309.9 | 250.07 | | +| Fastest | Zstd x1221 | 137,107,188 | 13.71 | 1205 | 310.0 | 251.38 | same | +| Smallest | Zstd x1221 | 137,107,188 | 13.71 | 2651 | 309.9 | 250.29 | same | +| None | Canonical x1221 | 725,468,444 | 72.55 | 391 | 79.7 | 41.42 | reads faster above 2,556 MB/s | +| hint Dictionary | Zstd x1221 | 137,107,188 | 13.71 | 5632 | 308.4 | 248.70 | same | +| hint Fsst | Fsst x1221 | 220,058,684 | 22.01 | 1710 | 112.8 | 7.06 | reads faster above 421 MB/s | +| hint Zstd | Zstd x1221 | 137,107,188 | 13.71 | 1200 | 307.5 | 249.53 | same | @@ -808,18 +808,18 @@ GB/s; and the fastest take, when it halves `Auto`'s. | shape | `Auto` writes | B/value | write ms | scan ms | take ms | worth knowing | |---|---|---:|---:|---:|---:|---| -| half true | Canonical x10 | 0.13 | 3 | 0.2 | 0.15 | | -| 1 % true | RunEnd x10 | 0.05 | 3 | 1.7 | 0.30 | None: Canonical x10, 0.13 B/value, scan 0.2 ms, take 0.18 ms; reads faster above 488 MB/s | +| half true | Canonical x10 | 0.13 | 2 | 0.2 | 0.18 | | +| 1 % true | RunEnd x10 | 0.05 | 3 | 1.6 | 0.30 | None: Canonical x10, 0.13 B/value, scan 0.2 ms, take 0.21 ms; reads faster above 506 MB/s |
half true: every configuration | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | Canonical x10 | 1,262,556 | 0.13 | 3 | 0.2 | 0.15 | | -| Fastest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.1 | 0.13 | reads faster at any throughput | -| Smallest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.22 | reads slower at any throughput | -| None | Canonical x10 | 1,262,556 | 0.13 | 1 | 0.2 | 0.19 | reads slower at any throughput | -| hint RunEnd | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.21 | reads slower at any throughput | +| Auto | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.18 | | +| Fastest | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.19 | same | +| Smallest | Canonical x10 | 1,262,556 | 0.13 | 3 | 0.2 | 0.19 | same | +| None | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.20 | reads slower at any throughput | +| hint RunEnd | Canonical x10 | 1,262,556 | 0.13 | 3 | 0.2 | 0.18 | same |
@@ -827,11 +827,11 @@ GB/s; and the fastest take, when it halves `Auto`'s. | configuration | written as | bytes | B/value | write ms | scan ms | take ms | against Auto | |---|---|---:|---:|---:|---:|---:|---| -| Auto | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.30 | | -| Fastest | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.31 | same | -| Smallest | RunEnd x10 | 164,692 | 0.02 | 10 | 2.9 | 1.46 | reads faster below 314 MB/s | -| None | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.18 | reads faster above 488 MB/s | -| hint RunEnd | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.28 | same | +| Auto | RunEnd x10 | 548,092 | 0.05 | 3 | 1.6 | 0.30 | | +| Fastest | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.30 | same | +| Smallest | RunEnd x10 | 164,692 | 0.02 | 10 | 2.9 | 1.47 | reads faster below 307 MB/s | +| None | Canonical x10 | 1,262,556 | 0.13 | 2 | 0.2 | 0.21 | reads faster above 506 MB/s | +| hint RunEnd | RunEnd x10 | 548,092 | 0.05 | 3 | 1.7 | 0.32 | same | @@ -840,7 +840,7 @@ GB/s; and the fastest take, when it halves `Auto`'s. `Fastest` wrote what `Auto` wrote on 20 of 20 columns. `Smallest` wrote something else on *i64, sorted runs of 1 000*, *i64, timestamps (ms, increasing, jittered)*, *i64, random in 0..999*, *i64, 16 distinct, random order*, *i64, 100 003 distinct, repeating*, *i64, random in 0..999, 10 % null*, *f64, 16 distinct, random order*, *f64, uniform in [0, 1)*, *utf8, 10 000 distinct ids*, *utf8, 10 000 distinct ids, 10 % null*, and took up to 17.6 times `Auto`'s write, 709 ms against 40 ms on *f64, prices (2 decimals)*, since it tries every scheme on every chunk. `None`, the plain form, made the largest file on 20 of 20 columns. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 32b3a82b with uncommitted changes, 2026-10-10 02:22 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 6403ae1a with uncommitted changes, 2026-10-10 04:21 UTC.* @@ -866,7 +866,7 @@ does not; a column it never departs on is left out. It keeps the writer's choice under every goal on the 10 other columns. -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 32b3a82b with uncommitted changes, 2026-10-10 02:24 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; commit 6403ae1a with uncommitted changes, 2026-10-10 04:23 UTC.* ## Kernels, against their baselines @@ -883,14 +883,14 @@ reference profile on that class, the one to quote a small difference from. | Method | BitWidth | Mean | Error | StdDev | Speedup | MannWhitney(5%) | ns/row | GB/s | Allocated | Alloc Ratio | |------------- |--------- |----------:|----------:|----------:|--------:|---------------- |-------:|------:|----------:|------------:| -| 'i64 scalar' | 17 | 76.387 μs | 1.8615 μs | 0.4834 μs | 1.00x | Baseline | 1.17 | 6.86 | - | NA | -| 'i64 vector' | 17 | 8.280 μs | 0.1418 μs | 0.0368 μs | 9.23x | Faster | 0.13 | 63.32 | - | NA | -| 'i32 scalar' | 17 | 86.997 μs | 1.7466 μs | 0.4536 μs | 0.88x | Same | 1.33 | 3.01 | - | NA | -| 'i64 pack' | 17 | 22.053 μs | 0.4604 μs | 0.1196 μs | 3.46x | Faster | 0.34 | 23.77 | - | NA | -| 'i32 pack' | 17 | 11.711 μs | 0.3273 μs | 0.0507 μs | 6.52x | Faster | 0.18 | 22.38 | - | NA | -| 'i32 vector' | 17 | 4.745 μs | 0.0168 μs | 0.0043 μs | 16.1x | Faster | 0.07 | 55.25 | - | NA | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* +| 'i64 scalar' | 17 | 75.748 μs | 0.8728 μs | 0.2267 μs | 1.00x | Baseline | 1.16 | 6.92 | - | NA | +| 'i64 vector' | 17 | 8.295 μs | 0.0571 μs | 0.0148 μs | 9.13x | Faster | 0.13 | 63.20 | - | NA | +| 'i32 scalar' | 17 | 87.020 μs | 1.3047 μs | 0.3388 μs | 0.87x | Same | 1.33 | 3.01 | - | NA | +| 'i64 pack' | 17 | 20.993 μs | 0.1760 μs | 0.0272 μs | 3.61x | Faster | 0.32 | 24.97 | - | NA | +| 'i32 pack' | 17 | 11.194 μs | 0.2068 μs | 0.0320 μs | 6.77x | Faster | 0.17 | 23.42 | - | NA | +| 'i32 vector' | 17 | 5.545 μs | 0.0665 μs | 0.0173 μs | 13.7x | Faster | 0.08 | 47.28 | - | NA | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 6403ae1a with uncommitted changes, 2026-10-10 04:24 UTC.* @@ -898,43 +898,43 @@ reference profile on that class, the one to quote a small difference from. | Method | Entry | Mean | Error | StdDev | ns/row | GB/s | Allocated | |----------------------------------------- |--------------------- |-----------:|-----------:|----------:|-------:|-----:|----------:| -| **'encode a batch to row keys'** | **conta(...)nical [33]** | **48.246 μs** | **0.4884 μs** | **0.1268 μs** | **11.78** | **3.65** | **48 B** | -| 'encode a batch to row keys, descending' | conta(...)nical [33] | 45.855 μs | 0.2506 μs | 0.0388 μs | 11.19 | 3.84 | 48 B | -| 'encode, then sort rows by key' | conta(...)nical [33] | 344.474 μs | 27.3616 μs | 4.2342 μs | 84.10 | 0.51 | 48 B | -| **'encode a batch to row keys'** | **conta(...)nulls [33]** | **14.714 μs** | **0.0875 μs** | **0.0227 μs** | **14.37** | **4.59** | **48 B** | -| 'encode a batch to row keys, descending' | conta(...)nulls [33] | 14.614 μs | 0.0758 μs | 0.0117 μs | 14.27 | 4.62 | 48 B | -| 'encode, then sort rows by key' | conta(...)nulls [33] | 47.087 μs | 3.1135 μs | 0.8086 μs | 45.98 | 1.44 | 48 B | -| **'encode a batch to row keys'** | **encod(...)r1025 [31]** | **6.381 μs** | **0.1115 μs** | **0.0290 μs** | **6.22** | **1.45** | **48 B** | -| 'encode a batch to row keys, descending' | encod(...)r1025 [31] | 6.400 μs | 0.1260 μs | 0.0327 μs | 6.24 | 1.44 | 48 B | -| 'encode, then sort rows by key' | encod(...)r1025 [31] | 36.293 μs | 2.3583 μs | 0.6125 μs | 35.41 | 0.25 | 48 B | - -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* +| **'encode a batch to row keys'** | **conta(...)nical [33]** | **47.741 μs** | **0.6224 μs** | **0.1616 μs** | **11.66** | **3.69** | **48 B** | +| 'encode a batch to row keys, descending' | conta(...)nical [33] | 45.581 μs | 0.6575 μs | 0.1708 μs | 11.13 | 3.86 | 48 B | +| 'encode, then sort rows by key' | conta(...)nical [33] | 348.210 μs | 14.9578 μs | 3.8845 μs | 85.01 | 0.51 | 48 B | +| **'encode a batch to row keys'** | **conta(...)nulls [33]** | **14.585 μs** | **0.1354 μs** | **0.0352 μs** | **14.24** | **4.63** | **48 B** | +| 'encode a batch to row keys, descending' | conta(...)nulls [33] | 14.773 μs | 0.1222 μs | 0.0317 μs | 14.43 | 4.57 | 48 B | +| 'encode, then sort rows by key' | conta(...)nulls [33] | 46.320 μs | 2.0464 μs | 0.5314 μs | 45.23 | 1.46 | 48 B | +| **'encode a batch to row keys'** | **encod(...)r1025 [31]** | **6.296 μs** | **0.0900 μs** | **0.0139 μs** | **6.14** | **1.47** | **48 B** | +| 'encode a batch to row keys, descending' | encod(...)r1025 [31] | 6.382 μs | 0.0572 μs | 0.0149 μs | 6.23 | 1.45 | 48 B | +| 'encode, then sort rows by key' | encod(...)r1025 [31] | 36.079 μs | 2.2510 μs | 0.5846 μs | 35.20 | 0.26 | 48 B | + +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 6403ae1a with uncommitted changes, 2026-10-10 04:24 UTC.* ### `CompressorBenchmarks` -| Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | -|----------------------------- |------- |--------------:|------------:|------------:|--------:|---------------- |----------:|------------:| -| 'Choose, the whole decision' | i64 | 43,085.478 ns | 792.0855 ns | 205.7021 ns | 1.00x | Baseline | - | NA | -| 'candidate: sequence' | i64 | 7.948 ns | 0.0725 ns | 0.0188 ns | 5,421x | Faster | - | NA | -| 'candidate: bit packing' | i64 | 6,606.360 ns | 310.5553 ns | 80.6502 ns | 6.52x | Faster | - | NA | -| 'candidate: FSST' | i64 | 1.550 ns | 0.0173 ns | 0.0045 ns | 27,793x | Faster | - | NA | -| 'candidate: zstd' | i64 | 8,612.868 ns | 124.5499 ns | 19.2742 ns | 5.00x | Faster | - | NA | -| 'candidate: ALP' | i64 | 1.760 ns | 0.0394 ns | 0.0102 ns | 24,480x | Faster | - | NA | +| Method | Column | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Allocated | Alloc Ratio | +|----------------------------- |------- |--------------:|------------:|-----------:|--------:|---------------- |----------:|------------:| +| 'Choose, the whole decision' | i64 | 42,651.392 ns | 332.2405 ns | 86.2818 ns | 1.00x | Baseline | - | NA | +| 'candidate: sequence' | i64 | 6.961 ns | 0.1711 ns | 0.0444 ns | 6,127x | Faster | - | NA | +| 'candidate: bit packing' | i64 | 6,606.545 ns | 87.0589 ns | 22.6089 ns | 6.46x | Faster | - | NA | +| 'candidate: FSST' | i64 | 1.546 ns | 0.0351 ns | 0.0054 ns | 27,591x | Faster | - | NA | +| 'candidate: zstd' | i64 | 8,591.549 ns | 178.2004 ns | 46.2781 ns | 4.96x | Faster | - | NA | +| 'candidate: ALP' | i64 | 1.770 ns | 0.0225 ns | 0.0058 ns | 24,095x | Faster | - | NA | -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 6403ae1a with uncommitted changes, 2026-10-10 04:24 UTC.* ### `LanesBenchmarks` -| Method | Lanes | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Gen0 | Completed Work Items | Lock Contentions | Allocated | Alloc Ratio | -|----------------------- |------ |---------:|---------:|--------:|--------:|---------------- |-------:|---------------------:|-----------------:|----------:|------------:| -| 'ours, n lanes' | 4 | 344.5 μs | 9.00 μs | 2.34 μs | 1.00x | Baseline | 3.4722 | 103.5903 | - | 46720 B | 1.00 | -| 'reference, n workers' | 4 | 787.7 μs | 30.70 μs | 7.97 μs | 0.44x | Slower | - | - | - | - | 0.00 | +| Method | Lanes | Mean | Error | StdDev | Speedup | MannWhitney(5%) | Gen0 | Completed Work Items | Lock Contentions | Allocated | Alloc Ratio | +|----------------------- |------ |---------:|---------:|---------:|--------:|---------------- |-------:|---------------------:|-----------------:|----------:|------------:| +| 'ours, n lanes' | 4 | 347.0 μs | 11.41 μs | 1.77 μs | 1.00x | Baseline | 3.4722 | 104.1458 | - | 46721 B | 1.00 | +| 'reference, n workers' | 4 | 794.6 μs | 68.49 μs | 17.79 μs | 0.44x | Slower | - | - | - | - | 0.00 | -*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 32b3a82b with uncommitted changes, 2026-10-10 02:25 UTC.* +*Measured on Apple M4 Pro (Arm64), 14 processors, macOS 26.7.1; .NET 11.0.0-rc.1.26425.128; BenchmarkDotNet 0.16.0-preview.2, the fast profile; commit 6403ae1a with uncommitted changes, 2026-10-10 04:24 UTC.* ## What this does not measure From 511497ac8c03cbdbc4a31a254b851c0ca4fa3678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20DUCOM?= Date: Sat, 10 Oct 2026 11:01:59 +0200 Subject: [PATCH 56/56] docs: the README opens on what the library does fast, with the latest speedups against Rust and DuckDB --- README.md | 59 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 413adfe12..516b49d20 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,38 @@ # Vorticity -A **pure .NET, dependency-free** implementation of the [Vortex](https://vortex.dev) columnar -file format (LF AI & Data). +Vorticity is a **high-performance**, pure .NET implementation of the [Vortex](https://vortex.dev) +columnar file format, with no dependencies: it reads and writes Vortex files faster than Vortex's +own Rust implementation, and its group bys run faster than DuckDB's. ## Performance -**Faster than Vortex's Rust implementation on most of what is measured**, process against process -on the same machine, both sides doing the same work. A table of four columns and 1,048,576 rows, -Native AOT on an Apple M4 Pro, Rust's time over Vorticity's: +Each speedup below is the other side's time divided by Vorticity's, so anything above 1.00x means +Vorticity is faster. Both sides run as separate processes on the same machine, an Apple M4 Pro with +Native AOT, and do the same work. -| scenario | one core | all 14 cores | +Against Vortex's own Rust implementation, Vorticity is faster in every scenario, and often twice as +fast. On a table of four columns and a million rows: + +| scenario | speedup, one core | speedup, 14 cores | |---|---:|---:| -| read every column | 1.98x | 2.31x | -| read one column of four | 2.23x | 2.24x | -| filter, 1 % of the rows | 1.35x | 2.89x | -| filter, half the rows | 2.05x | 2.31x | -| take 1,000 scattered rows | 1.59x | 1.17x | -| write the table back out | 2.39x | 1.37x | - -Per encoding, on one core: decoding faster on 50 of 57 files (median 1.27x), taking rows faster -from all 57 (median 1.14x), writing faster on all 56 (median 3.82x). A scan allocates next to -nothing per batch. [How it is measured](#how-performance-is-measured), and every figure on [the -benchmark page](docs/guide/benchmarks.md). +| read every column | 1.99x | 2.48x | +| read one column of four | 2.31x | 2.51x | +| filter, 1 % of the rows | 1.45x | 3.01x | +| filter, half the rows | 2.08x | 2.37x | +| take 1,000 scattered rows | 1.64x | 1.11x | +| write the table back out | 2.37x | 1.35x | + +Against DuckDB, its group bys win all 13 queries we measured, on the same files and with the same +number of threads: + +| DuckDB reading | speedup, one thread | speedup, 14 threads | +|---|---:|---:| +| its own table, loaded beforehand | 1.43x to 6.15x | 1.18x to 3.50x | +| the file, through its Vortex extension | 1.58x to 5.65x | 1.39x to 4.47x | + +You can read [how it is measured](#how-performance-is-measured) below, and find every figure on the +pages [against Rust](docs/guide/benchmarks.md), [against DuckDB](docs/guide/benchmarks-duckdb.md) +and [on x64](docs/guide/benchmarks-x64.md). ## Features @@ -154,7 +165,7 @@ at the same version as the core. | [docs/guide/](docs/guide/README.md) | **Using the library**: pages named by what you are trying to do, each one a program the samples project compiles and runs | | [docs/guide/benchmarks.md](docs/guide/benchmarks.md) | Every figure the bench publishes: what this costs against the Rust implementation, and each kernel against its baseline | | [docs/guide/benchmarks-x64.md](docs/guide/benchmarks-x64.md) | The same page from an x64 machine, a Zen 4 processor under Windows, against a Rust built there | -| [docs/guide/benchmarks-duckdb.md](docs/guide/benchmarks-duckdb.md) | Our group bys against DuckDB's on the same files, through its Vortex reader and from its own table: where we lead and where we trail | +| [docs/guide/benchmarks-duckdb.md](docs/guide/benchmarks-duckdb.md) | Our group bys against DuckDB's on the same files, through its Vortex reader and from its own table, query by query, on one thread and on fourteen | | [docs/design/](docs/design/README.md) | **Why it is shaped this way**: the design documents, from the scope to the byte layout to the public API | ## How performance is measured @@ -167,12 +178,12 @@ and hands what it writes to a sink that keeps nothing; on one core, Rust is time of its two ways of splitting a scan. The speedup is Rust's time over Vorticity's: above 1.00x, Vorticity is faster. The table above is read from the file Vorticity writes. -Per encoding, Vorticity is well ahead on text, nested and compressed integer columns (`varbin` -5.70x, `struct` 5.10x, `map` 3.11x, `pco` 2.96x), and level or behind on 7 files by a fifth at -most: booleans that start at a bit offset (0.80x and 0.99x), masked columns (0.93x to 0.99x), -`onpair` (0.97x) and `decimal` (1.00x). Its file is no more than 5 % larger than Rust's, or -smaller, on 45 of 56. Both readers warmed up in one process, Rust is faster on 2 of 19 cases: a -prefix filter on FSST strings (0.93x) and a band over bit-packed integers (0.98x). +Per encoding, Vorticity is well ahead on nested, text and compressed integer columns (`struct` +6.92x, `varbin` 6.08x, `map` 3.28x, `pco` 3.06x), and level or behind on 10 files by a tenth at +most: plain booleans (0.90x), `onpair` (0.95x), `decimal` (0.96x), masked columns (0.96x to +1.00x), patched bit-packed integers (0.98x), `alprd` and `sequence` (1.00x). Its file is no more +than 5 % larger than Rust's, or smaller, on 44 of 56. Both readers warmed up in one process, Rust +is faster on 1 of 19 cases: a prefix filter on FSST strings (0.93x). Every figure is on [the benchmark page](docs/guide/benchmarks.md), each section with the machine and the commit it was measured on, and [its x64 twin](docs/guide/benchmarks-x64.md) holds the