From e4b12173337d8728fdb5c596985936b677fb03c7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 29 Aug 2026 08:27:51 +0000 Subject: [PATCH] =?UTF-8?q?refactor(cpp):=20=E2=99=BB=EF=B8=8F=20extract?= =?UTF-8?q?=20the=20row=20hash=20table=20from=20OperatorIndex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OperatorIndex was two things: a packed position-list row representation and a keyless open-addressing index over those rows. RowHashTable is the second half, lifted out whole -- power-of-2 slots, linear probing, load factor 0.7, a 32-bit folded hash per slot used only as an equality pre-filter. Keyless is what makes the split possible: the table never stores or compares a key, so the hash and the equality test arrive as callables and the row representation stays entirely on the caller's side. find_batch keeps its pipeline by taking the row prefetch as a third callable -- deferring confirmation past the probe is the whole reason that prefetch has somewhere to go. The table's slot layout fixes for_each_slot's iteration order, which is the order of a propagator's evolved-term list and therefore its floating-point accumulation order, so this had to come out inert. Checked byte-wise against the previous implementation over 4000 random monomials: identical iteration order, find and find_batch results, clone order and memory_bytes. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/operator/CMakeLists.txt | 1 + cpp/monoprop/detail/operator/OperatorIndex.h | 221 ++------------- cpp/monoprop/detail/operator/RowHashTable.h | 269 +++++++++++++++++++ 3 files changed, 296 insertions(+), 195 deletions(-) create mode 100644 cpp/monoprop/detail/operator/RowHashTable.h diff --git a/cpp/monoprop/detail/operator/CMakeLists.txt b/cpp/monoprop/detail/operator/CMakeLists.txt index ea0266a1..939009ff 100644 --- a/cpp/monoprop/detail/operator/CMakeLists.txt +++ b/cpp/monoprop/detail/operator/CMakeLists.txt @@ -8,4 +8,5 @@ target_sources( "MPOperator.h" "OperatorIndex.h" "RowAccess.h" + "RowHashTable.h" ) diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 8a3c29b2..397b22d4 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -15,30 +15,24 @@ #pragma once #include -#include -#include #include #include #include #include #include -#include #include #include -#include #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/operator/RowHashTable.h" namespace monoprop::detail { -class TermIndexCeilingReached : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -// Operator-term store: entropy-packed position-list rows plus a keyless open-addressing hash index over -// those rows. Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), slots 1..c = +// Operator-term store: entropy-packed position-list rows plus a RowHashTable keyed over them. The rows +// are this class's business and the index is not: nothing below reads a slot, and nothing in +// RowHashTable reads a row -- the two meet only through the hash and equality callables passed in. +// Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), slots 1..c = // ascending set-bit positions; stride_ is fixed for the container's life so row offsets stay stable. // inline_width_ is a free parameter -- any width is correct, over-long rows spill losslessly to overflow. // Single-writer: one partition, one thread; parallelism is cross-partition. @@ -63,13 +57,7 @@ class OperatorIndex { static_assert(kMaxInlinePositions < std::numeric_limits::max(), "kOverflowMarker sentinel must not collide with a valid popcount"); - // Valid term indices are < kIndexCeiling (check_index_fits throws at the ceiling), so the - // all-ones TermIndex is free to mark an empty slot. - static constexpr size_t kIndexCeiling = static_cast(std::numeric_limits::max()); - static constexpr TermIndex kEmptySlot = std::numeric_limits::max(); - // find_batch's "absent" result; same value as detail::kMissingIndex (not included here — the - // operator store must not depend on evolution headers). - static constexpr size_t kNotFound = std::numeric_limits::max(); + static constexpr size_t kNotFound = RowHashTable::kNotFound; explicit OperatorIndex(size_t inline_width = kDefaultInlinePositions) : inline_width_(std::clamp(inline_width, 1, kMaxInlinePositions)), @@ -85,12 +73,8 @@ class OperatorIndex { out->rows_ = rows_; out->size_ = size_; out->overflow_ = overflow_; - out->reserve_index(table_.count); - for (const Slot &e : table_.slots) { - if (e.idx != kEmptySlot) { - out->insert_slot_(e.idx, e.h); - } - } + out->reserve_index(table_.count()); + table_.for_each_slot([&out](TermIndex idx, uint32_t h) { out->table_.insert_distinct(idx, h); }); return out; } @@ -177,195 +161,49 @@ class OperatorIndex { } auto find(const key_type &key) const -> std::optional { - const uint32_t h = fold_hash(key); - if (table_.count == 0) { - return std::nullopt; - } - size_t s = spread(h) & table_.mask; - for (;; s = (s + 1) & table_.mask) { - const Slot &e = table_.slots[s]; - if (e.idx == kEmptySlot) { - return std::nullopt; - } - if (e.h == h && row_eq_key(static_cast(e.idx), key)) { - return static_cast(e.idx); - } - } + return table_.find(fold_hash(key), [this, &key](size_t i) { return row_eq_key(i, key); }); } - // Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. Same result as n - // find() calls, but overlaps dram misses via a per-group hash/probe/confirm pipeline. An h - // collision falls back to an exact find. must not run concurrently with inserts. + // Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. The prefetch the pipeline + // is built around is the row prefetch below -- the table issues it between probe and confirm. auto find_batch(const key_type *keys, size_t n, size_t *out) const -> void { - static constexpr size_t G = 16; // keys prefetched together per pipeline pass - std::array hh; - std::array sp; - std::array cand; - for (size_t base = 0; base < n; base += G) { - const size_t g = std::min(G, n - base); - for (size_t j = 0; j < g; ++j) { - hh[j] = fold_hash(keys[base + j]); - sp[j] = spread(hh[j]); - __builtin_prefetch(&table_.slots[sp[j] & table_.mask], 0, 0); - } - for (size_t j = 0; j < g; ++j) { - cand[j] = kEmptySlot; - if (table_.count == 0) { - continue; - } - cand[j] = probe_hash_match_(hh[j], sp[j] & table_.mask); - if (cand[j] != kEmptySlot) { - __builtin_prefetch(&rows_[static_cast(cand[j]) * stride_], 0, 0); - } - } - for (size_t j = 0; j < g; ++j) { - if (cand[j] != kEmptySlot && row_eq_key(static_cast(cand[j]), keys[base + j])) { - out[base + j] = static_cast(cand[j]); - } - else if (cand[j] != kEmptySlot) { - const auto v = find(keys[base + j]); - out[base + j] = v ? *v : kNotFound; - } - else { - out[base + j] = kNotFound; - } - } - } + table_.find_batch( + keys, + n, + out, + [](const key_type &k) { return fold_hash(k); }, + [this](size_t i) { __builtin_prefetch(&rows_[i * stride_], 0, 0); }, + [this](size_t i, const key_type &k) { return row_eq_key(i, k); }); } // Insert-or-no-op. Row at `value` must already be written (the confirm reads dense rows). auto emplace(const key_type &key, mapped_type value) -> void { - check_index_fits(value); - const uint32_t h = fold_hash(key); - table_.rehash_if_needed(); - size_t s = spread(h) & table_.mask; - while (table_.slots[s].idx != kEmptySlot) { - if (table_.slots[s].h == h && row_eq_key(static_cast(table_.slots[s].idx), key)) { - return; - } - s = (s + 1) & table_.mask; - } - table_.slots[s] = Slot{static_cast(value), h}; - ++table_.count; + table_.emplace(fold_hash(key), value, [this, &key](size_t i) { return row_eq_key(i, key); }); } // Insert n distinct rows with consecutive indices [base, base+n). Rows must already be written. template auto bulk_insert(size_t n, mapped_type base, KeyFn &&key_at) -> void { - if (n == 0) { - return; - } - check_index_fits(base + n - 1); - for (size_t k = 0; k < n; ++k) { - insert_slot_(static_cast(base + k), fold_hash(key_at(k))); - } + table_.insert_distinct_range(base, n, [&key_at](size_t k) { return fold_hash(key_at(k)); }); } template auto for_each(Func &&fn) const -> void { - for (const Slot &e : table_.slots) { - if (e.idx != kEmptySlot) { - fn(row(static_cast(e.idx)), static_cast(e.idx)); - } - } + table_.for_each_slot([this, &fn](TermIndex idx, uint32_t) { fn(row(idx), static_cast(idx)); }); } // Diagnostic: the part of memory_bytes() that is unused geometric-growth capacity. [[nodiscard]] auto slack_bytes() const -> size_t { return (rows_.capacity() * sizeof(PosT)) - (std::min(rows_.capacity(), size_ * stride_) * sizeof(PosT)); } - auto index_estimated_memory_bytes() const -> size_t { - return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); - } + auto index_estimated_memory_bytes() const -> size_t { return sizeof(OperatorIndex) + table_.slot_bytes(); } private: - struct Slot { - TermIndex idx = kEmptySlot; - uint32_t h = 0; - }; - - // First slot on h's probe chain whose stored hash matches, or kEmptySlot if the chain ends first. - // Matches on h alone and leaves the dense-row comparison to the caller — that deferral is what lets - // find_batch prefetch the row between probe and confirm, so do not fold row_eq_key in here (find() - // deliberately keeps its own confirming variant). `start` must already be masked; the table must not - // be mutated concurrently. - [[gnu::always_inline]] auto probe_hash_match_(uint32_t h, size_t start) const -> TermIndex { - for (size_t s = start;; s = (s + 1) & table_.mask) { - const Slot &e = table_.slots[s]; - if (e.idx == kEmptySlot) { - return kEmptySlot; - } - if (e.h == h) { - return e.idx; - } - } - } - - static uint32_t fold_hash(const key_type &q) noexcept { - const size_t full = MonomialHash{}(q); - return static_cast(full ^ (static_cast(full) >> 32)); - } - // Avalanche the cached 32-bit fold into a full-width hash (splitmix64 finalizer): the stored h - // is only an equality pre-filter, so it must be re-mixed before its low bits drive table bucketing. - static size_t spread(uint32_t h) noexcept { - uint64_t x = static_cast(h) * 0x9E3779B97F4A7C15ULL; - x ^= x >> 30; - x *= 0xBF58476D1CE4E5B9ULL; - x ^= x >> 27; - x *= 0x94D049BB133111EBULL; - x ^= x >> 31; - return static_cast(x); + static auto fold_hash(const key_type &q) noexcept -> uint32_t { + return RowHashTable::fold(MonomialHash{}(q)); } - // One open-addressing table: power-of-2 slot count, linear probing, max load factor 0.7 - // (the group-prefetch win erodes at high load — longer probe chains add un-prefetched reads). - struct Table { - std::vector slots = std::vector(kMinSlots, Slot{}); - size_t mask = kMinSlots - 1; - size_t count = 0; - - auto rehash_if_needed() -> void { - if ((count + 1) * 10 >= slots.size() * 7) { - rehash_to(slots.size() * 2); - } - } - auto rehash_to(size_t new_cap) -> void { - new_cap = std::bit_ceil(std::max(new_cap, kMinSlots)); - if (new_cap <= slots.size()) { - return; - } - std::vector old = std::move(slots); - slots.assign(new_cap, Slot{}); - mask = new_cap - 1; - for (const Slot &e : old) { - if (e.idx == kEmptySlot) { - continue; - } - size_t s = spread(e.h) & mask; - while (slots[s].idx != kEmptySlot) { - s = (s + 1) & mask; - } - slots[s] = e; - } - } - }; - static constexpr size_t kMinSlots = 16; - // Slot count for `n` entries at ≤0.7 load. - static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } - [[nodiscard]] auto capacity() const -> size_t { return rows_.capacity() / stride_; } auto reserve_rows(size_t n) -> void { rows_.reserve(n * stride_); } - auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } - - // Insert (idx, h) into the table with no duplicate probe — callers on this path insert provably distinct - // keys (⊕G-injective miss batches, clone re-insertion). - auto insert_slot_(TermIndex idx, uint32_t h) -> void { - table_.rehash_if_needed(); - size_t s = spread(h) & table_.mask; - while (table_.slots[s].idx != kEmptySlot) { - s = (s + 1) & table_.mask; - } - table_.slots[s] = Slot{idx, h}; - ++table_.count; - } + auto reserve_index(size_t n) -> void { table_.reserve(n); } // Compare row i against key q without materializing the row (the find confirm). Reads the // popcount byte first, so a false h prefilter match usually costs one byte compare. @@ -386,20 +224,13 @@ class OperatorIndex { return true; } - static auto check_index_fits(size_t value) -> void { - if (value >= kIndexCeiling) { - throw TermIndexCeilingReached("OperatorIndex: operator index reached the TermIndex ceiling; rebuild with " - "-Dmonoprop_WIDE_TERM_INDEX (this partition's term count exceeded ~2^32)."); - } - } - DefaultInitVector rows_ = {}; size_t size_ = 0; size_t inline_width_ = kMaxInlinePositions; size_t stride_ = 1 + kMaxInlinePositions; // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; - Table table_ = {}; + RowHashTable table_ = {}; }; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/operator/RowHashTable.h b/cpp/monoprop/detail/operator/RowHashTable.h new file mode 100644 index 00000000..acb86332 --- /dev/null +++ b/cpp/monoprop/detail/operator/RowHashTable.h @@ -0,0 +1,269 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" + +namespace monoprop::detail { + +class TermIndexCeilingReached : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// The keyless open-addressing index a row store puts over its rows: power-of-2 slot count, linear +// probing, max load factor 0.7 (the group-prefetch win erodes at higher load -- longer probe chains add +// un-prefetched reads). A slot holds a row index plus a 32-bit hash used only as an equality +// pre-filter, so the table never stores or compares a key itself. +// +// Keyless is why the hash and the equality test arrive as callables rather than as members: the whole +// point is that the caller owns the row representation. `eq(row_index)` confirms a pre-filter hit +// against the caller's rows, and no operation here reads a row. +// +// The layout this produces is load-bearing, not an implementation detail: it fixes the iteration order +// of for_each_slot(), which sets the order of a propagator's user-visible evolved-term list and +// therefore its floating-point accumulation order. A store that keyed rows through its own copy of this +// logic could diverge on that while still looking correct, which is the reason the index lives apart +// from the row representation rather than inside one. +// +// Single-writer, matching its owners: one partition, one thread; parallelism is cross-partition. +class RowHashTable { +public: + // Valid row indices are < kIndexCeiling (check_index_fits throws at the ceiling), so the all-ones + // TermIndex is free to mark an empty slot. + static constexpr size_t kIndexCeiling = static_cast(std::numeric_limits::max()); + static constexpr TermIndex kEmptySlot = std::numeric_limits::max(); + // find_batch's "absent" result; same value as detail::kMissingIndex (not included here -- the + // operator store must not depend on evolution headers). + static constexpr size_t kNotFound = std::numeric_limits::max(); + + [[nodiscard]] auto count() const noexcept -> size_t { return count_; } + [[nodiscard]] auto slot_bytes() const -> size_t { return slots_.capacity() * sizeof(Slot); } + + // Slot capacity for `n` rows at <= 0.7 load. The +1 keeps a table reserved for exactly n rows off + // the rehash threshold on the n-th insert. + auto reserve(size_t n) -> void { rehash_to(slots_for_(n + 1)); } + + // The 32-bit form of a full-width row hash, which is what a slot stores as its equality + // pre-filter. Each store supplies its own full-width hash and folds it here, so the two agree on + // the pre-filter's format by construction. + [[nodiscard]] static constexpr auto fold(size_t full) noexcept -> uint32_t { + return static_cast(full ^ (static_cast(full) >> 32)); + } + + static auto check_index_fits(size_t value) -> void { + if (value >= kIndexCeiling) { + throw TermIndexCeilingReached("operator index reached the TermIndex ceiling; rebuild with " + "-Dmonoprop_WIDE_TERM_INDEX (this partition's term count exceeded ~2^32)."); + } + } + + // eq(row_index) -> bool confirms a hash pre-filter hit. + template + auto find(uint32_t h, Eq &&eq) const -> std::optional { + if (count_ == 0) { + return std::nullopt; + } + size_t s = spread(h) & mask_; + for (;; s = (s + 1) & mask_) { + const Slot &e = slots_[s]; + if (e.idx == kEmptySlot) { + return std::nullopt; + } + if (e.h == h && eq(static_cast(e.idx))) { + return static_cast(e.idx); + } + } + } + + // Insert-or-no-op. The row at `value` must already be written -- eq reads it. + template + auto emplace(uint32_t h, size_t value, Eq &&eq) -> void { + check_index_fits(value); + rehash_if_needed(); + size_t s = spread(h) & mask_; + while (slots_[s].idx != kEmptySlot) { + if (slots_[s].h == h && eq(static_cast(slots_[s].idx))) { + return; + } + s = (s + 1) & mask_; + } + slots_[s] = Slot{static_cast(value), h}; + ++count_; + } + + // Insert with no duplicate probe -- callers on this path insert provably distinct keys + // (+G-injective miss batches, clone re-insertion). + // insert_distinct over consecutive row indices [base, base + n), hashing each through hash_at(k). + // The stores' bulk_insert is this and nothing else, so it lives here rather than once per backend. + template + auto insert_distinct_range(size_t base, size_t n, HashFn &&hash_at) -> void { + if (n == 0) { + return; + } + check_index_fits(base + n - 1); + for (size_t k = 0; k < n; ++k) { + insert_distinct(static_cast(base + k), hash_at(k)); + } + } + + auto insert_distinct(TermIndex idx, uint32_t h) -> void { + rehash_if_needed(); + size_t s = spread(h) & mask_; + while (slots_[s].idx != kEmptySlot) { + s = (s + 1) & mask_; + } + slots_[s] = Slot{idx, h}; + ++count_; + } + + // Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. Same result as n find() + // calls, but overlaps dram misses via a per-group hash/probe/confirm pipeline. An h collision falls + // back to an exact find. Must not run concurrently with inserts. + // + // The three callables are what make the pipeline possible without the table knowing a row: + // hash(key) -> uint32_t, prefetch_row(row_index) issued between probe and confirm (which is the + // whole reason confirmation is deferred rather than folded into the probe), and + // eq(row_index, key) -> bool. + template + auto find_batch(const Key *keys, size_t n, size_t *out, Hash &&hash, PrefetchRow &&prefetch_row, Eq &&eq) const + -> void { + static constexpr size_t G = 16; // keys prefetched together per pipeline pass + std::array hh; + std::array sp; + std::array cand; + for (size_t base = 0; base < n; base += G) { + const size_t g = std::min(G, n - base); + for (size_t j = 0; j < g; ++j) { + hh[j] = hash(keys[base + j]); + sp[j] = spread(hh[j]); + __builtin_prefetch(&slots_[sp[j] & mask_], 0, 0); + } + for (size_t j = 0; j < g; ++j) { + cand[j] = kEmptySlot; + if (count_ == 0) { + continue; + } + cand[j] = probe_hash_match_(hh[j], sp[j] & mask_); + if (cand[j] != kEmptySlot) { + prefetch_row(static_cast(cand[j])); + } + } + for (size_t j = 0; j < g; ++j) { + if (cand[j] != kEmptySlot && eq(static_cast(cand[j]), keys[base + j])) { + out[base + j] = static_cast(cand[j]); + } + else if (cand[j] != kEmptySlot) { + const auto v = find(hh[j], [&eq, &keys, &base, &j](size_t i) { return eq(i, keys[base + j]); }); + out[base + j] = v ? *v : kNotFound; + } + else { + out[base + j] = kNotFound; + } + } + } + } + + // Occupied slots in table order, as fn(row_index, stored_hash). That order is the store's iteration + // order; see the class comment on why it must not drift between stores. + template + auto for_each_slot(Fn &&fn) const -> void { + for (const Slot &e : slots_) { + if (e.idx != kEmptySlot) { + fn(e.idx, e.h); + } + } + } + +private: + struct Slot { + TermIndex idx = kEmptySlot; + uint32_t h = 0; + }; + + static constexpr size_t kMinSlots = 16; + + static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } + + // Avalanche the cached 32-bit fold into a full-width hash (splitmix64 finalizer): the stored h is + // only an equality pre-filter, so it must be re-mixed before its low bits drive table bucketing. + static auto spread(uint32_t h) noexcept -> size_t { + uint64_t x = static_cast(h) * 0x9E3779B97F4A7C15ULL; + x ^= x >> 30; + x *= 0xBF58476D1CE4E5B9ULL; + x ^= x >> 27; + x *= 0x94D049BB133111EBULL; + x ^= x >> 31; + return static_cast(x); + } + + // First slot on h's probe chain whose stored hash matches, or kEmptySlot if the chain ends first. + // Matches on h alone and leaves confirmation to the caller -- that deferral is what lets find_batch + // prefetch the row between probe and confirm, so do not fold eq in here (find() deliberately keeps + // its own confirming variant). `start` must already be masked; the table must not be mutated + // concurrently. + [[gnu::always_inline]] auto probe_hash_match_(uint32_t h, size_t start) const -> TermIndex { + for (size_t s = start;; s = (s + 1) & mask_) { + const Slot &e = slots_[s]; + if (e.idx == kEmptySlot) { + return kEmptySlot; + } + if (e.h == h) { + return e.idx; + } + } + } + + auto rehash_if_needed() -> void { + if ((count_ + 1) * 10 >= slots_.size() * 7) { + rehash_to(slots_.size() * 2); + } + } + + auto rehash_to(size_t new_cap) -> void { + new_cap = std::bit_ceil(std::max(new_cap, kMinSlots)); + if (new_cap <= slots_.size()) { + return; + } + std::vector old = std::move(slots_); + slots_.assign(new_cap, Slot{}); + mask_ = new_cap - 1; + for (const Slot &e : old) { + if (e.idx == kEmptySlot) { + continue; + } + size_t s = spread(e.h) & mask_; + while (slots_[s].idx != kEmptySlot) { + s = (s + 1) & mask_; + } + slots_[s] = e; + } + } + + std::vector slots_ = std::vector(kMinSlots, Slot{}); + size_t mask_ = kMinSlots - 1; + size_t count_ = 0; +}; + +} // namespace monoprop::detail