Skip to content

refactor(cpp): ♻️ extract the row hash table from OperatorIndex #303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
robertodr wants to merge 1 commit into from
+296 −195
Open

refactor(cpp): ♻️ extract the row hash table from OperatorIndex #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpp/monoprop/detail/operator/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@ target_sources(
"MPOperator.h"
"OperatorIndex.h"
"RowAccess.h"
"RowHashTable.h"
)
221 changes: 26 additions & 195 deletions cpp/monoprop/detail/operator/OperatorIndex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,24 @@
#pragma once

#include <algorithm>
#include <array>
#include <bit>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <memory>
#include <optional>
#include <stdexcept>
#include <unordered_map>
#include <utility>
#include <vector>

#include "monoprop/TypeAliases.h"
#include "monoprop/core/Monomial.h"
#include "monoprop/detail/operator/RowHashTable.h"

namespace monoprop::detail {

class TermIndexCeilingReached : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};

// Operator-term store: entropy-packed position-list rows plus a keyless open-addressing hash index over
// those rows. Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), slots 1..c =
// Operator-term store: entropy-packed position-list rows plus a RowHashTable keyed over them. The rows
// are this class's business and the index is not: nothing below reads a slot, and nothing in
// RowHashTable reads a row -- the two meet only through the hash and equality callables passed in.
// Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), slots 1..c =
// ascending set-bit positions; stride_ is fixed for the container's life so row offsets stay stable.
// inline_width_ is a free parameter -- any width is correct, over-long rows spill losslessly to overflow.
// Single-writer: one partition, one thread; parallelism is cross-partition.
Expand All @@ -63,13 +57,7 @@ class OperatorIndex {
static_assert(kMaxInlinePositions < std::numeric_limits<PosT>::max(),
"kOverflowMarker sentinel must not collide with a valid popcount");

// Valid term indices are < kIndexCeiling (check_index_fits throws at the ceiling), so the
// all-ones TermIndex is free to mark an empty slot.
static constexpr size_t kIndexCeiling = static_cast<size_t>(std::numeric_limits<TermIndex>::max());
static constexpr TermIndex kEmptySlot = std::numeric_limits<TermIndex>::max();
// find_batch's "absent" result; same value as detail::kMissingIndex (not included here — the
// operator store must not depend on evolution headers).
static constexpr size_t kNotFound = std::numeric_limits<size_t>::max();
static constexpr size_t kNotFound = RowHashTable::kNotFound;

explicit OperatorIndex(size_t inline_width = kDefaultInlinePositions)
: inline_width_(std::clamp<size_t>(inline_width, 1, kMaxInlinePositions)),
Expand All @@ -85,12 +73,8 @@ class OperatorIndex {
out->rows_ = rows_;
out->size_ = size_;
out->overflow_ = overflow_;
out->reserve_index(table_.count);
for (const Slot &e : table_.slots) {
if (e.idx != kEmptySlot) {
out->insert_slot_(e.idx, e.h);
}
}
out->reserve_index(table_.count());
table_.for_each_slot([&out](TermIndex idx, uint32_t h) { out->table_.insert_distinct(idx, h); });
return out;
}

Expand Down Expand Up @@ -177,195 +161,49 @@ class OperatorIndex {
}

auto find(const key_type &key) const -> std::optional<size_t> {
const uint32_t h = fold_hash(key);
if (table_.count == 0) {
return std::nullopt;
}
size_t s = spread(h) & table_.mask;
for (;; s = (s + 1) & table_.mask) {
const Slot &e = table_.slots[s];
if (e.idx == kEmptySlot) {
return std::nullopt;
}
if (e.h == h && row_eq_key(static_cast<size_t>(e.idx), key)) {
return static_cast<size_t>(e.idx);
}
}
return table_.find(fold_hash(key), [this, &key](size_t i) { return row_eq_key(i, key); });
}

// Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. Same result as n
// find() calls, but overlaps dram misses via a per-group hash/probe/confirm pipeline. An h
// collision falls back to an exact find. must not run concurrently with inserts.
// Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. The prefetch the pipeline
// is built around is the row prefetch below -- the table issues it between probe and confirm.
auto find_batch(const key_type *keys, size_t n, size_t *out) const -> void {
static constexpr size_t G = 16; // keys prefetched together per pipeline pass
std::array<uint32_t, G> hh;
std::array<size_t, G> sp;
std::array<TermIndex, G> cand;
for (size_t base = 0; base < n; base += G) {
const size_t g = std::min(G, n - base);
for (size_t j = 0; j < g; ++j) {
hh[j] = fold_hash(keys[base + j]);
sp[j] = spread(hh[j]);
__builtin_prefetch(&table_.slots[sp[j] & table_.mask], 0, 0);
}
for (size_t j = 0; j < g; ++j) {
cand[j] = kEmptySlot;
if (table_.count == 0) {
continue;
}
cand[j] = probe_hash_match_(hh[j], sp[j] & table_.mask);
if (cand[j] != kEmptySlot) {
__builtin_prefetch(&rows_[static_cast<size_t>(cand[j]) * stride_], 0, 0);
}
}
for (size_t j = 0; j < g; ++j) {
if (cand[j] != kEmptySlot && row_eq_key(static_cast<size_t>(cand[j]), keys[base + j])) {
out[base + j] = static_cast<size_t>(cand[j]);
}
else if (cand[j] != kEmptySlot) {
const auto v = find(keys[base + j]);
out[base + j] = v ? *v : kNotFound;
}
else {
out[base + j] = kNotFound;
}
}
}
table_.find_batch(
keys,
n,
out,
[](const key_type &k) { return fold_hash(k); },
[this](size_t i) { __builtin_prefetch(&rows_[i * stride_], 0, 0); },
[this](size_t i, const key_type &k) { return row_eq_key(i, k); });
}

// Insert-or-no-op. Row at `value` must already be written (the confirm reads dense rows).
auto emplace(const key_type &key, mapped_type value) -> void {
check_index_fits(value);
const uint32_t h = fold_hash(key);
table_.rehash_if_needed();
size_t s = spread(h) & table_.mask;
while (table_.slots[s].idx != kEmptySlot) {
if (table_.slots[s].h == h && row_eq_key(static_cast<size_t>(table_.slots[s].idx), key)) {
return;
}
s = (s + 1) & table_.mask;
}
table_.slots[s] = Slot{static_cast<TermIndex>(value), h};
++table_.count;
table_.emplace(fold_hash(key), value, [this, &key](size_t i) { return row_eq_key(i, key); });
}
// Insert n distinct rows with consecutive indices [base, base+n). Rows must already be written.
template <typename KeyFn>
auto bulk_insert(size_t n, mapped_type base, KeyFn &&key_at) -> void {
if (n == 0) {
return;
}
check_index_fits(base + n - 1);
for (size_t k = 0; k < n; ++k) {
insert_slot_(static_cast<TermIndex>(base + k), fold_hash(key_at(k)));
}
table_.insert_distinct_range(base, n, [&key_at](size_t k) { return fold_hash(key_at(k)); });
}
template <typename Func>
auto for_each(Func &&fn) const -> void {
for (const Slot &e : table_.slots) {
if (e.idx != kEmptySlot) {
fn(row(static_cast<size_t>(e.idx)), static_cast<size_t>(e.idx));
}
}
table_.for_each_slot([this, &fn](TermIndex idx, uint32_t) { fn(row(idx), static_cast<size_t>(idx)); });
}
// Diagnostic: the part of memory_bytes() that is unused geometric-growth capacity.
[[nodiscard]] auto slack_bytes() const -> size_t {
return (rows_.capacity() * sizeof(PosT)) - (std::min(rows_.capacity(), size_ * stride_) * sizeof(PosT));
}

auto index_estimated_memory_bytes() const -> size_t {
return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot));
}
auto index_estimated_memory_bytes() const -> size_t { return sizeof(OperatorIndex) + table_.slot_bytes(); }

private:
struct Slot {
TermIndex idx = kEmptySlot;
uint32_t h = 0;
};

// First slot on h's probe chain whose stored hash matches, or kEmptySlot if the chain ends first.
// Matches on h alone and leaves the dense-row comparison to the caller — that deferral is what lets
// find_batch prefetch the row between probe and confirm, so do not fold row_eq_key in here (find()
// deliberately keeps its own confirming variant). `start` must already be masked; the table must not
// be mutated concurrently.
[[gnu::always_inline]] auto probe_hash_match_(uint32_t h, size_t start) const -> TermIndex {
for (size_t s = start;; s = (s + 1) & table_.mask) {
const Slot &e = table_.slots[s];
if (e.idx == kEmptySlot) {
return kEmptySlot;
}
if (e.h == h) {
return e.idx;
}
}
}

static uint32_t fold_hash(const key_type &q) noexcept {
const size_t full = MonomialHash<NumModes>{}(q);
return static_cast<uint32_t>(full ^ (static_cast<uint64_t>(full) >> 32));
}
// Avalanche the cached 32-bit fold into a full-width hash (splitmix64 finalizer): the stored h
// is only an equality pre-filter, so it must be re-mixed before its low bits drive table bucketing.
static size_t spread(uint32_t h) noexcept {
uint64_t x = static_cast<uint64_t>(h) * 0x9E3779B97F4A7C15ULL;
x ^= x >> 30;
x *= 0xBF58476D1CE4E5B9ULL;
x ^= x >> 27;
x *= 0x94D049BB133111EBULL;
x ^= x >> 31;
return static_cast<size_t>(x);
static auto fold_hash(const key_type &q) noexcept -> uint32_t {
return RowHashTable::fold(MonomialHash<NumModes>{}(q));
}

// One open-addressing table: power-of-2 slot count, linear probing, max load factor 0.7
// (the group-prefetch win erodes at high load — longer probe chains add un-prefetched reads).
struct Table {
std::vector<Slot> slots = std::vector<Slot>(kMinSlots, Slot{});
size_t mask = kMinSlots - 1;
size_t count = 0;

auto rehash_if_needed() -> void {
if ((count + 1) * 10 >= slots.size() * 7) {
rehash_to(slots.size() * 2);
}
}
auto rehash_to(size_t new_cap) -> void {
new_cap = std::bit_ceil(std::max<size_t>(new_cap, kMinSlots));
if (new_cap <= slots.size()) {
return;
}
std::vector<Slot> old = std::move(slots);
slots.assign(new_cap, Slot{});
mask = new_cap - 1;
for (const Slot &e : old) {
if (e.idx == kEmptySlot) {
continue;
}
size_t s = spread(e.h) & mask;
while (slots[s].idx != kEmptySlot) {
s = (s + 1) & mask;
}
slots[s] = e;
}
}
};
static constexpr size_t kMinSlots = 16;
// Slot count for `n` entries at ≤0.7 load.
static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max<size_t>(kMinSlots, (n * 10 / 7) + 1)); }

[[nodiscard]] auto capacity() const -> size_t { return rows_.capacity() / stride_; }
auto reserve_rows(size_t n) -> void { rows_.reserve(n * stride_); }
auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); }

// Insert (idx, h) into the table with no duplicate probe — callers on this path insert provably distinct
// keys (⊕G-injective miss batches, clone re-insertion).
auto insert_slot_(TermIndex idx, uint32_t h) -> void {
table_.rehash_if_needed();
size_t s = spread(h) & table_.mask;
while (table_.slots[s].idx != kEmptySlot) {
s = (s + 1) & table_.mask;
}
table_.slots[s] = Slot{idx, h};
++table_.count;
}
auto reserve_index(size_t n) -> void { table_.reserve(n); }

// Compare row i against key q without materializing the row (the find confirm). Reads the
// popcount byte first, so a false h prefilter match usually costs one byte compare.
Expand All @@ -386,20 +224,13 @@ class OperatorIndex {
return true;
}

static auto check_index_fits(size_t value) -> void {
if (value >= kIndexCeiling) {
throw TermIndexCeilingReached("OperatorIndex: operator index reached the TermIndex ceiling; rebuild with "
"-Dmonoprop_WIDE_TERM_INDEX (this partition's term count exceeded ~2^32).");
}
}

DefaultInitVector<PosT> rows_ = {};
size_t size_ = 0;
size_t inline_width_ = kMaxInlinePositions;
size_t stride_ = 1 + kMaxInlinePositions;
// Lossless side-map for rows whose popcount exceeds inline_width_.
std::unordered_map<size_t, value_type> overflow_ = {};
Table table_ = {};
RowHashTable table_ = {};
};

} // namespace monoprop::detail
Loading
Loading