From 3a261506797a4c4a70b418e9f63eac0b16e36577 Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Thu, 8 Oct 2026 13:39:40 -0700 Subject: [PATCH] base: shrink the code FlatHashMapV2 generates FlatHashMapV2 inlined its whole probe loop into every Find() and Insert() call site, costing ~750 bytes of code per insert+find site vs ~150 for FlatHashMapV1, so moving code over to it would have grown binaries. Insert() now inlines only constructing the key and value; finding or claiming the slot is out of line in FindOrPrepareInsert(), compiled once per key type, as is Find(). The state and the operations which don't depend on the key and value types (allocating, growing, probing for a free slot) move to an untemplated FlatHashMapV2Base, defined in flat_hash_map.cc and shared by all maps, and rehashing moves entries straight into their new slots rather than going through Insert(). Lookups now work as in absl: Find(), Erase(), Insert() and operator[] take a Key, which the caller converts to, and only a transparent Hasher enables other (heterogeneous) key types. MurmurHash is now transparent only for string-like keys: it also was for integers, hashing an int looked up in a map keyed by uint32_t as an int, so e.g. a negative one could miss. Each heterogeneous key type also compiled the probing again. For 16 map types with 8 insert+find sites each (x64): 121KB -> 35KB of code, vs 44KB for FlatHashMapV1. Inserts are unchanged; lookups through the out of line Find() are 4-14% slower in microbenchmarks. --- Android.bp | 1 + BUILD | 1 + include/perfetto/ext/base/flat_hash_map.h | 643 ++++++++++++---------- include/perfetto/ext/base/murmur_hash.h | 33 +- src/base/BUILD.gn | 1 + src/base/flat_hash_map.cc | 74 +++ src/base/flat_hash_map_unittest.cc | 24 +- src/base/murmur_hash_unittest.cc | 19 + 8 files changed, 489 insertions(+), 307 deletions(-) create mode 100644 src/base/flat_hash_map.cc diff --git a/Android.bp b/Android.bp index 88a1395f0e1..3ab8fcfb7d6 100644 --- a/Android.bp +++ b/Android.bp @@ -15975,6 +15975,7 @@ filegroup { "src/base/dynamic_string_writer.cc", "src/base/event_fd.cc", "src/base/file_utils.cc", + "src/base/flat_hash_map.cc", "src/base/futex.cc", "src/base/getopt_compat.cc", "src/base/intrusive_list.cc", diff --git a/BUILD b/BUILD index e544312f72c..5845c08bf9d 100644 --- a/BUILD +++ b/BUILD @@ -1996,6 +1996,7 @@ perfetto_cc_library( "src/base/dynamic_string_writer.cc", "src/base/event_fd.cc", "src/base/file_utils.cc", + "src/base/flat_hash_map.cc", "src/base/futex.cc", "src/base/getopt_compat.cc", "src/base/intrusive_list.cc", diff --git a/include/perfetto/ext/base/flat_hash_map.h b/include/perfetto/ext/base/flat_hash_map.h index e7a3021b5d9..14d66c969ca 100644 --- a/include/perfetto/ext/base/flat_hash_map.h +++ b/include/perfetto/ext/base/flat_hash_map.h @@ -135,6 +135,240 @@ static constexpr uint8_t kTombstone = 0xFE; // Deleted slot // The default load limit percent before growing the table. static constexpr int kDefaultLoadLimitPct = 75; +// Abstraction over a group of control bytes that enables batch operations. +// On x64, uses SSE to match 16 control bytes in parallel. +// On other platforms, uses SWAR (SIMD Within A Register) for 8 bytes. +// +// Match() returns an iterator over slots whose control byte matches h2. +// MatchEmpty() returns an iterator over empty slots (kFreeSlot). +// MatchEmptyOrDeleted() returns an iterator over empty or deleted slots. +#if PERFETTO_BUILDFLAG(PERFETTO_X64_CPU_OPT) +struct Group { + public: + // Group size 16 for x64 SSE + static constexpr size_t kSize = 16; + + // Iterates over set bits in a match mask. Each set bit indicates a slot + // in the group that matched the search criteria. Call Next() to get the + // index of each matching slot. + struct Iterator { + public: + PERFETTO_ALWAYS_INLINE explicit Iterator(uint16_t mask) : mask_(mask) {} + PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } + + PERFETTO_ALWAYS_INLINE size_t Next() { + auto idx = static_cast(CountTrailZeros(mask_)); + mask_ &= static_cast(mask_ - 1); + return idx; + } + + private: + uint16_t mask_; + }; + + PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { + ctrl_ = _mm_loadu_si128(reinterpret_cast(pos)); + } + + PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { + auto match = _mm_cmpeq_epi8(ctrl_, _mm_set1_epi8(static_cast(h2))); + return Iterator(static_cast(_mm_movemask_epi8(match))); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { + return Iterator( + static_cast(_mm_movemask_epi8(_mm_sign_epi8(ctrl_, ctrl_)))); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { + return Iterator(static_cast(_mm_movemask_epi8(ctrl_))); + } + + private: + __m128i ctrl_; +}; +#else +// SWAR fallback: processes 8 control bytes at a time using 64-bit arithmetic. +struct Group { + public: + // Group size 8 for ARM and other platforms + static constexpr size_t kSize = 8; + + // Iterates over set bits in a sparse 64-bit mask. Each set MSB indicates + // a matching byte in the group. Call Next() to get the index of each + // matching slot. + struct Iterator { + public: + PERFETTO_ALWAYS_INLINE explicit Iterator(uint64_t mask) : mask_(mask) {} + PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } + PERFETTO_ALWAYS_INLINE size_t Next() { + // Count zeros and divide by 8 (shift 3) + // 0x80 (Byte 0) -> CTZ 7 -> 7>>3 = 0 + // 0x8000 (Byte 1) -> CTZ 15 -> 15>>3 = 1 + size_t idx = static_cast(CountTrailZeros(mask_) >> 3); + mask_ &= mask_ - 1; // Clear lowest set bit + return idx; + } + + private: + uint64_t mask_; + }; + + PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { + memcpy(&ctrl_, pos, sizeof(ctrl_)); + } + + PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { + uint64_t x = ctrl_ ^ (kLsbs * h2); + return Iterator((x - kLsbs) & ~x & kMsbs); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { + // 0x80 check (Empty) + return Iterator((ctrl_ & ~(ctrl_ << 6)) & kMsbs); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { + // 0x80 or 0xFE check (Empty or Deleted) + return Iterator((ctrl_ & ~(ctrl_ << 7)) & kMsbs); + } + + private: + static constexpr uint64_t kLsbs = 0x0101010101010101ULL; + static constexpr uint64_t kMsbs = 0x8080808080808080ULL; + + uint64_t ctrl_; +}; +#endif + +// The state and the operations of FlatHashMapV2 which don't depend on the key +// and value types. Not templated, and the heavier operations are defined in +// flat_hash_map.cc, so that all maps share one copy of the code. +class FlatHashMapV2Base { + protected: + // Tracks growth capacity and whether any deletions have occurred. + // Using bitfields like absl's GrowthInfo to avoid manual bit manipulation. + struct GrowthInfo { + uint64_t growth_left : 63; + uint64_t has_tombstones : 1; + }; + + // The table being moved out of while growing. + struct OldTable { + std::unique_ptr storage; + const uint8_t* ctrl; + uint8_t* slots; + size_t capacity; + size_t size; + }; + + // The number of cloned control bytes after the main control byte array. + static constexpr size_t kNumClones = Group::kSize - 1; + + explicit FlatHashMapV2Base(int load_limit_pct) + : load_limit_percent_(load_limit_pct) {} + + FlatHashMapV2Base(FlatHashMapV2Base&& other) noexcept + : storage_(std::move(other.storage_)), + capacity_(other.capacity_), + size_(other.size_), + growth_info_(other.growth_info_), + load_limit_percent_(other.load_limit_percent_), + ctrl_(other.ctrl_), + slots_(other.slots_) { + other.capacity_ = 0; + other.size_ = 0; + other.growth_info_ = {0, 0}; + other.load_limit_percent_ = kDefaultLoadLimitPct; + other.ctrl_ = nullptr; + other.slots_ = nullptr; + } + + // Swiss Table hash splitting (matching absl): + // H1 = upper bits for bucket index + // H2 = lower 7 bits for tag + // This ensures H1 and H2 are independent, avoiding tag collisions within + // buckets. The seed XOR prevents clustering when hash values have patterns + // (e.g., sequential keys) + static constexpr size_t H1(size_t hash) { return (hash >> 7); } + static constexpr uint8_t H2(size_t hash) { return hash & 0x7F; } + + // Doesn't call destructors. + void Reset(size_t n, bool reallocate, size_t slot_size, size_t slot_align); + + // Find first empty OR tombstone slot for insertion. + // Called when has_tombstones is set to find an earlier tombstone that can + // be reused instead of taking a new empty slot. + size_t FindFirstEmptyOrTombstone(size_t key_hash) const; + + PERFETTO_ALWAYS_INLINE size_t + FindFirstEmptyOrTombstoneImpl(size_t key_hash) const { + const size_t cap_mask = capacity_ - 1; + size_t offset = H1(key_hash) & cap_mask; + size_t probe_size = 0; + while (true) { + Group group(ctrl_ + offset); + if (auto it = group.MatchEmptyOrDeleted(); PERFETTO_LIKELY(it)) { + return (offset + it.Next()) & cap_mask; + } + probe_size += Group::kSize; + offset = (offset + probe_size) & cap_mask; + } + } + + // Swaps in an empty table twice as large and returns the old one, from + // which the caller must move the entries before calling FinishGrow(). + OldTable Grow(size_t slot_size, size_t slot_align); + + void FinishGrow(size_t new_size) { + PERFETTO_CHECK(growth_info_.growth_left >= new_size); + size_ = new_size; + growth_info_.growth_left -= new_size; + } + + // Set control byte and update clone if needed + PERFETTO_ALWAYS_INLINE void SetCtrl(size_t i, uint8_t h) { + ctrl_[i] = h; + // Update clone if this is one of the first kNumClones entries + if (PERFETTO_UNLIKELY(i < kNumClones)) { + ctrl_[capacity_ + i] = h; + } + } + + // Owns the actual memory with the following layout: + // + // [Control bytes] + // |capacity_| bytes for control bytes. + // kNumClones (15 or 7) bytes for control byte clones (*). + // No alignment required (accessed at arbitrary byte offsets). + // + // [Padding for Slot alignment] + // + // [Slots] + // capacity_ * sizeof(Slot): contains key-value pairs. + // Must be aligned to alignof(Slot). + // + // (*) Control byte clones: The first kNumClones control bytes are duplicated + // at the end of the control array. This allows SIMD operations to read a full + // group (16 or 8 bytes) starting from any position without bounds checking, + // even near the end of the array. + std::unique_ptr storage_; + + size_t capacity_ = 0; + size_t size_ = 0; + + // Slots remaining + has_deleted flag + GrowthInfo growth_info_{0, 0}; + + // Load factor limit in % of |capacity_|. + int load_limit_percent_ = kDefaultLoadLimitPct; + + // Cached pointers for fast access (like absl::flat_hash_map) + // These are updated whenever storage is allocated/reallocated. + uint8_t* ctrl_ = nullptr; // Points to control bytes + uint8_t* slots_ = nullptr; // Points to slot array +}; + } // namespace flat_hash_map_v2_internal // Hash and equality functors which treat ASCII strings case-insensitively. @@ -164,7 +398,7 @@ template , typename Eq = flat_hash_map_v2_internal::HashEq> -class FlatHashMapV2 { +class FlatHashMapV2 : private flat_hash_map_v2_internal::FlatHashMapV2Base { private: // Import constants from internal namespace. static constexpr uint8_t kFreeSlot = flat_hash_map_v2_internal::kFreeSlot; @@ -178,6 +412,21 @@ class FlatHashMapV2 { Value value; }; + // Whether |K| is a type other than Key which, with a transparent Hasher, + // is hashed and compared to Key as is (e.g. a std::string_view looked up in + // a map keyed by std::string), sparing the construction of a Key. + template + static constexpr bool IsHeterogeneousKey() { + using RawK = std::remove_cv_t>; + if constexpr (std::is_same_v) { + return false; + } else { + return flat_hash_map_v2_internal::HasIsTransparent::value && + flat_hash_map_v2_internal::IsLookupKeyAllowed(); + } + } + public: class Iterator { public: @@ -220,7 +469,7 @@ class FlatHashMapV2 { explicit FlatHashMapV2(size_t initial_capacity = 0, int load_limit_pct = kDefaultLoadLimitPct) - : load_limit_percent_(load_limit_pct) { + : FlatHashMapV2Base(load_limit_pct) { if (initial_capacity > 0) { Reset(initial_capacity, true); } @@ -231,15 +480,7 @@ class FlatHashMapV2 { ~FlatHashMapV2() { Clear(); } FlatHashMapV2(FlatHashMapV2&& other) noexcept - : storage_(std::move(other.storage_)), - capacity_(other.capacity_), - size_(other.size_), - growth_info_(other.growth_info_), - load_limit_percent_(other.load_limit_percent_), - ctrl_(other.ctrl_), - slots_(other.slots_) { - new (&other) FlatHashMapV2(); - } + : FlatHashMapV2Base(std::move(other)) {} FlatHashMapV2& operator=(FlatHashMapV2&& other) noexcept { this->~FlatHashMapV2(); @@ -250,50 +491,45 @@ class FlatHashMapV2 { FlatHashMapV2(const FlatHashMapV2&) = delete; FlatHashMapV2& operator=(const FlatHashMapV2&) = delete; - template + // Find(), Erase(), Insert() and operator[] take a Key, which the caller + // converts to as needed. With a transparent Hasher, they also take any type + // it can hash and compare to Key as is (see IsHeterogeneousKey()). + + PERFETTO_ALWAYS_INLINE Value* Find(const Key& key) const { + return FindImpl(key); + } + template ()>> PERFETTO_ALWAYS_INLINE Value* Find(const K& key) const { - size_t key_hash = Hasher{}(key); - uint8_t h2 = H2(key_hash); - FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); - if (PERFETTO_UNLIKELY(res.needs_insert)) { - return nullptr; - } - return &slots_[res.idx].value; + return FindImpl(key); } - template + bool Erase(const Key& key) { return EraseImpl(key); } + template ()>> bool Erase(const K& key) { - size_t key_hash = Hasher{}(key); - uint8_t h2 = H2(key_hash); - FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); - if (PERFETTO_UNLIKELY(res.needs_insert)) { - return false; - } - PERFETTO_DCHECK(size_ > 0); - SetCtrl(res.idx, kTombstone); - slots_[res.idx].key.~Key(); - slots_[res.idx].value.~Value(); - size_--; - growth_info_.has_tombstones = 1; - return true; + return EraseImpl(key); } - // With a transparent Hasher, construct the stored Key only if the key is - // absent. Hashing and equality must agree for K and Key. - template + PERFETTO_ALWAYS_INLINE std::pair Insert(const Key& key, + Value value) { + return InsertImpl(key, std::move(value)); + } + PERFETTO_ALWAYS_INLINE std::pair Insert(Key&& key, + Value value) { + return InsertImpl(std::move(key), std::move(value)); + } + // Constructs the Key only if |key| is absent. + template ()>> PERFETTO_ALWAYS_INLINE std::pair Insert(K&& key, Value value) { - if constexpr (flat_hash_map_v2_internal::IsLookupKeyAllowed< - std::remove_reference_t, Key, Hasher, Eq>()) { - return InsertImpl(std::forward(key), std::move(value)); - } else { - return Insert(Key(std::forward(key)), std::move(value)); - } + return InsertImpl(std::forward(key), std::move(value)); } - template + Value& operator[](const Key& key) { return *Insert(key, Value{}).first; } + Value& operator[](Key&& key) { + return *Insert(std::move(key), Value{}).first; + } + template ()>> Value& operator[](K&& key) { - auto it_and_inserted = Insert(std::forward(key), Value{}); - return *it_and_inserted.first; + return *Insert(std::forward(key), Value{}).first; } void Clear() { @@ -306,15 +542,15 @@ class FlatHashMapV2 { if (tag == kFreeSlot || tag == kTombstone) { continue; } - slots_[i].key.~Key(); - slots_[i].value.~Value(); + slots()[i].key.~Key(); + slots()[i].value.~Value(); } Reset(capacity_, false); } - Iterator GetIterator() { return Iterator(ctrl_, ctrl_ + capacity_, slots_); } + Iterator GetIterator() { return Iterator(ctrl_, ctrl_ + capacity_, slots()); } Iterator GetIterator() const { - return Iterator(ctrl_, ctrl_ + capacity_, slots_); + return Iterator(ctrl_, ctrl_ + capacity_, slots()); } size_t size() const { return size_; } @@ -327,124 +563,10 @@ class FlatHashMapV2 { uint64_t needs_insert : 1; }; - // Tracks growth capacity and whether any deletions have occurred. - // Using bitfields like absl's GrowthInfo to avoid manual bit manipulation. - struct GrowthInfo { - uint64_t growth_left : 63; - uint64_t has_tombstones : 1; - }; - // Not found sentinel (must fit in 63-bit FindResult.idx) static constexpr size_t kNotFound = std::numeric_limits::max() >> 1; - // Abstraction over a group of control bytes that enables batch operations. - // On x64, uses SSE to match 16 control bytes in parallel. - // On other platforms, uses SWAR (SIMD Within A Register) for 8 bytes. - // - // Match() returns an iterator over slots whose control byte matches h2. - // MatchEmpty() returns an iterator over empty slots (kFreeSlot). - // MatchEmptyOrDeleted() returns an iterator over empty or deleted slots. -#if PERFETTO_BUILDFLAG(PERFETTO_X64_CPU_OPT) - struct Group { - public: - // Group size 16 for x64 SSE - static constexpr size_t kSize = 16; - - // Iterates over set bits in a match mask. Each set bit indicates a slot - // in the group that matched the search criteria. Call Next() to get the - // index of each matching slot. - struct Iterator { - public: - PERFETTO_ALWAYS_INLINE explicit Iterator(uint16_t mask) : mask_(mask) {} - PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } - - PERFETTO_ALWAYS_INLINE size_t Next() { - auto idx = static_cast(CountTrailZeros(mask_)); - mask_ &= static_cast(mask_ - 1); - return idx; - } - - private: - uint16_t mask_; - }; - - PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { - ctrl_ = _mm_loadu_si128(reinterpret_cast(pos)); - } - - PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { - auto match = _mm_cmpeq_epi8(ctrl_, _mm_set1_epi8(static_cast(h2))); - return Iterator(static_cast(_mm_movemask_epi8(match))); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { - return Iterator(static_cast( - _mm_movemask_epi8(_mm_sign_epi8(ctrl_, ctrl_)))); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { - return Iterator(static_cast(_mm_movemask_epi8(ctrl_))); - } - - private: - __m128i ctrl_; - }; -#else - // SWAR fallback: processes 8 control bytes at a time using 64-bit arithmetic. - struct Group { - public: - // Group size 8 for ARM and other platforms - static constexpr size_t kSize = 8; - - // Iterates over set bits in a sparse 64-bit mask. Each set MSB indicates - // a matching byte in the group. Call Next() to get the index of each - // matching slot. - struct Iterator { - public: - PERFETTO_ALWAYS_INLINE explicit Iterator(uint64_t mask) : mask_(mask) {} - PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } - PERFETTO_ALWAYS_INLINE size_t Next() { - // Count zeros and divide by 8 (shift 3) - // 0x80 (Byte 0) -> CTZ 7 -> 7>>3 = 0 - // 0x8000 (Byte 1) -> CTZ 15 -> 15>>3 = 1 - size_t idx = static_cast(CountTrailZeros(mask_) >> 3); - mask_ &= mask_ - 1; // Clear lowest set bit - return idx; - } - - private: - uint64_t mask_; - }; - - PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { - memcpy(&ctrl_, pos, sizeof(ctrl_)); - } - - PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { - uint64_t x = ctrl_ ^ (kLsbs * h2); - return Iterator((x - kLsbs) & ~x & kMsbs); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { - // 0x80 check (Empty) - return Iterator((ctrl_ & ~(ctrl_ << 6)) & kMsbs); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { - // 0x80 or 0xFE check (Empty or Deleted) - return Iterator((ctrl_ & ~(ctrl_ << 7)) & kMsbs); - } - - private: - static constexpr uint64_t kLsbs = 0x0101010101010101ULL; - static constexpr uint64_t kMsbs = 0x8080808080808080ULL; - - uint64_t ctrl_; - }; -#endif - - // The number of cloned control bytes after the main control byte array. - static constexpr size_t kNumClones = Group::kSize - 1; + using Group = flat_hash_map_v2_internal::Group; // Searches for a key in the table. This function IGNORES tombstones during // the search - it only stops at empty slots (kFreeSlot) or matching keys. @@ -490,7 +612,7 @@ class FlatHashMapV2 { while (true) { // Prefetch slots at current probe offset (like Absl). - __builtin_prefetch(slots_ + offset, 0, 3); + __builtin_prefetch(slots() + offset, 0, 3); Group group(ctrl + offset); @@ -499,7 +621,7 @@ class FlatHashMapV2 { // Must mask because offset + it.Next() can exceed capacity when // group straddles the table boundary (using cloned control bytes). size_t idx = (offset + it.Next()) & cap_mask; - if (PERFETTO_LIKELY(Eq{}(slots_[idx].key, key))) { + if (PERFETTO_LIKELY(Eq{}(slots()[idx].key, key))) { return {idx, false}; // Found } } @@ -524,31 +646,61 @@ class FlatHashMapV2 { } } - // Find first empty OR tombstone slot for insertion. - // Called when has_tombstones is set to find an earlier tombstone that can - // be reused instead of taking a new empty slot. - size_t FindFirstEmptyOrTombstone(size_t key_hash) const { - const size_t cap_mask = capacity_ - 1; - size_t offset = H1(key_hash) & cap_mask; - size_t probe_size = 0; - while (true) { - Group group(ctrl_ + offset); - if (auto it = group.MatchEmptyOrDeleted(); PERFETTO_LIKELY(it)) { - return (offset + it.Next()) & cap_mask; - } - probe_size += Group::kSize; - offset = (offset + probe_size) & cap_mask; + // Only constructing the key and value is inlined: finding the slot is out of + // line in FindOrPrepareInsert(), compiled once per lookup key type rather + // than per call site or per value category of |key|. + template + PERFETTO_ALWAYS_INLINE std::pair InsertImpl(K&& key, + Value&& value) { + const std::remove_reference_t& lookup_key = key; + FindResult res = FindOrPrepareInsert(lookup_key); + Slot* slot = &slots()[res.idx]; + const bool inserted = res.needs_insert; + if (inserted) { + new (&slot->key) Key(std::forward(key)); + new (&slot->value) Value(std::move(value)); } + return {&slot->value, inserted}; } + // Out of line, compiled once per lookup key type. template - PERFETTO_ALWAYS_INLINE std::pair InsertImpl(K&& key, - Value&& value) { + PERFETTO_NO_INLINE Value* FindImpl(const K& key) const { + size_t key_hash = Hasher{}(key); + uint8_t h2 = H2(key_hash); + FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); + if (PERFETTO_UNLIKELY(res.needs_insert)) { + return nullptr; + } + return &slots()[res.idx].value; + } + + template + PERFETTO_NO_INLINE bool EraseImpl(const K& key) { + size_t key_hash = Hasher{}(key); + uint8_t h2 = H2(key_hash); + FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); + if (PERFETTO_UNLIKELY(res.needs_insert)) { + return false; + } + PERFETTO_DCHECK(size_ > 0); + SetCtrl(res.idx, kTombstone); + slots()[res.idx].key.~Key(); + slots()[res.idx].value.~Value(); + size_--; + growth_info_.has_tombstones = 1; + return true; + } + + // Returns the slot holding |key| if present. Otherwise claims a slot for it, + // growing the table if needed, and the caller must construct into it. + template + PERFETTO_NO_INLINE FindResult FindOrPrepareInsert(const K& key) { size_t key_hash = Hasher{}(key); uint8_t h2 = H2(key_hash); FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); if (PERFETTO_UNLIKELY(!res.needs_insert)) { - return {&slots_[res.idx].value, false}; + return res; } if (PERFETTO_UNLIKELY(growth_info_.growth_left == 0)) { GrowAndRehash(); @@ -563,127 +715,48 @@ class FlatHashMapV2 { insert_idx = FindFirstEmptyOrTombstone(key_hash); is_freeslot = ctrl_[insert_idx] != kTombstone; } - new (&slots_[insert_idx].key) Key(std::forward(key)); - new (&slots_[insert_idx].value) Value(std::move(value)); SetCtrl(insert_idx, h2); size_++; if (is_freeslot) { growth_info_.growth_left--; } - return {&slots_[insert_idx].value, true}; + return {insert_idx, true}; } + // Allocating the new table is shared by all maps in Grow(): only moving the + // entries over, which needs their types, is compiled per map. PERFETTO_NO_INLINE void GrowAndRehash() { - // Grow factor must be a power of 2 because probing uses bitwise AND - // for modulo arithmetic (capacity must remain a power of 2). - static constexpr size_t kGrowFactor = 2; - static_assert((kGrowFactor & (kGrowFactor - 1)) == 0, - "kGrowFactor must be a power of 2"); - - PERFETTO_DCHECK(size_ <= capacity_); - - size_t old_capacity = capacity_; - size_t old_size = size_; - uint8_t* old_ctrl = ctrl_; - Slot* old_slots = slots_; - std::unique_ptr old_storage(std::move(storage_)); - - // This must be a CHECK (i.e. not just a DCHECK) to prevent UAF attacks on - // 32-bit archs that try to double the size of the table until wrapping. - size_t new_capacity = old_capacity * kGrowFactor; - PERFETTO_CHECK(new_capacity >= old_capacity); - Reset(new_capacity, true); - + OldTable old = Grow(sizeof(Slot), alignof(Slot)); + Slot* old_slots = reinterpret_cast(old.slots); + // Keys are unique and the new table has no tombstones, so each entry goes + // straight into the first empty slot of its probe sequence. size_t new_size = 0; - for (size_t i = 0; i < old_capacity; ++i) { - if (uint8_t t = old_ctrl[i]; t == kFreeSlot || t == kTombstone) { + for (size_t i = 0; i < old.capacity; ++i) { + if (uint8_t t = old.ctrl[i]; t == kFreeSlot || t == kTombstone) { continue; } - Insert(std::move(old_slots[i].key), std::move(old_slots[i].value)); - old_slots[i].key.~Key(); // Destroy the old objects. - old_slots[i].value.~Value(); + Slot& old_slot = old_slots[i]; + size_t key_hash = Hasher{}(old_slot.key); + size_t idx = FindFirstEmptyOrTombstoneImpl(key_hash); + new (&slots()[idx].key) Key(std::move(old_slot.key)); + new (&slots()[idx].value) Value(std::move(old_slot.value)); + SetCtrl(idx, H2(key_hash)); + old_slot.key.~Key(); // Destroy the old objects. + old_slot.value.~Value(); new_size++; } - PERFETTO_DCHECK(new_size == old_size); - size_ = new_size; + PERFETTO_DCHECK(new_size == old.size); + FinishGrow(new_size); } // Doesn't call destructors. Use Clear() for that. - PERFETTO_NO_INLINE void Reset(size_t n, bool reallocate) { - // Must be a pow2. - PERFETTO_CHECK((n & (n - 1)) == 0); - - // Always ensure at least 128 capacity to avoid too frequent growths. - capacity_ = std::max(n, 128u); - size_ = 0; - growth_info_.growth_left = - (capacity_ * static_cast(load_limit_percent_)) / 100; - growth_info_.has_tombstones = 0; - - if (reallocate) { - // See memory layout comment above |storage_|. - size_t slots_offset = - base::AlignUp(capacity_ + kNumClones, alignof(Slot)); - storage_.reset(new uint8_t[slots_offset + (capacity_ * sizeof(Slot))]); - ctrl_ = storage_.get(); - slots_ = reinterpret_cast(storage_.get() + slots_offset); - } - if (ctrl_) { - // Initialize all control bytes (including clones) to empty (kFreeSlot) - memset(ctrl_, kFreeSlot, capacity_ + kNumClones); - } + void Reset(size_t n, bool reallocate) { + FlatHashMapV2Base::Reset(n, reallocate, sizeof(Slot), alignof(Slot)); } - // Swiss Table hash splitting (matching absl): - // H1 = upper bits for bucket index - // H2 = lower 7 bits for tag - // This ensures H1 and H2 are independent, avoiding tag collisions within - // buckets. The seed XOR prevents clustering when hash values have patterns - // (e.g., sequential keys) - static constexpr size_t H1(size_t hash) { return (hash >> 7); } - static constexpr uint8_t H2(size_t hash) { return hash & 0x7F; } - - // Set control byte and update clone if needed - PERFETTO_ALWAYS_INLINE void SetCtrl(size_t i, uint8_t h) { - ctrl_[i] = h; - // Update clone if this is one of the first kNumClones entries - if (PERFETTO_UNLIKELY(i < kNumClones)) { - ctrl_[capacity_ + i] = h; - } + PERFETTO_ALWAYS_INLINE Slot* slots() const { + return reinterpret_cast(slots_); } - - // Owns the actual memory with the following layout: - // - // [Control bytes] - // |capacity_| bytes for control bytes. - // kNumClones (15 or 7) bytes for control byte clones (*). - // No alignment required (accessed at arbitrary byte offsets). - // - // [Padding for Slot alignment] - // - // [Slots] - // capacity_ * sizeof(Slot): contains key-value pairs. - // Must be aligned to alignof(Slot). - // - // (*) Control byte clones: The first kNumClones control bytes are duplicated - // at the end of the control array. This allows SIMD operations to read a full - // group (16 or 8 bytes) starting from any position without bounds checking, - // even near the end of the array. - std::unique_ptr storage_; - - size_t capacity_ = 0; - size_t size_ = 0; - - // Slots remaining + has_deleted flag - GrowthInfo growth_info_{0, 0}; - - // Load factor limit in % of |capacity_|. - int load_limit_percent_ = kDefaultLoadLimitPct; - - // Cached pointers for fast access (like absl::flat_hash_map) - // These are updated whenever storage is allocated/reallocated. - uint8_t* ctrl_ = nullptr; // Points to control bytes - Slot* slots_ = nullptr; // Points to slot array }; // Alias FlatHashMap to FlatHashMapV1 for backward compatibility. diff --git a/include/perfetto/ext/base/murmur_hash.h b/include/perfetto/ext/base/murmur_hash.h index 309bf6f623a..80b82427a98 100644 --- a/include/perfetto/ext/base/murmur_hash.h +++ b/include/perfetto/ext/base/murmur_hash.h @@ -223,13 +223,6 @@ constexpr bool HasMurmurHashBuiltinValue() { uint64_t>; } -// Helper to check if two types are integeral and U is convertible to T. -template -constexpr bool IsConvertibleIntegral() { - return std::is_integral_v && std::is_integral_v && - std::is_convertible_v; -} - // Helper to check if a type is string-like (i.e. string, c-string or string // views). template @@ -239,14 +232,25 @@ constexpr bool IsStringLike() { std::is_same_v || std::is_same_v; } -// Helper to check if heterogeneous lookup is allowed between T and U. -// Only allows it for convertible integral types and string-like types. +// Helper to check if heterogeneous lookup is allowed between T and U: only +// between string-like types. Other types (e.g. integers) are converted to T by +// the caller instead, as converting them is cheap, and hashing them as is +// could differ (e.g. a negative int hashed for a uint32_t key). template constexpr bool AllowsHeterogeneousLookup() { - return IsConvertibleIntegral() || - (IsStringLike() && IsStringLike()); + return IsStringLike() && IsStringLike(); } +// MurmurHash is transparent (i.e. supports heterogeneous lookups) only for +// string-like T. +template ()> +struct MurmurHashTransparency {}; + +template +struct MurmurHashTransparency { + using is_transparent = void; +}; + // Helper to detect pointers in Combine(...). // Hashing pointers directly is prohibited as it often indicates a misuse // (e.g., passing ptr and len instead of a single std::string_view). @@ -358,13 +362,10 @@ uint64_t MurmurHashValue(const T& value) { // std::hash drop-in class which uses MurmurHashValue as the primitive. // All specializations consistently delegate to MurmurHashValue. template -struct MurmurHash { - using is_transparent = void; - +struct MurmurHash : murmur_internal::MurmurHashTransparency { uint64_t operator()(const T& value) const { return MurmurHashValue(value); } - // Heterogeneous lookup support. Only allowed for types where it makes sense - // (e.g. string-like types and convertible integral types). + // Heterogeneous lookup support, between string-like types only. template auto operator()(const U& value) const -> std::enable_if_t< murmur_internal::AllowsHeterogeneousLookup>(), diff --git a/src/base/BUILD.gn b/src/base/BUILD.gn index 08d1af80863..e8df7435685 100644 --- a/src/base/BUILD.gn +++ b/src/base/BUILD.gn @@ -43,6 +43,7 @@ perfetto_component("base") { "dynamic_string_writer.cc", "event_fd.cc", "file_utils.cc", + "flat_hash_map.cc", "futex.cc", "getopt_compat.cc", "intrusive_list.cc", diff --git a/src/base/flat_hash_map.cc b/src/base/flat_hash_map.cc new file mode 100644 index 00000000000..5c1c064439c --- /dev/null +++ b/src/base/flat_hash_map.cc @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "perfetto/ext/base/flat_hash_map.h" + +#include +#include +#include +#include +#include +#include + +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/utils.h" + +namespace perfetto::base::flat_hash_map_v2_internal { + +void FlatHashMapV2Base::Reset(size_t n, + bool reallocate, + size_t slot_size, + size_t slot_align) { + // Must be a pow2. + PERFETTO_CHECK((n & (n - 1)) == 0); + + // Always ensure at least 128 capacity to avoid too frequent growths. + capacity_ = std::max(n, 128u); + size_ = 0; + growth_info_.growth_left = + (capacity_ * static_cast(load_limit_percent_)) / 100; + growth_info_.has_tombstones = 0; + + if (reallocate) { + // See memory layout comment above |storage_| in flat_hash_map.h. + size_t slots_offset = base::AlignUp(capacity_ + kNumClones, slot_align); + storage_.reset(new uint8_t[slots_offset + (capacity_ * slot_size)]); + ctrl_ = storage_.get(); + slots_ = storage_.get() + slots_offset; + } + if (ctrl_) { + // Initialize all control bytes (including clones) to empty (kFreeSlot) + memset(ctrl_, kFreeSlot, capacity_ + kNumClones); + } +} + +size_t FlatHashMapV2Base::FindFirstEmptyOrTombstone(size_t key_hash) const { + return FindFirstEmptyOrTombstoneImpl(key_hash); +} + +FlatHashMapV2Base::OldTable FlatHashMapV2Base::Grow(size_t slot_size, + size_t slot_align) { + PERFETTO_DCHECK(size_ <= capacity_); + OldTable old{std::move(storage_), ctrl_, slots_, capacity_, size_}; + // This must be a CHECK (i.e. not just a DCHECK) to prevent UAF attacks on + // 32-bit archs that try to double the size of the table until wrapping. + size_t new_capacity = capacity_ * 2; + PERFETTO_CHECK(new_capacity >= capacity_); + Reset(new_capacity, true, slot_size, slot_align); + return old; +} + +} // namespace perfetto::base::flat_hash_map_v2_internal diff --git a/src/base/flat_hash_map_unittest.cc b/src/base/flat_hash_map_unittest.cc index 993424ebdce..eadccf943c5 100644 --- a/src/base/flat_hash_map_unittest.cc +++ b/src/base/flat_hash_map_unittest.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -119,6 +120,15 @@ int Key::instances = 0; int Key::constructions = 0; int Value::instances = 0; +// Whether Map::Find() can be called with a K. +template +struct CanFind : std::false_type {}; +template +struct CanFind().Find( + std::declval()))>> : std::true_type {}; + TEST(FlatHashMapV2Test, HeterogeneousInsert) { struct Hasher { using is_transparent = void; @@ -191,7 +201,7 @@ TEST(FlatHashMapV2Test, InsertConvertibleKey) { EXPECT_EQ(*int_map.Find(int64_t{42}), 100); } -TEST(FlatHashMapV2Test, InsertConvertsForNonHeterogeneousEquality) { +TEST(FlatHashMapV2Test, NonHeterogeneousEqualityNeedsOwningKey) { struct Hasher { using is_transparent = void; size_t operator()(const Key& key) const { @@ -199,11 +209,13 @@ TEST(FlatHashMapV2Test, InsertConvertsForNonHeterogeneousEquality) { } size_t operator()(int key) const { return static_cast(key); } }; - // Key is explicitly constructible from int, but its equality only accepts - // Key. Insertion must convert to Key before probing the map. - FlatHashMapV2 map; - ASSERT_TRUE(map.Insert(42, 100).second); - auto duplicate = map.Insert(42, 200); + // The Hasher takes int, but Key's equality only accepts Key, so an int isn't + // a heterogeneous key: the caller has to construct a Key. + using Map = FlatHashMapV2; + static_assert(!CanFind::value); + Map map; + ASSERT_TRUE(map.Insert(Key(42), 100).second); + auto duplicate = map.Insert(Key(42), 200); EXPECT_FALSE(duplicate.second); EXPECT_EQ(*duplicate.first, 100); ASSERT_NE(map.Find(Key(42)), nullptr); diff --git a/src/base/murmur_hash_unittest.cc b/src/base/murmur_hash_unittest.cc index 0afcb9e64f9..466a281f4b6 100644 --- a/src/base/murmur_hash_unittest.cc +++ b/src/base/murmur_hash_unittest.cc @@ -16,8 +16,10 @@ #include "perfetto/ext/base/murmur_hash.h" +#include #include #include +#include #include "perfetto/ext/base/string_view.h" #include "test/gtest_and_gmock.h" @@ -44,5 +46,22 @@ TEST(MurmurHashTest, HeterogeneousStringHash) { EXPECT_EQ(hasher.operator()("abc"), expected); } +template +struct IsTransparent : std::false_type {}; +template +struct IsTransparent> + : std::true_type {}; + +TEST(MurmurHashTest, OnlyStringLikeKeysAreTransparent) { + static_assert(IsTransparent>::value); + static_assert(IsTransparent>::value); + static_assert(IsTransparent>::value); + static_assert(IsTransparent>::value); + // Other keys (e.g. integers) are converted by the caller instead: hashing + // an int as is could differ from hashing it as a uint32_t key. + static_assert(!IsTransparent>::value); + static_assert(!IsTransparent>::value); +} + } // namespace } // namespace perfetto::base