diff --git a/Android.bp b/Android.bp index 88a1395f0e..3ab8fcfb7d 100644 --- a/Android.bp +++ b/Android.bp @@ -15975,6 +15975,7 @@ filegroup { "src/base/dynamic_string_writer.cc", "src/base/event_fd.cc", "src/base/file_utils.cc", + "src/base/flat_hash_map.cc", "src/base/futex.cc", "src/base/getopt_compat.cc", "src/base/intrusive_list.cc", diff --git a/BUILD b/BUILD index e544312f72..5845c08bf9 100644 --- a/BUILD +++ b/BUILD @@ -1996,6 +1996,7 @@ perfetto_cc_library( "src/base/dynamic_string_writer.cc", "src/base/event_fd.cc", "src/base/file_utils.cc", + "src/base/flat_hash_map.cc", "src/base/futex.cc", "src/base/getopt_compat.cc", "src/base/intrusive_list.cc", diff --git a/include/perfetto/ext/base/flat_hash_map.h b/include/perfetto/ext/base/flat_hash_map.h index e7a3021b5d..14d66c969c 100644 --- a/include/perfetto/ext/base/flat_hash_map.h +++ b/include/perfetto/ext/base/flat_hash_map.h @@ -135,6 +135,240 @@ static constexpr uint8_t kTombstone = 0xFE; // Deleted slot // The default load limit percent before growing the table. static constexpr int kDefaultLoadLimitPct = 75; +// Abstraction over a group of control bytes that enables batch operations. +// On x64, uses SSE to match 16 control bytes in parallel. +// On other platforms, uses SWAR (SIMD Within A Register) for 8 bytes. +// +// Match() returns an iterator over slots whose control byte matches h2. +// MatchEmpty() returns an iterator over empty slots (kFreeSlot). +// MatchEmptyOrDeleted() returns an iterator over empty or deleted slots. +#if PERFETTO_BUILDFLAG(PERFETTO_X64_CPU_OPT) +struct Group { + public: + // Group size 16 for x64 SSE + static constexpr size_t kSize = 16; + + // Iterates over set bits in a match mask. Each set bit indicates a slot + // in the group that matched the search criteria. Call Next() to get the + // index of each matching slot. + struct Iterator { + public: + PERFETTO_ALWAYS_INLINE explicit Iterator(uint16_t mask) : mask_(mask) {} + PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } + + PERFETTO_ALWAYS_INLINE size_t Next() { + auto idx = static_cast(CountTrailZeros(mask_)); + mask_ &= static_cast(mask_ - 1); + return idx; + } + + private: + uint16_t mask_; + }; + + PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { + ctrl_ = _mm_loadu_si128(reinterpret_cast(pos)); + } + + PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { + auto match = _mm_cmpeq_epi8(ctrl_, _mm_set1_epi8(static_cast(h2))); + return Iterator(static_cast(_mm_movemask_epi8(match))); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { + return Iterator( + static_cast(_mm_movemask_epi8(_mm_sign_epi8(ctrl_, ctrl_)))); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { + return Iterator(static_cast(_mm_movemask_epi8(ctrl_))); + } + + private: + __m128i ctrl_; +}; +#else +// SWAR fallback: processes 8 control bytes at a time using 64-bit arithmetic. +struct Group { + public: + // Group size 8 for ARM and other platforms + static constexpr size_t kSize = 8; + + // Iterates over set bits in a sparse 64-bit mask. Each set MSB indicates + // a matching byte in the group. Call Next() to get the index of each + // matching slot. + struct Iterator { + public: + PERFETTO_ALWAYS_INLINE explicit Iterator(uint64_t mask) : mask_(mask) {} + PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } + PERFETTO_ALWAYS_INLINE size_t Next() { + // Count zeros and divide by 8 (shift 3) + // 0x80 (Byte 0) -> CTZ 7 -> 7>>3 = 0 + // 0x8000 (Byte 1) -> CTZ 15 -> 15>>3 = 1 + size_t idx = static_cast(CountTrailZeros(mask_) >> 3); + mask_ &= mask_ - 1; // Clear lowest set bit + return idx; + } + + private: + uint64_t mask_; + }; + + PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { + memcpy(&ctrl_, pos, sizeof(ctrl_)); + } + + PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { + uint64_t x = ctrl_ ^ (kLsbs * h2); + return Iterator((x - kLsbs) & ~x & kMsbs); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { + // 0x80 check (Empty) + return Iterator((ctrl_ & ~(ctrl_ << 6)) & kMsbs); + } + + PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { + // 0x80 or 0xFE check (Empty or Deleted) + return Iterator((ctrl_ & ~(ctrl_ << 7)) & kMsbs); + } + + private: + static constexpr uint64_t kLsbs = 0x0101010101010101ULL; + static constexpr uint64_t kMsbs = 0x8080808080808080ULL; + + uint64_t ctrl_; +}; +#endif + +// The state and the operations of FlatHashMapV2 which don't depend on the key +// and value types. Not templated, and the heavier operations are defined in +// flat_hash_map.cc, so that all maps share one copy of the code. +class FlatHashMapV2Base { + protected: + // Tracks growth capacity and whether any deletions have occurred. + // Using bitfields like absl's GrowthInfo to avoid manual bit manipulation. + struct GrowthInfo { + uint64_t growth_left : 63; + uint64_t has_tombstones : 1; + }; + + // The table being moved out of while growing. + struct OldTable { + std::unique_ptr storage; + const uint8_t* ctrl; + uint8_t* slots; + size_t capacity; + size_t size; + }; + + // The number of cloned control bytes after the main control byte array. + static constexpr size_t kNumClones = Group::kSize - 1; + + explicit FlatHashMapV2Base(int load_limit_pct) + : load_limit_percent_(load_limit_pct) {} + + FlatHashMapV2Base(FlatHashMapV2Base&& other) noexcept + : storage_(std::move(other.storage_)), + capacity_(other.capacity_), + size_(other.size_), + growth_info_(other.growth_info_), + load_limit_percent_(other.load_limit_percent_), + ctrl_(other.ctrl_), + slots_(other.slots_) { + other.capacity_ = 0; + other.size_ = 0; + other.growth_info_ = {0, 0}; + other.load_limit_percent_ = kDefaultLoadLimitPct; + other.ctrl_ = nullptr; + other.slots_ = nullptr; + } + + // Swiss Table hash splitting (matching absl): + // H1 = upper bits for bucket index + // H2 = lower 7 bits for tag + // This ensures H1 and H2 are independent, avoiding tag collisions within + // buckets. The seed XOR prevents clustering when hash values have patterns + // (e.g., sequential keys) + static constexpr size_t H1(size_t hash) { return (hash >> 7); } + static constexpr uint8_t H2(size_t hash) { return hash & 0x7F; } + + // Doesn't call destructors. + void Reset(size_t n, bool reallocate, size_t slot_size, size_t slot_align); + + // Find first empty OR tombstone slot for insertion. + // Called when has_tombstones is set to find an earlier tombstone that can + // be reused instead of taking a new empty slot. + size_t FindFirstEmptyOrTombstone(size_t key_hash) const; + + PERFETTO_ALWAYS_INLINE size_t + FindFirstEmptyOrTombstoneImpl(size_t key_hash) const { + const size_t cap_mask = capacity_ - 1; + size_t offset = H1(key_hash) & cap_mask; + size_t probe_size = 0; + while (true) { + Group group(ctrl_ + offset); + if (auto it = group.MatchEmptyOrDeleted(); PERFETTO_LIKELY(it)) { + return (offset + it.Next()) & cap_mask; + } + probe_size += Group::kSize; + offset = (offset + probe_size) & cap_mask; + } + } + + // Swaps in an empty table twice as large and returns the old one, from + // which the caller must move the entries before calling FinishGrow(). + OldTable Grow(size_t slot_size, size_t slot_align); + + void FinishGrow(size_t new_size) { + PERFETTO_CHECK(growth_info_.growth_left >= new_size); + size_ = new_size; + growth_info_.growth_left -= new_size; + } + + // Set control byte and update clone if needed + PERFETTO_ALWAYS_INLINE void SetCtrl(size_t i, uint8_t h) { + ctrl_[i] = h; + // Update clone if this is one of the first kNumClones entries + if (PERFETTO_UNLIKELY(i < kNumClones)) { + ctrl_[capacity_ + i] = h; + } + } + + // Owns the actual memory with the following layout: + // + // [Control bytes] + // |capacity_| bytes for control bytes. + // kNumClones (15 or 7) bytes for control byte clones (*). + // No alignment required (accessed at arbitrary byte offsets). + // + // [Padding for Slot alignment] + // + // [Slots] + // capacity_ * sizeof(Slot): contains key-value pairs. + // Must be aligned to alignof(Slot). + // + // (*) Control byte clones: The first kNumClones control bytes are duplicated + // at the end of the control array. This allows SIMD operations to read a full + // group (16 or 8 bytes) starting from any position without bounds checking, + // even near the end of the array. + std::unique_ptr storage_; + + size_t capacity_ = 0; + size_t size_ = 0; + + // Slots remaining + has_deleted flag + GrowthInfo growth_info_{0, 0}; + + // Load factor limit in % of |capacity_|. + int load_limit_percent_ = kDefaultLoadLimitPct; + + // Cached pointers for fast access (like absl::flat_hash_map) + // These are updated whenever storage is allocated/reallocated. + uint8_t* ctrl_ = nullptr; // Points to control bytes + uint8_t* slots_ = nullptr; // Points to slot array +}; + } // namespace flat_hash_map_v2_internal // Hash and equality functors which treat ASCII strings case-insensitively. @@ -164,7 +398,7 @@ template , typename Eq = flat_hash_map_v2_internal::HashEq> -class FlatHashMapV2 { +class FlatHashMapV2 : private flat_hash_map_v2_internal::FlatHashMapV2Base { private: // Import constants from internal namespace. static constexpr uint8_t kFreeSlot = flat_hash_map_v2_internal::kFreeSlot; @@ -178,6 +412,21 @@ class FlatHashMapV2 { Value value; }; + // Whether |K| is a type other than Key which, with a transparent Hasher, + // is hashed and compared to Key as is (e.g. a std::string_view looked up in + // a map keyed by std::string), sparing the construction of a Key. + template + static constexpr bool IsHeterogeneousKey() { + using RawK = std::remove_cv_t>; + if constexpr (std::is_same_v) { + return false; + } else { + return flat_hash_map_v2_internal::HasIsTransparent::value && + flat_hash_map_v2_internal::IsLookupKeyAllowed(); + } + } + public: class Iterator { public: @@ -220,7 +469,7 @@ class FlatHashMapV2 { explicit FlatHashMapV2(size_t initial_capacity = 0, int load_limit_pct = kDefaultLoadLimitPct) - : load_limit_percent_(load_limit_pct) { + : FlatHashMapV2Base(load_limit_pct) { if (initial_capacity > 0) { Reset(initial_capacity, true); } @@ -231,15 +480,7 @@ class FlatHashMapV2 { ~FlatHashMapV2() { Clear(); } FlatHashMapV2(FlatHashMapV2&& other) noexcept - : storage_(std::move(other.storage_)), - capacity_(other.capacity_), - size_(other.size_), - growth_info_(other.growth_info_), - load_limit_percent_(other.load_limit_percent_), - ctrl_(other.ctrl_), - slots_(other.slots_) { - new (&other) FlatHashMapV2(); - } + : FlatHashMapV2Base(std::move(other)) {} FlatHashMapV2& operator=(FlatHashMapV2&& other) noexcept { this->~FlatHashMapV2(); @@ -250,50 +491,45 @@ class FlatHashMapV2 { FlatHashMapV2(const FlatHashMapV2&) = delete; FlatHashMapV2& operator=(const FlatHashMapV2&) = delete; - template + // Find(), Erase(), Insert() and operator[] take a Key, which the caller + // converts to as needed. With a transparent Hasher, they also take any type + // it can hash and compare to Key as is (see IsHeterogeneousKey()). + + PERFETTO_ALWAYS_INLINE Value* Find(const Key& key) const { + return FindImpl(key); + } + template ()>> PERFETTO_ALWAYS_INLINE Value* Find(const K& key) const { - size_t key_hash = Hasher{}(key); - uint8_t h2 = H2(key_hash); - FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); - if (PERFETTO_UNLIKELY(res.needs_insert)) { - return nullptr; - } - return &slots_[res.idx].value; + return FindImpl(key); } - template + bool Erase(const Key& key) { return EraseImpl(key); } + template ()>> bool Erase(const K& key) { - size_t key_hash = Hasher{}(key); - uint8_t h2 = H2(key_hash); - FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); - if (PERFETTO_UNLIKELY(res.needs_insert)) { - return false; - } - PERFETTO_DCHECK(size_ > 0); - SetCtrl(res.idx, kTombstone); - slots_[res.idx].key.~Key(); - slots_[res.idx].value.~Value(); - size_--; - growth_info_.has_tombstones = 1; - return true; + return EraseImpl(key); } - // With a transparent Hasher, construct the stored Key only if the key is - // absent. Hashing and equality must agree for K and Key. - template + PERFETTO_ALWAYS_INLINE std::pair Insert(const Key& key, + Value value) { + return InsertImpl(key, std::move(value)); + } + PERFETTO_ALWAYS_INLINE std::pair Insert(Key&& key, + Value value) { + return InsertImpl(std::move(key), std::move(value)); + } + // Constructs the Key only if |key| is absent. + template ()>> PERFETTO_ALWAYS_INLINE std::pair Insert(K&& key, Value value) { - if constexpr (flat_hash_map_v2_internal::IsLookupKeyAllowed< - std::remove_reference_t, Key, Hasher, Eq>()) { - return InsertImpl(std::forward(key), std::move(value)); - } else { - return Insert(Key(std::forward(key)), std::move(value)); - } + return InsertImpl(std::forward(key), std::move(value)); } - template + Value& operator[](const Key& key) { return *Insert(key, Value{}).first; } + Value& operator[](Key&& key) { + return *Insert(std::move(key), Value{}).first; + } + template ()>> Value& operator[](K&& key) { - auto it_and_inserted = Insert(std::forward(key), Value{}); - return *it_and_inserted.first; + return *Insert(std::forward(key), Value{}).first; } void Clear() { @@ -306,15 +542,15 @@ class FlatHashMapV2 { if (tag == kFreeSlot || tag == kTombstone) { continue; } - slots_[i].key.~Key(); - slots_[i].value.~Value(); + slots()[i].key.~Key(); + slots()[i].value.~Value(); } Reset(capacity_, false); } - Iterator GetIterator() { return Iterator(ctrl_, ctrl_ + capacity_, slots_); } + Iterator GetIterator() { return Iterator(ctrl_, ctrl_ + capacity_, slots()); } Iterator GetIterator() const { - return Iterator(ctrl_, ctrl_ + capacity_, slots_); + return Iterator(ctrl_, ctrl_ + capacity_, slots()); } size_t size() const { return size_; } @@ -327,124 +563,10 @@ class FlatHashMapV2 { uint64_t needs_insert : 1; }; - // Tracks growth capacity and whether any deletions have occurred. - // Using bitfields like absl's GrowthInfo to avoid manual bit manipulation. - struct GrowthInfo { - uint64_t growth_left : 63; - uint64_t has_tombstones : 1; - }; - // Not found sentinel (must fit in 63-bit FindResult.idx) static constexpr size_t kNotFound = std::numeric_limits::max() >> 1; - // Abstraction over a group of control bytes that enables batch operations. - // On x64, uses SSE to match 16 control bytes in parallel. - // On other platforms, uses SWAR (SIMD Within A Register) for 8 bytes. - // - // Match() returns an iterator over slots whose control byte matches h2. - // MatchEmpty() returns an iterator over empty slots (kFreeSlot). - // MatchEmptyOrDeleted() returns an iterator over empty or deleted slots. -#if PERFETTO_BUILDFLAG(PERFETTO_X64_CPU_OPT) - struct Group { - public: - // Group size 16 for x64 SSE - static constexpr size_t kSize = 16; - - // Iterates over set bits in a match mask. Each set bit indicates a slot - // in the group that matched the search criteria. Call Next() to get the - // index of each matching slot. - struct Iterator { - public: - PERFETTO_ALWAYS_INLINE explicit Iterator(uint16_t mask) : mask_(mask) {} - PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } - - PERFETTO_ALWAYS_INLINE size_t Next() { - auto idx = static_cast(CountTrailZeros(mask_)); - mask_ &= static_cast(mask_ - 1); - return idx; - } - - private: - uint16_t mask_; - }; - - PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { - ctrl_ = _mm_loadu_si128(reinterpret_cast(pos)); - } - - PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { - auto match = _mm_cmpeq_epi8(ctrl_, _mm_set1_epi8(static_cast(h2))); - return Iterator(static_cast(_mm_movemask_epi8(match))); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { - return Iterator(static_cast( - _mm_movemask_epi8(_mm_sign_epi8(ctrl_, ctrl_)))); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { - return Iterator(static_cast(_mm_movemask_epi8(ctrl_))); - } - - private: - __m128i ctrl_; - }; -#else - // SWAR fallback: processes 8 control bytes at a time using 64-bit arithmetic. - struct Group { - public: - // Group size 8 for ARM and other platforms - static constexpr size_t kSize = 8; - - // Iterates over set bits in a sparse 64-bit mask. Each set MSB indicates - // a matching byte in the group. Call Next() to get the index of each - // matching slot. - struct Iterator { - public: - PERFETTO_ALWAYS_INLINE explicit Iterator(uint64_t mask) : mask_(mask) {} - PERFETTO_ALWAYS_INLINE explicit operator bool() const { return mask_; } - PERFETTO_ALWAYS_INLINE size_t Next() { - // Count zeros and divide by 8 (shift 3) - // 0x80 (Byte 0) -> CTZ 7 -> 7>>3 = 0 - // 0x8000 (Byte 1) -> CTZ 15 -> 15>>3 = 1 - size_t idx = static_cast(CountTrailZeros(mask_) >> 3); - mask_ &= mask_ - 1; // Clear lowest set bit - return idx; - } - - private: - uint64_t mask_; - }; - - PERFETTO_ALWAYS_INLINE explicit Group(const uint8_t* pos) { - memcpy(&ctrl_, pos, sizeof(ctrl_)); - } - - PERFETTO_ALWAYS_INLINE Iterator Match(uint8_t h2) const { - uint64_t x = ctrl_ ^ (kLsbs * h2); - return Iterator((x - kLsbs) & ~x & kMsbs); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmpty() const { - // 0x80 check (Empty) - return Iterator((ctrl_ & ~(ctrl_ << 6)) & kMsbs); - } - - PERFETTO_ALWAYS_INLINE Iterator MatchEmptyOrDeleted() const { - // 0x80 or 0xFE check (Empty or Deleted) - return Iterator((ctrl_ & ~(ctrl_ << 7)) & kMsbs); - } - - private: - static constexpr uint64_t kLsbs = 0x0101010101010101ULL; - static constexpr uint64_t kMsbs = 0x8080808080808080ULL; - - uint64_t ctrl_; - }; -#endif - - // The number of cloned control bytes after the main control byte array. - static constexpr size_t kNumClones = Group::kSize - 1; + using Group = flat_hash_map_v2_internal::Group; // Searches for a key in the table. This function IGNORES tombstones during // the search - it only stops at empty slots (kFreeSlot) or matching keys. @@ -490,7 +612,7 @@ class FlatHashMapV2 { while (true) { // Prefetch slots at current probe offset (like Absl). - __builtin_prefetch(slots_ + offset, 0, 3); + __builtin_prefetch(slots() + offset, 0, 3); Group group(ctrl + offset); @@ -499,7 +621,7 @@ class FlatHashMapV2 { // Must mask because offset + it.Next() can exceed capacity when // group straddles the table boundary (using cloned control bytes). size_t idx = (offset + it.Next()) & cap_mask; - if (PERFETTO_LIKELY(Eq{}(slots_[idx].key, key))) { + if (PERFETTO_LIKELY(Eq{}(slots()[idx].key, key))) { return {idx, false}; // Found } } @@ -524,31 +646,61 @@ class FlatHashMapV2 { } } - // Find first empty OR tombstone slot for insertion. - // Called when has_tombstones is set to find an earlier tombstone that can - // be reused instead of taking a new empty slot. - size_t FindFirstEmptyOrTombstone(size_t key_hash) const { - const size_t cap_mask = capacity_ - 1; - size_t offset = H1(key_hash) & cap_mask; - size_t probe_size = 0; - while (true) { - Group group(ctrl_ + offset); - if (auto it = group.MatchEmptyOrDeleted(); PERFETTO_LIKELY(it)) { - return (offset + it.Next()) & cap_mask; - } - probe_size += Group::kSize; - offset = (offset + probe_size) & cap_mask; + // Only constructing the key and value is inlined: finding the slot is out of + // line in FindOrPrepareInsert(), compiled once per lookup key type rather + // than per call site or per value category of |key|. + template + PERFETTO_ALWAYS_INLINE std::pair InsertImpl(K&& key, + Value&& value) { + const std::remove_reference_t& lookup_key = key; + FindResult res = FindOrPrepareInsert(lookup_key); + Slot* slot = &slots()[res.idx]; + const bool inserted = res.needs_insert; + if (inserted) { + new (&slot->key) Key(std::forward(key)); + new (&slot->value) Value(std::move(value)); } + return {&slot->value, inserted}; } + // Out of line, compiled once per lookup key type. template - PERFETTO_ALWAYS_INLINE std::pair InsertImpl(K&& key, - Value&& value) { + PERFETTO_NO_INLINE Value* FindImpl(const K& key) const { + size_t key_hash = Hasher{}(key); + uint8_t h2 = H2(key_hash); + FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); + if (PERFETTO_UNLIKELY(res.needs_insert)) { + return nullptr; + } + return &slots()[res.idx].value; + } + + template + PERFETTO_NO_INLINE bool EraseImpl(const K& key) { + size_t key_hash = Hasher{}(key); + uint8_t h2 = H2(key_hash); + FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); + if (PERFETTO_UNLIKELY(res.needs_insert)) { + return false; + } + PERFETTO_DCHECK(size_ > 0); + SetCtrl(res.idx, kTombstone); + slots()[res.idx].key.~Key(); + slots()[res.idx].value.~Value(); + size_--; + growth_info_.has_tombstones = 1; + return true; + } + + // Returns the slot holding |key| if present. Otherwise claims a slot for it, + // growing the table if needed, and the caller must construct into it. + template + PERFETTO_NO_INLINE FindResult FindOrPrepareInsert(const K& key) { size_t key_hash = Hasher{}(key); uint8_t h2 = H2(key_hash); FindResult res = FindSlotIgnoringTombstones(key, key_hash, h2); if (PERFETTO_UNLIKELY(!res.needs_insert)) { - return {&slots_[res.idx].value, false}; + return res; } if (PERFETTO_UNLIKELY(growth_info_.growth_left == 0)) { GrowAndRehash(); @@ -563,127 +715,48 @@ class FlatHashMapV2 { insert_idx = FindFirstEmptyOrTombstone(key_hash); is_freeslot = ctrl_[insert_idx] != kTombstone; } - new (&slots_[insert_idx].key) Key(std::forward(key)); - new (&slots_[insert_idx].value) Value(std::move(value)); SetCtrl(insert_idx, h2); size_++; if (is_freeslot) { growth_info_.growth_left--; } - return {&slots_[insert_idx].value, true}; + return {insert_idx, true}; } + // Allocating the new table is shared by all maps in Grow(): only moving the + // entries over, which needs their types, is compiled per map. PERFETTO_NO_INLINE void GrowAndRehash() { - // Grow factor must be a power of 2 because probing uses bitwise AND - // for modulo arithmetic (capacity must remain a power of 2). - static constexpr size_t kGrowFactor = 2; - static_assert((kGrowFactor & (kGrowFactor - 1)) == 0, - "kGrowFactor must be a power of 2"); - - PERFETTO_DCHECK(size_ <= capacity_); - - size_t old_capacity = capacity_; - size_t old_size = size_; - uint8_t* old_ctrl = ctrl_; - Slot* old_slots = slots_; - std::unique_ptr old_storage(std::move(storage_)); - - // This must be a CHECK (i.e. not just a DCHECK) to prevent UAF attacks on - // 32-bit archs that try to double the size of the table until wrapping. - size_t new_capacity = old_capacity * kGrowFactor; - PERFETTO_CHECK(new_capacity >= old_capacity); - Reset(new_capacity, true); - + OldTable old = Grow(sizeof(Slot), alignof(Slot)); + Slot* old_slots = reinterpret_cast(old.slots); + // Keys are unique and the new table has no tombstones, so each entry goes + // straight into the first empty slot of its probe sequence. size_t new_size = 0; - for (size_t i = 0; i < old_capacity; ++i) { - if (uint8_t t = old_ctrl[i]; t == kFreeSlot || t == kTombstone) { + for (size_t i = 0; i < old.capacity; ++i) { + if (uint8_t t = old.ctrl[i]; t == kFreeSlot || t == kTombstone) { continue; } - Insert(std::move(old_slots[i].key), std::move(old_slots[i].value)); - old_slots[i].key.~Key(); // Destroy the old objects. - old_slots[i].value.~Value(); + Slot& old_slot = old_slots[i]; + size_t key_hash = Hasher{}(old_slot.key); + size_t idx = FindFirstEmptyOrTombstoneImpl(key_hash); + new (&slots()[idx].key) Key(std::move(old_slot.key)); + new (&slots()[idx].value) Value(std::move(old_slot.value)); + SetCtrl(idx, H2(key_hash)); + old_slot.key.~Key(); // Destroy the old objects. + old_slot.value.~Value(); new_size++; } - PERFETTO_DCHECK(new_size == old_size); - size_ = new_size; + PERFETTO_DCHECK(new_size == old.size); + FinishGrow(new_size); } // Doesn't call destructors. Use Clear() for that. - PERFETTO_NO_INLINE void Reset(size_t n, bool reallocate) { - // Must be a pow2. - PERFETTO_CHECK((n & (n - 1)) == 0); - - // Always ensure at least 128 capacity to avoid too frequent growths. - capacity_ = std::max(n, 128u); - size_ = 0; - growth_info_.growth_left = - (capacity_ * static_cast(load_limit_percent_)) / 100; - growth_info_.has_tombstones = 0; - - if (reallocate) { - // See memory layout comment above |storage_|. - size_t slots_offset = - base::AlignUp(capacity_ + kNumClones, alignof(Slot)); - storage_.reset(new uint8_t[slots_offset + (capacity_ * sizeof(Slot))]); - ctrl_ = storage_.get(); - slots_ = reinterpret_cast(storage_.get() + slots_offset); - } - if (ctrl_) { - // Initialize all control bytes (including clones) to empty (kFreeSlot) - memset(ctrl_, kFreeSlot, capacity_ + kNumClones); - } + void Reset(size_t n, bool reallocate) { + FlatHashMapV2Base::Reset(n, reallocate, sizeof(Slot), alignof(Slot)); } - // Swiss Table hash splitting (matching absl): - // H1 = upper bits for bucket index - // H2 = lower 7 bits for tag - // This ensures H1 and H2 are independent, avoiding tag collisions within - // buckets. The seed XOR prevents clustering when hash values have patterns - // (e.g., sequential keys) - static constexpr size_t H1(size_t hash) { return (hash >> 7); } - static constexpr uint8_t H2(size_t hash) { return hash & 0x7F; } - - // Set control byte and update clone if needed - PERFETTO_ALWAYS_INLINE void SetCtrl(size_t i, uint8_t h) { - ctrl_[i] = h; - // Update clone if this is one of the first kNumClones entries - if (PERFETTO_UNLIKELY(i < kNumClones)) { - ctrl_[capacity_ + i] = h; - } + PERFETTO_ALWAYS_INLINE Slot* slots() const { + return reinterpret_cast(slots_); } - - // Owns the actual memory with the following layout: - // - // [Control bytes] - // |capacity_| bytes for control bytes. - // kNumClones (15 or 7) bytes for control byte clones (*). - // No alignment required (accessed at arbitrary byte offsets). - // - // [Padding for Slot alignment] - // - // [Slots] - // capacity_ * sizeof(Slot): contains key-value pairs. - // Must be aligned to alignof(Slot). - // - // (*) Control byte clones: The first kNumClones control bytes are duplicated - // at the end of the control array. This allows SIMD operations to read a full - // group (16 or 8 bytes) starting from any position without bounds checking, - // even near the end of the array. - std::unique_ptr storage_; - - size_t capacity_ = 0; - size_t size_ = 0; - - // Slots remaining + has_deleted flag - GrowthInfo growth_info_{0, 0}; - - // Load factor limit in % of |capacity_|. - int load_limit_percent_ = kDefaultLoadLimitPct; - - // Cached pointers for fast access (like absl::flat_hash_map) - // These are updated whenever storage is allocated/reallocated. - uint8_t* ctrl_ = nullptr; // Points to control bytes - Slot* slots_ = nullptr; // Points to slot array }; // Alias FlatHashMap to FlatHashMapV1 for backward compatibility. diff --git a/include/perfetto/ext/base/murmur_hash.h b/include/perfetto/ext/base/murmur_hash.h index 309bf6f623..80b82427a9 100644 --- a/include/perfetto/ext/base/murmur_hash.h +++ b/include/perfetto/ext/base/murmur_hash.h @@ -223,13 +223,6 @@ constexpr bool HasMurmurHashBuiltinValue() { uint64_t>; } -// Helper to check if two types are integeral and U is convertible to T. -template -constexpr bool IsConvertibleIntegral() { - return std::is_integral_v && std::is_integral_v && - std::is_convertible_v; -} - // Helper to check if a type is string-like (i.e. string, c-string or string // views). template @@ -239,14 +232,25 @@ constexpr bool IsStringLike() { std::is_same_v || std::is_same_v; } -// Helper to check if heterogeneous lookup is allowed between T and U. -// Only allows it for convertible integral types and string-like types. +// Helper to check if heterogeneous lookup is allowed between T and U: only +// between string-like types. Other types (e.g. integers) are converted to T by +// the caller instead, as converting them is cheap, and hashing them as is +// could differ (e.g. a negative int hashed for a uint32_t key). template constexpr bool AllowsHeterogeneousLookup() { - return IsConvertibleIntegral() || - (IsStringLike() && IsStringLike()); + return IsStringLike() && IsStringLike(); } +// MurmurHash is transparent (i.e. supports heterogeneous lookups) only for +// string-like T. +template ()> +struct MurmurHashTransparency {}; + +template +struct MurmurHashTransparency { + using is_transparent = void; +}; + // Helper to detect pointers in Combine(...). // Hashing pointers directly is prohibited as it often indicates a misuse // (e.g., passing ptr and len instead of a single std::string_view). @@ -358,13 +362,10 @@ uint64_t MurmurHashValue(const T& value) { // std::hash drop-in class which uses MurmurHashValue as the primitive. // All specializations consistently delegate to MurmurHashValue. template -struct MurmurHash { - using is_transparent = void; - +struct MurmurHash : murmur_internal::MurmurHashTransparency { uint64_t operator()(const T& value) const { return MurmurHashValue(value); } - // Heterogeneous lookup support. Only allowed for types where it makes sense - // (e.g. string-like types and convertible integral types). + // Heterogeneous lookup support, between string-like types only. template auto operator()(const U& value) const -> std::enable_if_t< murmur_internal::AllowsHeterogeneousLookup>(), diff --git a/src/base/BUILD.gn b/src/base/BUILD.gn index 08d1af8086..e8df743568 100644 --- a/src/base/BUILD.gn +++ b/src/base/BUILD.gn @@ -43,6 +43,7 @@ perfetto_component("base") { "dynamic_string_writer.cc", "event_fd.cc", "file_utils.cc", + "flat_hash_map.cc", "futex.cc", "getopt_compat.cc", "intrusive_list.cc", diff --git a/src/base/flat_hash_map.cc b/src/base/flat_hash_map.cc new file mode 100644 index 0000000000..5c1c064439 --- /dev/null +++ b/src/base/flat_hash_map.cc @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "perfetto/ext/base/flat_hash_map.h" + +#include +#include +#include +#include +#include +#include + +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/utils.h" + +namespace perfetto::base::flat_hash_map_v2_internal { + +void FlatHashMapV2Base::Reset(size_t n, + bool reallocate, + size_t slot_size, + size_t slot_align) { + // Must be a pow2. + PERFETTO_CHECK((n & (n - 1)) == 0); + + // Always ensure at least 128 capacity to avoid too frequent growths. + capacity_ = std::max(n, 128u); + size_ = 0; + growth_info_.growth_left = + (capacity_ * static_cast(load_limit_percent_)) / 100; + growth_info_.has_tombstones = 0; + + if (reallocate) { + // See memory layout comment above |storage_| in flat_hash_map.h. + size_t slots_offset = base::AlignUp(capacity_ + kNumClones, slot_align); + storage_.reset(new uint8_t[slots_offset + (capacity_ * slot_size)]); + ctrl_ = storage_.get(); + slots_ = storage_.get() + slots_offset; + } + if (ctrl_) { + // Initialize all control bytes (including clones) to empty (kFreeSlot) + memset(ctrl_, kFreeSlot, capacity_ + kNumClones); + } +} + +size_t FlatHashMapV2Base::FindFirstEmptyOrTombstone(size_t key_hash) const { + return FindFirstEmptyOrTombstoneImpl(key_hash); +} + +FlatHashMapV2Base::OldTable FlatHashMapV2Base::Grow(size_t slot_size, + size_t slot_align) { + PERFETTO_DCHECK(size_ <= capacity_); + OldTable old{std::move(storage_), ctrl_, slots_, capacity_, size_}; + // This must be a CHECK (i.e. not just a DCHECK) to prevent UAF attacks on + // 32-bit archs that try to double the size of the table until wrapping. + size_t new_capacity = capacity_ * 2; + PERFETTO_CHECK(new_capacity >= capacity_); + Reset(new_capacity, true, slot_size, slot_align); + return old; +} + +} // namespace perfetto::base::flat_hash_map_v2_internal diff --git a/src/base/flat_hash_map_unittest.cc b/src/base/flat_hash_map_unittest.cc index 993424ebdc..eadccf943c 100644 --- a/src/base/flat_hash_map_unittest.cc +++ b/src/base/flat_hash_map_unittest.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -119,6 +120,15 @@ int Key::instances = 0; int Key::constructions = 0; int Value::instances = 0; +// Whether Map::Find() can be called with a K. +template +struct CanFind : std::false_type {}; +template +struct CanFind().Find( + std::declval()))>> : std::true_type {}; + TEST(FlatHashMapV2Test, HeterogeneousInsert) { struct Hasher { using is_transparent = void; @@ -191,7 +201,7 @@ TEST(FlatHashMapV2Test, InsertConvertibleKey) { EXPECT_EQ(*int_map.Find(int64_t{42}), 100); } -TEST(FlatHashMapV2Test, InsertConvertsForNonHeterogeneousEquality) { +TEST(FlatHashMapV2Test, NonHeterogeneousEqualityNeedsOwningKey) { struct Hasher { using is_transparent = void; size_t operator()(const Key& key) const { @@ -199,11 +209,13 @@ TEST(FlatHashMapV2Test, InsertConvertsForNonHeterogeneousEquality) { } size_t operator()(int key) const { return static_cast(key); } }; - // Key is explicitly constructible from int, but its equality only accepts - // Key. Insertion must convert to Key before probing the map. - FlatHashMapV2 map; - ASSERT_TRUE(map.Insert(42, 100).second); - auto duplicate = map.Insert(42, 200); + // The Hasher takes int, but Key's equality only accepts Key, so an int isn't + // a heterogeneous key: the caller has to construct a Key. + using Map = FlatHashMapV2; + static_assert(!CanFind::value); + Map map; + ASSERT_TRUE(map.Insert(Key(42), 100).second); + auto duplicate = map.Insert(Key(42), 200); EXPECT_FALSE(duplicate.second); EXPECT_EQ(*duplicate.first, 100); ASSERT_NE(map.Find(Key(42)), nullptr); diff --git a/src/base/murmur_hash_unittest.cc b/src/base/murmur_hash_unittest.cc index 0afcb9e64f..466a281f4b 100644 --- a/src/base/murmur_hash_unittest.cc +++ b/src/base/murmur_hash_unittest.cc @@ -16,8 +16,10 @@ #include "perfetto/ext/base/murmur_hash.h" +#include #include #include +#include #include "perfetto/ext/base/string_view.h" #include "test/gtest_and_gmock.h" @@ -44,5 +46,22 @@ TEST(MurmurHashTest, HeterogeneousStringHash) { EXPECT_EQ(hasher.operator()("abc"), expected); } +template +struct IsTransparent : std::false_type {}; +template +struct IsTransparent> + : std::true_type {}; + +TEST(MurmurHashTest, OnlyStringLikeKeysAreTransparent) { + static_assert(IsTransparent>::value); + static_assert(IsTransparent>::value); + static_assert(IsTransparent>::value); + static_assert(IsTransparent>::value); + // Other keys (e.g. integers) are converted by the caller instead: hashing + // an int as is could differ from hashing it as a uint32_t key. + static_assert(!IsTransparent>::value); + static_assert(!IsTransparent>::value); +} + } // namespace } // namespace perfetto::base