From e2acb8a7edde17977fe389744eac854c36dd3c98 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 10:19:11 +0800 Subject: [PATCH 01/15] perf(storage): prefetch flush lookups and filter address tags on main --- include/lavik/storage/scan_hash_map.h | 38 ++++++++++++++++++++++----- src/storage/engine/flush.cpp | 24 ++++++++++++++++- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index 09433a698..373e3f146 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -1221,6 +1221,21 @@ class ScanHashMap { return FindAddressImpl(address, hash, 0); } + // Owner-only hints for batches of independent address lookups. These do not + // resolve entries or establish membership; the eventual FindAddress must + // still validate the current table population and exact live address. + void PrefetchAddressMetadata() const noexcept { + __builtin_prefetch(&tables_[0], 0, 3); + } + + void PrefetchAddress(std::uint32_t hash) const noexcept { + for (int t = 0, count = Rehashing() ? 2 : 1; t < count; ++t) { + const Table& table = tables_[t]; + if (table.buckets_) + __builtin_prefetch(&table.buckets_[hash & BucketMask(table)], 0, 3); + } + } + // Optional bucket fingerprint for FindAddress. Capture it from the same // immutable key digest as AddressHash while the entry is still live. static std::uint8_t AddressTag(const Digest& digest) noexcept { @@ -2320,15 +2335,24 @@ class ScanHashMap { } const Bucket* bucket = &table.buckets_[hash & BucketMask(table)]; while (bucket != nullptr) { - for (std::size_t slot = 0; slot < kEntriesPerBucket; ++slot) { - if (Occupied(*bucket, slot)) { - if constexpr (FilterTag) { - if (bucket->hashes_[slot] != tag) continue; - } + if constexpr (FilterTag) { + // Flush receipts already retain the key's tag. Use the same SIMD + // filter as key lookup, but still validate a live handle and exact + // address: tags collide and holes retain stale fingerprint bytes. + for (std::uint32_t matches = MatchingTags(*bucket, tag); matches != 0; + matches &= matches - 1) { + const std::size_t slot = std::countr_zero(matches); + if (!Occupied(*bucket, slot)) continue; const Entry* entry = Resolve(bucket->entries_[slot]); - if (reinterpret_cast(entry) == address) { + if (reinterpret_cast(entry) == address) + return entry; + } + } else { + for (std::size_t slot = 0; slot < kEntriesPerBucket; ++slot) { + if (!Occupied(*bucket, slot)) continue; + const Entry* entry = Resolve(bucket->entries_[slot]); + if (reinterpret_cast(entry) == address) return entry; - } } } bucket = Chained(*bucket) ? Child(table, bucket) : nullptr; diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index f7f19ce2f..eca10a772 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -487,7 +487,29 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { // header both fdatasync'd above), so the versions they superseded are no // longer anyone's durable copy and can leave their blocks' accounting. std::vector retired_records; - for (const RecordIdentity& identity : pending->staged_records_) { + const auto prefetch_index = [&](std::size_t position, bool bucket) { + if (position >= pending->staged_records_.size()) return; + const auto& future = pending->staged_records_[position]; + if (future.entry_address_ == 0 || + future.index_generation_ != store->index_generations_[future.db_id_]) + return; + const auto& index = + PartitionFor(*store, future.partition_id_).indexes_[future.db_id_]; + if (bucket) + index.PrefetchAddress(future.entry_hash_); + else + index.PrefetchAddressMetadata(); + }; + for (std::size_t position = 0; position < pending->staged_records_.size(); + ++position) { + // Batch receipts expose independent random lookups. Warm fixed map + // metadata first, then its bucket while processing + // earlier receipts. All hints stay on the owning worker and retain no + // entry pointer; generation, live membership and physical-boundary + // checks still follow. + prefetch_index(position + 16, false); + prefetch_index(position + 8, true); + const RecordIdentity& identity = pending->staged_records_[position]; if (identity.retired_extents_ != nullptr) { SpawnExtentReclaim(*store, identity.retired_extents_); } From adb53b9403fbaabb41dff5d2f4db01b26a6a8544 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 02:38:26 +0000 Subject: [PATCH 02/15] Preallocate worker index arena directories for concurrent resolution Keep primary arena directories at a stable address and publish page descriptors atomically. Concurrent resolution requires callers to protect entry and arena lifetimes; private auxiliary arenas retain incremental directory growth. --- docs/architecture/04-storage-and-recovery.md | 20 ++- include/lavik/storage/scan_hash_map.h | 71 +++++++--- src/storage/engine/init.cpp | 7 + tests/scan_hash_map_test.cpp | 139 +++++++++++++++++++ 4 files changed, 212 insertions(+), 25 deletions(-) diff --git a/docs/architecture/04-storage-and-recovery.md b/docs/architecture/04-storage-and-recovery.md index 56a06b7e7..700eef66d 100644 --- a/docs/architecture/04-storage-and-recovery.md +++ b/docs/architecture/04-storage-and-recovery.md @@ -519,11 +519,19 @@ seed and top-level digests appear only in the one-use checkpoint, never in authoritative records or replication protocols. All partition and logical-database indexes on a worker allocate entries from a -shared worker-local arena, so sparse indexes share capacity instead of -stranding it at partition boundaries. A population detached by `FLUSHDB`, a -replica reset, or a replica abort retains ownership of that arena until -reclamation finishes. Replica promotion and abort reuse the same worker-local -detached-index drain as synchronous database flushes, so a completed lifecycle +shared arena, so sparse indexes share capacity instead of stranding it at +partition boundaries. Startup allocates its complete page directory, charging +16 MiB of descriptor storage to each worker before recovery; entry pages remain +demand-allocated. The directory stays at one address and publishes page layout +atomically. Handle-to-address resolution can run concurrently with owner-side +allocation when the caller protects the resolved entry and arena lifetimes; +it does not pin entries or make the hash tables safe for concurrent access. +Allocator mutation and index access remain owner-local. Private auxiliary +arenas retain incremental directory allocation and owner-only access. +A population detached by `FLUSHDB`, a replica reset, or a replica abort retains +ownership of that arena until reclamation finishes. Replica promotion and +abort reuse the same worker-local detached-index drain as synchronous database +flushes, so a completed lifecycle cannot leave old index arenas or record-block live-byte accounting queued for a later rebuild. The drain covers the worker FIFO; it is not partitioned by replication attempt. @@ -1156,7 +1164,7 @@ current source code are authoritative for present storage behavior. |---|---| | Public lifecycle, routing, typed operations, locked transaction contract, snapshots, epochs, maintenance, and durability interfaces | `include/lavik/storage/engine.h` | | Worker, partition, block, append-stream, allocator, recovery, and background-maintenance state | `src/storage/engine/impl.h` | -| Runtime index representation, shared entry arena, runtime key digests, and asynchronous entry-identity validation | `include/lavik/storage/scan_hash_map.h`, `include/lavik/storage/format.h`, `src/storage/format.cpp`, `src/storage/engine/impl.h`, `src/storage/engine/write.cpp`, `src/storage/engine/flush.cpp` | +| Runtime index representation, shared entry arena, runtime key digests, and asynchronous entry-identity validation | `include/lavik/storage/scan_hash_map.h`, `include/lavik/storage/format.h`, `src/storage/format.cpp`, `src/storage/engine/impl.h`, `src/storage/engine/init.cpp`, `src/storage/engine/write.cpp`, `src/storage/engine/flush.cpp` | | Compact and grouped Hash/Set serving, group identity and object side index | `src/storage/engine/hash_tree.cpp`, `src/storage/engine/hash_codec.cpp`, `src/storage/engine/grouped/hash.cpp`, `src/storage/engine/grouped/object_index.cpp`, [Grouped collections](09-grouped-collections.md) | | Compact physical index representation shared by user-key and group-location indexes | `include/lavik/storage/detail/record_index.h` | | Persistent constants, device and block IDs, A/B metadata pages, record and extent layouts, and checksums | `include/lavik/storage/format.h`, `src/storage/format.cpp` | diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index 09433a698..7db66ca08 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -47,7 +48,7 @@ namespace lavik::storage { -// Worker-local backing store for compact hash-table entry handles. Small pages +// Single-owner backing store for compact hash-table entry handles. Small pages // are carved from 1 MiB spans so mimalloc pays the 64 KiB alignment overhead // once per 16 logical pages instead of once per page. Large entries still own // dedicated 64 KiB-aligned allocations. Mimalloc remains responsible for OS @@ -61,6 +62,9 @@ namespace lavik::storage { // directory entry and the low 11 bits select a slot in that page. Page IDs may // be reused only after the last live slot is gone. Callers must therefore prove // bucket membership before resolving an identity retained across suspension. +// PreallocateDirectory enables concurrent address resolution of protected live +// entries; allocation, reclamation, and all allocator bookkeeping stay local to +// the owner. Directory publication alone does not protect an entry's lifetime. class ScanHashMapEntryArena { public: using Handle = std::uint32_t; @@ -92,6 +96,13 @@ class ScanHashMapEntryArena { ScanHashMapEntryArena& operator=(const ScanHashMapEntryArena&) = delete; ~ScanHashMapEntryArena(); + // Allocate the complete page directory before publishing this arena to + // readers. Success fixes its address until destruction (16 MiB at the default + // handle limit); pages remain demand-allocated. Owner-only and idempotent. + // Returns false on admission rejection, preserving the existing directory. + // Private, owner-only arenas may omit this and retain incremental growth. + bool PreallocateDirectory(); + // Returns false only when this allocation would need a new page and every // encodable page ID is already live. Physical allocator failure is fatal. bool CanAllocate(std::size_t bytes) const noexcept; @@ -104,12 +115,16 @@ class ScanHashMapEntryArena { Allocation Allocate(std::size_t bytes); void Deallocate(Handle handle) noexcept; - // Resolve translates a handle read from an occupied live bucket. Debug - // builds validate that ownership invariant; release builds rely on it and - // do not turn every lookup into defensive arena bookkeeping. + // Resolve translates a handle read from an occupied live bucket. After a + // successful PreallocateDirectory, it may run concurrently with owner-side + // allocation/reclamation of OTHER entries. The caller must publish the handle + // safely and keep its entry and the arena alive throughout the call and use + // of the returned pointer. This is not a stale-handle validation or pin API. void* Resolve(Handle handle) const noexcept; // HandleOf accepts an address owned by this arena and returns zero when the - // aligned page belongs to a different arena. + // aligned page belongs to a different arena. The same concurrent-access and + // lifetime requirements as Resolve apply; arbitrary freed pointers are + // unsafe. Handle HandleOf(const void* pointer) const noexcept; std::size_t allocated_pages() const noexcept { return allocated_pages_; } @@ -183,6 +198,7 @@ class ScanHashMapEntryArena { std::uint64_t descriptor) noexcept; std::size_t NextDirectoryCapacity() const noexcept; bool EnsureDirectoryCapacity(); + bool ResizeDirectory(std::size_t capacity); std::size_t DirectoryGrowthBytesForNextPage() const noexcept; std::uint32_t AllocatePageId(); void ReleasePageId(std::uint32_t page_id) noexcept; @@ -205,11 +221,15 @@ class ScanHashMapEntryArena { // Directory slot zero stays empty so handle zero is the null reference. // A normal descriptor packs the 64 KiB-aligned page address with its small // block size in the low 16 bits; the large marker reads its variable size - // from the page header. The directory is a raw, deliberately uninitialized - // array with an explicit growth policy. Fresh IDs advance monotonically, so - // only the copied prefix and newly published slot are ever read. Released - // slots encode the next free ID in-band, eliminating a second - // capacity-growing array while keeping ReleasePage noexcept. + // from the page header. PreallocateDirectory fixes this array before remote + // readers are admitted; private arenas can instead grow on their owner. + // Descriptors use atomic_ref for release/acquire publication of immutable + // page layout. Unused slots are zero so HandleOf can reject foreign pages + // without reading the owner's mutable page-ID watermark. Freed slots encode + // the next free ID in-band. A stable directory does not pin pages. + static_assert(std::atomic_ref::is_always_lock_free); + static_assert(std::atomic_ref::required_alignment <= + alignof(std::uint64_t)); std::uint64_t* directory_ = nullptr; std::size_t directory_capacity_ = 0; std::uint32_t next_page_id_ = 1; @@ -304,6 +324,11 @@ inline ScanHashMapEntryArena::~ScanHashMapEntryArena() { } } +inline bool ScanHashMapEntryArena::PreallocateDirectory() { + const std::size_t capacity = static_cast(maximum_page_id_) + 1; + return directory_capacity_ == capacity || ResizeDirectory(capacity); +} + inline std::uint8_t ScanHashMapEntryArena::ClassFor( std::size_t bytes) noexcept { if (bytes > 4096) return kLargeClassMarker; @@ -331,16 +356,19 @@ inline const std::byte* ScanHashMapEntryArena::SlotAddress( inline std::uint64_t ScanHashMapEntryArena::DirectoryDescriptor( std::uint32_t page_id) const noexcept { - assert(page_id != 0 && page_id < next_page_id_); + // next_page_id_ is owner-only and can advance during a remote Resolve. + // Validate against the fixed capacity, not that mutable high watermark. + assert(page_id != 0 && page_id <= maximum_page_id_); assert(directory_ != nullptr && page_id < directory_capacity_); - return directory_[page_id]; + return std::atomic_ref(directory_[page_id]).load(std::memory_order_acquire); } inline void ScanHashMapEntryArena::SetDirectoryDescriptor( std::uint32_t page_id, std::uint64_t descriptor) noexcept { assert(page_id != 0 && page_id < next_page_id_); assert(directory_ != nullptr && page_id < directory_capacity_); - directory_[page_id] = descriptor; + std::atomic_ref(directory_[page_id]) + .store(descriptor, std::memory_order_release); } inline std::size_t ScanHashMapEntryArena::NextDirectoryCapacity() @@ -356,7 +384,10 @@ inline std::size_t ScanHashMapEntryArena::NextDirectoryCapacity() inline bool ScanHashMapEntryArena::EnsureDirectoryCapacity() { if (next_page_id_ < directory_capacity_) return true; - const std::size_t next_capacity = NextDirectoryCapacity(); + return ResizeDirectory(NextDirectoryCapacity()); +} + +inline bool ScanHashMapEntryArena::ResizeDirectory(std::size_t next_capacity) { if (next_capacity <= directory_capacity_ || next_capacity > std::numeric_limits::max() / sizeof(std::uint64_t)) { @@ -366,9 +397,11 @@ inline bool ScanHashMapEntryArena::EnsureDirectoryCapacity() { allocation_domain_, next_capacity * sizeof(std::uint64_t), alignof(std::uint64_t))); if (next == nullptr) return false; - // Slot zero is permanently invalid. Copy only descriptors that have been - // published; the unused suffix stays uninitialized until fresh IDs consume - // it, avoiding vector value-initialization and a capacity-sized memset. + // Zero descriptors let HandleOf reject a live pointer owned by another arena + // without consulting next_page_id_, which can change on the allocating owner. + // This initialization precedes publication of the directory to any reader. + std::uninitialized_value_construct_n(next, next_capacity); + // Slot zero is permanently invalid; retain all published and recycled IDs. if (directory_ != nullptr && next_page_id_ > 1) { std::memcpy(next + 1, directory_ + 1, (next_page_id_ - 1) * sizeof(std::uint64_t)); @@ -728,7 +761,7 @@ inline void ScanHashMapEntryArena::Deallocate(Handle handle) noexcept { inline void* ScanHashMapEntryArena::Resolve(Handle handle) const noexcept { const std::uint32_t page_id = handle >> kSlotBits; - assert(page_id != 0 && page_id < next_page_id_); + assert(page_id != 0 && page_id <= maximum_page_id_); const std::uint64_t descriptor = DirectoryDescriptor(page_id); const std::uint64_t marker = descriptor & std::uint64_t{0xffff}; assert(descriptor != 0 && marker != kFreePageMarker); @@ -764,7 +797,7 @@ inline ScanHashMapEntryArena::Handle ScanHashMapEntryArena::HandleOf( (address - data) % page->block_size_ != 0) { return 0; } - if (page->page_id_ >= next_page_id_ || + if (page->page_id_ >= directory_capacity_ || (DirectoryDescriptor(page->page_id_) & ~std::uint64_t{0xffff}) != page_address) { return 0; diff --git a/src/storage/engine/init.cpp b/src/storage/engine/init.cpp index 7c70e1765..3c5bb1dfe 100644 --- a/src/storage/engine/init.cpp +++ b/src/storage/engine/init.cpp @@ -911,6 +911,13 @@ absl::Status StorageEngine::Impl::Prepare(unsigned worker_count) { /*externally_admitted=*/true, /*externally_accounted=*/false, /*owner_shard=*/i + 1); + // Charge the fixed directory to its worker before recovery/admission. Its + // address must never change once protected handles can be resolved + // remotely. + if (!store.record_index_entry_arena_->PreallocateDirectory()) { + return absl::ResourceExhaustedError( + "cannot preallocate index page directory"); + } store.indirect_keys_.SetEntryArena(store.record_index_entry_arena_); const std::size_t partition_count = (kLogicalStorageShards + worker_count - 1 - i) / worker_count; diff --git a/tests/scan_hash_map_test.cpp b/tests/scan_hash_map_test.cpp index e5e70c229..ad92422ff 100644 --- a/tests/scan_hash_map_test.cpp +++ b/tests/scan_hash_map_test.cpp @@ -19,8 +19,10 @@ #include #include +#include #include #include +#include #include #include "lavik/memory.h" @@ -590,6 +592,143 @@ TEST(ScanHashMapTest, DirectoryCapacityDoesNotInflateEmptyArena) { EXPECT_LT(sizeof(ScanHashMapEntryArena), 1024); } +TEST(ScanHashMapTest, PreallocatedDirectoryIsChargedBeforeAllocatingPages) { + ASSERT_TRUE(lavik::InitMemoryLimit(64 * 1024 * 1024, 1).ok()); + lavik::BindMemoryAccountingShard(0); + struct RestoreLimit { + ~RestoreLimit() { + (void)lavik::InitMemoryLimit(1024ULL * 1024 * 1024, 1); + lavik::BindMemoryAccountingShard(lavik::kMaxMemoryWorkers); + } + } restore; + const auto before = lavik::WorkerMemoryAccountingBytes(0); + { + ScanHashMapEntryArena arena; + ASSERT_TRUE(arena.PreallocateDirectory()); + const auto directory_bytes = lavik::WorkerMemoryAccountingBytes(0) - before; + EXPECT_GE(directory_bytes, 16 * 1024 * 1024); + EXPECT_EQ(arena.allocated_pages(), 0); + EXPECT_EQ(arena.allocated_spans(), 0); + ASSERT_TRUE(arena.PreallocateDirectory()); + EXPECT_EQ(lavik::WorkerMemoryAccountingBytes(0) - before, directory_bytes); + + const auto page_bytes = arena.AllocationBytesIfNewPage(64); + auto allocation = arena.Allocate(64); + ASSERT_NE(allocation.pointer_, nullptr); + EXPECT_EQ(arena.Resolve(allocation.handle_), allocation.pointer_); + arena.Deallocate(allocation.handle_); + EXPECT_EQ(arena.AllocationBytesIfNewPage(64), page_bytes); + EXPECT_EQ(lavik::WorkerMemoryAccountingBytes(0) - before, directory_bytes); + } + EXPECT_EQ(lavik::WorkerMemoryAccountingBytes(0), before); +} + +TEST(ScanHashMapTest, + HandleOfRejectsForeignPagesWithUnpublishedDirectorySlots) { + ScanHashMapEntryArena owner(2); + ScanHashMapEntryArena fixed(2); + ScanHashMapEntryArena growing; + ASSERT_TRUE(fixed.PreallocateDirectory()); + auto first = owner.Allocate(8192); + auto second = owner.Allocate(8192); + ASSERT_NE(first.pointer_, nullptr); + ASSERT_NE(second.pointer_, nullptr); + EXPECT_EQ(fixed.HandleOf(first.pointer_), 0); + EXPECT_EQ(fixed.HandleOf(second.pointer_), 0); + EXPECT_EQ(growing.HandleOf(second.pointer_), 0); + auto local = growing.Allocate(32); + ASSERT_NE(local.pointer_, nullptr); + EXPECT_EQ(growing.HandleOf(first.pointer_), 0); + EXPECT_EQ(growing.HandleOf(second.pointer_), 0); + growing.Deallocate(local.handle_); + owner.Deallocate(first.handle_); + owner.Deallocate(second.handle_); +} + +TEST(ScanHashMapTest, ConcurrentResolveSurvivesOtherPageAllocationAndReuse) { + ScanHashMapEntryArena arena(1023); + ASSERT_TRUE(arena.PreallocateDirectory()); + std::vector live; + for (std::size_t bytes : {32, 256, 4096, 8192}) { + auto allocation = arena.Allocate(bytes); + ASSERT_NE(allocation.pointer_, nullptr); + std::construct_at(static_cast(allocation.pointer_), bytes); + live.push_back(allocation); + } + + // These handles remain live until both readers join. The owner mutates page + // high watermarks, free lists, and other slots (including slots in the same + // small pages); readers may only inspect the protected entries and layout. + std::latch ready(2); + std::atomic mismatch{false}; + std::atomic reads{0}; + auto read = [&](std::stop_token stop) { + ready.count_down(); + while (!stop.stop_requested()) { + for (const auto& entry : live) { + if (arena.Resolve(entry.handle_) != entry.pointer_ || + arena.HandleOf(entry.pointer_) != entry.handle_) { + mismatch.store(true, std::memory_order_relaxed); + } + } + reads.fetch_add(1, std::memory_order_relaxed); + } + }; + std::jthread first(read); + std::jthread second(read); + ready.wait(); + for (unsigned round = 0; round < 32; ++round) { + std::vector transient; + for (unsigned i = 0; i < 2048; ++i) { + auto allocation = arena.Allocate(i % 16 == 0 ? 8192 : 4096); + ASSERT_NE(allocation.pointer_, nullptr); + transient.push_back(allocation.handle_); + } + for (auto handle : transient) arena.Deallocate(handle); + } + first.request_stop(); + second.request_stop(); + first.join(); + second.join(); + EXPECT_FALSE(mismatch.load()); + EXPECT_GT(reads.load(), 0); + for (const auto& entry : live) arena.Deallocate(entry.handle_); +} + +TEST(ScanHashMapTest, ConcurrentResolveObservesNewAndReusedPageLayouts) { + ScanHashMapEntryArena arena(3); + ASSERT_TRUE(arena.PreallocateDirectory()); + std::atomic published{0}; + std::atomic mismatch{false}; + std::jthread reader([&](std::stop_token stop) { + while (!stop.stop_requested()) { + auto handle = published.load(std::memory_order_acquire); + if (handle == 0) { + std::this_thread::yield(); + continue; + } + auto* value = static_cast(arena.Resolve(handle)); + if (*value != handle || arena.HandleOf(value) != handle) + mismatch.store(true, std::memory_order_relaxed); + // The owner may reclaim this entry only after this acknowledgement. + published.store(0, std::memory_order_release); + } + }); + for (unsigned i = 0; i < 1024; ++i) { + auto allocation = arena.Allocate(i % 2 == 0 ? 32 : 8192); + ASSERT_NE(allocation.pointer_, nullptr); + std::construct_at(static_cast(allocation.pointer_), + allocation.handle_); + published.store(allocation.handle_, std::memory_order_release); + while (published.load(std::memory_order_acquire) != 0) + std::this_thread::yield(); + arena.Deallocate(allocation.handle_); + } + reader.request_stop(); + reader.join(); + EXPECT_FALSE(mismatch.load()); +} + TEST(ScanHashMapTest, PreallocatesSettledTableForKnownPopulation) { ScanHashMap map; ASSERT_TRUE(map.PreallocateForExpectedSize(91)); From 53cd0e3819e2b5b1efc9640f2356138d1e4d7f8f Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 04:20:07 +0000 Subject: [PATCH 03/15] Reserve cache-line read states and minimum buckets for record indexes Select 32 read stripes and a 32-bucket floor through a compile-time parameter only for RecordIndex. Preserve empty bucket allocation, recovery sizing, shrink limits, and first-insert admission estimates. Read states remain closed pending integration of the concurrent reader protocol. --- docs/architecture/04-storage-and-recovery.md | 6 + include/lavik/storage/detail/record_index.h | 5 +- include/lavik/storage/scan_hash_map.h | 164 ++++++++++++++----- tests/record_location_test.cpp | 68 ++++++++ 4 files changed, 198 insertions(+), 45 deletions(-) diff --git a/docs/architecture/04-storage-and-recovery.md b/docs/architecture/04-storage-and-recovery.md index 700eef66d..ba14e3340 100644 --- a/docs/architecture/04-storage-and-recovery.md +++ b/docs/architecture/04-storage-and-recovery.md @@ -528,6 +528,12 @@ allocation when the caller protects the resolved entry and arena lifetimes; it does not pin entries or make the hash tables safe for concurrent access. Allocator mutation and index access remain owner-local. Private auxiliary arenas retain incremental directory allocation and owner-only access. +Each primary key index reserves 32 cache-line-isolated atomic read-admission +states, including while empty. Its bucket arrays remain demand-allocated, +start at 32 buckets, and never shrink below 32 while nonempty; checkpoint +recovery applies the same floor. The states remain closed and do not yet admit +remote index readers. Grouped-object and auxiliary maps retain their existing +owner-only layout and bucket sizing. A population detached by `FLUSHDB`, a replica reset, or a replica abort retains ownership of that arena until reclamation finishes. Replica promotion and abort reuse the same worker-local detached-index drain as synchronous database diff --git a/include/lavik/storage/detail/record_index.h b/include/lavik/storage/detail/record_index.h index a5006c89c..f8a16562e 100644 --- a/include/lavik/storage/detail/record_index.h +++ b/include/lavik/storage/detail/record_index.h @@ -559,9 +559,12 @@ struct RecordIndexEntryPolicy { } }; +// Only the primary key index reserves read stripes. Keeping every allocated +// table at least this large makes hash & 31 a stable stripe across resizing; +// auxiliary and grouped-object maps retain their owner-only layout. using RecordIndex = ScanHashMap::digits, - RecordIndexEntryPolicy>; + RecordIndexEntryPolicy, 32>; static_assert(static_cast(ValueType::kStream) < (1U << 3)); static_assert(sizeof(RecordLocationCore) == 32); diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index 7db66ca08..a835a9f08 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -827,13 +827,38 @@ struct ScanHashMapInlineEntryPolicy { } }; +// One cache line per stripe prevents unrelated buckets' readers from writing +// the same line. The high bit closes admission; lower bits are reserved for +// short-lived index readers. Storage starts closed: reserving these words does +// not by itself make ScanHashMap operations safe for concurrent access. +struct alignas(64) ScanHashMapReadState { + static constexpr std::uint32_t kClosed = std::uint32_t{1} << 31; + static constexpr std::uint32_t kReadersMask = kClosed - 1; + std::atomic state_{kClosed}; +}; +static_assert(sizeof(ScanHashMapReadState) == 64); +static_assert(std::atomic::is_always_lock_free); + +template +struct ScanHashMapReadStates { + std::array stripes_{}; +}; + +// Owner-only maps retain their original footprint and alignment. +template <> +struct ScanHashMapReadStates<0> {}; + // Cached hashes retain the 32 address bits needed by the largest direct bucket // table. MaxBucketExponent is configurable so tests can exercise saturation // without allocating 2^32 buckets; production uses the full uint32_t range. +// ReadStripeCount reserves optional fixed admission state and sets the minimum +// nonempty table size so each bucket belongs to exactly one hash-selected +// stripe. This is layout support only; callers still serialize all map access. template ::digits, - typename EntryPolicy = ScanHashMapInlineEntryPolicy> + typename EntryPolicy = ScanHashMapInlineEntryPolicy, + std::size_t ReadStripeCount = 0> class ScanHashMap { public: using EntryHandle = ScanHashMapEntryArena::Handle; @@ -843,6 +868,13 @@ class ScanHashMap { static_assert(MaxBucketExponent <= std::numeric_limits::digits); static_assert(MaxBucketExponent < std::numeric_limits::digits); + static_assert(ReadStripeCount == 0 || std::has_single_bit(ReadStripeCount)); + static constexpr std::size_t kReadStripeCount = ReadStripeCount; + static constexpr std::size_t kMinimumBucketCount = + ReadStripeCount == 0 ? 1 : ReadStripeCount; + static constexpr unsigned kMinimumBucketExponent = + std::countr_zero(kMinimumBucketCount); + static_assert(kMinimumBucketExponent <= MaxBucketExponent); using StoredValue = typename EntryPolicy::StoredValue; using EntryExtra = typename EntryPolicy::Extra; @@ -1044,6 +1076,21 @@ class ScanHashMap { ~ScanHashMap() { Clear(); } + // Fixed-address state belongs to this map object, not its entry population; + // it is not transferred by move or Detach. All stripes remain closed until + // the storage layer integrates reader admission and every mutation path. + // Move, clear, detach, and destruction require quiescent readers. + auto& read_states() noexcept + requires(ReadStripeCount != 0) + { + return read_states_.stripes_; + } + const auto& read_states() const noexcept + requires(ReadStripeCount != 0) + { + return read_states_.stripes_; + } + // Binds an empty map to a worker-shared arena. Sharing keeps slab occupancy // independent of the 16,384 partition and 16 database boundaries. Detached // maps retain shared ownership so asynchronous FLUSHDB reclamation cannot @@ -1091,7 +1138,8 @@ class ScanHashMap { if (!inserting) return required; if (!tables_[0].buckets_) { - return add(required, sizeof(Bucket) + kSlowPathBookkeepingAllowance); + return add(required, kMinimumBucketCount * sizeof(Bucket) + + kSlowPathBookkeepingAllowance); } if (!Rehashing() && tables_[0].used_ + 1 > @@ -1140,7 +1188,8 @@ class ScanHashMap { return tables_[0].buckets_ || tables_[1].buckets_; } - // Allocates the final direct-bucket table for an empty map whose population + // Allocates the final direct-bucket table (respecting the configured minimum) + // for an empty map whose population // is known exactly, such as an offline recovery checkpoint. Normal inserts // deliberately grow incrementally because their eventual population is not // known; recovery can avoid repeatedly allocating, walking, and retaining @@ -1158,8 +1207,9 @@ class ScanHashMap { const std::size_t required_buckets = expected_entries / kTargetEntriesPerBucket + (expected_entries % kTargetEntriesPerBucket != 0); - const std::size_t exponent = - required_buckets <= 1 ? 0 : std::bit_width(required_buckets - 1); + const std::size_t exponent = std::max( + kMinimumBucketExponent, + required_buckets <= 1 ? 0 : std::bit_width(required_buckets - 1)); if (exponent > MaxBucketExponent || exponent >= std::numeric_limits::digits) { return false; @@ -1374,7 +1424,8 @@ class ScanHashMap { // emits a whole chain per cursor position, so an owner-serialized cursor // scan does not miss an entry that exists throughout while erases occur // between Scan calls. Scan and mutation are not thread-safe concurrently. - // Actual removal enables best-effort shrinking below 25% occupancy; a + // Actual removal enables best-effort shrinking below 25% occupancy, down to + // the configured bucket floor (or no buckets when empty); a // value replacement (including a tombstone) does not. Maintain() finishes // reclamation when foreground traffic stops. Entry addresses stay stable. bool Erase(const Digest& digest, std::string_view key) { @@ -2040,8 +2091,8 @@ class ScanHashMap { if (!tables_[0].buckets_) { assert(arena_ != nullptr); tables_[0] = Table(arena_->allocation_domain()); - if (!tables_[0].AllocateBuckets(1)) std::terminate(); - tables_[0].exponent_ = 0; + if (!tables_[0].AllocateBuckets(kMinimumBucketCount)) std::terminate(); + tables_[0].exponent_ = kMinimumBucketExponent; } } @@ -2079,7 +2130,8 @@ class ScanHashMap { return released; } const std::size_t buckets = BucketCount(tables_[0]); - if (buckets <= 1 || size() >= buckets * kShrinkEntriesPerBucket) { + if (buckets <= kMinimumBucketCount || + size() >= buckets * kShrinkEntriesPerBucket) { shrink_pending_ = false; return false; } @@ -2656,10 +2708,13 @@ class ScanHashMap { std::size_t rehash_index_ = kNotRehashing; bool shrink_pending_ = false; std::shared_ptr arena_; + [[no_unique_address]] ScanHashMapReadStates read_states_; }; -template -std::uint8_t ScanHashMap::Entry:: +template +std::uint8_t +ScanHashMap::Entry:: EncodedKeyMetadataBytes(std::uint32_t encoded) noexcept { std::uint8_t bytes = 1; while (encoded >= 0x80) { @@ -2669,10 +2724,13 @@ std::uint8_t ScanHashMap::Entry:: return bytes; } -template -std::uint8_t -ScanHashMap::Entry::EncodeKeyMetadata( - std::byte* output, std::uint32_t logical_size, bool key_complete) noexcept { +template +std::uint8_t ScanHashMap< + Value, MaxBucketExponent, EntryPolicy, + ReadStripeCount>::Entry::EncodeKeyMetadata(std::byte* output, + std::uint32_t logical_size, + bool key_complete) noexcept { assert(output != nullptr); assert(logical_size <= kMaxLogicalKeySize); std::uint32_t encoded = @@ -2689,10 +2747,12 @@ ScanHashMap::Entry::EncodeKeyMetadata( return bytes; } -template -typename ScanHashMap::Entry::KeyMetadata -ScanHashMap::Entry::DecodeKeyMetadata( - const std::byte* input) noexcept { +template +typename ScanHashMap::Entry::KeyMetadata +ScanHashMap::Entry:: + DecodeKeyMetadata(const std::byte* input) noexcept { assert(input != nullptr); std::uint32_t encoded = 0; for (std::uint8_t index = 0; index < 5; ++index) { @@ -2713,19 +2773,25 @@ ScanHashMap::Entry::DecodeKeyMetadata( return {}; } -template -std::size_t ScanHashMap::Entry::tail_bytes() const noexcept { +template +std::size_t ScanHashMap::Entry::tail_bytes() const noexcept { const KeyMetadata metadata = DecodeKeyMetadata(tail()); return metadata.encoded_bytes_ + (metadata.key_complete_ ? metadata.logical_size_ : sizeof(Digest)); } -template -typename ScanHashMap::Entry::Allocation -ScanHashMap::Entry::Create( - ScanHashMapEntryArena& arena, const Digest& digest, std::string_view key, - const Value& value, bool key_complete) { +template +typename ScanHashMap::Entry::Allocation +ScanHashMap::Entry::Create(ScanHashMapEntryArena& arena, + const Digest& digest, + std::string_view key, + const Value& value, + bool key_complete) { if (key.size() > kMaxLogicalKeySize) { return {}; } @@ -2771,10 +2837,13 @@ ScanHashMap::Entry::Create( return {.entry_ = entry, .handle_ = allocation.handle_}; } -template -typename ScanHashMap::Entry::Allocation -ScanHashMap::Entry::CreateReplacement( - ScanHashMapEntryArena& arena, const Entry& source, const Value& value) { +template +typename ScanHashMap::Entry::Allocation +ScanHashMap::Entry:: + CreateReplacement(ScanHashMapEntryArena& arena, const Entry& source, + const Value& value) { const std::size_t tail_bytes = source.tail_bytes(); const bool extended = EntryPolicy::HasExtraValue(value); const std::size_t header_bytes = @@ -2804,29 +2873,36 @@ ScanHashMap::Entry::CreateReplacement( return {.entry_ = entry, .handle_ = allocation.handle_}; } -template -typename ScanHashMap::EntryExtra* -ScanHashMap::Entry::extra() noexcept { +template +typename ScanHashMap::EntryExtra* +ScanHashMap::Entry::extra() noexcept { return &static_cast(this)->extra_; } -template -const typename ScanHashMap::EntryExtra* -ScanHashMap::Entry::extra() - const noexcept { +template +const typename ScanHashMap::EntryExtra* +ScanHashMap::Entry::extra() const noexcept { return &static_cast(this)->extra_; } -template -std::byte* -ScanHashMap::Entry::tail() noexcept { +template +std::byte* ScanHashMap::Entry::tail() noexcept { return reinterpret_cast(this) + (has_extra() ? sizeof(ExtendedEntry) : sizeof(Entry)); } -template -const std::byte* ScanHashMap::Entry::tail() const noexcept { +template +const std::byte* ScanHashMap::Entry::tail() const noexcept { return reinterpret_cast(this) + (has_extra() ? sizeof(ExtendedEntry) : sizeof(Entry)); } diff --git a/tests/record_location_test.cpp b/tests/record_location_test.cpp index 95f860c3c..98d87c1ba 100644 --- a/tests/record_location_test.cpp +++ b/tests/record_location_test.cpp @@ -24,6 +24,74 @@ namespace lavik::storage { namespace { +TEST(RecordIndexTest, BucketFloorSurvivesGrowthShrinkAndPopulationReplacement) { + RecordIndex index; + auto* states = index.read_states().data(); + EXPECT_EQ(index.allocated_bucket_count(), 0); + ASSERT_TRUE(index.PreallocateForExpectedSize(0)); + EXPECT_EQ(index.allocated_bucket_count(), 0); + + // Admission must cover the larger first bucket array, even before the arena + // or any entry has been allocated. + ScanHashMap::digits, + RecordIndexEntryPolicy> + owner_only; + const auto digest = ComputeDigest("keep"); + EXPECT_EQ( + index.RequiredAllocationBytes(digest, "keep", true, false, true), + owner_only.RequiredAllocationBytes(digest, "keep", true, false, true) + + 31 * 64); + ASSERT_NE(index.InsertNew(digest, "keep", RecordLocation{}), nullptr); + ASSERT_EQ(index.allocated_bucket_count(), 32); + for (unsigned i = 0; i < 400; ++i) { + const auto key = "grow-" + std::to_string(i); + ASSERT_NE(index.InsertNew(ComputeDigest(key), key, RecordLocation{}), + nullptr); + } + for (unsigned step = 0; step < 1024 && index.Maintain(); ++step) { + } + ASSERT_FALSE(index.rehashing()); + EXPECT_EQ(index.allocated_bucket_count(), 64); + for (unsigned i = 0; i < 400; ++i) { + const auto key = "grow-" + std::to_string(i); + ASSERT_TRUE(index.Erase(ComputeDigest(key), key)); + } + for (unsigned step = 0; step < 1024 && index.Maintain(); ++step) { + } + EXPECT_FALSE(index.rehashing()); + EXPECT_EQ(index.allocated_bucket_count(), 32); + EXPECT_NE(index.Find(digest, "keep"), nullptr); + EXPECT_EQ(index.read_states().data(), states); + + auto detached = index.Detach(); + EXPECT_EQ(index.allocated_bucket_count(), 0); + EXPECT_EQ(index.read_states().data(), states); + RecordIndex moved(std::move(detached)); + EXPECT_EQ(moved.allocated_bucket_count(), 32); + EXPECT_NE(moved.Find(digest, "keep"), nullptr); + ASSERT_TRUE(index.PreallocateForExpectedSize(1)); + EXPECT_EQ(index.allocated_bucket_count(), 32); + index.Clear(); + EXPECT_EQ(index.allocated_bucket_count(), 0); + EXPECT_EQ(index.read_states().data(), states); +} + +TEST(RecordIndexTest, ReadStatesAreClosedAndCacheLineIsolated) { + // Worker startup keeps indexes in a vector: its allocator must preserve the + // over-alignment as well as spacing within each fixed stripe array. + std::vector indexes(2); + for (const auto& index : indexes) { + ASSERT_EQ(index.read_states().size(), 32); + for (const auto& stripe : index.read_states()) { + EXPECT_EQ(reinterpret_cast(&stripe) % 64, 0); + EXPECT_EQ(stripe.state_.load(std::memory_order_relaxed), + ScanHashMapReadState::kClosed); + } + EXPECT_EQ(sizeof(index.read_states()), 32 * 64); + EXPECT_EQ(index.allocated_bucket_count(), 0); + } +} + TEST(RecoveryMemoryTest, DividesTemporaryBatchTargetAcrossWorkers) { EXPECT_EQ(RecoveryWorkerBatchTargetBytes(0), kRecoveryProcessBatchTargetBytes); From c99083719f236ffe6665271554ce2dcbb10e47ef Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 06:10:16 +0000 Subject: [PATCH 04/15] Add admitted connection-worker disk GET with index reader and block pin protocols --- CMakeLists.txt | 8 + docs/architecture/02-request-serving.md | 3 + .../03-transaction-coordination.md | 13 +- docs/architecture/04-storage-and-recovery.md | 58 ++- include/lavik/storage/engine.h | 6 + include/lavik/storage/scan_hash_map.h | 374 +++++++++++++++++- include/lavik/tx/fingerprint.h | 2 + include/lavik/tx/transaction.h | 4 +- include/lavik/tx/tx_shard.h | 67 +++- src/redis/command.cpp | 54 ++- src/redis/list_command.cpp | 3 +- src/redis/set_command.cpp | 3 +- src/redis/sort_command.cpp | 2 +- src/redis/stream_command.cpp | 8 +- src/redis/string_command.cpp | 15 +- src/redis/zset_command.cpp | 13 +- src/storage/engine.cpp | 5 + src/storage/engine/backup.cpp | 23 +- src/storage/engine/collection_ingest.cpp | 3 +- src/storage/engine/compact_api.cpp | 3 +- src/storage/engine/defrag.cpp | 77 +++- src/storage/engine/expire.cpp | 4 +- src/storage/engine/flush.cpp | 13 +- src/storage/engine/flush_db.cpp | 17 +- src/storage/engine/hash_api.cpp | 6 +- src/storage/engine/impl.h | 111 +++++- src/storage/engine/init.cpp | 67 ++++ src/storage/engine/list_api.cpp | 6 +- src/storage/engine/read.cpp | 141 +++++-- src/storage/engine/replica_collection.cpp | 3 +- src/storage/engine/replication.cpp | 34 +- src/storage/engine/sorted_set_api.cpp | 3 +- src/storage/engine/tomb_raider.cpp | 16 +- src/storage/engine/tx_cleaner.cpp | 15 +- src/storage/engine/write.cpp | 44 ++- src/tx/transaction.cpp | 7 +- src/tx/tx_shard.cpp | 1 + tests/multikey_e2e_test.cpp | 9 +- tests/record_location_test.cpp | 184 +++++++++ tests/remote_get_e2e.py | 145 +++++++ tests/replica_abort_reclaim_e2e_test.cpp | 8 + tests/storage_capacity_test.cpp | 58 +++ tests/tomb_raider_e2e_test.cpp | 21 +- tests/tx_lock_test.cpp | 53 +++ 44 files changed, 1540 insertions(+), 170 deletions(-) create mode 100644 tests/remote_get_e2e.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fbf20472..b8dc53220 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2149,6 +2149,14 @@ endforeach() ) set_tests_properties(lavik_extent_recovery_e2e PROPERTIES TIMEOUT 600) + add_test( + NAME lavik_remote_get_e2e + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/remote_get_e2e.py + $ + ) + set_tests_properties(lavik_remote_get_e2e PROPERTIES TIMEOUT 180) + add_test( NAME lavik_defrag_payload_e2e COMMAND ${Python3_EXECUTABLE} diff --git a/docs/architecture/02-request-serving.md b/docs/architecture/02-request-serving.md index d1ef4f05b..bcdebc6fb 100644 --- a/docs/architecture/02-request-serving.md +++ b/docs/architecture/02-request-serving.md @@ -198,6 +198,9 @@ owner routing. The selected database is request data, never ambient worker state. Storage chooses an owner from the key's Redis hash-slot partition. Commands with one owner can execute locally or through one Bycorf cross-worker submission. +Eligible standalone disk GETs first try an admitted index snapshot and physical +read on the connection worker; contention or unsupported storage representations +fall back to owner dispatch. Transaction callbacks retain owner routing. Commands needing atomic access to several keys build a `tx::Transaction` and execute one or more shard callbacks. Global commands explicitly collect from or coordinate all workers. diff --git a/docs/architecture/03-transaction-coordination.md b/docs/architecture/03-transaction-coordination.md index fd1b17900..fbb9f8e34 100644 --- a/docs/architecture/03-transaction-coordination.md +++ b/docs/architecture/03-transaction-coordination.md @@ -74,6 +74,15 @@ has two counter layers: - holds cover callbacks that are currently executing or suspended and tell the queue when a conflicting predecessor has actually drained. +Storage binds admission hooks to each shard. Recording a key intent closes its +index reader stripe before hold acquisition, and releasing or cancelling that +intent releases the closure. Closures nest across keys sharing a stripe and +structural index writers. Existing remote index readers must drain before the +callback takes its holds; the same ordered and bypass queues handle this wait. +The last reader of a closed stripe sends a worker notification that repolls +eligibility rather than resuming all transaction frames. An already pinned disk +read holds no index reader registration, so it does not delay a later writer. + The queue contains both plain acquisition waiters and persistent transaction entries. It is sorted by scheduling id. Plain waiters are removed before their coroutine resumes; transaction entries retain their position until a releasing @@ -86,9 +95,9 @@ reaches the front. allocating a scheduling id because only one owner participates. 2. `Execute` submits to that owner and acquires the deduplicated key set through `TxShard::AcquireKeys` if the transaction does not already retain a guard. -3. Acquisition records every intent. If all intents are immediately compatible, +3. Acquisition records every intent. If all intents are immediately compatible and index readers have drained, it acquires holds and continues without touching the global id counter or - shard queue. If any intent conflicts, the awaiter keeps all recorded intents, + shard queue. If an intent conflicts or readers remain, the awaiter keeps all recorded intents, allocates an id, inserts a plain waiter, and suspends until it is the hold-compatible queue head. 4. The shard entry hook runs once, then the callback runs. A releasing execution diff --git a/docs/architecture/04-storage-and-recovery.md b/docs/architecture/04-storage-and-recovery.md index ba14e3340..2b5664cff 100644 --- a/docs/architecture/04-storage-and-recovery.md +++ b/docs/architecture/04-storage-and-recovery.md @@ -88,12 +88,12 @@ Every worker has one ordinary and one transaction append stream. Physical streams are per worker, not per logical partition, so active 8 MiB staging buffers scale with workers rather than with 16,384 slots. A worker also owns the `BlockState` objects assigned to it, including committed and live byte counts, -pins, staging identity, flush state, and defrag state. The runtime owner and -allocation epoch form immutable identity after publication and are the only -fields a key-index owner may read directly. The owner is published atomically -after epoch initialization, so an acquire owner read makes the immutable epoch -visible without an extra worker hop. Other state remains owner-local and other -workers use Bycorf cross-core submissions to read or mutate it. +staging identity, flush state, and defrag state. Runtime owner and allocation +epoch form immutable identity after publication. A shared atomic pin word +protects physical lifetime and closes admission to connection-worker reads +while maintenance owns the block. The owner is published atomically after +epoch initialization. Other mutable block fields remain owner-local; remote +workers submit accounting and maintenance work to that owner. Append-stream creation and rollover are single-flight within each stream. A writer may release owner-local store state while physical allocation waits, @@ -526,14 +526,28 @@ demand-allocated. The directory stays at one address and publishes page layout atomically. Handle-to-address resolution can run concurrently with owner-side allocation when the caller protects the resolved entry and arena lifetimes; it does not pin entries or make the hash tables safe for concurrent access. -Allocator mutation and index access remain owner-local. Private auxiliary -arenas retain incremental directory allocation and owner-only access. +Allocator mutation remains owner-local. Private auxiliary arenas retain +incremental directory allocation and owner-only access. Each primary key index reserves 32 cache-line-isolated atomic read-admission states, including while empty. Its bucket arrays remain demand-allocated, start at 32 buckets, and never shrink below 32 while nonempty; checkpoint -recovery applies the same floor. The states remain closed and do not yet admit -remote index readers. Grouped-object and auxiliary maps retain their existing -owner-only layout and bucket sizing. +recovery applies the same floor. Reader admission is explicitly enabled only +for a published map with a fixed arena directory and an externally protected +map lifetime. A remote reader registers atomically in the low-hash stripe, +performs only const lookup, and releases its guard before suspending. Writers +close the affected stripe and suspend their owner coroutine until existing +readers drain, then resolve entry identities again before mutation. The last +reader of a closed stripe notifies the owner; transaction arbitration and +structural writers recheck readiness there. +Shared overflow-directory growth and population replacement close all stripes. +Resizing closes admission before changing table layout, defers behind existing +readers, and stays closed through the final table switch and old-table release. +Owner lookups and periodic maintenance retry deferred resizing; nesting keeps +resize completion from reopening another writer's scope. Recovery and detached +populations remain private, and grouped-object and auxiliary maps retain their +owner-only layout and bucket sizing. Transaction intents retain stripe closure +through queued and executing work, so a remote index snapshot cannot observe +an intermediate multi-key mutation. A population detached by `FLUSHDB`, a replica reset, or a replica abort retains ownership of that arena until reclamation finishes. Replica promotion and abort reuse the same worker-local detached-index drain as synchronous database @@ -648,7 +662,27 @@ need a staging buffer to commit. ### Reads and pins -An ordinary single-key GET may resolve its complete inline-key index entry +A standalone GET can read directly on its connection worker after ordinary +request, database and serving-generation admission. It takes a short index +reader guard on the key owner, snapshots a complete-key plain String location, +then pins the block and validates its allocation identity before dropping the +index guard. Only durable, non-grouped, non-external records whose block owner +is also their key owner qualify, and the connection worker must have device +I/O access. Closed index/block admission, unsupported representations, expired +entries, and concurrent admission contention return to the normal owner route. +Index misses use the same transaction and population admission checks. + +The index snapshot orders the GET before a subsequently admitted writer. The +block pin survives I/O and cancellation cleanup, so writes can append and retire +old live-byte accounting while the read finishes. Defrag closes remote pin +admission before inspecting the count; the last pin on a pending block asks its +owner to reconsider reclamation. Local fallback reads retain their existing pin +rules. Database/replication epoch and population-generation changes invalidate +an in-flight fast read. Worker shutdown drains I/O and destroys suspended frames +on all workers before destroying any remotely referenced index, arena or block +state. + +An owner-local ordinary single-key GET may resolve its complete inline-key index entry without registering a shared transaction lock when that logical database's lock table is empty. The empty-table observation and index lookup do not suspend, so the lookup is the read's linearization point. A multi-shard writer diff --git a/include/lavik/storage/engine.h b/include/lavik/storage/engine.h index 47deb7580..5ab135b4a 100644 --- a/include/lavik/storage/engine.h +++ b/include/lavik/storage/engine.h @@ -1603,6 +1603,12 @@ class StorageEngine { bycorf::Task AbortReplicaRoot(std::uint64_t session_id); void SetReplicaLoading(bool loading) noexcept; + // Connection-worker optimization for a standalone GET. The caller holds + // request/DB population admission through completion. nullopt requests the + // ordinary owner path; a present result is final (including not-found). + bycorf::Task>> TryRemoteGet( + std::uint8_t db_id, std::string_view key, std::uint16_t slot); + // These operations must execute on OwnerForKey(key), normally through // SubmitTaskTo. Only digest/location metadata is retained after completion. // routed_partition_id reuses a route computed for this exact key before its diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index a835a9f08..1b4d9cab9 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -103,6 +103,12 @@ class ScanHashMapEntryArena { // Private, owner-only arenas may omit this and retain incremental growth. bool PreallocateDirectory(); + // Owner-only capability check before publishing concurrent arena users. + bool directory_preallocated() const noexcept { + return directory_capacity_ == + static_cast(maximum_page_id_) + 1; + } + // Returns false only when this allocation would need a new page and every // encodable page ID is already live. Physical allocator failure is fatal. bool CanAllocate(std::size_t bytes) const noexcept; @@ -827,33 +833,236 @@ struct ScanHashMapInlineEntryPolicy { } }; -// One cache line per stripe prevents unrelated buckets' readers from writing -// the same line. The high bit closes admission; lower bits are reserved for -// short-lived index readers. Storage starts closed: reserving these words does -// not by itself make ScanHashMap operations safe for concurrent access. +// Reader registration and closure share one word, so a writer cannot miss a +// reader entering between its zero-count check and the first mutation. struct alignas(64) ScanHashMapReadState { static constexpr std::uint32_t kClosed = std::uint32_t{1} << 31; static constexpr std::uint32_t kReadersMask = kClosed - 1; std::atomic state_{kClosed}; + // Owner-only nesting includes suspended writer scopes. Releasing one scope + // must not reopen a stripe still closed by another writer or a resize. + std::uint32_t writers_ = 0; + // Immutable after reader admission is published. + void* notify_context_ = nullptr; + void (*notify_)(void*) noexcept = nullptr; }; static_assert(sizeof(ScanHashMapReadState) == 64); static_assert(std::atomic::is_always_lock_free); +class ScanHashMapReadGuard { + public: + ScanHashMapReadGuard() = default; + explicit ScanHashMapReadGuard(ScanHashMapReadState* state) : state_(state) {} + ScanHashMapReadGuard(const ScanHashMapReadGuard&) = delete; + ScanHashMapReadGuard& operator=(const ScanHashMapReadGuard&) = delete; + ScanHashMapReadGuard(ScanHashMapReadGuard&& other) noexcept + : state_(std::exchange(other.state_, nullptr)) {} + ScanHashMapReadGuard& operator=(ScanHashMapReadGuard&& other) noexcept { + Reset(); + state_ = std::exchange(other.state_, nullptr); + return *this; + } + ~ScanHashMapReadGuard() { Reset(); } + explicit operator bool() const noexcept { return state_ != nullptr; } + void Reset() noexcept { + if (auto* state = std::exchange(state_, nullptr)) { + const auto notify = state->notify_; + void* context = state->notify_context_; + const auto old = state->state_.fetch_sub(1, std::memory_order_acq_rel); + assert((old & ScanHashMapReadState::kReadersMask) != 0); + if (old == (ScanHashMapReadState::kClosed | 1) && notify) notify(context); + } + } + + private: + ScanHashMapReadState* state_ = nullptr; +}; + template struct ScanHashMapReadStates { + static_assert(Count > 0 && Count <= 64 && std::has_single_bit(Count)); + static constexpr std::uint64_t kAll = ~std::uint64_t{0} >> (64 - Count); + static constexpr auto kClosed = ScanHashMapReadState::kClosed; + static constexpr auto kReadersMask = ScanHashMapReadState::kReadersMask; + + class WriteGuard { + public: + WriteGuard() = default; + WriteGuard(ScanHashMapReadStates* states, std::uint64_t mask, bool all) + : states_(states), mask_(mask), all_(all) { + if (all_) ++states_->structural_writers_; + states_->PublishDisabled(); + states_->Close(mask_); + } + WriteGuard(const WriteGuard&) = delete; + WriteGuard& operator=(const WriteGuard&) = delete; + WriteGuard(WriteGuard&& other) noexcept + : states_(std::exchange(other.states_, nullptr)), + mask_(other.mask_), + all_(other.all_) {} + WriteGuard& operator=(WriteGuard&& other) noexcept { + Reset(); + states_ = std::exchange(other.states_, nullptr); + mask_ = other.mask_; + all_ = other.all_; + return *this; + } + ~WriteGuard() { Reset(); } + bool ready() const noexcept { + return states_ == nullptr || states_->Drained(mask_); + } + void Reset() noexcept { + if (auto* states = std::exchange(states_, nullptr)) { + if (all_) --states->structural_writers_; + states->Open(mask_); + states->PublishDisabled(); + } + } + + private: + ScanHashMapReadStates* states_ = nullptr; + std::uint64_t mask_ = 0; + bool all_ = false; + }; + std::array stripes_{}; + + ScanHashMapReadGuard TryRead(std::uint64_t hash) noexcept { + if (disabled_.load(std::memory_order_acquire)) return {}; + auto& state = stripes_[hash & (Count - 1)].state_; + auto old = state.load(std::memory_order_relaxed); + if ((old & kClosed) != 0 || (old & kReadersMask) == kReadersMask) return {}; + // One attempt: reader contention falls back rather than spinning on a + // connection worker. No table or entry may be touched before this CAS. + if (!state.compare_exchange_strong(old, old + 1, std::memory_order_acquire, + std::memory_order_relaxed)) + return {}; + return ScanHashMapReadGuard(&stripes_[hash & (Count - 1)]); + } + void CloseIntent(std::uint64_t hash) noexcept { + Close(std::uint64_t{1} << (hash & (Count - 1))); + } + void OpenIntent(std::uint64_t hash) noexcept { + Open(std::uint64_t{1} << (hash & (Count - 1))); + } + bool IntentReady(std::uint64_t hash) const noexcept { + return Drained(std::uint64_t{1} << (hash & (Count - 1))); + } + void SetNotifier(void* context, void (*notify)(void*) noexcept) { + assert(!enabled_); + for (auto& stripe : stripes_) { + stripe.notify_context_ = context; + stripe.notify_ = notify; + } + } + WriteGuard Write(std::uint64_t hash) noexcept { + if (!enabled_) return {}; + return WriteGuard(this, std::uint64_t{1} << (hash & (Count - 1)), false); + } + WriteGuard WriteAll() noexcept { + if (!enabled_) return {}; + return WriteGuard(this, kAll, true); + } + // Enable only on a published, lifetime-protected map and a fixed arena. + // Initial recovery and detached populations remain owner-only. + void Enable() noexcept { + assert(!enabled_ && Drained(kAll)); + enabled_ = true; + for (auto& stripe : stripes_) { + if (stripe.writers_ == 0) + stripe.state_.fetch_and(~kClosed, std::memory_order_release); + } + PublishDisabled(); + } + void SetPopulated(bool populated) noexcept { + populated_ = populated; + PublishDisabled(); + } + void BlockResize() noexcept { + if (resize_blocked_) return; + resize_blocked_ = true; + ++structural_writers_; + PublishDisabled(); + Close(kAll); + } + void EndResize() noexcept { + if (!resize_blocked_) return; + resize_blocked_ = false; + --structural_writers_; + Open(kAll); + PublishDisabled(); + } + bool Drained(std::uint64_t mask = kAll) const noexcept { + while (mask != 0) { + const auto index = std::countr_zero(mask); + if ((stripes_[index].state_.load(std::memory_order_acquire) & + kReadersMask) != 0) + return false; + mask &= mask - 1; + } + return true; + } + + private: + void PublishDisabled() noexcept { + const bool disabled = !enabled_ || !populated_ || structural_writers_ != 0; + // Ordinary single-stripe writes must not invalidate the global probe's + // cache line. Only population/structural admission transitions publish it. + if (disabled != published_disabled_) { + disabled_.store(disabled, std::memory_order_release); + published_disabled_ = disabled; + } + } + void Close(std::uint64_t mask) noexcept { + while (mask != 0) { + auto& stripe = stripes_[std::countr_zero(mask)]; + if (stripe.writers_++ == 0) + stripe.state_.fetch_or(kClosed, std::memory_order_acq_rel); + mask &= mask - 1; + } + } + void Open(std::uint64_t mask) noexcept { + while (mask != 0) { + auto& stripe = stripes_[std::countr_zero(mask)]; + assert(stripe.writers_ != 0); + if (--stripe.writers_ == 0 && enabled_) + stripe.state_.fetch_and(~kClosed, std::memory_order_release); + mask &= mask - 1; + } + } + // Keep the global probe off the per-stripe reader/writer counter lines. + alignas(64) std::atomic disabled_{true}; + bool enabled_ = false; + bool populated_ = false; + bool resize_blocked_ = false; + bool published_disabled_ = true; + unsigned structural_writers_ = 0; }; -// Owner-only maps retain their original footprint and alignment. +// Owner-only maps retain their original footprint and alignment. Their write +// guards compile to empty objects and perform no atomic operations. template <> -struct ScanHashMapReadStates<0> {}; +struct ScanHashMapReadStates<0> { + struct WriteGuard { + bool ready() const noexcept { return true; } + void Reset() noexcept {} + }; + WriteGuard Write(std::uint64_t) noexcept { return {}; } + WriteGuard WriteAll() noexcept { return {}; } + void SetPopulated(bool) noexcept {} + void BlockResize() noexcept {} + void EndResize() noexcept {} + bool Drained() const noexcept { return true; } +}; // Cached hashes retain the 32 address bits needed by the largest direct bucket // table. MaxBucketExponent is configurable so tests can exercise saturation // without allocating 2^32 buckets; production uses the full uint32_t range. // ReadStripeCount reserves optional fixed admission state and sets the minimum // nonempty table size so each bucket belongs to exactly one hash-selected -// stripe. This is layout support only; callers still serialize all map access. +// stripe. Mutations remain owner-only; remote readers use TryRead and const +// lookup while holding the returned guard. Writers close and drain before +// calling synchronous mutations; missed integration fails closed. template ::digits, @@ -862,6 +1071,8 @@ template ::WriteGuard; // Digest arguments for inline keys must equal ComputeDigest(key). Entries do // not retain a bucket hash, so rehash and pointer-only mutation reconstruct // it from the live key. External entries retain the supplied digest tail. @@ -1076,14 +1287,75 @@ class ScanHashMap { ~ScanHashMap() { Clear(); } - // Fixed-address state belongs to this map object, not its entry population; - // it is not transferred by move or Detach. All stripes remain closed until - // the storage layer integrates reader admission and every mutation path. - // Move, clear, detach, and destruction require quiescent readers. - auto& read_states() noexcept + // Admission belongs to the map object, not a detached entry population. + // The caller protects this object's lifetime before TryRead and through the + // guard's destruction. Read guards must not cross a coroutine suspension. + ScanHashMapReadGuard TryRead(const Digest& digest) noexcept requires(ReadStripeCount != 0) { - return read_states_.stripes_; + return read_states_.TryRead(Hash(digest)); + } + // Recovery/private maps stay disabled. Enable only after fixing the arena + // directory and publishing a stable, externally protected map lifetime. + void EnableConcurrentReads() noexcept + requires(ReadStripeCount != 0) + { + assert(arena_ != nullptr && arena_->directory_preallocated()); + read_states_.Enable(); + } + // Owner-only transaction intent admission; paired through cancellation and + // final release, including time spent queued or suspended between hops. + void CloseReadIntent(std::uint64_t hash) + requires(ReadStripeCount != 0) + { + read_states_.CloseIntent(hash); + } + void ReleaseReadIntent(std::uint64_t hash) + requires(ReadStripeCount != 0) + { + read_states_.OpenIntent(hash); + } + bool ReadIntentReady(std::uint64_t hash) const + requires(ReadStripeCount != 0) + { + return read_states_.IntentReady(hash); + } + void SetReaderNotifier(void* context, void (*notify)(void*) noexcept) + requires(ReadStripeCount != 0) + { + read_states_.SetNotifier(context, notify); + } + + // Owner-only. The guard closes admission immediately; ready() is polled by + // the caller's coroutine before it re-finds any entry or mutates the map. + // Potential insertions that allocate shared directory state need all stripes. + WriteGuard CloseForWrite(const Digest& digest) noexcept { + return NeedsStructuralWrite(digest) ? read_states_.WriteAll() + : read_states_.Write(Hash(digest)); + } + WriteGuard CloseForValueUpdate(std::uint64_t hash) noexcept { + return read_states_.Write(hash); + } + WriteGuard CloseAllReaders() noexcept { return read_states_.WriteAll(); } + bool NeedsStructuralWrite(const Digest& digest) const noexcept { + if constexpr (ReadStripeCount == 0) return false; + // Keep an enclosing mutation globally closed if its Find finishes rehash. + // Otherwise that final step could reopen unrelated readers before a later + // insertion grows the shared overflow directory. + if (Rehashing() || resize_pending_ || !tables_[0].buckets_) return true; + if (!Rehashing() && + tables_[0].used_ + 1 > + BucketCount(tables_[0]) * kTargetEntriesPerBucket && + tables_[0].exponent_ < MaxBucketExponent) + return true; + const Table& table = Rehashing() ? tables_[1] : tables_[0]; + const Bucket* bucket = &table.buckets_[Hash(digest) & BucketMask(table)]; + while (bucket != nullptr) { + for (auto handle : bucket->entries_) + if (handle == 0) return false; + bucket = Chained(*bucket) ? Child(table, bucket) : nullptr; + } + return true; // AllocateChild may change the shared overflow vector. } const auto& read_states() const noexcept requires(ReadStripeCount != 0) @@ -1201,6 +1473,8 @@ class ScanHashMap { // deliberately not converted into this validation result and remains fatal to // the process. bool PreallocateForExpectedSize(std::size_t expected_entries) { + auto write = CloseAllReaders(); + RequireDrained(write); assert(empty() && !has_allocated_storage()); if (expected_entries == 0) return true; @@ -1220,6 +1494,7 @@ class ScanHashMap { prepared.exponent_ = static_cast(exponent); if (!prepared.AllocateBuckets(std::size_t{1} << exponent)) return false; tables_[0] = std::move(prepared); + read_states_.SetPopulated(true); return true; } @@ -1334,6 +1609,8 @@ class ScanHashMap { Entry** replaced) { assert(existing != nullptr); assert(replaced != nullptr); + auto write = read_states_.Write(Hash(digest)); + RequireDrained(write); *replaced = nullptr; if (existing->can_assign(value)) { existing->assign(value); @@ -1370,6 +1647,8 @@ class ScanHashMap { InsertResult InsertOrAssign(const Digest& digest, std::string_view key, const Value& value, bool key_complete = true) { + auto write = CloseForWrite(digest); + RequireDrained(write); if (Entry* existing = Find(digest, key); existing != nullptr) { // The inline policy never changes representation. Specialized policies // use ReplaceValue when a new value changes the concrete entry type. @@ -1399,6 +1678,8 @@ class ScanHashMap { Entry* InsertNew(const Digest& digest, std::string_view key, const Value& value, bool key_complete = true) { + auto write = CloseForWrite(digest); + RequireDrained(write); EnsureArena(); EnsureTable(); MaybeStartExpansion(); @@ -1429,6 +1710,8 @@ class ScanHashMap { // value replacement (including a tombstone) does not. Maintain() finishes // reclamation when foreground traffic stops. Entry addresses stay stable. bool Erase(const Digest& digest, std::string_view key) { + auto write = read_states_.Write(Hash(digest)); + RequireDrained(write); AdvanceRehashIfNeeded(); const std::uint64_t hash = Hash(digest); const std::uint8_t tag = HashTag(hash); @@ -1468,6 +1751,8 @@ class ScanHashMap { } AdvanceRehashIfNeeded(); const std::uint64_t hash = EntryHash(*entry); + auto write = read_states_.Write(hash); + RequireDrained(write); const int tables = Rehashing() ? 2 : 1; for (int t = 0; t < tables; ++t) { Table& table = tables_[t]; @@ -1497,10 +1782,16 @@ class ScanHashMap { // Performs at most one source-chain migration, or starts a smaller table. // Owners may call this between Scan calls to reclaim buckets without new - // requests. Returns false when settled or blocked by memory admission; a + // requests. Returns false when settled or blocked by readers or memory + // admission; a // blocked attempt leaves entries intact and can be retried later. Like a // mutation, this invalidates StableScanCursor but preserves Entry addresses. bool Maintain() { + if (ReadStripeCount != 0 && resize_pending_ && !shrink_pending_ && + !Rehashing()) { + if (!read_states_.Drained()) return false; + MaybeStartExpansion(); + } if (Rehashing()) return RehashStep(); return shrink_pending_ && MaybeStartShrink(); } @@ -1634,10 +1925,15 @@ class ScanHashMap { } void Clear() noexcept { + auto write = CloseAllReaders(); + RequireDrained(write); + read_states_.SetPopulated(false); DestroyTable(tables_[0], true); DestroyTable(tables_[1], true); rehash_index_ = kNotRehashing; shrink_pending_ = false; + resize_pending_ = false; + read_states_.EndResize(); } // Moves every entry out into the returned map and leaves *this empty and @@ -1652,10 +1948,18 @@ class ScanHashMap { // before dereferencing; this class does not track that, since the useful // granularity is whatever set of maps the caller detaches together. ScanHashMap Detach() noexcept { + auto write = CloseAllReaders(); + RequireDrained(write); ScanHashMap detached(arena_); detached.tables_ = std::exchange(tables_, {}); detached.rehash_index_ = std::exchange(rehash_index_, kNotRehashing); detached.shrink_pending_ = std::exchange(shrink_pending_, false); + detached.resize_pending_ = std::exchange(resize_pending_, false); + if (detached.Rehashing() || detached.resize_pending_) + detached.read_states_.BlockResize(); + detached.read_states_.SetPopulated(detached.has_allocated_storage()); + read_states_.SetPopulated(false); + read_states_.EndResize(); return detached; } @@ -2072,6 +2376,21 @@ class ScanHashMap { return ReverseBits(cursor); } + static void RequireDrained(const WriteGuard& guard) noexcept { + // Synchronous map methods cannot suspend or spin. Storage must await its + // outer guard before calling them; a missed mutation boundary fails closed. + if (!guard.ready()) std::terminate(); + } + bool BeginResize() noexcept { + read_states_.BlockResize(); + resize_pending_ = !read_states_.Drained(); + return !resize_pending_; + } + void EndResize() noexcept { + resize_pending_ = false; + read_states_.EndResize(); + } + bool Rehashing() const noexcept { return rehash_index_ != kNotRehashing; } // RehashStep is intentionally large and stays out of line. Keep its common @@ -2082,6 +2401,9 @@ class ScanHashMap { [[gnu::always_inline]] void AdvanceRehashIfNeeded() { if (Rehashing()) [[unlikely]] { RehashStep(); + } else if (ReadStripeCount != 0 && resize_pending_ && !shrink_pending_) + [[unlikely]] { + MaybeStartExpansion(); } else if (shrink_pending_) [[unlikely]] { MaybeStartShrink(); } @@ -2093,6 +2415,7 @@ class ScanHashMap { tables_[0] = Table(arena_->allocation_domain()); if (!tables_[0].AllocateBuckets(kMinimumBucketCount)) std::terminate(); tables_[0].exponent_ = kMinimumBucketExponent; + read_states_.SetPopulated(true); } } @@ -2111,6 +2434,7 @@ class ScanHashMap { if (tables_[0].exponent_ >= MaxBucketExponent) { return; } + if (!BeginResize()) return; assert(arena_ != nullptr); tables_[1] = Table(arena_->allocation_domain()); tables_[1].exponent_ = tables_[0].exponent_ + 1; @@ -2124,23 +2448,31 @@ class ScanHashMap { bool MaybeStartShrink() { if (Rehashing()) return false; if (empty()) { + if (!BeginResize()) return false; const bool released = has_allocated_storage(); tables_ = {}; shrink_pending_ = false; + read_states_.SetPopulated(false); + EndResize(); return released; } const std::size_t buckets = BucketCount(tables_[0]); if (buckets <= kMinimumBucketCount || size() >= buckets * kShrinkEntriesPerBucket) { shrink_pending_ = false; + EndResize(); return false; } + if (!BeginResize()) return false; Table prepared(arena_->allocation_domain()); prepared.exponent_ = tables_[0].exponent_ - 1; // Deletion has no enclosing growth permit, even for a primary index. // Reserve the temporary old+new footprint before publishing the target; // rejection must never turn a successful Erase into a failed command. - if (!prepared.AllocateBuckets(buckets / 2, true)) return false; + if (!prepared.AllocateBuckets(buckets / 2, true)) { + EndResize(); + return false; + } tables_[1] = std::move(prepared); rehash_index_ = 0; return true; @@ -2148,6 +2480,7 @@ class ScanHashMap { bool RehashStep() { if (!Rehashing()) return false; + assert(read_states_.Drained()); // A source chain can split into only two target chains when the direct // table doubles, or merge into one target chain when it halves. Prepare @@ -2169,6 +2502,7 @@ class ScanHashMap { tables_[0] = std::move(tables_[1]); tables_[1] = Table(arena_->allocation_domain()); rehash_index_ = kNotRehashing; + EndResize(); } return true; } @@ -2697,16 +3031,26 @@ class ScanHashMap { } void MoveFrom(ScanHashMap&& other) noexcept { + auto write = CloseAllReaders(); + auto source_write = other.CloseAllReaders(); + RequireDrained(write); + RequireDrained(source_write); arena_ = std::move(other.arena_); tables_ = std::move(other.tables_); rehash_index_ = std::exchange(other.rehash_index_, kNotRehashing); shrink_pending_ = std::exchange(other.shrink_pending_, false); + resize_pending_ = std::exchange(other.resize_pending_, false); + if (Rehashing() || resize_pending_) read_states_.BlockResize(); + read_states_.SetPopulated(has_allocated_storage()); other.tables_ = {}; + other.read_states_.SetPopulated(false); + other.read_states_.EndResize(); } std::array tables_{}; std::size_t rehash_index_ = kNotRehashing; bool shrink_pending_ = false; + bool resize_pending_ = false; std::shared_ptr arena_; [[no_unique_address]] ScanHashMapReadStates read_states_; }; diff --git a/include/lavik/tx/fingerprint.h b/include/lavik/tx/fingerprint.h index 43982d1c1..cfac859f1 100644 --- a/include/lavik/tx/fingerprint.h +++ b/include/lavik/tx/fingerprint.h @@ -51,6 +51,8 @@ struct KeyRef { LockFp fp_ = 0; LockMode mode_ = LockMode::kShared; std::uint8_t db_ = 0; + // Unknown-slot callers conservatively gate every local slot of this DB. + std::uint16_t slot_ = storage::kLogicalStorageShards; }; } // namespace lavik::tx diff --git a/include/lavik/tx/transaction.h b/include/lavik/tx/transaction.h index 23d0dda02..14fe3dc1e 100644 --- a/include/lavik/tx/transaction.h +++ b/include/lavik/tx/transaction.h @@ -44,6 +44,7 @@ struct TxKey { std::uint32_t arg_index_ = 0; LockMode mode_ = LockMode::kShared; std::uint8_t db_ = 0; + std::uint16_t slot_ = storage::kLogicalStorageShards; }; // The per-shard view handed to a shard callback: this shard's keys in @@ -95,7 +96,8 @@ class Transaction { // Owner is the key's home worker (StorageEngine::OwnerForKey). void AddKey(unsigned owner, std::uint8_t db, const storage::Digest& digest, - std::uint32_t arg_index, LockMode mode); + std::uint32_t arg_index, LockMode mode, + std::uint16_t slot = storage::kLogicalStorageShards); // Groups keys by shard and builds the deduplicated lock sets. No keys may // be added afterwards (spans into the internal vectors are handed out). diff --git a/include/lavik/tx/tx_shard.h b/include/lavik/tx/tx_shard.h index 70638c167..99545c065 100644 --- a/include/lavik/tx/tx_shard.h +++ b/include/lavik/tx/tx_shard.h @@ -28,6 +28,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/inlined_vector.h" +#include "bycorf/runtime/sync.h" #include "lavik/storage/format.h" #include "lavik/tx/intent_lock.h" #include "lavik/tx/tx_queue.h" @@ -41,8 +42,10 @@ namespace lavik::tx { // Per-worker transaction scheduling context: the VLL lock tables (one per // logical DB), the txid-ordered queue, and the poll loop that starts the // queue head once it is hold-compatible. All state is single-threaded on the -// owning worker; the only shared word in the system is TxRuntime::next_txid, -// touched exclusively by contended or multi-shard acquisitions. +// owning worker. Storage admission hooks close atomic index reader stripes +// for each intent; drain notifications return here to resume eligible work. +// TxRuntime::next_txid is touched only by contended or multi-shard +// acquisitions. class TxShard { public: class Awaiter; @@ -103,7 +106,21 @@ class TxShard { Awaiter(const Awaiter&) = delete; Awaiter& operator=(const Awaiter&) = delete; + ~Awaiter() { + if (!owns_intents_) return; + // Cancellation may destroy the frame while readers still block its + // queue entry, or after Poll granted holds but before it resumed. + if (granted_ || waiter_.holds_acquired_) { + shard_->Release(keys_); + } else { + if (waiter_.txid_ != 0) shard_->queue_.Remove(&waiter_); + shard_->ReleaseIntents(keys_); + shard_->Poll(); + } + } + bool await_ready() { + owns_intents_ = true; granted_ = shard_->TryFastPath(keys_); return granted_; } @@ -115,7 +132,10 @@ class TxShard { shard_->Enqueue(&waiter_); } - Guard await_resume() noexcept { return Guard(shard_, keys_); } + Guard await_resume() noexcept { + owns_intents_ = false; + return Guard(shard_, keys_); + } private: TxShard* shard_; @@ -123,8 +143,18 @@ class TxShard { std::span keys_; TxWaiter waiter_; bool granted_ = false; + bool owns_intents_ = false; }; + // Owner-local hooks bind storage reader admission to the full intent + // lifetime. + struct ReadAdmission { + void* context_ = nullptr; + void (*close_)(void*, const KeyRef&) = nullptr; + void (*open_)(void*, const KeyRef&) = nullptr; + bool (*ready_)(void*, const KeyRef&) = nullptr; + }; + void BindReadAdmission(ReadAdmission admission) { admission_ = admission; } TxShard() = default; TxShard(const TxShard&) = delete; TxShard& operator=(const TxShard&) = delete; @@ -141,9 +171,10 @@ class TxShard { Awaiter AcquireKeys(std::span keys) { return Awaiter(this, keys); } - Awaiter AcquireKey(std::uint8_t db_id, LockFp fp, LockMode mode) { + Awaiter AcquireKey(std::uint8_t db_id, LockFp fp, LockMode mode, + std::uint16_t slot = storage::kLogicalStorageShards) { assert(db_id < storage::kLogicalDatabaseCount); - return Awaiter(this, KeyRef{fp, mode, db_id}); + return Awaiter(this, KeyRef{fp, mode, db_id, slot}); } bool CanReadOptimistically(std::uint8_t db_id) const noexcept { @@ -163,6 +194,12 @@ class TxShard { // while it is hold-compatible. Called after every release and by the queue // machinery; safe to call at any time. void Poll(); + // Last-reader messages are hints. Only Poll decides which transactions run. + void ReadersDrained() { + if (worker_) reader_progress_.NotifyAll(*worker_); + Poll(); + } + auto WaitForReaders() { return reader_progress_.Wait(); } // Shard-local WATCH registrations (push model): every real keyspace // modification marks the watchers of that fingerprint; EXEC checks its own @@ -244,6 +281,7 @@ class TxShard { bool AcquireIntents(std::span keys) { bool granted = true; for (const KeyRef& key : keys) { + if (admission_.close_) admission_.close_(admission_.context_, key); granted &= locks_[key.db_].AcquireIntent(key.fp_, key.mode_); } return granted; @@ -251,11 +289,13 @@ class TxShard { void ReleaseIntents(std::span keys) { for (const KeyRef& key : keys) { locks_[key.db_].ReleaseIntent(key.fp_, key.mode_); + if (admission_.open_) admission_.open_(admission_.context_, key); } } bool CanHoldAll(std::span keys) const { for (const KeyRef& key : keys) { - if (!locks_[key.db_].CanHold(key.fp_, key.mode_)) { + if ((admission_.ready_ && !admission_.ready_(admission_.context_, key)) || + !locks_[key.db_].CanHold(key.fp_, key.mode_)) { return false; } } @@ -284,12 +324,22 @@ class TxShard { private: friend class Awaiter; + ReadAdmission admission_; + bycorf::AsyncNotification reader_progress_; bool TryFastPath(std::span keys) { if (keys.size() == 1) { const KeyRef& key = keys.front(); + if (admission_.close_) admission_.close_(admission_.context_, key); + // A reader drain uses the existing intent/queue path without installing + // a hold early. Schedule-time intent compatibility remains unchanged. + const bool readers_ready = + !admission_.ready_ || admission_.ready_(admission_.context_, key); const bool granted = - locks_[key.db_].AcquireIntentAndHoldIfGranted(key.fp_, key.mode_); + readers_ready + ? locks_[key.db_].AcquireIntentAndHoldIfGranted(key.fp_, + key.mode_) + : (locks_[key.db_].AcquireIntent(key.fp_, key.mode_), false); if (granted) { ++fastpath_runs_; } @@ -297,7 +347,7 @@ class TxShard { } // Intents are recorded even when not granted: they block later barging // while this acquisition waits in the queue. - if (!AcquireIntents(keys)) { + if (!AcquireIntents(keys) || !CanHoldAll(keys)) { return false; } // Granted => sole/compatible intent owner => (holds ⊆ intents) no @@ -321,6 +371,7 @@ class TxShard { if (keys.size() == 1) { const KeyRef& key = keys.front(); locks_[key.db_].ReleaseHoldAndIntent(key.fp_, key.mode_); + if (admission_.open_) admission_.open_(admission_.context_, key); Poll(); return; } diff --git a/src/redis/command.cpp b/src/redis/command.cpp index df2289cb3..b9972acbb 100644 --- a/src/redis/command.cpp +++ b/src/redis/command.cpp @@ -3653,7 +3653,8 @@ Task> BeginNegativeRandomStream( const std::shared_ptr& state) { auto begin = [state]() -> Task> { tx::TxShard::Guard guard = co_await tx::CurrentTxShard().AcquireKey( - state->db_, tx::FingerprintOf(state->digest_), tx::LockMode::kShared); + state->db_, tx::FingerprintOf(state->digest_), tx::LockMode::kShared, + lavik::storage::RedisSlot(state->key_)); storage::HashOperation operation; operation.kind_ = state->options_.hash_ && state->options_.with_values_ ? storage::HashOperationKind::kGetAll @@ -4154,7 +4155,8 @@ Task> PrepareCollectionDump( tx::TxShard::Guard guard; if (!locked) guard = co_await tx::CurrentTxShard().AcquireKey( - db, tx::FingerprintOf(digest), tx::LockMode::kShared); + db, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); auto value = co_await g_storage->ReadValueForTransferLocked(db, key, digest); guard = {}; if (!value.ok()) co_return value.status(); @@ -5807,7 +5809,8 @@ Task ExecuteRename(const CommandRequest& request, transaction.AddKey(ShardForKey(args[argument]), request.db_id_, storage::ComputeDigest(args[argument]), static_cast(argument), - tx::LockMode::kExclusive); + tx::LockMode::kExclusive, + storage::RedisSlot(args[argument])); } transaction.Seal(); ClusterShardValidatorContext cluster_validator; @@ -6000,10 +6003,11 @@ Task ExecuteCopy(const CommandRequest& request, tx::Transaction transaction; transaction.AddKey(ShardForKey(args[1]), request.db_id_, - storage::ComputeDigest(args[1]), 1, tx::LockMode::kShared); + storage::ComputeDigest(args[1]), 1, tx::LockMode::kShared, + storage::RedisSlot(args[1])); transaction.AddKey(ShardForKey(args[2]), options->destination_db_, storage::ComputeDigest(args[2]), 2, - tx::LockMode::kExclusive); + tx::LockMode::kExclusive, storage::RedisSlot(args[2])); transaction.Seal(); ClusterShardValidatorContext cluster_validator; InstallClusterShardValidator(transaction, request, cluster_validator); @@ -6114,7 +6118,8 @@ Task ExecuteMSetNx(const CommandRequest& request, transaction.AddKey(ShardForKey(args[argument]), request.db_id_, storage::ComputeDigest(args[argument]), static_cast(argument), - tx::LockMode::kExclusive); + tx::LockMode::kExclusive, + storage::RedisSlot(args[argument])); } transaction.Seal(); ClusterShardValidatorContext cluster_validator; @@ -6272,7 +6277,8 @@ Task ExecuteMultiKey( for (std::size_t i = keys->first_; i <= keys->last_; i += keys->step_) { txn.AddKey(ShardForKey(args[i]), request.db_id_, storage::ComputeDigest(args[i]), static_cast(i), - write ? tx::LockMode::kExclusive : tx::LockMode::kShared); + write ? tx::LockMode::kExclusive : tx::LockMode::kShared, + storage::RedisSlot(args[i])); } txn.Seal(); ClusterShardValidatorContext cluster_validator; @@ -8318,7 +8324,8 @@ Task ExecuteEval(const CommandRequest& request, transaction->AddKey( ShardForKey(key), request.db_id_, storage::ComputeDigest(key), static_cast(i), - read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive); + read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive, + storage::RedisSlot(key)); } transaction->Seal(); if (!read_only) { @@ -10318,9 +10325,11 @@ Task ExecuteExecBody( } } else { tx::Transaction txn; - for (const auto& keys : cmd_keys) { - for (const ExecKey& key : keys) { - txn.AddKey(key.owner_, key.db_, key.digest_, key.arg_, key.mode_); + for (std::size_t command_index = 0; command_index < cmd_keys.size(); + ++command_index) { + for (const ExecKey& key : cmd_keys[command_index]) { + txn.AddKey(key.owner_, key.db_, key.digest_, key.arg_, key.mode_, + storage::RedisSlot(queued[command_index].args_[key.arg_])); } } for (const auto& watched : ctx.watched_) { @@ -10335,7 +10344,7 @@ Task ExecuteExecBody( } if (!covered) { txn.AddKey(watched.owner_, watched.db_, watched.digest_, 0, - tx::LockMode::kShared); + tx::LockMode::kShared, storage::RedisSlot(watched.key_)); } } txn.Seal(); @@ -13107,6 +13116,27 @@ Task ExecuteCommandBody( const unsigned target = routed ? request.RoutedPartitionId() % g_storage->worker_count() : ShardForKey(args[1]); + if (request.kind_ == CommandKind::kGet && args.size() == 2 && + target != ThisWorker().id_) { + auto fast = co_await g_storage->TryRemoteGet( + request.db_id_, args[1], + routed ? request.RoutedPartitionId() + : storage::RedisSlot(args[1])); + if (fast) { + if (const char* error = CommandServingGenerationError(request); + error) + co_return BuiltReply(reply_builder.AppendError(error)); + CommandReply reply; + if (fast->ok()) + reply.disk_value_ = std::move(**fast); + else if (absl::IsNotFound(fast->status())) + reply.encoded_ = reply_builder.AppendNull(); + else + reply.encoded_ = + AppendStorageError(reply_builder, fast->status()); + co_return reply; + } + } #if LAVIK_ENABLE_TRACE if (request.kind_ == CommandKind::kGet) { ReadLatencyTrace trace; diff --git a/src/redis/list_command.cpp b/src/redis/list_command.cpp index 40beac8a9..8ce380d7d 100644 --- a/src/redis/list_command.cpp +++ b/src/redis/list_command.cpp @@ -733,7 +733,8 @@ Task ExecuteListMultiKey(const CommandRequest& request, for (std::size_t arg : key_args) { txn.AddKey(ShardForKey(args[arg]), request.db_id_, storage::ComputeDigest(args[arg]), - static_cast(arg), tx::LockMode::kExclusive); + static_cast(arg), tx::LockMode::kExclusive, + storage::RedisSlot(args[arg])); } txn.Seal(); ClusterShardValidatorContext cluster_validator; diff --git a/src/redis/set_command.cpp b/src/redis/set_command.cpp index 6a14e8011..f20056cc2 100644 --- a/src/redis/set_command.cpp +++ b/src/redis/set_command.cpp @@ -714,7 +714,8 @@ Task ExecuteSetMultiKey(const CommandRequest& request, : tx::LockMode::kShared; transaction.AddKey(g_storage->OwnerForKey(args[i]), request.db_id_, storage::ComputeDigest(args[i]), - static_cast(i), lock_mode); + static_cast(i), lock_mode, + storage::RedisSlot(args[i])); } transaction.Seal(); // Cluster owner-side re-check: the admission was captured at dispatch time; diff --git a/src/redis/sort_command.cpp b/src/redis/sort_command.cpp index cd4b38cbe..1b16c415f 100644 --- a/src/redis/sort_command.cpp +++ b/src/redis/sort_command.cpp @@ -528,7 +528,7 @@ Task ExecuteSortCommand(const CommandRequest& request, tx::Transaction transaction; for (std::size_t i = 0; i < keys.size(); ++i) { transaction.AddKey(keys[i].owner_, request.db_id_, keys[i].digest_, i, - keys[i].mode_); + keys[i].mode_, storage::RedisSlot(keys[i].name_)); } transaction.Seal(); std::unique_ptr replication; diff --git a/src/redis/stream_command.cpp b/src/redis/stream_command.cpp index c5c88d622..8ec319027 100644 --- a/src/redis/stream_command.cpp +++ b/src/redis/stream_command.cpp @@ -1273,7 +1273,8 @@ PrepareStreamRangeReply(std::uint8_t db, std::string_view key, tx::TxShard::Guard guard; if (!locked_digest) guard = co_await tx::CurrentTxShard().AcquireKey( - db, tx::FingerprintOf(digest), tx::LockMode::kShared); + db, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); auto source = co_await g_storage->ReadValueForTransferLocked(db, key, digest, range); if (!source.ok()) co_return source.status(); @@ -1393,7 +1394,8 @@ Task> ReadOneLocal( if (group_read && !locked_digest) { delivery_digest = storage::ComputeDigest(key); delivery_guard = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(delivery_digest), tx::LockMode::kExclusive); + db_id, tx::FingerprintOf(delivery_digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); locked_digest = &delivery_digest; } const storage::MutationPrecondition mutation_precondition = @@ -2153,7 +2155,7 @@ Task ExecuteImpl(const CommandRequest& request, inspection_digest = storage::ComputeDigest(a[2]); inspection_guard = co_await tx::CurrentTxShard().AcquireKey( request.db_id_, tx::FingerprintOf(inspection_digest), - tx::LockMode::kShared); + tx::LockMode::kShared, lavik::storage::RedisSlot(a[2])); digest = &inspection_digest; } auto prepared = co_await PrepareStreamRangeReply( diff --git a/src/redis/string_command.cpp b/src/redis/string_command.cpp index 30fe85849..c30db0920 100644 --- a/src/redis/string_command.cpp +++ b/src/redis/string_command.cpp @@ -1339,7 +1339,8 @@ bycorf::Task ExecuteStringCommand(const CommandRequest& request, request.kind_ == CommandKind::kSubstr; auto key_lock = co_await tx::CurrentTxShard().AcquireKey( request.db_id_, tx::FingerprintOf(digest), - read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive); + read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive, + lavik::storage::RedisSlot(request.args_[1])); const storage::MutationPrecondition mutation_precondition = ClusterMutationPrecondition(request); co_return co_await ExecuteStringCommandLocked( @@ -1382,7 +1383,8 @@ bycorf::Task ExecuteBitmapCommand(const CommandRequest& request, request.kind_ != CommandKind::kBitField; auto key_lock = co_await tx::CurrentTxShard().AcquireKey( request.db_id_, tx::FingerprintOf(digest), - read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive); + read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive, + lavik::storage::RedisSlot(request.args_[1])); const storage::MutationPrecondition mutation_precondition = ClusterMutationPrecondition(request); co_return co_await ExecuteBitmapCommandLocked( @@ -1408,12 +1410,14 @@ bycorf::Task ExecuteBitOpCommand(const CommandRequest& request, tx::Transaction transaction; transaction.AddKey(g_storage->OwnerForKey(request.args_[2]), request.db_id_, storage::ComputeDigest(request.args_[2]), 2, - tx::LockMode::kExclusive); + tx::LockMode::kExclusive, + storage::RedisSlot(request.args_[2])); for (std::size_t argument = 3; argument < request.args_.size(); ++argument) { transaction.AddKey( g_storage->OwnerForKey(request.args_[argument]), request.db_id_, storage::ComputeDigest(request.args_[argument]), - static_cast(argument), tx::LockMode::kShared); + static_cast(argument), tx::LockMode::kShared, + storage::RedisSlot(request.args_[argument])); } transaction.Seal(); // The transaction validator rejects stale authority before each mutating @@ -1549,7 +1553,8 @@ bycorf::Task ExecuteLcsCommand(const CommandRequest& request, transaction.AddKey( g_storage->OwnerForKey(request.args_[argument]), request.db_id_, storage::ComputeDigest(request.args_[argument]), - static_cast(argument), tx::LockMode::kShared); + static_cast(argument), tx::LockMode::kShared, + storage::RedisSlot(request.args_[argument])); } transaction.Seal(); absl::Status status = co_await transaction.Schedule(); diff --git a/src/redis/zset_command.cpp b/src/redis/zset_command.cpp index f1b8fc3aa..1a43b10e1 100644 --- a/src/redis/zset_command.cpp +++ b/src/redis/zset_command.cpp @@ -665,7 +665,8 @@ Task ExecuteZSetMultiPopAttempt(const CommandRequest& request, transaction.AddKey( g_storage->OwnerForKey(request.args_[argument]), request.db_id_, storage::ComputeDigest(request.args_[argument]), - static_cast(argument), tx::LockMode::kExclusive); + static_cast(argument), tx::LockMode::kExclusive, + storage::RedisSlot(request.args_[argument])); } transaction.Seal(); // Cluster owner-side re-check: pops mutate, so the validator @@ -2664,15 +2665,15 @@ Task ExecuteZSetMultiKey(const CommandRequest& request, tx::Transaction transaction; if (context.store_) { const std::size_t destination = context.store_shape_->destination_arg_; - transaction.AddKey(g_storage->OwnerForKey(args[destination]), - request.db_id_, - storage::ComputeDigest(args[destination]), destination, - tx::LockMode::kExclusive); + transaction.AddKey( + g_storage->OwnerForKey(args[destination]), request.db_id_, + storage::ComputeDigest(args[destination]), destination, + tx::LockMode::kExclusive, storage::RedisSlot(args[destination])); } for (std::size_t i = context.first_source_; i <= context.last_source_; ++i) { transaction.AddKey(g_storage->OwnerForKey(args[i]), request.db_id_, storage::ComputeDigest(args[i]), i, - tx::LockMode::kShared); + tx::LockMode::kShared, storage::RedisSlot(args[i])); } transaction.Seal(); // Cluster owner-side re-check: only STORE forms mutate. diff --git a/src/storage/engine.cpp b/src/storage/engine.cpp index d75de3183..5c7047da8 100644 --- a/src/storage/engine.cpp +++ b/src/storage/engine.cpp @@ -634,6 +634,11 @@ void StorageEngine::SetReplicaLoading(bool loading) noexcept { impl_->SetReplicaLoading(loading); } +Task>> StorageEngine::TryRemoteGet( + std::uint8_t db_id, std::string_view key, std::uint16_t slot) { + return impl_->TryRemoteGet(db_id, key, slot); +} + Task> StorageEngine::Get( std::uint8_t db_id, std::string_view key, ReadLatencyTrace* trace, std::optional routed_partition_id) { diff --git a/src/storage/engine/backup.cpp b/src/storage/engine/backup.cpp index 94da467da..937dc74c3 100644 --- a/src/storage/engine/backup.cpp +++ b/src/storage/engine/backup.cpp @@ -266,10 +266,10 @@ Task StorageEngine::Impl::PinRdbSnapshotValue( state->allocation_epoch_ != pin.allocation_epoch_ || (pin.extent_ && state->kind_ != BlockKind::kPayloadExtent) || (!pin.extent_ && state->kind_ == BlockKind::kPayloadExtent) || - state->pins_ == std::numeric_limits::max()) { + state->pin_count() == BlockState::kPinMask) { co_return absl::AbortedError("stale RDB snapshot block"); } - ++state->pins_; + state->PinLocal(); co_return absl::OkStatus(); }; absl::Status status; @@ -291,11 +291,8 @@ Task StorageEngine::Impl::PinRdbSnapshotValue( BlockState* state = FindBlockState(block_store, pin.block_id_); if (state != nullptr && state->allocated_ && state->allocation_epoch_ == pin.allocation_epoch_ && - state->pins_ != 0) { - --state->pins_; - if (state->pins_ == 0 && state->release_pending_) { - ReleaseStagingBuffer(block_store, *state); - } + state->pin_count() != 0) { + UnpinBlock(block_store, *state, pin.block_id_); } co_return absl::OkStatus(); }; @@ -342,11 +339,8 @@ Task StorageEngine::Impl::ReleaseRdbSnapshotValue( BlockState* state = FindBlockState(block_store, pin.block_id_); if (state != nullptr && state->allocated_ && state->allocation_epoch_ == pin.allocation_epoch_ && - state->pins_ != 0) { - --state->pins_; - if (state->pins_ == 0 && state->release_pending_) { - ReleaseStagingBuffer(block_store, *state); - } + state->pin_count() != 0) { + UnpinBlock(block_store, *state, pin.block_id_); } co_return absl::OkStatus(); }; @@ -564,7 +558,8 @@ StorageEngine::Impl::MaterializeRdbSnapshotKey( using Phase = SavedValue::Phase; const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); auto* capture = partition.rdb_snapshot_ ? &*partition.rdb_snapshot_ : nullptr; if (capture == nullptr || capture->session_id_ != session_id || !store.rdb_snapshot_ || store.rdb_snapshot_->id_ != session_id || @@ -1202,7 +1197,7 @@ Task> StorageEngine::Impl::ReadRdbCollectionPage( const auto* block = engine->FindBlockState(physical, pin.block_id_); co_return block != nullptr && block->allocated_ && block->allocation_epoch_ == pin.allocation_epoch_ - ? block->pins_ + ? block->pin_count() : 0; }; // if/else, not ?:, to keep the two co_awaits in separate full diff --git a/src/storage/engine/collection_ingest.cpp b/src/storage/engine/collection_ingest.cpp index fc800a4bf..8532561a6 100644 --- a/src/storage/engine/collection_ingest.cpp +++ b/src/storage/engine/collection_ingest.cpp @@ -97,7 +97,8 @@ StorageEngine::Impl::RestoreCollectionValue( const MutationPrecondition* mutation_precondition) { const auto digest = ComputeDigest(key); auto hold = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await RestoreCollectionValueLocked( db_id, key, digest, type, expire_at_ms, replace, expected_items, std::move(reader), nullptr, replication, mutation_precondition); diff --git a/src/storage/engine/compact_api.cpp b/src/storage/engine/compact_api.cpp index 4472be620..14d3293bc 100644 --- a/src/storage/engine/compact_api.cpp +++ b/src/storage/engine/compact_api.cpp @@ -63,7 +63,8 @@ Task StorageEngine::Impl::ExecuteCompact( const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( db_id, tx::FingerprintOf(digest), - read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive); + read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await ExecuteCompactLocked( db_id, key, digest, value_type, read_only, callback, nullptr, now_ms, replication, false, mutation_precondition, std::move(access)); diff --git a/src/storage/engine/defrag.cpp b/src/storage/engine/defrag.cpp index 6c1b3d017..e2ca12d3c 100644 --- a/src/storage/engine/defrag.cpp +++ b/src/storage/engine/defrag.cpp @@ -75,7 +75,7 @@ Task> StorageEngine::Impl::ReclaimExtentLocal( "extent reclaim found a record block"); } state->live_bytes_ = 0; - if (state->pins_ != 0 || state->freeing_) { + if (state->pin_count() != 0 || state->freeing_) { store.store_state_mutex_.Unlock(*store.worker_); absl::Status waited = co_await bycorf::SleepFor( *store.worker_, std::chrono::milliseconds(1)); @@ -143,16 +143,55 @@ Task StorageEngine::Impl::ReclaimExtents( co_return absl::OkStatus(); } +void StorageEngine::Impl::UnpinBlock(WorkerStore& store, BlockState& state, + std::uint64_t block_id) { + const auto old = state.pins_.fetch_sub(1, std::memory_order_acq_rel); + assert((old & BlockState::kPinMask) != 0); + if ((old & BlockState::kPinMask) != 1) return; + if (old & BlockState::kRemoteClosed) + store.block_pin_progress_.NotifyAll(*store.worker_); + if (state.release_pending_) ReleaseStagingBuffer(store, state); + MaybeQueueDefrag(store, block_id); +} + +void StorageEngine::Impl::UnpinRemoteBlock(BlockState& state, unsigned owner, + std::uint64_t block_id) { + const auto old = state.pins_.fetch_sub(1, std::memory_order_acq_rel); + assert((old & BlockState::kPinMask) != 0); + if (old != (BlockState::kRemoteClosed | 1)) return; + // This is only a maintenance hint, not an allocation-specific release. + // Re-evaluate the current allocation on its owner, so a delayed message is + // harmless even after reuse. The stable store context needs no heap receipt + // that could leak when teardown drains I/O after the event loops stop. + bycorf::PostNotification( + bycorf::ThisWorker().cross_core_, owner, + {stores_[owner].get(), block_id, + [](void* ctx, std::uint64_t id) noexcept { + auto& store = *static_cast(ctx); + auto& engine = *store.engine_; + store.block_pin_progress_.NotifyAll(*store.worker_); + auto* state = engine.FindBlockState(store, id); + if (!state || state->pin_count() != 0) return; + if (state->release_pending_) ReleaseStagingBuffer(store, *state); + engine.MaybeQueueDefrag(store, id); + }}); +} + void StorageEngine::Impl::MaybeQueueDefrag(WorkerStore& store, std::uint64_t block_id) { BlockState* state = FindBlockState(store, block_id); if (state == nullptr || !state->allocated_ || state->defrag_queued_ || - state->defragging_ || state->pins_ != 0 || state->in_memory_ || + state->defragging_ || state->freeing_ || state->release_pending_) + return; + if (state->in_memory_ || (state->kind_ != BlockKind::kRecords && state->kind_ != BlockKind::kIndirectKeys) || state->flush_queued_ || state->flush_in_progress_ || IsActiveBlock(store, block_id) || state->committed_bytes_ <= kBlockHeaderBytes) { + // A deferred staging-buffer release may have closed admission even for + // a block that is not a compaction candidate. Do not strand it closed. + state->OpenRemotePins(); return; } const std::uint64_t used = state->committed_bytes_ - kBlockHeaderBytes; @@ -160,8 +199,13 @@ void StorageEngine::Impl::MaybeQueueDefrag(WorkerStore& store, // is exactly 1000 * live < 501 * used for positive used. Both products fit // in uint64_t, so candidate checks need no division or rounding change. if (static_cast(state->live_bytes_) * 1000 >= used * 501) { + state->OpenRemotePins(); return; } + // Mark pending maintenance before testing the count. The last remote + // unpin then posts the retry that used to be lost for overwritten blocks. + state->CloseRemotePins(); + if (state->pin_count() != 0) return; state->defrag_queued_ = true; store.defrag_queue_.push_back(block_id); RequestDefrag(store); @@ -767,9 +811,23 @@ Task StorageEngine::Impl::CleanBlockLocked( co_return absl::OkStatus(); } BlockState& source = *source_ptr; + source.CloseRemotePins(); + struct Reopen { + Impl* engine_; + WorkerStore& store_; + std::uint64_t id_; + std::uint64_t epoch_; + ~Reopen() { + auto* state = engine_->FindBlockState(store_, id_); + if (state && state->allocation_epoch_ == epoch_ && !state->freeing_ && + !state->defragging_ && !state->defrag_queued_ && + state->pin_count() == 0) + state->OpenRemotePins(); + } + } reopen{this, store, block_id, source.allocation_epoch_}; if (!source.allocated_ || source.defragging_ || source.in_memory_ || - source.pins_ != 0 || source.flush_queued_ || source.flush_in_progress_ || - IsActiveBlock(store, block_id)) { + source.pin_count() != 0 || source.flush_queued_ || + source.flush_in_progress_ || IsActiveBlock(store, block_id)) { co_return absl::OkStatus(); } source.defragging_ = true; @@ -1102,15 +1160,8 @@ Task StorageEngine::Impl::SalvageBlockRecords( Task StorageEngine::Impl::ReleaseEmptyBlock( WorkerStore& store, std::uint64_t block_id, BlockState& source) { - while (source.pins_ != 0) { - absl::Status waited = - co_await bycorf::SleepFor(*store.worker_, std::chrono::milliseconds(1)); - if (!waited.ok()) { - source.freeing_ = false; - source.defragging_ = false; - co_return waited; - } - } + source.CloseRemotePins(); + while (source.pin_count() != 0) co_await store.block_pin_progress_.Wait(); // The durable allocation bitmap is authoritative during recovery. Retire // the runtime state before publishing the block to cold_free so another diff --git a/src/storage/engine/expire.cpp b/src/storage/engine/expire.cpp index 4b153886f..fe0fae576 100644 --- a/src/storage/engine/expire.cpp +++ b/src/storage/engine/expire.cpp @@ -336,7 +336,7 @@ Task StorageEngine::Impl::ExpireCandidate( auto& partition = PartitionFor(store, candidate.partition_id_); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( candidate.db_id_, tx::FingerprintOf(candidate.digest_), - tx::LockMode::kExclusive); + tx::LockMode::kExclusive, lavik::storage::RedisSlot(candidate.key_)); co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); auto resolved = @@ -385,6 +385,8 @@ Task StorageEngine::Impl::ExpireCandidate( // Append may release store_state_mutex_ while acquiring space or pinning // transaction dependencies. The key hold excludes logical writes, but GC // may replace the index entry and the group's physical coordinates. + auto index_write = co_await AwaitIndexWrite( + *store.worker_, partition.indexes_[candidate.db_id_], candidate.digest_); resolved = co_await FindVerifiedEntry(store, partition.indexes_[candidate.db_id_], candidate.digest_, candidate.key_); diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index f7f19ce2f..8fae190be 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -284,8 +284,9 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { if (state->kind_ == BlockKind::kTransaction) { tx_cleaner_dirty_.store(true, std::memory_order_release); } - if (state->pins_ > 0) { + if (state->pin_count() > 0) { state->release_pending_ = true; + state->CloseRemotePins(); } else { ReleaseStagingBuffer(*store, *state); MaybeQueueDefrag(*store, block_id); @@ -519,6 +520,13 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { // applies to the replacement entry's own location. RecordIndex& index = PartitionFor(*store, identity.partition_id_) .indexes_[identity.db_id_]; + auto index_write = co_await AwaitIndexValueWrite(*store->worker_, index, + identity.entry_hash_); + // Detach can run while the reader drain yields. Recheck the population + // before resolving the saved integer address into this index. + if (identity.index_generation_ != + store->index_generations_[identity.db_id_]) + continue; RecordIndex::Entry* current_entry = index.FindAddress( identity.entry_address_, identity.entry_hash_, identity.entry_tag_); if (current_entry == nullptr) { @@ -588,8 +596,9 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { if (state->kind_ == BlockKind::kTransaction) { tx_cleaner_dirty_.store(true, std::memory_order_release); } - if (state->pins_ > 0) { + if (state->pin_count() > 0) { state->release_pending_ = true; + state->CloseRemotePins(); continue; } ReleaseStagingBuffer(*store, *state); diff --git a/src/storage/engine/flush_db.cpp b/src/storage/engine/flush_db.cpp index 792a20bfa..1e58251c0 100644 --- a/src/storage/engine/flush_db.cpp +++ b/src/storage/engine/flush_db.cpp @@ -176,7 +176,7 @@ Task StorageEngine::Impl::DetachDbEpochs( co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); for (std::uint8_t db_id = 0; db_id < kLogicalDatabaseCount; ++db_id) { - if (changed[db_id]) DetachDbLocal(store, db_id); + if (changed[db_id]) co_await DetachDbLocal(store, db_id); } co_return absl::OkStatus(); }; @@ -220,7 +220,7 @@ Task StorageEngine::Impl::DetachDbEpoch( WorkerStore& store = *stores_[target]; co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); - DetachDbLocal(store, db_id); + co_await DetachDbLocal(store, db_id); co_return absl::OkStatus(); }; // if/else, not ?:, to keep the two co_awaits in separate full @@ -281,10 +281,16 @@ Task StorageEngine::Impl::AdvanceDbEpoch(std::uint8_t db_id, co_return co_await ReclaimDetachedAllWorkers(/*wait=*/true); } -void StorageEngine::Impl::DetachDbLocal(WorkerStore& store, - std::uint8_t db_id) { +Task StorageEngine::Impl::DetachDbLocal(WorkerStore& store, + std::uint8_t db_id) { // Epoch metadata covers all 16 DBs even when runtime indexes cover DB0 only. - if (db_id >= options_.database_count_) return; + if (db_id >= options_.database_count_) co_return absl::OkStatus(); + std::vector index_writes; + index_writes.reserve(store.partitions_.size()); + for (auto& partition : store.partitions_) + index_writes.push_back(partition.indexes_[db_id].CloseAllReaders()); + for (auto& guard : index_writes) + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); // FLUSHDB invalidates every watcher of this database, including watches // on keys that never existed (Redis semantics). tx::CurrentTxShard().MarkAllWatched(db_id); @@ -331,6 +337,7 @@ void StorageEngine::Impl::DetachDbLocal(WorkerStore& store, } } store.live_key_count_[db_id] = 0; + co_return absl::OkStatus(); } void StorageEngine::Impl::QueueDetachedIndex(WorkerStore& store, diff --git a/src/storage/engine/hash_api.cpp b/src/storage/engine/hash_api.cpp index 17d5c2f13..71d9c54e8 100644 --- a/src/storage/engine/hash_api.cpp +++ b/src/storage/engine/hash_api.cpp @@ -41,7 +41,8 @@ Task> StorageEngine::Impl::ExecuteHash( auto key_lock = co_await tx::CurrentTxShard().AcquireKey( db_id, tx::FingerprintOf(digest), IsHashLikeWrite(operation) ? tx::LockMode::kExclusive - : tx::LockMode::kShared); + : tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await ExecuteHashLocked(db_id, key, digest, operation, nullptr, replication, mutation_precondition); } @@ -55,7 +56,8 @@ Task> StorageEngine::Impl::ExecuteSet( auto key_lock = co_await tx::CurrentTxShard().AcquireKey( db_id, tx::FingerprintOf(digest), IsHashLikeWrite(operation) ? tx::LockMode::kExclusive - : tx::LockMode::kShared); + : tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await ExecuteSetLocked(db_id, key, digest, operation, nullptr, replication, mutation_precondition); } diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 83f39e248..747fa1c25 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -110,6 +110,37 @@ class UnlockGuard { using ExtentManifest = std::shared_ptr>; +// These waits yield only the owner coroutine. Remote readers never suspend +// while registered, so draining does not depend on store-state/key locks held +// by the caller. Call before resolving entries, and revalidate after any wait. +inline Task AwaitIndexWrite( + [[maybe_unused]] Worker& worker, RecordIndex& index, const Digest& digest) { + auto guard = index.CloseForWrite(digest); + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); + // Another owner coroutine may have changed capacity during the wait. + // No entry pointer has been obtained yet; upgrade before resolving it. + if (index.NeedsStructuralWrite(digest)) { + guard.Reset(); + guard = index.CloseAllReaders(); + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); + } + co_return std::move(guard); +} + +inline Task AwaitIndexValueWrite( + [[maybe_unused]] Worker& worker, RecordIndex& index, std::uint64_t hash) { + auto guard = index.CloseForValueUpdate(hash); + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); + co_return std::move(guard); +} + +inline Task AwaitAllIndexReaders( + [[maybe_unused]] Worker& worker, RecordIndex& index) { + auto guard = index.CloseAllReaders(); + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); + co_return std::move(guard); +} + // Metadata is changed only on the UUID's logical owner. Physical record blocks // hold shared references until their allocation bits are durably cleared, even // after DEL/expiry or rollback removes their records from the user index. @@ -300,7 +331,32 @@ struct alignas(32) BlockState { std::uint64_t allocation_epoch_ = 0; std::uint32_t committed_bytes_ = 0; std::uint32_t live_bytes_ = 0; - std::uint32_t pins_ = 0; + static constexpr std::uint32_t kRemoteClosed = std::uint32_t{1} << 31; + static constexpr std::uint32_t kPinMask = kRemoteClosed - 1; + std::atomic pins_{kRemoteClosed}; + std::uint32_t pin_count() const noexcept { + return pins_.load(std::memory_order_acquire) & kPinMask; + } + // The owner already validated allocation/freeing state without suspending. + // Local fallback reads may pin during defrag; only remote admission closes. + void PinLocal() noexcept { + [[maybe_unused]] const auto old = + pins_.fetch_add(1, std::memory_order_relaxed); + assert((old & kPinMask) != kPinMask); + } + bool TryRemotePin() noexcept { + auto state = pins_.load(std::memory_order_relaxed); + return !(state & kRemoteClosed) && (state & kPinMask) != kPinMask && + pins_.compare_exchange_strong(state, state + 1, + std::memory_order_acquire, + std::memory_order_relaxed); + } + void CloseRemotePins() noexcept { + pins_.fetch_or(kRemoteClosed, std::memory_order_acq_rel); + } + void OpenRemotePins() noexcept { + pins_.fetch_and(kPinMask, std::memory_order_release); + } // Written only by the owner as it claims or releases the block; read by // anyone that needs to know where to dispatch. kUnownedBlock means free. std::atomic owner_{kUnownedBlock}; @@ -329,7 +385,8 @@ struct alignas(32) BlockState { allocation_epoch_ = new_allocation_epoch; committed_bytes_ = 0; live_bytes_ = 0; - pins_ = 0; + pins_.store(new_owner == kUnownedBlock ? kRemoteClosed : 0, + std::memory_order_release); writer_id_ = 0; layout_worker_count_ = 0; staging_slot_ = 0; @@ -1559,6 +1616,26 @@ class StorageEngine::Impl { public: struct FullSyncCollection; struct WorkerStore { + Impl* engine_ = nullptr; + bycorf::AsyncNotification block_pin_progress_; + std::atomic remote_reads_ready_{false}; + std::atomic remote_population_generation_{0}; + unsigned remote_population_holds_ = 0; + // Worker-owned, nestable population fence. Snapshot readers compare this + // generation before/after lookup and after I/O; it never guards raw data. + struct PopulationReadFence { + WorkerStore& store_; + explicit PopulationReadFence(WorkerStore& store) : store_(store) { + if (store_.remote_population_holds_++ == 0) + store_.remote_population_generation_.fetch_add( + 1, std::memory_order_release); + } + ~PopulationReadFence() { + if (--store_.remote_population_holds_ == 0) + store_.remote_population_generation_.fetch_add( + 1, std::memory_order_release); + } + }; struct PendingTxCommit { std::uint64_t txid_ = 0; std::vector writes_; @@ -2108,6 +2185,16 @@ class StorageEngine::Impl { absl::Status Prepare(unsigned worker_count); Task InitializeWorker(Worker& worker); + static void NotifyIndexReaders(void* context) noexcept; + static void ChangeReadIntent(void* context, const tx::KeyRef& key, + bool close); + static bool ReadIntentReady(void* context, const tx::KeyRef& key); + void UnpinBlock(WorkerStore& store, BlockState& state, + std::uint64_t block_id); + void UnpinRemoteBlock(BlockState& state, unsigned owner, + std::uint64_t block_id); + Task>> TryRemoteGet( + std::uint8_t db_id, std::string_view key, std::uint16_t slot); void FinalizeWorker(unsigned worker_id) noexcept; bool AbandonWorkerStateForProcessExit() noexcept; @@ -3361,7 +3448,8 @@ class StorageEngine::Impl { LoadValueAwaitable(Impl& engine, WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, ReadLatencyTrace* trace, - std::optional expected_db_epoch); + std::optional expected_db_epoch, + bool remotely_pinned = false); ~LoadValueAwaitable(); bool await_ready() const noexcept { return ready_ && !slow_.valid(); } @@ -3397,6 +3485,7 @@ class StorageEngine::Impl { ReadLatencyTrace* trace_; std::uint64_t replication_epoch_; std::optional expected_db_epoch_; + bool remotely_pinned_ = false; std::uint64_t aligned_offset_ = 0; std::size_t file_id_ = 0; std::size_t record_headroom_ = 0; @@ -3555,8 +3644,11 @@ class StorageEngine::Impl { } void DestroyBlockState(WorkerStore& store, std::uint64_t block_id) { + auto& state = BlockStateAt(block_id); + state.CloseRemotePins(); + assert(state.pin_count() == 0); store.tx_blocks_.erase(block_id); - BlockStateAt(block_id).Reset(kUnownedBlock); + state.Reset(kUnownedBlock); } // Walks this worker's blocks. The array is shared, so ownership is filtered @@ -3661,10 +3753,10 @@ class StorageEngine::Impl { Task PersistCheckpointRoot(const CheckpointRoot& root); // Takes the database out of service on this worker. Everything here is O(the - // partition count) and runs without suspending, so the caller's FLUSHDB gate - // stays closed for a bounded time no matter how many keys the database holds. + // partition count). Close/drain remote readers asynchronously before the + // non-suspending population replacement, under the caller's FLUSHDB gate. // Retiring the detached entries is left to ReclaimDetachedIndexes. - void DetachDbLocal(WorkerStore& store, std::uint8_t db_id); + Task DetachDbLocal(WorkerStore& store, std::uint8_t db_id); // Retires entries detached by database flush or a replica population // transition: subtracts what they contributed to their blocks, then frees @@ -3892,7 +3984,8 @@ class StorageEngine::Impl { WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, ReadLatencyTrace* trace = nullptr, - std::optional expected_db_epoch = std::nullopt); + std::optional expected_db_epoch = std::nullopt, + bool remotely_pinned = false); std::uint64_t ForegroundBlocksForDevice( std::size_t device_index) const noexcept { @@ -4731,6 +4824,8 @@ class StorageEngine::Impl { std::int64_t recovery_started_ms_ = 0; static constexpr std::size_t kDefragReserveBlocksPerDevice = 8; std::array, kLogicalDatabaseCount> db_epochs_{}; + std::array, kLogicalStorageShards> + remote_replication_epochs_{}; // Runtime translation installed with a promoted replica root. Source DB // epochs can be lower than this node's pre-existing local epoch. std::array, kLogicalDatabaseCount> diff --git a/src/storage/engine/init.cpp b/src/storage/engine/init.cpp index 3c5bb1dfe..98070f612 100644 --- a/src/storage/engine/init.cpp +++ b/src/storage/engine/init.cpp @@ -946,6 +946,8 @@ absl::Status StorageEngine::Impl::Prepare(unsigned worker_count) { .subspan(offset, options_.database_count_); partition_store.replication_epoch_ = epoch_values_[kLogicalDatabaseCount + partition]; + remote_replication_epochs_[partition].store( + partition_store.replication_epoch_, std::memory_order_release); partition_store.replica_candidate_epoch_ = partition_store.replication_epoch_; } @@ -1084,6 +1086,7 @@ Task StorageEngine::Impl::ApplyRecoveryLiveReferenceBatches( Task StorageEngine::Impl::InitializeWorker(Worker& worker) { WorkerStore& store = *stores_[worker.id()]; store.worker_ = &worker; + store.engine_ = this; absl::Status status = store.buffers_.Init(worker, options_.buffers_); if (status.ok()) { @@ -1989,6 +1992,9 @@ Task StorageEngine::Impl::InitializeWorker(Worker& worker) { co_await store.store_state_mutex_.Lock(); { UnlockGuard unlock(&store.store_state_mutex_, store.worker_); + auto index_write = co_await AwaitIndexWrite( + *store.worker_, partition.indexes_[expired.db_id_], + expired.digest_); auto resolved = co_await FindVerifiedEntry( store, partition.indexes_[expired.db_id_], expired.digest_, expired.key_); @@ -2056,6 +2062,20 @@ Task StorageEngine::Impl::InitializeWorker(Worker& worker) { Fail(status); co_return status; } + auto& shard = tx::CurrentTxShard(); + shard.BindReadAdmission({&store, + [](void* ctx, const tx::KeyRef& key) { + ChangeReadIntent(ctx, key, true); + }, + [](void* ctx, const tx::KeyRef& key) { + ChangeReadIntent(ctx, key, false); + }, + &ReadIntentReady}); + for (auto& index : store.partition_indexes_) { + index.SetReaderNotifier(&store, &NotifyIndexReaders); + index.EnableConcurrentReads(); + } + store.remote_reads_ready_.store(true, std::memory_order_release); worker.SpawnRoot(PeriodicFlush(&store)); worker.SpawnBackground(ActiveExpiration(&store)); // The scheduler lives until shutdown, including OFF and incomplete FULL. @@ -2066,8 +2086,55 @@ Task StorageEngine::Impl::InitializeWorker(Worker& worker) { co_return absl::OkStatus(); } +void StorageEngine::Impl::NotifyIndexReaders(void* context) noexcept { + auto& store = *static_cast(context); + // Notifications name worker-lifetime state, never a suspended frame. The + // server drains all workers and I/O before destroying any WorkerStore. + bycorf::PostNotification(bycorf::ThisWorker().cross_core_, + store.worker_->id(), + {nullptr, 0, [](void*, std::uint64_t) noexcept { + tx::CurrentTxShard().ReadersDrained(); + }}); +} + +void StorageEngine::Impl::ChangeReadIntent(void* context, const tx::KeyRef& key, + bool close) { + auto& store = *static_cast(context); + auto change = [&](RecordIndex& index) { + if (close) + index.CloseReadIntent(key.fp_); + else + index.ReleaseReadIntent(key.fp_); + }; + if (key.db_ >= store.engine_->options_.database_count_) return; + if (key.slot_ < kLogicalStorageShards) { + change(store.engine_->PartitionFor(store, key.slot_).indexes_[key.db_]); + } else { + // Legacy/internal callers without a route are conservative. A full digest + // collision merged from different slots also takes this bounded path. + for (auto& partition : store.partitions_) + change(partition.indexes_[key.db_]); + } +} + +bool StorageEngine::Impl::ReadIntentReady(void* context, + const tx::KeyRef& key) { + auto& store = *static_cast(context); + if (key.db_ >= store.engine_->options_.database_count_) return true; + if (key.slot_ < kLogicalStorageShards) + return store.engine_->PartitionFor(store, key.slot_) + .indexes_[key.db_] + .ReadIntentReady(key.fp_); + for (const auto& partition : store.partitions_) + if (!partition.indexes_[key.db_].ReadIntentReady(key.fp_)) return false; + return true; +} + void StorageEngine::Impl::FinalizeWorker(unsigned worker_id) noexcept { assert(worker_id < stores_.size()); + tx::CurrentTxShard().BindReadAdmission({}); + stores_[worker_id]->remote_reads_ready_.store(false, + std::memory_order_release); if (abandon_worker_state_on_finalize_.load(std::memory_order_acquire)) { // Worker::Shutdown has already drained IO and closed io_uring/SPDK state, // and detached coroutine frames were destroyed before this callback. The diff --git a/src/storage/engine/list_api.cpp b/src/storage/engine/list_api.cpp index e746f4b0b..a976df001 100644 --- a/src/storage/engine/list_api.cpp +++ b/src/storage/engine/list_api.cpp @@ -26,7 +26,8 @@ Task> StorageEngine::Impl::ListPush( assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await ListPushLocked(db_id, key, digest, values, nullptr, replication, mutation_precondition); } @@ -60,7 +61,8 @@ Task> StorageEngine::Impl::ExecuteList( operation.kind_ == ListOperationKind::kPosition; auto key_lock = co_await tx::CurrentTxShard().AcquireKey( db_id, tx::FingerprintOf(digest), - read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive); + read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await ExecuteListLocked(db_id, key, digest, operation, nullptr, replication, mutation_precondition); } diff --git a/src/storage/engine/read.cpp b/src/storage/engine/read.cpp index 6d9401122..067b0cfa9 100644 --- a/src/storage/engine/read.cpp +++ b/src/storage/engine/read.cpp @@ -226,6 +226,88 @@ StorageEngine::Impl::RandomKeyLocal(std::uint8_t db_id) { co_return std::optional{}; } +Task>> +StorageEngine::Impl::TryRemoteGet(std::uint8_t db_id, std::string_view key, + std::uint16_t slot) { + const unsigned owner = slot % worker_count_; + auto& local = CurrentStore(); + if (owner == local.worker_->id() || db_id >= options_.database_count_ || + replica_loading_.load(std::memory_order_acquire) || + replica_recovery_fenced_.load(std::memory_order_acquire) || + request_serving_fenced_until_restart_.load(std::memory_order_acquire)) + co_return std::nullopt; + auto& source = *stores_[owner]; + if (!source.remote_reads_ready_.load(std::memory_order_acquire)) + co_return std::nullopt; + const auto generation = + source.remote_population_generation_.load(std::memory_order_acquire); + if (generation & 1) co_return std::nullopt; + const auto db_epoch = DbEpoch(db_id); + const auto replication_epoch = + remote_replication_epochs_[slot].load(std::memory_order_acquire); + const auto current = [&] { + return source.remote_population_generation_.load( + std::memory_order_acquire) == generation && + DbEpoch(db_id) == db_epoch && + remote_replication_epochs_[slot].load(std::memory_order_acquire) == + replication_epoch && + !replica_loading_.load(std::memory_order_acquire) && + !replica_recovery_fenced_.load(std::memory_order_acquire) && + !request_serving_fenced_until_restart_.load( + std::memory_order_acquire); + }; + const auto digest = ComputeDigest(key); + auto& index = PartitionFor(source, slot).indexes_[db_id]; + auto reader = index.TryRead(digest); + if (!reader) co_return std::nullopt; + const auto* entry = std::as_const(index).Find(digest, key); + if (!current()) co_return std::nullopt; + if (entry && !entry->key_complete()) co_return std::nullopt; + if (entry == nullptr || entry->value_.kind() == RecordKind::kTombstone) { + co_return std::optional>( + absl::NotFoundError("key not found")); + } + // Cold representations and expiration retain the complete owner state + // machine (including expiry scheduling and indirect-key collision checks). + if (!entry->key_complete() || entry->value_.grouped() || + entry->value_.external() || entry->value_.in_memory() || + entry->value_.value_type() != ValueType::kString || IsExpiredNow(*entry)) + co_return std::nullopt; + const RecordLocation location = MaterializeIndexLocation(*entry); + if (location.block_owner() != owner) co_return std::nullopt; + // io_uring opens every configured file on every worker. Only SPDK builds + // the controller-specific qpair owner directory. + if (bycorf::SpdkStorageEnabled()) { + const auto device = DeviceIndexForBlock(location.block_id()); + if (!std::binary_search(device_owners_[device].begin(), + device_owners_[device].end(), + static_cast(local.worker_->id()))) + co_return std::nullopt; + } + auto& block = BlockStateAt(location.block_id()); + if (!block.TryRemotePin()) co_return std::nullopt; + struct Pin { + Impl* engine; + BlockState& block; + unsigned owner; + std::uint64_t block_id; + ~Pin() { engine->UnpinRemoteBlock(block, owner, block_id); } + } pin{this, block, owner, location.block_id()}; + if (block.owner_.load(std::memory_order_acquire) != owner || + block.allocation_epoch_ != location.allocation_epoch() || !current()) + co_return std::nullopt; + // The only index-to-I/O handoff: no suspension or borrowed entry survives it. + reader.Reset(); + auto loaded = co_await LoadValueLocal( + local, db_id, key, location, replication_epoch, nullptr, db_epoch, true); + if (!current() || (!loaded.ok() && absl::IsAborted(loaded.status()))) + co_return std::nullopt; + if (!loaded.ok()) + co_return std::optional>(loaded.status()); + co_return std::optional>( + EncodeDiskValue(std::move(*loaded))); +} + // Keep optimistic-read admission out of the already large command dispatcher. // LTO otherwise duplicates this wrapper and expands unrelated command cases, // increasing their instruction-cache footprint even though only GET uses it. @@ -257,7 +339,8 @@ Task> StorageEngine::Impl::GetWithLockState( tx::TxShard::Guard key_lock; if (acquire_key_lock) { key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); } WorkerStore& store = CurrentStore(); auto& partition = routed_partition_id.has_value() @@ -271,7 +354,8 @@ Task> StorageEngine::Impl::GetWithLockState( // Verifying an out-of-index key suspends before we know this is the // requested key, so there is no lock-free linearization point yet. key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); optimistic_read = false; continue; } @@ -316,7 +400,8 @@ Task> StorageEngine::Impl::GetWithLockState( // retire its physical record before the I/O path pins it. Do not turn // that race into a spurious nil; take the original lock path and reread. key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); optimistic_read = false; continue; } @@ -404,17 +489,15 @@ Task> StorageEngine::Impl::BatchGetLocked( lease_(std::move(lease)), record_headroom_(record_headroom), read_bytes_(read_bytes) { - ++state_->pins_; + state_->PinLocal(); } PendingRead(const PendingRead&) = delete; PendingRead& operator=(const PendingRead&) = delete; ~PendingRead() { - --state_->pins_; - if (state_->pins_ == 0 && state_->release_pending_) { - StorageEngine::Impl::ReleaseStagingBuffer(*store_, *state_); - } + store_->engine_->UnpinBlock(*store_, *state_, + candidate_.location_.block_id()); } WorkerStore* store_ = nullptr; @@ -597,7 +680,8 @@ Task> StorageEngine::Impl::StringLength( assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await StringLengthLocked(db_id, key, digest); } @@ -649,7 +733,8 @@ Task> StorageEngine::Impl::ReadKeyMetadata( co_return absl::InvalidArgumentError("invalid logical database"); const auto digest = ComputeDigest(key); auto hold = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await ReadKeyMetadataLocked(db_id, key, digest); } @@ -693,7 +778,8 @@ Task StorageEngine::Impl::GetExpiration(std::uint8_t db_id, assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await GetExpirationLocked(db_id, key, digest); } @@ -782,7 +868,8 @@ Task> StorageEngine::Impl::ReadRawValue( assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await ReadRawValueLocked(db_id, key, digest); } @@ -810,7 +897,8 @@ Task StorageEngine::Impl::Exists(std::uint8_t db_id, assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_return co_await ExistsLocked(db_id, key, digest); } @@ -1030,7 +1118,7 @@ StorageEngine::Impl::ReadExtentAwaitable::ReadExtentAwaitable( ready_ = absl::AbortedError("stale or missing external extent"); return; } - ++state->pins_; + state->PinLocal(); pinned_ = state; read_bytes_ = AlignDirect(kBlockHeaderBytes + ref.payload_bytes_); buffer_wait_.emplace(&store.buffers_, read_bytes_); @@ -1038,7 +1126,7 @@ StorageEngine::Impl::ReadExtentAwaitable::ReadExtentAwaitable( } StorageEngine::Impl::ReadExtentAwaitable::~ReadExtentAwaitable() { - if (pinned_) --pinned_->pins_; + if (pinned_) engine_.UnpinBlock(store_, *pinned_, ref_.block_id_); } void StorageEngine::Impl::ReadExtentAwaitable::FinishBufferWait() { @@ -1335,9 +1423,10 @@ StorageEngine::Impl::LoadValueAwaitable StorageEngine::Impl::LoadValue( StorageEngine::Impl::LoadValueAwaitable StorageEngine::Impl::LoadValueLocal( WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, - ReadLatencyTrace* trace, std::optional expected_db_epoch) { + ReadLatencyTrace* trace, std::optional expected_db_epoch, + bool remotely_pinned) { return LoadValueAwaitable(*this, store, db_id, key, location, - replication_epoch, trace, expected_db_epoch); + replication_epoch, trace, expected_db_epoch, remotely_pinned); } StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( @@ -1374,7 +1463,8 @@ StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( Impl& engine, WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, - ReadLatencyTrace* trace, std::optional expected_db_epoch) + ReadLatencyTrace* trace, std::optional expected_db_epoch, + bool remotely_pinned) : engine_(engine), store_(store), key_(key), @@ -1382,6 +1472,7 @@ StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( trace_(trace), replication_epoch_(replication_epoch), expected_db_epoch_(expected_db_epoch), + remotely_pinned_(remotely_pinned), db_id_(db_id) { PrepareLocal(); } @@ -1391,9 +1482,7 @@ StorageEngine::Impl::LoadValueAwaitable::~LoadValueAwaitable() { Unpin(); } void StorageEngine::Impl::LoadValueAwaitable::Unpin() { auto* state = std::exchange(pinned_, nullptr); if (!state) return; - --state->pins_; - if (state->pins_ == 0 && state->release_pending_) - ReleaseStagingBuffer(store_, *state); + engine_.UnpinBlock(store_, *state, location_.block_id()); } void StorageEngine::Impl::LoadValueAwaitable::PrepareLocal() { @@ -1437,13 +1526,13 @@ void StorageEngine::Impl::LoadValueAwaitable::PrepareLocal() { // Resolve staging memory only after acquiring the buffer: waiting can let // flush/defrag retire it. The staged copy itself never suspends, so only // disk I/O needs a physical pin. - auto* state = engine_.FindBlockState(store_, location_.block_id()); - if (!state || !state->allocated_ || state->freeing_ || - state->allocation_epoch_ != location_.allocation_epoch()) { + auto* state = remotely_pinned_ ? nullptr : engine_.FindBlockState(store_, location_.block_id()); + if (!remotely_pinned_ && (!state || !state->allocated_ || state->freeing_ || + state->allocation_epoch_ != location_.allocation_epoch())) { ready_.emplace(absl::AbortedError("stale index block epoch")); return; } - if (location_.in_memory() && state->in_memory_) [[unlikely]] { + if (!remotely_pinned_ && location_.in_memory() && state->in_memory_) [[unlikely]] { auto buffer = engine_.StagingBufferFor(store_, *state); if (!buffer.data_ || location_.record_offset() + location_.total_disk_bytes() > @@ -1468,7 +1557,7 @@ void StorageEngine::Impl::LoadValueAwaitable::PrepareLocal() { "record exceeds registered read buffer capacity")); return; } - ++state->pins_; + if (state) state->PinLocal(); pinned_ = state; if (trace_) trace_->disk_read_ = true; } diff --git a/src/storage/engine/replica_collection.cpp b/src/storage/engine/replica_collection.cpp index 6cc3b70dd..4708db70b 100644 --- a/src/storage/engine/replica_collection.cpp +++ b/src/storage/engine/replica_collection.cpp @@ -146,7 +146,8 @@ Task StorageEngine::Impl::BeginReplicaCollection( state->decoder_.emplace(std::move(*decoder)); const auto digest = ComputeDigest(stage.key_); state->key_hold_ = co_await tx::CurrentTxShard().AcquireKey( - stage.db_id_, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + stage.db_id_, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(stage.key_)); co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); auto current = co_await FindVerifiedEntry( diff --git a/src/storage/engine/replication.cpp b/src/storage/engine/replication.cpp index 0a7cb961a..296abdd77 100644 --- a/src/storage/engine/replication.cpp +++ b/src/storage/engine/replication.cpp @@ -70,7 +70,7 @@ Task StorageEngine::Impl::PinFullSyncExtents( state->allocation_epoch_ != ref.allocation_epoch_) { co_return absl::AbortedError("stale full-sync value extent"); } - ++state->pins_; + state->PinLocal(); co_return absl::OkStatus(); }; absl::Status status; @@ -104,8 +104,8 @@ Task StorageEngine::Impl::ReleaseFullSyncExtents( BlockState* state = FindBlockState(extent_store, ref.block_id_); if (state != nullptr && state->allocated_ && state->allocation_epoch_ == ref.allocation_epoch_ && - state->pins_ != 0) { - --state->pins_; + state->pin_count() != 0) { + UnpinBlock(extent_store, *state, ref.block_id_); } co_return absl::OkStatus(); }; @@ -177,7 +177,8 @@ Task StorageEngine::Impl::ReadSnapshotRecord( { const Digest digest = ComputeDigest(*key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(*key)); const std::string& coverage_key = *key; auto capture = partition.fullsync_subscribers_.find(session_id); const auto active_session = store.fullsync_sessions_.find(session_id); @@ -346,7 +347,8 @@ StorageEngine::Impl::ReadFullSyncOverrideRecord( std::uint64_t session_id, const SnapshotRecord& requested) { const Digest digest = ComputeDigest(requested.key_); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - requested.db_id_, tx::FingerprintOf(digest), tx::LockMode::kShared); + requested.db_id_, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(requested.key_)); auto& index = partition.indexes_[requested.db_id_]; auto resolved = co_await FindVerifiedEntry(store, index, digest, requested.key_); @@ -1381,6 +1383,7 @@ Task> StorageEngine::Impl::ResetReplicaPartition( std::span source_db_epochs, std::uint64_t persisted_replication_epoch, bool replica_lock_held) { WorkerStore& store = CurrentStore(); + WorkerStore::PopulationReadFence population_fence(store); std::unique_ptr replica_unlock; if (!replica_lock_held) { co_await store.replica_apply_mutex_.Lock(); @@ -1457,6 +1460,8 @@ Task> StorageEngine::Impl::ResetReplicaPartition( } tomb_raider_index_generation_.fetch_add(1, std::memory_order_release); partition.replication_epoch_ = next_epoch; + remote_replication_epochs_[partition_id].store(next_epoch, + std::memory_order_release); partition.mutation_sequence_ = 0; for (auto& [session_id, capture] : partition.fullsync_subscribers_) { ClearFullSyncCapture(store, session_id, capture); @@ -1491,6 +1496,7 @@ StorageEngine::Impl::ResetReplicaPartitions( "replica reset batch is empty"); } WorkerStore& store = CurrentStore(); + WorkerStore::PopulationReadFence population_fence(store); co_await store.replica_apply_mutex_.Lock(); UnlockGuard replica_unlock(&store.replica_apply_mutex_, store.worker_); std::array seen{}; @@ -1571,6 +1577,8 @@ StorageEngine::Impl::ResetReplicaPartitions( sync->local_db_epochs_ = local_db_epochs; for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; + auto index_write = co_await AwaitAllIndexReaders( + *store.worker_, partition.indexes_[db_id]); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; @@ -1585,6 +1593,8 @@ StorageEngine::Impl::ResetReplicaPartitions( } partition.replica_candidate_epoch_ = epoch_updates[index].second; partition.replication_epoch_ = epoch_updates[index].second; + remote_replication_epochs_[reset.partition_id_].store( + partition.replication_epoch_, std::memory_order_release); partition.mutation_sequence_ = 0; partition.replica_value_stage_.reset(); partition.replica_sync_ = std::move(sync); @@ -1645,6 +1655,7 @@ Task StorageEngine::Impl::ResetPartitionsDetachLocal( std::span partition_ids) { if (partition_ids.empty()) co_return absl::OkStatus(); WorkerStore& store = CurrentStore(); + WorkerStore::PopulationReadFence population_fence(store); co_await store.replica_apply_mutex_.Lock(); UnlockGuard replica_unlock(&store.replica_apply_mutex_, store.worker_); @@ -1678,6 +1689,8 @@ Task StorageEngine::Impl::ResetPartitionsDetachLocal( auto& partition = PartitionFor(store, partition_ids[i]); for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; + auto index_write = co_await AwaitAllIndexReaders( + *store.worker_, partition.indexes_[db_id]); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; @@ -1691,6 +1704,8 @@ Task StorageEngine::Impl::ResetPartitionsDetachLocal( } partition.replica_candidate_epoch_ = epoch_updates[i].second; partition.replication_epoch_ = epoch_updates[i].second; + remote_replication_epochs_[partition_ids[i]].store( + partition.replication_epoch_, std::memory_order_release); partition.mutation_sequence_ = 0; partition.replica_value_stage_.reset(); } @@ -1798,7 +1813,8 @@ Task> StorageEngine::Impl::ReplicaCommandNeedsApply( } const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kShared); + db_id, tx::FingerprintOf(digest), tx::LockMode::kShared, + lavik::storage::RedisSlot(key)); co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); auto& partition = PartitionFor(store, partition_id); @@ -2031,7 +2047,8 @@ Task StorageEngine::Impl::ApplyReplicaRecordsLocked( const SnapshotRecord& applied = *effective; const Digest digest = ComputeDigest(applied.key_); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - applied.db_id_, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + applied.db_id_, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(applied.key_)); co_await store.store_state_mutex_.Lock(); UnlockGuard write_unlock(&store.store_state_mutex_, store.worker_); sync = partition.replica_sync_.get(); @@ -2384,6 +2401,7 @@ Task StorageEngine::Impl::AbortReplicaRoot( for (unsigned target = 0; target < worker_count_; ++target) { auto discard = [this, target, session_id]() -> Task { WorkerStore& store = *stores_[target]; + WorkerStore::PopulationReadFence population_fence(store); { co_await store.replica_apply_mutex_.Lock(); UnlockGuard replica_unlock(&store.replica_apply_mutex_, store.worker_); @@ -2397,6 +2415,8 @@ Task StorageEngine::Impl::AbortReplicaRoot( for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; + auto index_write = co_await AwaitAllIndexReaders( + *store.worker_, partition.indexes_[db_id]); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; diff --git a/src/storage/engine/sorted_set_api.cpp b/src/storage/engine/sorted_set_api.cpp index 67e861a1e..bd8f2d15e 100644 --- a/src/storage/engine/sorted_set_api.cpp +++ b/src/storage/engine/sorted_set_api.cpp @@ -48,7 +48,8 @@ Task> StorageEngine::Impl::ExecuteSortedSet( operation.kind_ != SortedSetOperationKind::kRemoveRange; auto key_lock = co_await tx::CurrentTxShard().AcquireKey( db_id, tx::FingerprintOf(digest), - read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive); + read_only ? tx::LockMode::kShared : tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await ExecuteSortedSetLocked(db_id, key, digest, operation, nullptr, replication, mutation_precondition); diff --git a/src/storage/engine/tomb_raider.cpp b/src/storage/engine/tomb_raider.cpp index e930f34d0..8fb318ea3 100644 --- a/src/storage/engine/tomb_raider.cpp +++ b/src/storage/engine/tomb_raider.cpp @@ -441,6 +441,8 @@ Task StorageEngine::Impl::TombMarkLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) { co_return absl::OkStatus(); // forfeit the round } + auto index_write = co_await AwaitAllIndexReaders(*store.worker_, index); + if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); cursor = index.Scan(cursor, [](RecordIndex::Entry& entry) { if (entry.value_.kind() == RecordKind::kTombstone || (entry.value_.kind() == RecordKind::kValue && @@ -448,6 +450,7 @@ Task StorageEngine::Impl::TombMarkLocal(WorkerStore& store, entry.value_.set_unclaimed(true); } }); + index_write.Reset(); if (++steps % 256 == 0) { co_await bycorf::Yield(*store.worker_); } @@ -466,6 +469,10 @@ Task StorageEngine::Impl::TombClaimLocal( } auto& partition = PartitionForKey(store, claim.key_); if (partition.replication_epoch_ == claim.replication_epoch_) { + auto index_write = co_await AwaitIndexValueWrite( + *store.worker_, partition.indexes_[claim.db_id_], + claim.digest_.value_); + if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto resolved = co_await FindVerifiedEntry( store, partition.indexes_[claim.db_id_], claim.digest_, claim.key_); #if LAVIK_FAULTS_ENABLED @@ -871,6 +878,8 @@ Task StorageEngine::Impl::TombReapLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) { co_return absl::OkStatus(); } + auto index_write = co_await AwaitAllIndexReaders(*store.worker_, index); + if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); cursor = index.Scan(cursor, [&](RecordIndex::Entry& entry) { if (!entry.value_.unclaimed()) { return; @@ -897,6 +906,7 @@ Task StorageEngine::Impl::TombReapLocal(WorkerStore& store, }); } }); + index_write.Reset(); if (++steps % 256 == 0) { co_await bycorf::Yield(*store.worker_); } @@ -947,12 +957,16 @@ Task StorageEngine::Impl::TombReapLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( candidate.db_id_, tx::FingerprintOf(candidate.digest_), - tx::LockMode::kExclusive); + tx::LockMode::kExclusive, lavik::storage::RedisSlot(candidate.key_)); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto& partition = PartitionForKey(store, candidate.key_); + auto index_write = co_await AwaitIndexWrite( + *store.worker_, partition.indexes_[candidate.db_id_], + candidate.digest_); + if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto resolved = co_await FindVerifiedEntry(store, partition.indexes_[candidate.db_id_], candidate.digest_, candidate.key_); diff --git a/src/storage/engine/tx_cleaner.cpp b/src/storage/engine/tx_cleaner.cpp index 08c22717d..8e1934071 100644 --- a/src/storage/engine/tx_cleaner.cpp +++ b/src/storage/engine/tx_cleaner.cpp @@ -511,11 +511,22 @@ Task StorageEngine::Impl::PromoteTxBlockLocal( IsActiveBlock(store, block.block_id_) || active_transaction || source->in_memory_ || source->flush_queued_ || source->flush_in_progress_ || source->defragging_ || source->freeing_ || - source->pins_ != 0) { + source->pin_count() != 0) { store.store_state_mutex_.Unlock(*store.worker_); co_return absl::FailedPreconditionError( "transaction block changed before promotion"); } + source->CloseRemotePins(); + if (source->pin_count() != 0) { + source->OpenRemotePins(); + store.store_state_mutex_.Unlock(*store.worker_); + co_return absl::FailedPreconditionError( + "transaction block has remote readers"); + } + struct RemoteAdmission { + BlockState& state_; + ~RemoteAdmission() { state_.OpenRemotePins(); } + } remote_admission{*source}; const bool has_live_records = tx_block->second.live_tagged_bytes_ != 0; source->defragging_ = true; const auto [file_id, offset] = FileOffset(block.block_id_); @@ -573,7 +584,7 @@ Task StorageEngine::Impl::RetireTxBlockLocal( // Reads and undo dependencies can still acquire references after // inspection. - if (state->pins_ != 0 || found->second.live_tagged_bytes_ != 0 || + if (state->pin_count() != 0 || found->second.live_tagged_bytes_ != 0 || found->second.dependency_pins_ != 0) co_return absl::FailedPreconditionError( "transaction block is still referenced"); diff --git a/src/storage/engine/write.cpp b/src/storage/engine/write.cpp index 15f76a0e3..c7d447a0a 100644 --- a/src/storage/engine/write.cpp +++ b/src/storage/engine/write.cpp @@ -150,7 +150,8 @@ Task> StorageEngine::Impl::SetWithLockState( // Transaction callers already own this guard and must not reacquire it. auto key_lock = acquire_key_lock ? co_await tx::CurrentTxShard().AcquireKey( db_id, tx::FingerprintOf(digest), - tx::LockMode::kExclusive) + tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)) : tx::TxShard::Guard{}; if (acquire_key_lock && trace != nullptr) { trace->key_lock_acquired_ns_ = SetTraceNowNanos(); @@ -255,7 +256,8 @@ Task> StorageEngine::Impl::UpdateExpiration( assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await UpdateExpirationLocked(db_id, key, digest, expire_at_ms, condition, nullptr, replication, mutation_precondition); @@ -357,7 +359,8 @@ Task> StorageEngine::Impl::Delete( assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await DeleteLocked(db_id, key, digest, nullptr, replication, mutation_precondition); } @@ -422,7 +425,8 @@ Task> StorageEngine::Impl::RestoreRawValue( assert(db_id < kLogicalDatabaseCount); const Digest digest = ComputeDigest(key); auto key_lock = co_await tx::CurrentTxShard().AcquireKey( - db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive); + db_id, tx::FingerprintOf(digest), tx::LockMode::kExclusive, + lavik::storage::RedisSlot(key)); co_return co_await RestoreRawValueLocked(db_id, key, digest, value, replace, nullptr, replication, mutation_precondition); @@ -1018,6 +1022,12 @@ Task StorageEngine::Impl::RollbackTxLocal( WorkerStore& store = CurrentStore(); co_await store.store_state_mutex_.Lock(); UnlockGuard unlock(&store.store_state_mutex_, store.worker_); + std::vector index_writes; + index_writes.reserve(store.partition_indexes_.size()); + for (auto& index : store.partition_indexes_) + index_writes.push_back(index.CloseAllReaders()); + for (auto& guard : index_writes) + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); auto found = store.tx_undo_.find(txid); if (found == store.tx_undo_.end()) { co_return absl::OkStatus(); @@ -1731,7 +1741,7 @@ StorageEngine::Impl::WriteExtentValueLocked( // and explicit pin keep allocation identity stable while this coroutine // exclusively owns the buffer/cursor. Only private bytes and device IO // are touched outside store state; body-before-header durability is intact. - ++state.pins_; + state.PinLocal(); auto write_extent = [&]() -> Task { LAVIK_FAULT_INJECT(if (extent_index == 0) { const auto paused = co_await PauseGroupedWriteForTest( @@ -1858,8 +1868,8 @@ StorageEngine::Impl::WriteExtentValueLocked( // allocate the IO coroutine frame. No destructor attempts an async lock. co_await store.store_state_mutex_.Lock(); assert(state.allocation_epoch_ == reserved->allocation_epoch_ && - state.pins_ != 0); - --state.pins_; + state.pin_count() != 0); + UnpinBlock(store, state, reserved->block_id_); release_buffer(); if (exception) { reclaim_allocated(); @@ -2920,11 +2930,18 @@ Task StorageEngine::Impl::WriteRecordLocked( // can replace this key, or FLUSHDB/replica reset can replace its index, // during that gap. Capture and validate the current physical record only // after the append stream is locked again and an active block is available. + RecordIndex::WriteGuard index_write; + if (index_ptr != nullptr) { + index_write = + co_await AwaitIndexWrite(*store.worker_, *index_ptr, request.digest_); + } RecordIndex::Entry* previous_entry = nullptr; if (index_ptr != nullptr) { previous_entry = index_ptr->Find(request.digest_, request.key_); if (previous_entry != nullptr && !previous_entry->key_complete()) [[unlikely]] { + index_write.Reset(); + index_write = co_await AwaitAllIndexReaders(*store.worker_, *index_ptr); auto resolved = co_await FindVerifiedEntry(store, *index_ptr, request.digest_, request.key_); if (!resolved.ok()) { @@ -3044,6 +3061,19 @@ Task StorageEngine::Impl::WriteRecordLocked( active_stream()->committed_bytes_ + total_disk_bytes > append_limit) { goto acquire_active_stream; } + // Earlier captures may have yielded and changed shared overflow capacity. + // If admission must widen, close it now and retry from stable identities + // after draining; no bytes or entry mutations have been published yet. + RecordIndex::WriteGuard structural_index_write; + if (index_ptr != nullptr && previous_entry == nullptr && + index_ptr->NeedsStructuralWrite(request.digest_)) { + structural_index_write = index_ptr->CloseAllReaders(); + if (!structural_index_write.ready()) { + while (!structural_index_write.ready()) + co_await tx::CurrentTxShard().WaitForReaders(); + goto acquire_active_stream; + } + } const bool has_index_extra = request.expire_at_ms_ != 0; const bool needs_index_allocation = index_ptr != nullptr && (previous_entry == nullptr || diff --git a/src/tx/transaction.cpp b/src/tx/transaction.cpp index 95b40326d..6866ff9e4 100644 --- a/src/tx/transaction.cpp +++ b/src/tx/transaction.cpp @@ -27,7 +27,7 @@ using bycorf::Task; void Transaction::AddKey(unsigned owner, std::uint8_t db, const storage::Digest& digest, std::uint32_t arg_index, - LockMode mode) { + LockMode mode, std::uint16_t slot) { assert(shards_.empty() && "AddKey after Seal"); keys_.push_back(TxKey{ .digest_ = digest, @@ -35,6 +35,7 @@ void Transaction::AddKey(unsigned owner, std::uint8_t db, .arg_index_ = arg_index, .mode_ = mode, .db_ = db, + .slot_ = slot, }); owners_.push_back(static_cast(owner)); } @@ -75,12 +76,14 @@ void Transaction::Seal() { if (key.mode_ == LockMode::kExclusive) { lock_refs_[j].mode_ = LockMode::kExclusive; } + if (lock_refs_[j].slot_ != key.slot_) + lock_refs_[j].slot_ = storage::kLogicalStorageShards; merged = true; break; } } if (!merged) { - lock_refs_.push_back(KeyRef{key.fp_, key.mode_, key.db_}); + lock_refs_.push_back(KeyRef{key.fp_, key.mode_, key.db_, key.slot_}); } } sd.lock_count_ = diff --git a/src/tx/tx_shard.cpp b/src/tx/tx_shard.cpp index c380b16da..ef6491bb3 100644 --- a/src/tx/tx_shard.cpp +++ b/src/tx/tx_shard.cpp @@ -127,6 +127,7 @@ void TxShard::Poll() { // point after resumption). committed_txid_ = std::max(committed_txid_, head->txid_); AcquireHolds(head->keys_); + head->holds_acquired_ = true; ++queued_runs_; // Remove before resuming: once resumed, the waiter (living in the // suspended coroutine's frame) is no longer referenced by the queue. diff --git a/tests/multikey_e2e_test.cpp b/tests/multikey_e2e_test.cpp index b1b8a1631..0f725795c 100644 --- a/tests/multikey_e2e_test.cpp +++ b/tests/multikey_e2e_test.cpp @@ -885,7 +885,7 @@ int main(int argc, char** argv) { } // Force one logical partition owned by each of the four server workers to - // grow beyond one bucket. Its stateless hash-table cursor uses low + // exercise multiple buckets. Its stateless hash-table cursor uses low // bucket-index bits, which the global SCAN cursor must preserve while // crossing worker ownership boundaries between client calls. { @@ -912,7 +912,7 @@ int main(int argc, char** argv) { std::string collected; std::string cursor = "0"; std::array saw_local_cursor{}; - std::size_t calls = 0; + const auto scan_deadline = std::chrono::steady_clock::now() + 10s; do { const std::string reply = client.Command( {"SCAN", cursor, "MATCH", "scan-packed:*", "COUNT", "1"}); @@ -935,7 +935,10 @@ int main(int argc, char** argv) { saw_local_cursor[partition_id % saw_local_cursor.size()] = true; } collected += reply.substr(cursor_end); - if (++calls >= 1000) { + // COUNT bounds visited buckets, not matching keys. Sparse indexes + // retain the reader-stripe bucket floor, so a fixed request-count + // bound would reject a valid terminating scan of the whole DB. + if (std::chrono::steady_clock::now() >= scan_deadline) { Fail("packed-cursor SCAN did not terminate"); } } while (cursor != "0"); diff --git a/tests/record_location_test.cpp b/tests/record_location_test.cpp index 98d87c1ba..3f1008e87 100644 --- a/tests/record_location_test.cpp +++ b/tests/record_location_test.cpp @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -92,6 +93,189 @@ TEST(RecordIndexTest, ReadStatesAreClosedAndCacheLineIsolated) { } } +// Production only enables primary RecordIndex readers. This compact test map +// exercises the same protocol with directly inspectable payloads. +using ConcurrentIndex = + ScanHashMap, + 32>; + +std::shared_ptr ConcurrentArena() { + auto arena = std::make_shared(63); + EXPECT_TRUE(arena->PreallocateDirectory()); + return arena; +} + +TEST(IndexReadersTest, LastReaderNotifiesOnceAndIntentSurvivesResizeScope) { + ConcurrentIndex map(ConcurrentArena()); + ASSERT_NE(map.InsertNew(Digest{0}, "key", 1, false), nullptr); + unsigned notifications = 0; + map.SetReaderNotifier(¬ifications, [](void* ctx) noexcept { + ++*static_cast(ctx); + }); + map.EnableConcurrentReads(); + auto first = map.TryRead(Digest{0}); + auto last = map.TryRead(Digest{0}); + ASSERT_TRUE(first); + ASSERT_TRUE(last); + map.CloseReadIntent(0); + { + auto structural = map.CloseAllReaders(); + EXPECT_FALSE(structural.ready()); + first.Reset(); + EXPECT_EQ(notifications, 0); + last.Reset(); + EXPECT_EQ(notifications, 1); + EXPECT_TRUE(structural.ready()); + } + EXPECT_FALSE(map.TryRead(Digest{0})); + map.ReleaseReadIntent(0); + EXPECT_TRUE(map.TryRead(Digest{0})); + EXPECT_EQ(notifications, 1); +} + +TEST(IndexReadersTest, + StripeClosureAndNestedGlobalClosureDrainExistingReaders) { + ConcurrentIndex map(ConcurrentArena()); + ASSERT_NE(map.InsertNew(Digest{0}, "external", 1, false), nullptr); + ASSERT_NE(map.InsertNew(Digest{1}, "external", 2, false), nullptr); + EXPECT_FALSE(map.TryRead(Digest{0})); + map.EnableConcurrentReads(); + auto read = map.TryRead(Digest{0}); + ASSERT_TRUE(read); + auto writer = map.CloseForValueUpdate(0); + EXPECT_FALSE(writer.ready()); + EXPECT_FALSE(map.TryRead(Digest{0})); + EXPECT_TRUE(map.TryRead(Digest{1})); + auto all = map.CloseAllReaders(); + EXPECT_FALSE(all.ready()); + EXPECT_FALSE(map.TryRead(Digest{1})); + auto moved = std::move(read); + EXPECT_FALSE(read); + moved.Reset(); + ASSERT_TRUE(writer.ready()); + ASSERT_TRUE(all.ready()); + all.Reset(); + EXPECT_FALSE(map.TryRead(Digest{0})); + EXPECT_TRUE(map.TryRead(Digest{1})); + map.InsertOrAssign(Digest{0}, "external", 3, false); + writer.Reset(); + auto verified = map.TryRead(Digest{0}); + ASSERT_TRUE(verified); + EXPECT_EQ(std::as_const(map).Find(Digest{0}, "external")->value_, 3); +} + +TEST(IndexReadersTest, ShrinkWaitsForReadersAndOwnerFindResumesIt) { + ConcurrentIndex map(ConcurrentArena()); + ASSERT_TRUE(map.PreallocateForExpectedSize(400)); // 64 buckets + for (unsigned i = 0; i < 4; ++i) + ASSERT_NE(map.InsertNew(Digest{i}, "external", i, false), nullptr); + map.EnableConcurrentReads(); + auto read = map.TryRead(Digest{0}); + ASSERT_TRUE(read); + const auto* pinned = std::as_const(map).Find(Digest{0}, "external"); + ASSERT_NE(pinned, nullptr); + // Erasing another stripe is allowed; shrinking the shared table must wait. + ASSERT_TRUE(map.Erase(Digest{1}, "external")); + EXPECT_FALSE(map.rehashing()); + EXPECT_EQ(map.allocated_bucket_count(), 64); + EXPECT_FALSE(map.Maintain()); + EXPECT_EQ(std::as_const(map).Find(Digest{0}, "external"), pinned); + EXPECT_FALSE(map.TryRead(Digest{2})); + read.Reset(); + EXPECT_NE(map.Find(Digest{2}, "external"), nullptr); + EXPECT_TRUE(map.rehashing()); + EXPECT_FALSE(map.TryRead(Digest{0})); + // A still-active ordinary writer must survive the end of global rehash. + auto writer = map.CloseForValueUpdate(0); + for (unsigned step = 0; step < 128 && map.Maintain(); ++step) { + } + ASSERT_FALSE(map.rehashing()); + EXPECT_EQ(map.allocated_bucket_count(), 32); + EXPECT_FALSE(map.TryRead(Digest{0})); + EXPECT_TRUE(map.TryRead(Digest{2})); + writer.Reset(); + EXPECT_TRUE(map.TryRead(Digest{0})); +} + +TEST(IndexReadersTest, SharedOverflowGrowthAndDetachRequireAllReadersToDrain) { + ConcurrentIndex map(ConcurrentArena()); + // Twelve entries fill bucket zero without triggering table growth. + for (unsigned i = 0; i < 12; ++i) + ASSERT_NE(map.InsertNew(Digest{i * 32}, "external", i, false), nullptr); + ASSERT_NE(map.InsertNew(Digest{1}, "external", 1, false), nullptr); + map.EnableConcurrentReads(); + auto reader = map.TryRead(Digest{1}); + ASSERT_TRUE(reader); + auto writer = map.CloseForWrite(Digest{12 * 32}); + EXPECT_FALSE(writer.ready()); + EXPECT_FALSE(map.TryRead(Digest{2})); + reader.Reset(); + ASSERT_TRUE(writer.ready()); + ASSERT_NE(map.InsertNew(Digest{12 * 32}, "external", 12, false), nullptr); + writer.Reset(); + reader = map.TryRead(Digest{1}); + ASSERT_TRUE(reader); + auto detach = map.CloseAllReaders(); + EXPECT_FALSE(detach.ready()); + reader.Reset(); + ASSERT_TRUE(detach.ready()); + auto old = map.Detach(); + detach.Reset(); + EXPECT_FALSE(map.TryRead(Digest{1})); // empty population + EXPECT_FALSE(old.TryRead(Digest{1})); // detached maps are private + ASSERT_NE(map.InsertNew(Digest{1}, "external", 99, false), nullptr); + EXPECT_TRUE(map.TryRead(Digest{1})); + old.Clear(); +} + +TEST(IndexReadersTest, ConcurrentReadersNeverObservePartialEntryReplacement) { + auto arena = ConcurrentArena(); + RecordIndex map(arena); + const auto digest = ComputeDigest("live"); + ASSERT_NE(map.InsertNew(digest, "live", RecordLocation{}), nullptr); + map.EnableConcurrentReads(); + std::atomic bad{false}; + std::latch ready(2); + auto loop = [&](std::stop_token stop) { + bool announced = false; + while (!stop.stop_requested()) { + auto read = map.TryRead(digest); + if (!read) continue; + const auto* entry = std::as_const(map).Find(digest, "live"); + if (entry == nullptr || + entry->value_.mutation_sequence_ != entry->value_.logical_size()) + bad.store(true); + if (!announced) { + ready.count_down(); + announced = true; + } + } + }; + std::jthread first(loop), second(loop); + ready.wait(); + for (unsigned i = 1; i <= 2000; ++i) { + auto write = map.CloseForWrite(digest); + while (!write.ready()) std::this_thread::yield(); + auto* entry = map.Find(digest, "live"); + RecordLocation next; + next.mutation_sequence_ = i; + next.logical_size_ = i; + next.expire_at_ms_ = i % 2 == 0 ? 123 : 0; + RecordIndex::Entry* retired = nullptr; + auto* replacement = map.ReplaceValue(entry, next, digest, &retired); + if (replacement == nullptr) { + bad.store(true); + break; + } + map.DestroyDetached(retired); + } + first.request_stop(); + second.request_stop(); + first.join(); + second.join(); + EXPECT_FALSE(bad.load()); +} + TEST(RecoveryMemoryTest, DividesTemporaryBatchTargetAcrossWorkers) { EXPECT_EQ(RecoveryWorkerBatchTargetBytes(0), kRecoveryProcessBatchTargetBytes); diff --git a/tests/remote_get_e2e.py b/tests/remote_get_e2e.py new file mode 100644 index 000000000..f3855921c --- /dev/null +++ b/tests/remote_get_e2e.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 EloqData Inc. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise connection-worker disk GET with writes, transactions and detach.""" + +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import sys +import tempfile +import time + +import defrag_payload_e2e as support + + +class Client(support.Client): + def response(self): + head = self.stream.readline() + assert head.endswith(b"\r\n"), head + assert not head.startswith(b"-"), head + if head.startswith(b"*"): + return [self.response() for _ in range(int(head[1:-2]))] + if head.startswith(b"$"): + size = int(head[1:-2]) + if size < 0: + return None + data = self.stream.read(size) + assert self.stream.read(2) == b"\r\n" + return data + return head[1:-2] + + +def stats(client): + return dict( + line.split(b":", 1) + for line in client.batch([[b"INFO", b"stats"]])[0].splitlines() + if b":" in line + ) + + +def drain(client): + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + values = stats(client) + if all( + values[key] == b"0" + for key in ( + b"storage_dirty_staging_bytes", + b"storage_flushes_pending", + b"storage_tx_commits_pending", + ) + ): + return + time.sleep(0.02) + raise AssertionError("durability did not drain") + + +def check(): + keys = [b"remote-" + str(i).encode() for i in range(6000)] + before = b"a" * 4096 + after = b"b" * 4096 + with support.server(2) as (admin, _): + port = admin.socket.getpeername()[1] + for batch in support.chunks(keys): + assert admin.batch([[b"SET", key, before] for key in batch]) == [ + b"OK" + ] * len(batch) + drain(admin) + for batch in support.chunks(keys): + assert admin.batch([[b"GET", key] for key in batch]) == [before] * len( + batch + ) + # Hash tags select populated indexes while the complete keys differ. + missing = [b"{" + key + b"}:missing" for key in keys[:100]] + assert admin.batch([[b"GET", key] for key in missing]) == [None] * len(missing) + + def read(): + client = Client(port) + try: + for i in range(6000): + assert client.batch([[b"GET", keys[i % 1000]]])[0] in ( + before, + after, + ) + finally: + client.close() + + def write(): + client = Client(port) + try: + # Retire enough of the first sealed blocks to overlap reads + # with the ordinary defrag admission and last-unpin retry. + for i in range(4500): + key = keys[i] + value = after if i % 2 else before + assert client.batch( + [[b"MULTI"], [b"SET", key, value], [b"GET", key], [b"EXEC"]] + ) == [b"OK", b"QUEUED", b"QUEUED", [b"OK", value]] + finally: + client.close() + + with ThreadPoolExecutor(max_workers=5) as pool: + tasks = [pool.submit(read) for _ in range(4)] + [pool.submit(write)] + for task in tasks: + task.result() + + # Clear while requests are arriving. Either generation is valid for + # an overlapping GET, but stale bytes must never reappear after FLUSHDB. + def read_during_flush(): + client = Client(port) + try: + for i in range(1000): + assert client.batch([[b"GET", keys[i]]])[0] in (None, before, after) + finally: + client.close() + + with ThreadPoolExecutor(max_workers=2) as pool: + task = pool.submit(read_during_flush) + assert admin.batch([[b"FLUSHDB"]]) == [b"OK"] + task.result() + for batch in support.chunks(keys[:1000]): + assert admin.batch([[b"GET", key] for key in batch]) == [None] * len(batch) + assert admin.batch([[b"SET", b"survivor", before]]) == [b"OK"] + drain(admin) + with support.server(2) as (admin, _): + assert admin.batch([[b"GET", b"survivor"]]) == [before] + for batch in support.chunks(keys[:1000]): + assert admin.batch([[b"GET", key] for key in batch]) == [None] * len(batch) + + +def main(): + support.BINARY = str(Path(sys.argv[1]).resolve()) + with tempfile.TemporaryDirectory( + prefix="lavik-remote-get-", dir=os.environ.get("LAVIK_TEST_DATA_DIR") + ) as directory: + support.WORKDIR = Path(directory) + with (support.WORKDIR / "data").open("wb") as file: + file.truncate(512 * 1024 * 1024) + check() + print("remote GET, concurrent transactions, FLUSHDB and cold recovery passed") + + +if __name__ == "__main__": + main() diff --git a/tests/replica_abort_reclaim_e2e_test.cpp b/tests/replica_abort_reclaim_e2e_test.cpp index c8a59c8c8..cd2ca37af 100644 --- a/tests/replica_abort_reclaim_e2e_test.cpp +++ b/tests/replica_abort_reclaim_e2e_test.cpp @@ -751,6 +751,14 @@ class ReplicaAbortReclaimService final : public bycorf::Service { session_id, partition, db_id, 0, 1, 1, lavik::storage::kReplicationTransferBytes); if (!batch.ok()) co_return batch.status(); + // A one-bucket scan may be empty: primary indexes retain the reader + // stripe floor even when this partition contains only one key. + while (batch->records_.empty() && batch->cursor_ != 0) { + batch = co_await storage_->SnapshotPartition( + session_id, partition, db_id, batch->cursor_, 1, 1, + lavik::storage::kReplicationTransferBytes); + if (!batch.ok()) co_return batch.status(); + } if (batch->records_.size() != 1 || batch->records_.front().key_ != key || batch->records_.front().source_id_ == 0) { co_return absl::FailedPreconditionError( diff --git a/tests/storage_capacity_test.cpp b/tests/storage_capacity_test.cpp index f2dd4abff..924588225 100644 --- a/tests/storage_capacity_test.cpp +++ b/tests/storage_capacity_test.cpp @@ -40,6 +40,11 @@ class ExpirationAuthorityTestPeer { public: using WorkerCache = StorageEngine::Impl::WorkerStore; + static RecordIndex& IndexFor(StorageEngine& storage, std::string_view key) { + auto* impl = storage.impl_.get(); + return impl->PartitionForKey(impl->CurrentStore(), key).indexes_[0]; + } + static std::shared_ptr CachedGrant(const StorageEngine& storage, WorkerCache& cache) { return storage.impl_->CurrentExpirationAuthority(cache); @@ -269,6 +274,9 @@ class FiniteExpirationAuthorityService final : public bycorf::Service { if (!result_.ok()) co_return Finish(); expiration_paused_ = true; + result_ = co_await ExerciseIndexReaderDrain(worker); + if (!result_.ok()) co_return Finish(); + result_ = storage_->SetExpirationAuthorityUntil(FarFutureExpirationDeadline()); if (!result_.ok()) co_return Finish(); @@ -313,6 +321,56 @@ class FiniteExpirationAuthorityService final : public bycorf::Service { const absl::Status& result() const noexcept { return result_; } private: + struct ReaderDrainProbe { + std::atomic finished{false}; + bool owner_progress = false; + }; + + static bycorf::Task ReleaseBlockedReader( + bycorf::Worker* worker, lavik::storage::RecordIndex* index, + lavik::storage::Digest digest, std::shared_ptr probe, + lavik::storage::ScanHashMapReadGuard reader) { + while (!probe->finished.load()) { + if ((index->read_states()[digest.value_ & 31].state_.load() & + lavik::storage::ScanHashMapReadState::kClosed) != 0) { + probe->owner_progress = true; + reader.Reset(); + co_return absl::OkStatus(); + } + co_await bycorf::Yield(*worker); + } + co_return absl::OkStatus(); + } + + bycorf::Task ExerciseIndexReaderDrain(bycorf::Worker& worker) { + constexpr std::string_view key = "index-reader-drain"; + auto seeded = co_await storage_->Set(0, key, "before"); + if (!seeded.ok()) co_return seeded.status(); + auto& index = + lavik::storage::ExpirationAuthorityTestPeer::IndexFor(*storage_, key); + const auto digest = lavik::storage::ComputeDigest(key); + auto probe = std::make_shared(); + auto reader = index.TryRead(digest); + if (!reader) co_return absl::InternalError("test reader admission failed"); + worker.Spawn(ReleaseBlockedReader(&worker, &index, digest, probe, + std::move(reader))); + auto replaced = co_await storage_->Set(0, key, "after"); + probe->finished.store(true); + if (!replaced.ok()) co_return replaced.status(); + if (!probe->owner_progress) + co_return absl::InternalError( + "index writer did not yield while draining readers"); + auto loaded = co_await storage_->Get(0, key); + if (!loaded.ok()) co_return loaded.status(); + const auto bytes = loaded->value_bytes(); + if (std::string_view(reinterpret_cast(bytes.data()), + bytes.size()) != "after") + co_return absl::InternalError( + "index writer did not publish after reader drain"); + auto removed = co_await storage_->Delete(0, key); + co_return removed.status(); + } + bycorf::Task SeedExpired(std::string_view key) { auto seeded = co_await storage_->Set( 0, key, "value", lavik::storage::SetOptions{.expire_at_ms_ = 1}); diff --git a/tests/tomb_raider_e2e_test.cpp b/tests/tomb_raider_e2e_test.cpp index 7f8b6c369..c69e92f15 100644 --- a/tests/tomb_raider_e2e_test.cpp +++ b/tests/tomb_raider_e2e_test.cpp @@ -160,6 +160,16 @@ class TombRaiderTestPeer { status = co_await SealForSweep(storage); if (!status.ok()) co_return status; auto* source = impl->FindBlockState(store, block); + // Flush completion schedules retired-record accounting on the owner; + // it does not join that detached task. Wait for the actual fixture + // invariant, especially for the multi-batch prefix-claim scenario. + const auto accounting_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (source != nullptr && source->live_bytes_ != 0 && + std::chrono::steady_clock::now() < accounting_deadline) { + co_await bycorf::Yield(worker); + source = impl->FindBlockState(store, block); + } if (source == nullptr || source->live_bytes_ != 0) co_return absl::FailedPreconditionError( "old records block is not empty"); @@ -182,15 +192,18 @@ class TombRaiderTestPeer { std::optional> retirement; bycorf::AsyncNotification retired; struct ReaderPin { + StorageEngine::Impl* impl_; + StorageEngine::Impl::WorkerStore* store_; + std::uint64_t block_; BlockState* state_ = nullptr; void Reset() { if (state_ != nullptr) { - --state_->pins_; + impl_->UnpinBlock(*store_, *state_, block_); state_ = nullptr; } } ~ReaderPin() { Reset(); } - } pin; + } pin{impl, &store, block}; const auto begin_retirement = [&]() -> absl::Status { if (retirement.has_value()) return absl::OkStatus(); auto task = impl->CleanBlockLocked(store, block); @@ -220,7 +233,7 @@ class TombRaiderTestPeer { co_await store.store_state_mutex_.Lock(); status = begin_retirement(); if (status.ok() && !retirement->done()) { - ++source->pins_; + source->PinLocal(); pin.state_ = source; } else if (status.ok()) { status = absl::FailedPreconditionError( @@ -234,7 +247,7 @@ class TombRaiderTestPeer { status = co_await bycorf::SleepFor(worker, std::chrono::milliseconds(1)); } - if (status.ok() && (!source->freeing_ || source->pins_ != 1 || + if (status.ok() && (!source->freeing_ || source->pin_count() != 1 || store.pending_record_block_retirements_ != 0)) status = absl::FailedPreconditionError( "reader pin did not hold the freeing window"); diff --git a/tests/tx_lock_test.cpp b/tests/tx_lock_test.cpp index 2ec0369d5..a4ba77a54 100644 --- a/tests/tx_lock_test.cpp +++ b/tests/tx_lock_test.cpp @@ -145,6 +145,59 @@ TestTask RunImmediate(TxShard& shard, LockFp fp, LockMode mode, order.push_back(std::string(name) + ":done"); } +TEST(TxLockTest, ReaderDrainResumesQueueAndRetainsAllIntentClosures) { + struct Admission { + std::array intents{}; + std::array readers{}; + } state; + TxShard shard; + std::atomic counter{1}; + shard.BindTxidCounter(&counter); + shard.BindReadAdmission( + {&state, + [](void* ctx, const KeyRef& key) { + ++static_cast(ctx)->intents[key.fp_ & 31]; + }, + [](void* ctx, const KeyRef& key) { + --static_cast(ctx)->intents[key.fp_ & 31]; + }, + [](void* ctx, const KeyRef& key) { + return static_cast(ctx)->readers[key.fp_ & 31] == 0; + }}); + state.readers[0] = 2; + std::vector order; + ManualEvent io; + RunWithIo(shard, 0, LockMode::kExclusive, io, order, "writer"); + RunImmediate(shard, 0, LockMode::kShared, order, "reader"); + EXPECT_TRUE(order.empty()); + EXPECT_EQ(state.intents[0], 2); + RunImmediate(shard, 1, LockMode::kExclusive, order, "unrelated"); + EXPECT_EQ(order.size(), 2); + --state.readers[0]; + shard.ReadersDrained(); + EXPECT_EQ(order.size(), 2); + --state.readers[0]; + shard.ReadersDrained(); + ASSERT_EQ(order.size(), 3); + EXPECT_EQ(order.back(), "writer:start"); + EXPECT_EQ(state.intents[0], 2); + io.Fire(); + EXPECT_EQ(order.back(), "reader:done"); + EXPECT_EQ(state.intents[0], 0); + EXPECT_TRUE(shard.CanReadOptimistically(0)); + state.readers[0] = 1; + { + auto cancelled = shard.AcquireKey(0, 0, LockMode::kExclusive); + ASSERT_FALSE(cancelled.await_ready()); + cancelled.await_suspend(std::noop_coroutine()); + EXPECT_EQ(state.intents[0], 1); + } + EXPECT_EQ(state.intents[0], 0); + EXPECT_TRUE(shard.CanReadOptimistically(0)); + state.readers[0] = 0; + shard.ReadersDrained(); // No pointer into the destroyed awaiter remains. +} + TEST(TxLockTest, QueuesConflictsWithoutBlockingOtherKeys) { TxShard shard; // unbound: Poll resumes inline std::atomic counter{1}; From 00573f08e70c276f7f2e187d67bb95913d71f515 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 08:55:19 +0000 Subject: [PATCH 05/15] Expand SPDK qpair access independently of home write placement --- docs/architecture/04-storage-and-recovery.md | 26 ++++--- docs/operations/multi-device-storage.md | 14 +++- src/storage/engine/device_affinity.h | 45 ++++++++++++- src/storage/engine/impl.h | 3 +- src/storage/engine/init.cpp | 38 ++++++----- tests/device_affinity_test.cpp | 71 ++++++++++++++++++++ 6 files changed, 169 insertions(+), 28 deletions(-) diff --git a/docs/architecture/04-storage-and-recovery.md b/docs/architecture/04-storage-and-recovery.md index 2b5664cff..9d86e1114 100644 --- a/docs/architecture/04-storage-and-recovery.md +++ b/docs/architecture/04-storage-and-recovery.md @@ -80,9 +80,9 @@ Storage separates three ownership domains: - Each device has one allocator owner, which serializes that device's ready and cold-free pools, scan bitmap, fixed metadata page generations, epoch mirrors, and allocation-epoch counter. io_uring initially assigns this role as - `device_index % worker_count`. SPDK selects it from the controller's qpair - owners, so metadata and allocation I/O never route to a worker that cannot - open the namespace. + `device_index % worker_count`. SPDK selects it from the controller's home + workers, all of which hold a qpair, so metadata and allocation I/O never + route to a worker that cannot open the namespace. Every worker has one ordinary and one transaction append stream. Physical streams are per worker, not per logical partition, so active 8 MiB staging @@ -122,6 +122,11 @@ best-effort subset registration, prioritizing read slots. Each lease retains its slot identity and immutable registration status; unregistered slots use the same aligned memory with plain asynchronous I/O. On SPDK, a worker opens only namespaces whose physical controller assigned it a qpair. +Weighted home-device assignment controls write placement independently of +qpair access. Each controller reserves qpairs for its home workers, then +assigns its remaining capacity to other workers, up to one qpair per worker. +A controller with enough qpairs is therefore directly accessible from every +connection worker. Namespaces on that controller share each worker's qpair. SPDK also requires DMA buffer registration and fails initialization when that registration or controller-qpair coverage is insufficient. @@ -368,10 +373,10 @@ zeroes stale checkpoint bitmap state when no checkpoint is usable. `InitializeWorker` creates the worker's aligned buffer pool, registers the fixed-file table, and opens the paths accessible on that backend. Before worker startup, io_uring assigns weighted home devices for foreground allocation. -SPDK groups namespaces by physical controller, weights controllers by usable -foreground blocks, and distributes available controller qpairs across workers; -startup fails unless every controller and every worker can be covered. Worker -barriers then coordinate one parallel recovery rather than independent +SPDK groups namespaces by physical controller, assigns weighted home devices, +and expands qpair access with the remaining controller capacity; startup fails +unless every controller and every worker can be covered. Worker barriers then +coordinate one parallel recovery rather than independent worker-local boots. Request readiness follows successful completion of the complete recovery and allocator-cleanup sequence; the exact moment a listening socket exists is not the readiness boundary. @@ -818,9 +823,10 @@ The persistent allocation bitmap is the recovery authority, not merely an allocation hint. Device allocators activate ready IDs in batches and preserve eight allocatable blocks per device for defrag. Each worker first balances foreground allocations across its weighted home devices. io_uring can then -fall back to every other configured device; SPDK cannot leave the namespaces of -controllers for which that worker owns qpairs. Foreground allocation may wait -while an active flush, extent reclaim, or runnable defrag can still return +fall back to every other configured device; SPDK allocates only from its home +devices, even when spare qpairs give it read access to other controllers. +Foreground allocation may wait while an active flush, extent reclaim, or +runnable defrag can still return space; it reports exhaustion only after a stable observation with no progress in flight. A queued defrag while defrag is paused is deliberately not counted as runnable progress, so a full-device write reports exhaustion instead of diff --git a/docs/operations/multi-device-storage.md b/docs/operations/multi-device-storage.md index f6bb3060c..f01b51caa 100644 --- a/docs/operations/multi-device-storage.md +++ b/docs/operations/multi-device-storage.md @@ -180,7 +180,19 @@ stream allocates on demand and does not reserve standby capacity. - With more usable devices than workers, devices are greedily grouped to keep aggregate group weights balanced. A worker chooses within its group by the lowest allocated-blocks/weight ratio. -- Allocation falls back to other devices when the preferred device is full. +- On io_uring, allocation falls back to other devices when the preferred device + is full. SPDK allocation stays within the worker's home devices. + +SPDK groups namespaces by physical controller. Home assignments follow the +foreground capacity weights, bounded by each controller's available qpair +count. Qpair access is broader than write affinity: after covering home +workers, each controller assigns spare qpairs to other workers, preferring +those with fewer qpairs, up to one per worker. For six controllers with 16 +available qpairs each and 16 workers, every worker opens all six controllers +(96 qpairs total), while retaining its weighted home write placement. Multiple +namespaces on one controller share a worker's qpair. Startup logs show +`home-workers` and `qpair-workers` separately. If a controller has fewer qpairs +than workers, connection-worker GET falls back when its worker lacks access. The standby request starts immediately after an ordinary active block is installed, and starts again after rollover consumes that standby. Appends do diff --git a/src/storage/engine/device_affinity.h b/src/storage/engine/device_affinity.h index 2dc25d954..07a9db0e9 100644 --- a/src/storage/engine/device_affinity.h +++ b/src/storage/engine/device_affinity.h @@ -35,12 +35,18 @@ struct ControllerAffinityInput { }; struct ControllerAffinityPlan { + // Home assignments control write placement independently of I/O access. std::vector controller_quotas_; std::vector> worker_controllers_; + // Sorted worker IDs with a qpair, including every home worker. Namespaces + // on the same controller share this assignment and one qpair per worker. + std::vector> controller_qpair_workers_; }; // Inputs must be in stable controller-id order. The returned indexes refer to -// that order, making the same topology deterministic across restarts. +// that order, making the same topology deterministic across restarts. Preserve +// weighted home placement, then use spare qpairs to widen direct I/O access up +// to min(controller capacity, worker count), without changing write placement. inline absl::StatusOr PlanControllerAffinity( std::span controllers, unsigned worker_count) { @@ -146,6 +152,43 @@ inline absl::StatusOr PlanControllerAffinity( *lightest += controllers[controller].foreground_weight_; } } + plan.controller_qpair_workers_.resize(controllers.size()); + std::vector worker_qpair_counts(worker_count, 0); + for (unsigned worker = 0; worker < worker_count; ++worker) { + for (const std::size_t controller : plan.worker_controllers_[worker]) { + plan.controller_qpair_workers_[controller].push_back(worker); + ++worker_qpair_counts[worker]; + } + } + for (std::size_t controller = 0; controller < controllers.size(); + ++controller) { + auto& workers = plan.controller_qpair_workers_[controller]; + const unsigned cap = + std::min(controllers[controller].io_qpair_count_, worker_count); + if (workers.size() == cap) continue; + + std::vector candidates; + for (unsigned worker = 0; worker < worker_count; ++worker) { + if (!std::binary_search(workers.begin(), workers.end(), worker)) { + candidates.push_back(worker); + } + } + // Home qpairs are reserved first, so additional access cannot starve a + // writer. Spread limited spare capacity across workers deterministically. + std::sort(candidates.begin(), candidates.end(), + [&worker_qpair_counts](unsigned left, unsigned right) { + if (worker_qpair_counts[left] != worker_qpair_counts[right]) { + return worker_qpair_counts[left] < worker_qpair_counts[right]; + } + return left < right; + }); + const std::size_t extra = cap - workers.size(); + for (std::size_t i = 0; i < extra; ++i) { + workers.push_back(candidates[i]); + ++worker_qpair_counts[candidates[i]]; + } + std::sort(workers.begin(), workers.end()); + } return plan; } diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 747fa1c25..1a516b8a8 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -4646,7 +4646,8 @@ class StorageEngine::Impl { unsigned worker_count_ = 0; std::uint64_t total_data_blocks_ = 0; std::vector devices_; - // Populated only for SPDK: workers owning a qpair for each controller. + // Populated only for SPDK: sorted qpair workers for each device's controller. + // Includes spare-qpair readers beyond the weighted home write assignment. std::vector> device_owners_; // Which worker owns each block, by device and local block id. A record // carries its block's owner in its index entry, but an extent reference has diff --git a/src/storage/engine/init.cpp b/src/storage/engine/init.cpp index 98070f612..3488e7e58 100644 --- a/src/storage/engine/init.cpp +++ b/src/storage/engine/init.cpp @@ -2290,8 +2290,6 @@ absl::Status StorageEngine::Impl::ConfigureWorkerDeviceAffinity() { controller.workers_.push_back(static_cast(worker)); for (const std::size_t device_index : controller.devices_) { stores_[worker]->home_devices_.push_back(device_index); - device_owners_[device_index].push_back( - static_cast(worker)); } }; @@ -2302,13 +2300,16 @@ absl::Status StorageEngine::Impl::ConfigureWorkerDeviceAffinity() { } } - for (std::size_t device_index = 0; device_index < devices_.size(); - ++device_index) { - auto& owners = device_owners_[device_index]; - std::sort(owners.begin(), owners.end()); - assert(!owners.empty()); - device_allocators_[device_index]->owner_ = - owners[devices_[device_index].id_ % owners.size()]; + for (std::size_t i = 0; i < controllers.size(); ++i) { + const auto& home_workers = controllers[i]->workers_; + assert(!home_workers.empty()); + for (const std::size_t device_index : controllers[i]->devices_) { + device_owners_[device_index] = planned->controller_qpair_workers_[i]; + // Extra read access must not move the device's allocator or alter its + // weighted home assignment. Every home worker also owns a qpair. + device_allocators_[device_index]->owner_ = + home_workers[devices_[device_index].id_ % home_workers.size()]; + } } for (auto& store : stores_) { if (store->home_devices_.empty()) { @@ -2317,16 +2318,23 @@ absl::Status StorageEngine::Impl::ConfigureWorkerDeviceAffinity() { } store->home_device_allocations_.assign(store->home_devices_.size(), 0); } - for (const ControllerPlan* controller : controllers) { - std::string workers; + for (std::size_t i = 0; i < controllers.size(); ++i) { + const ControllerPlan* controller = controllers[i]; + std::string home_workers; for (const std::uint16_t worker : controller->workers_) { - if (!workers.empty()) workers += ','; - workers += std::to_string(worker); + if (!home_workers.empty()) home_workers += ','; + home_workers += std::to_string(worker); + } + std::string qpair_workers; + for (const std::uint16_t worker : planned->controller_qpair_workers_[i]) { + if (!qpair_workers.empty()) qpair_workers += ','; + qpair_workers += std::to_string(worker); } spdlog::info( - "SPDK controller={} io-qpairs={} foreground-weight={} owners=[{}]", + "SPDK controller={} io-qpairs={} foreground-weight={} " + "home-workers=[{}] qpair-workers=[{}]", controller->id_, controller->io_queue_count_, controller->weight_, - workers); + home_workers, qpair_workers); } return absl::OkStatus(); } else { diff --git a/tests/device_affinity_test.cpp b/tests/device_affinity_test.cpp index 4ec612484..4fd080d9c 100644 --- a/tests/device_affinity_test.cpp +++ b/tests/device_affinity_test.cpp @@ -76,5 +76,76 @@ TEST(DeviceAffinityTest, AssignsEveryControllerWhenControllersExceedWorkers) { EXPECT_EQ(plan->worker_controllers_[1], (std::vector{1, 2})); } +TEST(DeviceAffinityTest, SixControllersGiveEveryWorkerDirectAccess) { + std::vector controllers; + for (unsigned controller = 0; controller < 6; ++controller) { + controllers.push_back({.id_ = std::to_string(controller), + .foreground_weight_ = 10, + .io_qpair_count_ = 16}); + } + auto plan = PlanControllerAffinity(controllers, 16); + ASSERT_TRUE(plan.ok()) << plan.status(); + // More read access must preserve the existing weighted write distribution. + EXPECT_EQ(plan->controller_quotas_, + (std::vector{3, 3, 3, 3, 2, 2})); + for (const auto& home : plan->worker_controllers_) { + EXPECT_EQ(home.size(), 1); + } + ASSERT_EQ(plan->controller_qpair_workers_.size(), 6); + for (const auto& workers : plan->controller_qpair_workers_) { + ASSERT_EQ(workers.size(), 16); + for (unsigned worker = 0; worker < 16; ++worker) { + EXPECT_EQ(workers[worker], worker); + } + } +} + +TEST(DeviceAffinityTest, SpareQpairsRespectCapacityAndRetainHomeAccess) { + const std::vector controllers{ + {.id_ = "0", .foreground_weight_ = 10, .io_qpair_count_ = 2}, + {.id_ = "1", .foreground_weight_ = 10, .io_qpair_count_ = 3}, + {.id_ = "2", .foreground_weight_ = 10, .io_qpair_count_ = 4}, + }; + auto plan = PlanControllerAffinity(controllers, 6); + ASSERT_TRUE(plan.ok()) << plan.status(); + EXPECT_EQ(plan->controller_quotas_, (std::vector{2, 2, 2})); + std::vector qpair_counts(6, 0); + for (std::size_t controller = 0; controller < controllers.size(); + ++controller) { + const auto& workers = plan->controller_qpair_workers_[controller]; + EXPECT_EQ(workers.size(), controllers[controller].io_qpair_count_); + EXPECT_TRUE(std::is_sorted(workers.begin(), workers.end())); + EXPECT_EQ(std::adjacent_find(workers.begin(), workers.end()), + workers.end()); + for (const auto worker : workers) { + ASSERT_LT(worker, qpair_counts.size()); + ++qpair_counts[worker]; + } + } + for (unsigned worker = 0; worker < 6; ++worker) { + EXPECT_GE(qpair_counts[worker], 1); + EXPECT_LE(qpair_counts[worker], 2); + for (const auto controller : plan->worker_controllers_[worker]) { + const auto& workers = plan->controller_qpair_workers_[controller]; + EXPECT_TRUE(std::binary_search(workers.begin(), workers.end(), worker)); + } + } +} + +TEST(DeviceAffinityTest, MoreControllersThanWorkersCapAccessAtWorkerCount) { + const std::vector controllers{ + {.id_ = "0", .foreground_weight_ = 10, .io_qpair_count_ = 16}, + {.id_ = "1", .foreground_weight_ = 5, .io_qpair_count_ = 16}, + {.id_ = "2", .foreground_weight_ = 1, .io_qpair_count_ = 16}, + }; + auto plan = PlanControllerAffinity(controllers, 2); + ASSERT_TRUE(plan.ok()) << plan.status(); + EXPECT_EQ(plan->worker_controllers_[0], (std::vector{0})); + EXPECT_EQ(plan->worker_controllers_[1], (std::vector{1, 2})); + for (const auto& workers : plan->controller_qpair_workers_) { + EXPECT_EQ(workers, (std::vector{0, 1})); + } +} + } // namespace } // namespace lavik::storage From ba11938b246e585ed1ac4266da2f45d6e3cd880b Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 09:07:07 +0000 Subject: [PATCH 06/15] Reuse drained index admission across nested rehash writes --- include/lavik/storage/scan_hash_map.h | 42 ++++++++++++++------ tests/record_location_test.cpp | 57 +++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index 1b4d9cab9..46a8d37ce 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -890,9 +890,10 @@ struct ScanHashMapReadStates { WriteGuard() = default; WriteGuard(ScanHashMapReadStates* states, std::uint64_t mask, bool all) : states_(states), mask_(mask), all_(all) { - if (all_) ++states_->structural_writers_; - states_->PublishDisabled(); - states_->Close(mask_); + if (all_) + states_->CloseStructural(); + else + states_->Close(mask_); } WriteGuard(const WriteGuard&) = delete; WriteGuard& operator=(const WriteGuard&) = delete; @@ -913,9 +914,10 @@ struct ScanHashMapReadStates { } void Reset() noexcept { if (auto* states = std::exchange(states_, nullptr)) { - if (all_) --states->structural_writers_; - states->Open(mask_); - states->PublishDisabled(); + if (all_) + states->OpenStructural(); + else + states->Open(mask_); } } @@ -981,18 +983,16 @@ struct ScanHashMapReadStates { void BlockResize() noexcept { if (resize_blocked_) return; resize_blocked_ = true; - ++structural_writers_; - PublishDisabled(); - Close(kAll); + CloseStructural(); } void EndResize() noexcept { if (!resize_blocked_) return; resize_blocked_ = false; - --structural_writers_; - Open(kAll); - PublishDisabled(); + OpenStructural(); } bool Drained(std::uint64_t mask = kAll) const noexcept { + if (structural_drained_) return true; + const bool all = mask == kAll; while (mask != 0) { const auto index = std::countr_zero(mask); if ((stripes_[index].state_.load(std::memory_order_acquire) & @@ -1000,10 +1000,25 @@ struct ScanHashMapReadStates { return false; mask &= mask - 1; } + // All stripes remain closed until the last structural scope exits. Once + // drained, owner mutations and rehash steps can reuse that observation. + if (all && structural_writers_ != 0) structural_drained_ = true; return true; } private: + void CloseStructural() noexcept { + if (structural_writers_++ != 0) return; + PublishDisabled(); + Close(kAll); + } + void OpenStructural() noexcept { + assert(structural_writers_ != 0); + if (--structural_writers_ != 0) return; + structural_drained_ = false; + Open(kAll); + PublishDisabled(); + } void PublishDisabled() noexcept { const bool disabled = !enabled_ || !populated_ || structural_writers_ != 0; // Ordinary single-stripe writes must not invalidate the global probe's @@ -1036,6 +1051,9 @@ struct ScanHashMapReadStates { bool populated_ = false; bool resize_blocked_ = false; bool published_disabled_ = true; + // Owner-only. Resize and enclosing mutations share one all-stripe closure; + // finishing the last rehash step cannot reopen an enclosing writer's scope. + mutable bool structural_drained_ = false; unsigned structural_writers_ = 0; }; diff --git a/tests/record_location_test.cpp b/tests/record_location_test.cpp index 3f1008e87..3e102c39f 100644 --- a/tests/record_location_test.cpp +++ b/tests/record_location_test.cpp @@ -164,6 +164,63 @@ TEST(IndexReadersTest, EXPECT_EQ(std::as_const(map).Find(Digest{0}, "external")->value_, 3); } +TEST(IndexReadersTest, NestedStructuralScopesRetainClosureAndRecheckNextDrain) { + ConcurrentIndex map(ConcurrentArena()); + ASSERT_NE(map.InsertNew(Digest{0}, "external", 1, false), nullptr); + map.EnableConcurrentReads(); + auto reader = map.TryRead(Digest{0}); + ASSERT_TRUE(reader); + auto outer = map.CloseAllReaders(); + auto inner = map.CloseAllReaders(); + EXPECT_FALSE(outer.ready()); + EXPECT_FALSE(inner.ready()); + reader.Reset(); + ASSERT_TRUE(outer.ready()); + ASSERT_TRUE(inner.ready()); + outer.Reset(); + // A suspended writer can outlive the scope that originally closed the map. + EXPECT_FALSE(map.TryRead(Digest{0})); + auto moved = std::move(inner); + EXPECT_TRUE(moved.ready()); + moved.Reset(); + reader = map.TryRead(Digest{0}); + ASSERT_TRUE(reader); + auto next = map.CloseAllReaders(); + EXPECT_FALSE(next.ready()); // A previous drain cannot authorize this scope. + reader.Reset(); + EXPECT_TRUE(next.ready()); + next.Reset(); + EXPECT_TRUE(map.TryRead(Digest{0})); +} + +TEST(IndexReadersTest, ExpansionCompletionKeepsEnclosingWriteClosed) { + ConcurrentIndex map(ConcurrentArena()); + for (unsigned i = 0; i < 288; ++i) + ASSERT_NE(map.InsertNew(Digest{i}, "external", i, false), nullptr); + ASSERT_FALSE(map.rehashing()); + map.EnableConcurrentReads(); + { + auto write = map.CloseForWrite(Digest{288}); + ASSERT_TRUE(write.ready()); + ASSERT_NE(map.InsertNew(Digest{288}, "external", 288, false), nullptr); + } + ASSERT_TRUE(map.rehashing()); + for (unsigned step = 0; step < 64 && map.rehashing(); ++step) { + auto write = map.CloseForWrite(Digest{0}); + ASSERT_TRUE(write.ready()); + ASSERT_NE(map.Find(Digest{0}, "external"), nullptr); + // Even Find's final migration must leave every stripe closed until the + // enclosing mutation finishes, not just the stripe of the written key. + EXPECT_FALSE(map.TryRead(Digest{1})); + map.InsertOrAssign(Digest{0}, "external", step, false); + EXPECT_TRUE(write.ready()); + EXPECT_FALSE(map.TryRead(Digest{1})); + } + ASSERT_FALSE(map.rehashing()); + EXPECT_TRUE(map.TryRead(Digest{1})); + EXPECT_EQ(map.allocated_bucket_count(), 64); +} + TEST(IndexReadersTest, ShrinkWaitsForReadersAndOwnerFindResumesIt) { ConcurrentIndex map(ConcurrentArena()); ASSERT_TRUE(map.PreallocateForExpectedSize(400)); // 64 buckets From 7fd3e38f43d0428fb126e9b5a5ee1e73a0e4bd57 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 21:21:04 +0800 Subject: [PATCH 07/15] perf(storage): skip uncontended index reader-drain coroutines --- src/storage/engine/flush.cpp | 9 +++++++-- src/storage/engine/impl.h | 10 ++++++++-- src/storage/engine/write.cpp | 9 +++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index 8fae190be..0c948c4de 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -520,8 +520,13 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { // applies to the replacement entry's own location. RecordIndex& index = PartitionFor(*store, identity.partition_id_) .indexes_[identity.db_id_]; - auto index_write = co_await AwaitIndexValueWrite(*store->worker_, index, - identity.entry_hash_); + auto index_write = index.CloseForValueUpdate(identity.entry_hash_); + if (!index_write.ready()) { + // Keep admission closed across the slow wait. Most flush updates + // have no registered reader and need no waiting coroutine. + index_write = co_await AwaitIndexValueWrite(*store->worker_, index, + identity.entry_hash_); + } // Detach can run while the reader drain yields. Recheck the population // before resolving the saved integer address into this index. if (identity.index_generation_ != diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 1a516b8a8..310d60cf4 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -116,10 +116,16 @@ using ExtentManifest = std::shared_ptr>; inline Task AwaitIndexWrite( [[maybe_unused]] Worker& worker, RecordIndex& index, const Digest& digest) { auto guard = index.CloseForWrite(digest); - while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); + bool waited = false; + while (!guard.ready()) { + waited = true; + co_await tx::CurrentTxShard().WaitForReaders(); + } // Another owner coroutine may have changed capacity during the wait. + // Without suspension CloseForWrite's decision is still current. Avoid + // walking the same bucket chain again on every uncontended write. // No entry pointer has been obtained yet; upgrade before resolving it. - if (index.NeedsStructuralWrite(digest)) { + if (waited && index.NeedsStructuralWrite(digest)) { guard.Reset(); guard = index.CloseAllReaders(); while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); diff --git a/src/storage/engine/write.cpp b/src/storage/engine/write.cpp index c7d447a0a..fcc83219e 100644 --- a/src/storage/engine/write.cpp +++ b/src/storage/engine/write.cpp @@ -2932,8 +2932,13 @@ Task StorageEngine::Impl::WriteRecordLocked( // after the append stream is locked again and an active block is available. RecordIndex::WriteGuard index_write; if (index_ptr != nullptr) { - index_write = - co_await AwaitIndexWrite(*store.worker_, *index_ptr, request.digest_); + index_write = index_ptr->CloseForWrite(request.digest_); + if (!index_write.ready()) { + // Retain the outer closure while the slow path drains readers and + // rechecks capacity. Ordinary writes need no extra coroutine frame. + index_write = + co_await AwaitIndexWrite(*store.worker_, *index_ptr, request.digest_); + } } RecordIndex::Entry* previous_entry = nullptr; if (index_ptr != nullptr) { From 1187e4d33890804234ca71ed5df70591b29ccf55 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 11 Oct 2026 14:39:06 +0000 Subject: [PATCH 08/15] Mark index reader drain waits as unlikely --- src/storage/engine/flush.cpp | 2 +- src/storage/engine/write.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index 0c948c4de..65bef19d3 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -521,7 +521,7 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { RecordIndex& index = PartitionFor(*store, identity.partition_id_) .indexes_[identity.db_id_]; auto index_write = index.CloseForValueUpdate(identity.entry_hash_); - if (!index_write.ready()) { + if (!index_write.ready()) [[unlikely]] { // Keep admission closed across the slow wait. Most flush updates // have no registered reader and need no waiting coroutine. index_write = co_await AwaitIndexValueWrite(*store->worker_, index, diff --git a/src/storage/engine/write.cpp b/src/storage/engine/write.cpp index fcc83219e..3fb867351 100644 --- a/src/storage/engine/write.cpp +++ b/src/storage/engine/write.cpp @@ -2933,7 +2933,7 @@ Task StorageEngine::Impl::WriteRecordLocked( RecordIndex::WriteGuard index_write; if (index_ptr != nullptr) { index_write = index_ptr->CloseForWrite(request.digest_); - if (!index_write.ready()) { + if (!index_write.ready()) [[unlikely]] { // Retain the outer closure while the slow path drains readers and // rechecks capacity. Ordinary writes need no extra coroutine frame. index_write = From 76e8ed298a0beb2d41a77d39c4c2504715e1774a Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 10:22:08 +0800 Subject: [PATCH 09/15] perf(redis): reuse routing slot and prefetch remote GET metadata --- include/lavik/storage/scan_hash_map.h | 12 ++++++++++++ src/redis/command.cpp | 13 ++++++------- src/storage/engine/read.cpp | 3 ++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index 46a8d37ce..e170791b9 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -929,6 +929,8 @@ struct ScanHashMapReadStates { std::array stripes_{}; + void Prefetch() const noexcept { __builtin_prefetch(&disabled_, 0, 3); } + ScanHashMapReadGuard TryRead(std::uint64_t hash) noexcept { if (disabled_.load(std::memory_order_acquire)) return {}; auto& state = stripes_[hash & (Count - 1)].state_; @@ -1308,6 +1310,16 @@ class ScanHashMap { // Admission belongs to the map object, not a detached entry population. // The caller protects this object's lifetime before TryRead and through the // guard's destruction. Read guards must not cross a coroutine suspension. + // Address-only hints on the lifetime-protected map. Do not follow bucket + // or arena pointers before reader admission: a concurrent resize may own + // those populations. Hashing the key can overlap these two metadata loads. + void PrefetchRead() const noexcept + requires(ReadStripeCount != 0) + { + __builtin_prefetch(&tables_[0], 0, 3); + read_states_.Prefetch(); + } + ScanHashMapReadGuard TryRead(const Digest& digest) noexcept requires(ReadStripeCount != 0) { diff --git a/src/redis/command.cpp b/src/redis/command.cpp index b9972acbb..c3914874e 100644 --- a/src/redis/command.cpp +++ b/src/redis/command.cpp @@ -13113,15 +13113,14 @@ Task ExecuteCommandBody( // cluster reads learned the same route during admission. Deriving the // worker from the retained slot is cheaper than hashing args[1] again. const bool routed = request.HasRoutedPartitionFor(1); - const unsigned target = - routed ? request.RoutedPartitionId() % g_storage->worker_count() - : ShardForKey(args[1]); + const std::uint16_t slot = + routed ? request.RoutedPartitionId() : storage::RedisSlot(args[1]); + const unsigned target = slot % g_storage->worker_count(); if (request.kind_ == CommandKind::kGet && args.size() == 2 && target != ThisWorker().id_) { - auto fast = co_await g_storage->TryRemoteGet( - request.db_id_, args[1], - routed ? request.RoutedPartitionId() - : storage::RedisSlot(args[1])); + // Reuse the route hash for admission on the connection worker. + auto fast = + co_await g_storage->TryRemoteGet(request.db_id_, args[1], slot); if (fast) { if (const char* error = CommandServingGenerationError(request); error) diff --git a/src/storage/engine/read.cpp b/src/storage/engine/read.cpp index 067b0cfa9..5dc826fa2 100644 --- a/src/storage/engine/read.cpp +++ b/src/storage/engine/read.cpp @@ -256,8 +256,9 @@ StorageEngine::Impl::TryRemoteGet(std::uint8_t db_id, std::string_view key, !request_serving_fenced_until_restart_.load( std::memory_order_acquire); }; - const auto digest = ComputeDigest(key); auto& index = PartitionFor(source, slot).indexes_[db_id]; + index.PrefetchRead(); + const auto digest = ComputeDigest(key); auto reader = index.TryRead(digest); if (!reader) co_return std::nullopt; const auto* entry = std::as_const(index).Find(digest, key); From 1f8377025dccfd9656abf0be46319612dd174f72 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 03:20:40 +0000 Subject: [PATCH 10/15] Avoid child coroutine frames in connection-worker disk GET --- CMakeLists.txt | 1 + docs/architecture/04-storage-and-recovery.md | 9 +- include/lavik/storage/detail/record_index.h | 294 +--------------- .../lavik/storage/detail/record_location.h | 320 ++++++++++++++++++ include/lavik/storage/engine.h | 64 +++- src/storage/engine.cpp | 4 +- src/storage/engine/impl.h | 19 +- src/storage/engine/read.cpp | 230 ++++++++++--- tests/load_value_awaitable_test.cpp | 23 +- tests/remote_get_awaitable_test.cpp | 251 ++++++++++++++ 10 files changed, 842 insertions(+), 373 deletions(-) create mode 100644 include/lavik/storage/detail/record_location.h create mode 100644 tests/remote_get_awaitable_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b8dc53220..e42de64a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1767,6 +1767,7 @@ if (BUILD_TESTING) tests/rdb_collection_test.cpp tests/record_location_test.cpp tests/load_value_awaitable_test.cpp + tests/remote_get_awaitable_test.cpp tests/request_gate_test.cpp tests/replication_command_test.cpp tests/replication_group_test.cpp diff --git a/docs/architecture/04-storage-and-recovery.md b/docs/architecture/04-storage-and-recovery.md index 9d86e1114..d431b6ec6 100644 --- a/docs/architecture/04-storage-and-recovery.md +++ b/docs/architecture/04-storage-and-recovery.md @@ -674,8 +674,11 @@ then pins the block and validates its allocation identity before dropping the index guard. Only durable, non-grouped, non-external records whose block owner is also their key owner qualify, and the connection worker must have device I/O access. Closed index/block admission, unsupported representations, expired -entries, and concurrent admission contention return to the normal owner route. -Index misses use the same transaction and population admission checks. +entries, concurrent admission contention, and an exhausted connection-worker +read-buffer pool return to the normal owner route. An admitted fast read submits +I/O directly from an awaitable in the requesting coroutine; completion resumes +that same coroutine for record and population validation, without a child read +coroutine. Index misses use the same transaction and population admission checks. The index snapshot orders the GET before a subsequently admitted writer. The block pin survives I/O and cancellation cleanup, so writes can append and retire @@ -1212,7 +1215,7 @@ current source code are authoritative for present storage behavior. | Worker, partition, block, append-stream, allocator, recovery, and background-maintenance state | `src/storage/engine/impl.h` | | Runtime index representation, shared entry arena, runtime key digests, and asynchronous entry-identity validation | `include/lavik/storage/scan_hash_map.h`, `include/lavik/storage/format.h`, `src/storage/format.cpp`, `src/storage/engine/impl.h`, `src/storage/engine/init.cpp`, `src/storage/engine/write.cpp`, `src/storage/engine/flush.cpp` | | Compact and grouped Hash/Set serving, group identity and object side index | `src/storage/engine/hash_tree.cpp`, `src/storage/engine/hash_codec.cpp`, `src/storage/engine/grouped/hash.cpp`, `src/storage/engine/grouped/object_index.cpp`, [Grouped collections](09-grouped-collections.md) | -| Compact physical index representation shared by user-key and group-location indexes | `include/lavik/storage/detail/record_index.h` | +| Compact physical index representation shared by user-key and group-location indexes | `include/lavik/storage/detail/record_location.h`, `include/lavik/storage/detail/record_index.h` | | Persistent constants, device and block IDs, A/B metadata pages, record and extent layouts, and checksums | `include/lavik/storage/format.h`, `src/storage/format.cpp` | | Checkpoint serialization, bitmap validation, generation publication and consumption, fallback, and block retirement | `src/storage/engine/checkpoint.cpp`, `src/storage/engine/flush.cpp`, `src/storage/engine/init.cpp`, `src/storage/engine/recovery.cpp` | | System-state manifest, catalog COW extents, full-sync fence, population token, promotion base, and one-use shutdown certificate | `src/storage/engine/system_state.cpp`, `include/lavik/storage/engine.h` | diff --git a/include/lavik/storage/detail/record_index.h b/include/lavik/storage/detail/record_index.h index f8a16562e..83fec8520 100644 --- a/include/lavik/storage/detail/record_index.h +++ b/include/lavik/storage/detail/record_index.h @@ -21,304 +21,12 @@ #include #include "lavik/memory.h" +#include "lavik/storage/detail/record_location.h" #include "lavik/storage/format.h" #include "lavik/storage/scan_hash_map.h" namespace lavik::storage { -class RecordIndexValue; - -struct RecordLocationCore { - // Runtime block identities need only the 27-bit local block id plus the - // 16-bit configured device id. Pairing those 43 bits with a 53-bit - // allocation epoch removes four bytes without weakening any practical - // reuse horizon: at 8 MiB per allocation, the epoch spans 64 ZiB per - // device. The durable format retains both original 64-bit fields. - static constexpr unsigned kBlockIdBits = kLocalBlockIdBits + 16; - static constexpr unsigned kAllocationEpochBits = 53; - static constexpr unsigned kAllocationEpochLowBits = 64 - kBlockIdBits; - static constexpr std::uint64_t kBlockIdMask = - (std::uint64_t{1} << kBlockIdBits) - 1; - static constexpr std::uint64_t kAllocationEpochMask = - (std::uint64_t{1} << kAllocationEpochBits) - 1; - - static_assert(kBlockIdBits == 43); - static_assert(kAllocationEpochLowBits == 21); - static_assert(kAllocationEpochBits - kAllocationEpochLowBits == 32); - - // Offsets and record lengths are always 8-byte aligned inside an 8 MiB - // block, so storing their alignment units preserves their complete range in - // 20 bits each. The owner is process-local and InitMemoryLimit caps a - // process at 1024 workers. Keep this as an explicitly masked runtime word, - // rather than C++ bit-fields, so layout and overflow behavior are auditable. - // Expiration and grouped-collection representation are independent bits. - // Grouped is not external: it selects a collection directory, whereas - // external selects the extent representation of one physical payload. - // Uncommon state belongs in sparse side tables, not in every key's value. - class PackedMetadata { - public: - static constexpr unsigned kOffsetBits = 20; - static constexpr unsigned kLengthBits = 20; - static constexpr unsigned kOwnerBits = 10; - static constexpr unsigned kStateBits = 11; - static constexpr unsigned kReservedBits = 3; - - static constexpr unsigned kLengthShift = kOffsetBits; - static constexpr unsigned kOwnerShift = kLengthShift + kLengthBits; - static constexpr unsigned kInMemoryShift = kOwnerShift + kOwnerBits; - static constexpr unsigned kExternalShift = kInMemoryShift + 1; - static constexpr unsigned kKeyIndirectShift = kExternalShift + 1; - static constexpr unsigned kShieldingShift = kKeyIndirectShift + 1; - static constexpr unsigned kUnclaimedShift = kShieldingShift + 1; - static constexpr unsigned kTxTaggedShift = kUnclaimedShift + 1; - static constexpr unsigned kTypeCodeShift = kTxTaggedShift + 1; - static constexpr unsigned kHasExpiryShift = kTypeCodeShift + 3; - static constexpr unsigned kGroupedShift = kHasExpiryShift + 1; - - static constexpr std::uint64_t kOffsetMask = - (std::uint64_t{1} << kOffsetBits) - 1; - static constexpr std::uint64_t kLengthMask = - (std::uint64_t{1} << kLengthBits) - 1; - static constexpr std::uint64_t kOwnerMask = - (std::uint64_t{1} << kOwnerBits) - 1; - static constexpr std::uint64_t kTypeCodeMask = 0x7; - static constexpr std::uint8_t kTombstoneTypeCode = 0x7; - - static_assert(kStorageBlockBytes / kRecordAlignment - 1 <= kOffsetMask); - static_assert((kStorageBlockBytes - kBlockHeaderBytes) / kRecordAlignment <= - kLengthMask); - static_assert(kMaxMemoryWorkers <= (std::uint64_t{1} << kOwnerBits)); - - static PackedMetadata Encode(std::uint32_t record_offset, - std::uint32_t total_disk_bytes, - std::uint16_t block_owner, bool in_memory, - bool external, bool key_indirect, - bool shielding, bool unclaimed, bool tx_tagged, - RecordKind kind, ValueType value_type, - bool has_expiry = false, - bool grouped = false) noexcept { - assert(record_offset % kRecordAlignment == 0); - assert(total_disk_bytes % kRecordAlignment == 0); - assert((record_offset / kRecordAlignment) <= kOffsetMask); - assert((total_disk_bytes / kRecordAlignment) <= kLengthMask); - assert(block_owner < kMaxMemoryWorkers); - assert(kind == RecordKind::kValue || kind == RecordKind::kTombstone); - assert(kind != RecordKind::kValue || - static_cast(value_type) < kTombstoneTypeCode); - assert(kind != RecordKind::kTombstone || value_type == ValueType::kNone); - - const std::uint8_t type_code = - kind == RecordKind::kTombstone - ? kTombstoneTypeCode - : static_cast(value_type); - std::uint64_t bits = - static_cast(record_offset / kRecordAlignment) | - (static_cast(total_disk_bytes / kRecordAlignment) - << kLengthShift) | - (static_cast(block_owner) << kOwnerShift) | - (static_cast(type_code) << kTypeCodeShift); - SetBit(&bits, kInMemoryShift, in_memory); - SetBit(&bits, kExternalShift, external); - SetBit(&bits, kKeyIndirectShift, key_indirect); - SetBit(&bits, kShieldingShift, shielding); - SetBit(&bits, kUnclaimedShift, unclaimed); - SetBit(&bits, kTxTaggedShift, tx_tagged); - SetBit(&bits, kHasExpiryShift, has_expiry); - SetBit(&bits, kGroupedShift, grouped); - return PackedMetadata(bits); - } - - std::uint32_t record_offset() const noexcept { - return static_cast(bits_ & kOffsetMask) * kRecordAlignment; - } - std::uint32_t total_disk_bytes() const noexcept { - return static_cast((bits_ >> kLengthShift) & kLengthMask) * - kRecordAlignment; - } - std::uint16_t block_owner() const noexcept { - return static_cast((bits_ >> kOwnerShift) & kOwnerMask); - } - bool in_memory() const noexcept { return Bit(kInMemoryShift); } - bool external() const noexcept { return Bit(kExternalShift); } - bool key_indirect() const noexcept { return Bit(kKeyIndirectShift); } - bool shielding() const noexcept { return Bit(kShieldingShift); } - bool unclaimed() const noexcept { return Bit(kUnclaimedShift); } - bool tx_tagged() const noexcept { return Bit(kTxTaggedShift); } - bool has_expiry() const noexcept { return Bit(kHasExpiryShift); } - bool grouped() const noexcept { return Bit(kGroupedShift); } - RecordKind kind() const noexcept { - return type_code() == kTombstoneTypeCode ? RecordKind::kTombstone - : RecordKind::kValue; - } - ValueType value_type() const noexcept { - return type_code() == kTombstoneTypeCode - ? ValueType::kNone - : static_cast(type_code()); - } - - void set_in_memory(bool value) noexcept { - SetBit(&bits_, kInMemoryShift, value); - } - void set_shielding(bool value) noexcept { - SetBit(&bits_, kShieldingShift, value); - } - void set_unclaimed(bool value) noexcept { - SetBit(&bits_, kUnclaimedShift, value); - } - void set_tx_tagged(bool value) noexcept { - SetBit(&bits_, kTxTaggedShift, value); - } - void set_has_expiry(bool value) noexcept { - SetBit(&bits_, kHasExpiryShift, value); - } - - private: - friend class RecordIndexValue; - - explicit constexpr PackedMetadata(std::uint64_t bits) noexcept - : bits_(bits) {} - - static void SetBit(std::uint64_t* bits, unsigned shift, - bool value) noexcept { - const std::uint64_t mask = std::uint64_t{1} << shift; - *bits = value ? (*bits | mask) : (*bits & ~mask); - } - bool Bit(unsigned shift) const noexcept { - return (bits_ & (std::uint64_t{1} << shift)) != 0; - } - std::uint8_t type_code() const noexcept { - return static_cast((bits_ >> kTypeCodeShift) & - kTypeCodeMask); - } - - std::uint64_t bits_ = 0; - }; - - static_assert(PackedMetadata::kOffsetBits + PackedMetadata::kLengthBits + - PackedMetadata::kOwnerBits + PackedMetadata::kStateBits + - PackedMetadata::kReservedBits == - 64); - - RecordLocationCore() noexcept = default; - - RecordLocationCore(std::uint64_t block_id, std::uint64_t mutation_sequence, - std::uint64_t allocation_epoch, std::uint32_t logical_size, - PackedMetadata metadata) noexcept - : mutation_sequence_(mutation_sequence), - block_and_epoch_low_( - EncodeBlockAndEpochLow(block_id, allocation_epoch)), - allocation_epoch_high_(EncodeAllocationEpochHigh(allocation_epoch)), - logical_size_(logical_size), - metadata_(metadata) { - assert(CanEncodeBlockIdentity(block_id, allocation_epoch)); - } - - static constexpr bool CanEncodeBlockIdentity( - std::uint64_t block_id, std::uint64_t allocation_epoch) noexcept { - return block_id <= kBlockIdMask && allocation_epoch <= kAllocationEpochMask; - } - - std::uint64_t mutation_sequence_ = 0; - // Low word: block id in bits [0, 42], low allocation-epoch bits in - // [43, 63]. The remaining 32 epoch bits sit beside logical_size_, filling - // what would otherwise be alignment padding before metadata_. - std::uint64_t block_and_epoch_low_ = 0; - std::uint32_t allocation_epoch_high_ = 0; - // Exact Redis-visible bytes/cardinality. - std::uint32_t logical_size_ = 0; - PackedMetadata metadata_ = - PackedMetadata::Encode(0, 0, 0, false, false, false, false, false, false, - RecordKind::kValue, ValueType::kNone); - - std::uint64_t block_id() const noexcept { - return block_and_epoch_low_ & kBlockIdMask; - } - std::uint64_t allocation_epoch() const noexcept { - return (block_and_epoch_low_ >> kBlockIdBits) | - (static_cast(allocation_epoch_high_) - << kAllocationEpochLowBits); - } - - std::uint32_t record_offset() const noexcept { - return metadata_.record_offset(); - } - std::uint32_t total_disk_bytes() const noexcept { - return metadata_.total_disk_bytes(); - } - std::uint16_t block_owner() const noexcept { return metadata_.block_owner(); } - bool in_memory() const noexcept { return metadata_.in_memory(); } - bool external() const noexcept { return metadata_.external(); } - bool key_indirect() const noexcept { return metadata_.key_indirect(); } - // True while an older, still-unexpired value of this key may survive on - // disk. Erasing this entry then would un-suppress that copy: recovery - // picks the newest surviving record, so the key would resurrect with the - // stale value. Propagates through every overwrite — tombstones included, - // since a superseded tombstone leaves the disk like any dead record — and - // is rebuilt exactly during recovery, which sees every surviving record. - bool shielding() const noexcept { return metadata_.shielding(); } - // Tomb-raider round state: set on candidates (tombstones, shielded values) - // when a round begins, cleared when the sweep finds an older on-disk - // record the entry still suppresses. Whatever survives the sweep - // unclaimed proved nothing on disk needs it. False outside rounds, and - // any overwrite resets it, exempting concurrently-touched keys. - bool unclaimed() const noexcept { return metadata_.unclaimed(); } - // The on-disk record carries a nonzero transaction id. Retirement uses the - // bit to remove its bytes from transaction-block accounting; the - // 32-byte index core deliberately does not retain the full txid. - bool tx_tagged() const noexcept { return metadata_.tx_tagged(); } - bool has_expiry() const noexcept { return metadata_.has_expiry(); } - bool grouped() const noexcept { return metadata_.grouped(); } - RecordKind kind() const noexcept { return metadata_.kind(); } - ValueType value_type() const noexcept { return metadata_.value_type(); } - - void set_in_memory(bool value) noexcept { metadata_.set_in_memory(value); } - void set_shielding(bool value) noexcept { metadata_.set_shielding(value); } - void set_unclaimed(bool value) noexcept { metadata_.set_unclaimed(value); } - void set_tx_tagged(bool value) noexcept { metadata_.set_tx_tagged(value); } - - bool SamePhysicalRecord(const RecordLocationCore& other) const noexcept { - return block_id() == other.block_id() && - record_offset() == other.record_offset() && - allocation_epoch() == other.allocation_epoch(); - } - - private: - static constexpr std::uint64_t EncodeBlockAndEpochLow( - std::uint64_t block_id, std::uint64_t allocation_epoch) noexcept { - return (block_id & kBlockIdMask) | - ((allocation_epoch & - ((std::uint64_t{1} << kAllocationEpochLowBits) - 1)) - << kBlockIdBits); - } - - static constexpr std::uint32_t EncodeAllocationEpochHigh( - std::uint64_t allocation_epoch) noexcept { - return static_cast(allocation_epoch >> - kAllocationEpochLowBits); - } -}; - -struct RecordLocation final : RecordLocationCore { - RecordLocation() noexcept = default; - - RecordLocation(std::uint64_t block_id, std::uint64_t mutation_sequence, - std::uint64_t allocation_epoch, std::uint64_t expire_at_ms, - std::uint32_t logical_size, PackedMetadata metadata) noexcept - : RecordLocationCore(block_id, mutation_sequence, allocation_epoch, - logical_size, metadata), - expire_at_ms_(expire_at_ms) { - metadata_.set_has_expiry(expire_at_ms != 0); - } - - RecordLocation(const RecordLocationCore& core, - std::uint64_t expire_at_ms) noexcept - : RecordLocationCore(core), expire_at_ms_(expire_at_ms) { - assert(has_expiry() == (expire_at_ms != 0)); - } - - std::uint64_t expire_at_ms_ = 0; -}; - // The index retains only key-specific location state. A block's allocation // epoch and runtime owner are already stored once in its dense BlockState; // repeating them in every record from the same 8 MiB allocation would cost diff --git a/include/lavik/storage/detail/record_location.h b/include/lavik/storage/detail/record_location.h new file mode 100644 index 000000000..980ff432d --- /dev/null +++ b/include/lavik/storage/detail/record_location.h @@ -0,0 +1,320 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "lavik/memory.h" +#include "lavik/storage/format.h" + +namespace lavik::storage { + +class RecordIndexValue; + +struct RecordLocationCore { + // Runtime block identities need only the 27-bit local block id plus the + // 16-bit configured device id. Pairing those 43 bits with a 53-bit + // allocation epoch removes four bytes without weakening any practical + // reuse horizon: at 8 MiB per allocation, the epoch spans 64 ZiB per + // device. The durable format retains both original 64-bit fields. + static constexpr unsigned kBlockIdBits = kLocalBlockIdBits + 16; + static constexpr unsigned kAllocationEpochBits = 53; + static constexpr unsigned kAllocationEpochLowBits = 64 - kBlockIdBits; + static constexpr std::uint64_t kBlockIdMask = + (std::uint64_t{1} << kBlockIdBits) - 1; + static constexpr std::uint64_t kAllocationEpochMask = + (std::uint64_t{1} << kAllocationEpochBits) - 1; + + static_assert(kBlockIdBits == 43); + static_assert(kAllocationEpochLowBits == 21); + static_assert(kAllocationEpochBits - kAllocationEpochLowBits == 32); + + // Offsets and record lengths are always 8-byte aligned inside an 8 MiB + // block, so storing their alignment units preserves their complete range in + // 20 bits each. The owner is process-local and InitMemoryLimit caps a + // process at 1024 workers. Keep this as an explicitly masked runtime word, + // rather than C++ bit-fields, so layout and overflow behavior are auditable. + // Expiration and grouped-collection representation are independent bits. + // Grouped is not external: it selects a collection directory, whereas + // external selects the extent representation of one physical payload. + // Uncommon state belongs in sparse side tables, not in every key's value. + class PackedMetadata { + public: + static constexpr unsigned kOffsetBits = 20; + static constexpr unsigned kLengthBits = 20; + static constexpr unsigned kOwnerBits = 10; + static constexpr unsigned kStateBits = 11; + static constexpr unsigned kReservedBits = 3; + + static constexpr unsigned kLengthShift = kOffsetBits; + static constexpr unsigned kOwnerShift = kLengthShift + kLengthBits; + static constexpr unsigned kInMemoryShift = kOwnerShift + kOwnerBits; + static constexpr unsigned kExternalShift = kInMemoryShift + 1; + static constexpr unsigned kKeyIndirectShift = kExternalShift + 1; + static constexpr unsigned kShieldingShift = kKeyIndirectShift + 1; + static constexpr unsigned kUnclaimedShift = kShieldingShift + 1; + static constexpr unsigned kTxTaggedShift = kUnclaimedShift + 1; + static constexpr unsigned kTypeCodeShift = kTxTaggedShift + 1; + static constexpr unsigned kHasExpiryShift = kTypeCodeShift + 3; + static constexpr unsigned kGroupedShift = kHasExpiryShift + 1; + + static constexpr std::uint64_t kOffsetMask = + (std::uint64_t{1} << kOffsetBits) - 1; + static constexpr std::uint64_t kLengthMask = + (std::uint64_t{1} << kLengthBits) - 1; + static constexpr std::uint64_t kOwnerMask = + (std::uint64_t{1} << kOwnerBits) - 1; + static constexpr std::uint64_t kTypeCodeMask = 0x7; + static constexpr std::uint8_t kTombstoneTypeCode = 0x7; + + static_assert(kStorageBlockBytes / kRecordAlignment - 1 <= kOffsetMask); + static_assert((kStorageBlockBytes - kBlockHeaderBytes) / kRecordAlignment <= + kLengthMask); + static_assert(kMaxMemoryWorkers <= (std::uint64_t{1} << kOwnerBits)); + + static PackedMetadata Encode(std::uint32_t record_offset, + std::uint32_t total_disk_bytes, + std::uint16_t block_owner, bool in_memory, + bool external, bool key_indirect, + bool shielding, bool unclaimed, bool tx_tagged, + RecordKind kind, ValueType value_type, + bool has_expiry = false, + bool grouped = false) noexcept { + assert(record_offset % kRecordAlignment == 0); + assert(total_disk_bytes % kRecordAlignment == 0); + assert((record_offset / kRecordAlignment) <= kOffsetMask); + assert((total_disk_bytes / kRecordAlignment) <= kLengthMask); + assert(block_owner < kMaxMemoryWorkers); + assert(kind == RecordKind::kValue || kind == RecordKind::kTombstone); + assert(kind != RecordKind::kValue || + static_cast(value_type) < kTombstoneTypeCode); + assert(kind != RecordKind::kTombstone || value_type == ValueType::kNone); + + const std::uint8_t type_code = + kind == RecordKind::kTombstone + ? kTombstoneTypeCode + : static_cast(value_type); + std::uint64_t bits = + static_cast(record_offset / kRecordAlignment) | + (static_cast(total_disk_bytes / kRecordAlignment) + << kLengthShift) | + (static_cast(block_owner) << kOwnerShift) | + (static_cast(type_code) << kTypeCodeShift); + SetBit(&bits, kInMemoryShift, in_memory); + SetBit(&bits, kExternalShift, external); + SetBit(&bits, kKeyIndirectShift, key_indirect); + SetBit(&bits, kShieldingShift, shielding); + SetBit(&bits, kUnclaimedShift, unclaimed); + SetBit(&bits, kTxTaggedShift, tx_tagged); + SetBit(&bits, kHasExpiryShift, has_expiry); + SetBit(&bits, kGroupedShift, grouped); + return PackedMetadata(bits); + } + + std::uint32_t record_offset() const noexcept { + return static_cast(bits_ & kOffsetMask) * kRecordAlignment; + } + std::uint32_t total_disk_bytes() const noexcept { + return static_cast((bits_ >> kLengthShift) & kLengthMask) * + kRecordAlignment; + } + std::uint16_t block_owner() const noexcept { + return static_cast((bits_ >> kOwnerShift) & kOwnerMask); + } + bool in_memory() const noexcept { return Bit(kInMemoryShift); } + bool external() const noexcept { return Bit(kExternalShift); } + bool key_indirect() const noexcept { return Bit(kKeyIndirectShift); } + bool shielding() const noexcept { return Bit(kShieldingShift); } + bool unclaimed() const noexcept { return Bit(kUnclaimedShift); } + bool tx_tagged() const noexcept { return Bit(kTxTaggedShift); } + bool has_expiry() const noexcept { return Bit(kHasExpiryShift); } + bool grouped() const noexcept { return Bit(kGroupedShift); } + RecordKind kind() const noexcept { + return type_code() == kTombstoneTypeCode ? RecordKind::kTombstone + : RecordKind::kValue; + } + ValueType value_type() const noexcept { + return type_code() == kTombstoneTypeCode + ? ValueType::kNone + : static_cast(type_code()); + } + + void set_in_memory(bool value) noexcept { + SetBit(&bits_, kInMemoryShift, value); + } + void set_shielding(bool value) noexcept { + SetBit(&bits_, kShieldingShift, value); + } + void set_unclaimed(bool value) noexcept { + SetBit(&bits_, kUnclaimedShift, value); + } + void set_tx_tagged(bool value) noexcept { + SetBit(&bits_, kTxTaggedShift, value); + } + void set_has_expiry(bool value) noexcept { + SetBit(&bits_, kHasExpiryShift, value); + } + + private: + friend class RecordIndexValue; + + explicit constexpr PackedMetadata(std::uint64_t bits) noexcept + : bits_(bits) {} + + static void SetBit(std::uint64_t* bits, unsigned shift, + bool value) noexcept { + const std::uint64_t mask = std::uint64_t{1} << shift; + *bits = value ? (*bits | mask) : (*bits & ~mask); + } + bool Bit(unsigned shift) const noexcept { + return (bits_ & (std::uint64_t{1} << shift)) != 0; + } + std::uint8_t type_code() const noexcept { + return static_cast((bits_ >> kTypeCodeShift) & + kTypeCodeMask); + } + + std::uint64_t bits_ = 0; + }; + + static_assert(PackedMetadata::kOffsetBits + PackedMetadata::kLengthBits + + PackedMetadata::kOwnerBits + PackedMetadata::kStateBits + + PackedMetadata::kReservedBits == + 64); + + RecordLocationCore() noexcept = default; + + RecordLocationCore(std::uint64_t block_id, std::uint64_t mutation_sequence, + std::uint64_t allocation_epoch, std::uint32_t logical_size, + PackedMetadata metadata) noexcept + : mutation_sequence_(mutation_sequence), + block_and_epoch_low_( + EncodeBlockAndEpochLow(block_id, allocation_epoch)), + allocation_epoch_high_(EncodeAllocationEpochHigh(allocation_epoch)), + logical_size_(logical_size), + metadata_(metadata) { + assert(CanEncodeBlockIdentity(block_id, allocation_epoch)); + } + + static constexpr bool CanEncodeBlockIdentity( + std::uint64_t block_id, std::uint64_t allocation_epoch) noexcept { + return block_id <= kBlockIdMask && allocation_epoch <= kAllocationEpochMask; + } + + std::uint64_t mutation_sequence_ = 0; + // Low word: block id in bits [0, 42], low allocation-epoch bits in + // [43, 63]. The remaining 32 epoch bits sit beside logical_size_, filling + // what would otherwise be alignment padding before metadata_. + std::uint64_t block_and_epoch_low_ = 0; + std::uint32_t allocation_epoch_high_ = 0; + // Exact Redis-visible bytes/cardinality. + std::uint32_t logical_size_ = 0; + PackedMetadata metadata_ = + PackedMetadata::Encode(0, 0, 0, false, false, false, false, false, false, + RecordKind::kValue, ValueType::kNone); + + std::uint64_t block_id() const noexcept { + return block_and_epoch_low_ & kBlockIdMask; + } + std::uint64_t allocation_epoch() const noexcept { + return (block_and_epoch_low_ >> kBlockIdBits) | + (static_cast(allocation_epoch_high_) + << kAllocationEpochLowBits); + } + + std::uint32_t record_offset() const noexcept { + return metadata_.record_offset(); + } + std::uint32_t total_disk_bytes() const noexcept { + return metadata_.total_disk_bytes(); + } + std::uint16_t block_owner() const noexcept { return metadata_.block_owner(); } + bool in_memory() const noexcept { return metadata_.in_memory(); } + bool external() const noexcept { return metadata_.external(); } + bool key_indirect() const noexcept { return metadata_.key_indirect(); } + // True while an older, still-unexpired value of this key may survive on + // disk. Erasing this entry then would un-suppress that copy: recovery + // picks the newest surviving record, so the key would resurrect with the + // stale value. Propagates through every overwrite — tombstones included, + // since a superseded tombstone leaves the disk like any dead record — and + // is rebuilt exactly during recovery, which sees every surviving record. + bool shielding() const noexcept { return metadata_.shielding(); } + // Tomb-raider round state: set on candidates (tombstones, shielded values) + // when a round begins, cleared when the sweep finds an older on-disk + // record the entry still suppresses. Whatever survives the sweep + // unclaimed proved nothing on disk needs it. False outside rounds, and + // any overwrite resets it, exempting concurrently-touched keys. + bool unclaimed() const noexcept { return metadata_.unclaimed(); } + // The on-disk record carries a nonzero transaction id. Retirement uses the + // bit to remove its bytes from transaction-block accounting; the + // 32-byte index core deliberately does not retain the full txid. + bool tx_tagged() const noexcept { return metadata_.tx_tagged(); } + bool has_expiry() const noexcept { return metadata_.has_expiry(); } + bool grouped() const noexcept { return metadata_.grouped(); } + RecordKind kind() const noexcept { return metadata_.kind(); } + ValueType value_type() const noexcept { return metadata_.value_type(); } + + void set_in_memory(bool value) noexcept { metadata_.set_in_memory(value); } + void set_shielding(bool value) noexcept { metadata_.set_shielding(value); } + void set_unclaimed(bool value) noexcept { metadata_.set_unclaimed(value); } + void set_tx_tagged(bool value) noexcept { metadata_.set_tx_tagged(value); } + + bool SamePhysicalRecord(const RecordLocationCore& other) const noexcept { + return block_id() == other.block_id() && + record_offset() == other.record_offset() && + allocation_epoch() == other.allocation_epoch(); + } + + private: + static constexpr std::uint64_t EncodeBlockAndEpochLow( + std::uint64_t block_id, std::uint64_t allocation_epoch) noexcept { + return (block_id & kBlockIdMask) | + ((allocation_epoch & + ((std::uint64_t{1} << kAllocationEpochLowBits) - 1)) + << kBlockIdBits); + } + + static constexpr std::uint32_t EncodeAllocationEpochHigh( + std::uint64_t allocation_epoch) noexcept { + return static_cast(allocation_epoch >> + kAllocationEpochLowBits); + } +}; + +struct RecordLocation final : RecordLocationCore { + RecordLocation() noexcept = default; + + RecordLocation(std::uint64_t block_id, std::uint64_t mutation_sequence, + std::uint64_t allocation_epoch, std::uint64_t expire_at_ms, + std::uint32_t logical_size, PackedMetadata metadata) noexcept + : RecordLocationCore(block_id, mutation_sequence, allocation_epoch, + logical_size, metadata), + expire_at_ms_(expire_at_ms) { + metadata_.set_has_expiry(expire_at_ms != 0); + } + + RecordLocation(const RecordLocationCore& core, + std::uint64_t expire_at_ms) noexcept + : RecordLocationCore(core), expire_at_ms_(expire_at_ms) { + assert(has_expiry() == (expire_at_ms != 0)); + } + + std::uint64_t expire_at_ms_ = 0; +}; + +} // namespace lavik::storage diff --git a/include/lavik/storage/engine.h b/include/lavik/storage/engine.h index 5ab135b4a..b2a77dd8e 100644 --- a/include/lavik/storage/engine.h +++ b/include/lavik/storage/engine.h @@ -40,6 +40,7 @@ #include "lavik/set_trace.h" #include "lavik/storage/buffer_pool.h" #include "lavik/storage/collection_page.h" +#include "lavik/storage/detail/record_location.h" #include "lavik/storage/format.h" #include "lavik/storage/sorted_set.h" @@ -1603,11 +1604,16 @@ class StorageEngine { bycorf::Task AbortReplicaRoot(std::uint64_t session_id); void SetReplicaLoading(bool loading) noexcept; + class RemoteGetAwaitable; + // Connection-worker optimization for a standalone GET. The caller holds // request/DB population admission through completion. nullopt requests the // ordinary owner path; a present result is final (including not-found). - bycorf::Task>> TryRemoteGet( - std::uint8_t db_id, std::string_view key, std::uint16_t slot); + // Admission runs synchronously; immediately co_await the result on this + // worker, keeping key and request admission alive through completion. The + // fast path submits I/O without a child Task; buffer contention falls back. + RemoteGetAwaitable TryRemoteGet(std::uint8_t db_id, std::string_view key, + std::uint16_t slot); // These operations must execute on OwnerForKey(key), normally through // SubmitTaskTo. Only digest/location metadata is retained after completion. @@ -1951,8 +1957,62 @@ class StorageEngine { friend class ExpirationAuthorityTestPeer; friend class TombRaiderTestPeer; friend class LoadValueAwaitableTestPeer; + friend class RemoteGetTestPeer; class Impl; std::unique_ptr impl_; }; +// Owns a remote block pin and the I/O completion inline in the caller's frame. +// No child coroutine or separately allocated continuation is needed. Like the +// runtime's one-shot I/O awaitables, it cannot move after publishing its +// address to the device, and shutdown must drain I/O before destroying its +// parent. +class StorageEngine::RemoteGetAwaitable { + public: + using Result = std::optional>; + + RemoteGetAwaitable(RemoteGetAwaitable&&) = delete; + RemoteGetAwaitable& operator=(RemoteGetAwaitable&&) = delete; + RemoteGetAwaitable(const RemoteGetAwaitable&) = delete; + RemoteGetAwaitable& operator=(const RemoteGetAwaitable&) = delete; + + bool await_ready() const noexcept { return !io_.has_value(); } + bool await_suspend(std::coroutine_handle<> awaiting) { + return io_->await_suspend(awaiting); + } + Result await_resume(); + + private: + friend class StorageEngine; + friend class StorageEngine::Impl; + friend class RemoteGetTestPeer; + + struct Pin { + Impl* engine_ = nullptr; + std::uint64_t block_id_ = 0; + unsigned owner_ = 0; + ~Pin(); + }; + + RemoteGetAwaitable(Impl& engine, std::uint8_t db_id, std::string_view key, + std::uint16_t slot); + bool Current(const Impl& engine) const noexcept; + + Result ready_; + std::string_view key_; + RecordLocation location_; + std::uint64_t generation_ = 0; + std::uint64_t db_epoch_ = 0; + std::uint64_t replication_epoch_ = 0; + std::size_t record_headroom_ = 0; + std::size_t read_bytes_ = 0; + std::uint16_t slot_ = 0; + std::uint8_t db_id_ = 0; + // Reverse destruction releases the completion and buffer before the pin, + // including when an admitted awaitable is destroyed without being awaited. + Pin pin_; + ReadBufferLease lease_; + std::optional io_; +}; + } // namespace lavik::storage diff --git a/src/storage/engine.cpp b/src/storage/engine.cpp index 5c7047da8..6e6c031e9 100644 --- a/src/storage/engine.cpp +++ b/src/storage/engine.cpp @@ -634,9 +634,9 @@ void StorageEngine::SetReplicaLoading(bool loading) noexcept { impl_->SetReplicaLoading(loading); } -Task>> StorageEngine::TryRemoteGet( +StorageEngine::RemoteGetAwaitable StorageEngine::TryRemoteGet( std::uint8_t db_id, std::string_view key, std::uint16_t slot) { - return impl_->TryRemoteGet(db_id, key, slot); + return RemoteGetAwaitable(*impl_, db_id, key, slot); } Task> StorageEngine::Get( diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 310d60cf4..402c88638 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -2199,8 +2199,7 @@ class StorageEngine::Impl { std::uint64_t block_id); void UnpinRemoteBlock(BlockState& state, unsigned owner, std::uint64_t block_id); - Task>> TryRemoteGet( - std::uint8_t db_id, std::string_view key, std::uint16_t slot); + void PrepareRemoteGet(RemoteGetAwaitable& result); void FinalizeWorker(unsigned worker_id) noexcept; bool AbandonWorkerStateForProcessExit() noexcept; @@ -3351,6 +3350,8 @@ class StorageEngine::Impl { friend class ExpirationAuthorityTestPeer; friend class TombRaiderTestPeer; friend class LoadValueAwaitableTestPeer; + friend class RemoteGetTestPeer; + friend class StorageEngine::RemoteGetAwaitable; struct DurableSystemState { std::uint64_t generation_ = 0; @@ -3454,8 +3455,7 @@ class StorageEngine::Impl { LoadValueAwaitable(Impl& engine, WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, ReadLatencyTrace* trace, - std::optional expected_db_epoch, - bool remotely_pinned = false); + std::optional expected_db_epoch); ~LoadValueAwaitable(); bool await_ready() const noexcept { return ready_ && !slow_.valid(); } @@ -3491,7 +3491,6 @@ class StorageEngine::Impl { ReadLatencyTrace* trace_; std::uint64_t replication_epoch_; std::optional expected_db_epoch_; - bool remotely_pinned_ = false; std::uint64_t aligned_offset_ = 0; std::size_t file_id_ = 0; std::size_t record_headroom_ = 0; @@ -3569,6 +3568,13 @@ class StorageEngine::Impl { absl::StatusOr EncodeDiskValue(LoadedValue loaded); + // Validates the pinned remote record in the connection task after I/O. + // Remote admission/fallback is separate from owner-local relocation retries. + absl::StatusOr DecodeRemoteValue( + ReadBufferLease lease, std::uint8_t db_id, std::string_view key, + RecordLocation location, std::uint64_t replication_epoch, + std::size_t record_headroom, std::uint64_t expected_db_epoch); + void QueueExpiredCandidate(WorkerStore& store, std::uint16_t partition_id, std::uint8_t db_id, const RecordIndex::Entry& entry, @@ -3990,8 +3996,7 @@ class StorageEngine::Impl { WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, ReadLatencyTrace* trace = nullptr, - std::optional expected_db_epoch = std::nullopt, - bool remotely_pinned = false); + std::optional expected_db_epoch = std::nullopt); std::uint64_t ForegroundBlocksForDevice( std::size_t device_index) const noexcept { diff --git a/src/storage/engine/read.cpp b/src/storage/engine/read.cpp index 5dc826fa2..6827a524f 100644 --- a/src/storage/engine/read.cpp +++ b/src/storage/engine/read.cpp @@ -226,56 +226,57 @@ StorageEngine::Impl::RandomKeyLocal(std::uint8_t db_id) { co_return std::optional{}; } -Task>> -StorageEngine::Impl::TryRemoteGet(std::uint8_t db_id, std::string_view key, - std::uint16_t slot) { +StorageEngine::RemoteGetAwaitable::RemoteGetAwaitable(Impl& engine, + std::uint8_t db_id, + std::string_view key, + std::uint16_t slot) + : key_(key), slot_(slot), db_id_(db_id) { + engine.PrepareRemoteGet(*this); +} + +void StorageEngine::Impl::PrepareRemoteGet(RemoteGetAwaitable& result) { + const auto db_id = result.db_id_; + const auto key = result.key_; + const auto slot = result.slot_; const unsigned owner = slot % worker_count_; auto& local = CurrentStore(); if (owner == local.worker_->id() || db_id >= options_.database_count_ || replica_loading_.load(std::memory_order_acquire) || replica_recovery_fenced_.load(std::memory_order_acquire) || request_serving_fenced_until_restart_.load(std::memory_order_acquire)) - co_return std::nullopt; + return; auto& source = *stores_[owner]; - if (!source.remote_reads_ready_.load(std::memory_order_acquire)) - co_return std::nullopt; + if (!source.remote_reads_ready_.load(std::memory_order_acquire)) return; const auto generation = source.remote_population_generation_.load(std::memory_order_acquire); - if (generation & 1) co_return std::nullopt; + if (generation & 1) return; const auto db_epoch = DbEpoch(db_id); const auto replication_epoch = remote_replication_epochs_[slot].load(std::memory_order_acquire); - const auto current = [&] { - return source.remote_population_generation_.load( - std::memory_order_acquire) == generation && - DbEpoch(db_id) == db_epoch && - remote_replication_epochs_[slot].load(std::memory_order_acquire) == - replication_epoch && - !replica_loading_.load(std::memory_order_acquire) && - !replica_recovery_fenced_.load(std::memory_order_acquire) && - !request_serving_fenced_until_restart_.load( - std::memory_order_acquire); - }; + result.pin_.owner_ = owner; + result.generation_ = generation; + result.db_epoch_ = db_epoch; + result.replication_epoch_ = replication_epoch; auto& index = PartitionFor(source, slot).indexes_[db_id]; index.PrefetchRead(); const auto digest = ComputeDigest(key); auto reader = index.TryRead(digest); - if (!reader) co_return std::nullopt; + if (!reader) return; const auto* entry = std::as_const(index).Find(digest, key); - if (!current()) co_return std::nullopt; - if (entry && !entry->key_complete()) co_return std::nullopt; + if (!result.Current(*this)) return; + if (entry && !entry->key_complete()) return; if (entry == nullptr || entry->value_.kind() == RecordKind::kTombstone) { - co_return std::optional>( - absl::NotFoundError("key not found")); + result.ready_.emplace(absl::NotFoundError("key not found")); + return; } // Cold representations and expiration retain the complete owner state // machine (including expiry scheduling and indirect-key collision checks). if (!entry->key_complete() || entry->value_.grouped() || entry->value_.external() || entry->value_.in_memory() || entry->value_.value_type() != ValueType::kString || IsExpiredNow(*entry)) - co_return std::nullopt; + return; const RecordLocation location = MaterializeIndexLocation(*entry); - if (location.block_owner() != owner) co_return std::nullopt; + if (location.block_owner() != owner) return; // io_uring opens every configured file on every worker. Only SPDK builds // the controller-specific qpair owner directory. if (bycorf::SpdkStorageEnabled()) { @@ -283,30 +284,97 @@ StorageEngine::Impl::TryRemoteGet(std::uint8_t db_id, std::string_view key, if (!std::binary_search(device_owners_[device].begin(), device_owners_[device].end(), static_cast(local.worker_->id()))) - co_return std::nullopt; + return; } auto& block = BlockStateAt(location.block_id()); - if (!block.TryRemotePin()) co_return std::nullopt; - struct Pin { - Impl* engine; - BlockState& block; - unsigned owner; - std::uint64_t block_id; - ~Pin() { engine->UnpinRemoteBlock(block, owner, block_id); } - } pin{this, block, owner, location.block_id()}; + if (!block.TryRemotePin()) return; + result.pin_.engine_ = this; + result.pin_.block_id_ = location.block_id(); + result.location_ = location; if (block.owner_.load(std::memory_order_acquire) != owner || - block.allocation_epoch_ != location.allocation_epoch() || !current()) - co_return std::nullopt; + block.allocation_epoch_ != location.allocation_epoch() || + !result.Current(*this)) + return; // The only index-to-I/O handoff: no suspension or borrowed entry survives it. reader.Reset(); - auto loaded = co_await LoadValueLocal( - local, db_id, key, location, replication_epoch, nullptr, db_epoch, true); - if (!current() || (!loaded.ok() && absl::IsAborted(loaded.status()))) - co_return std::nullopt; - if (!loaded.ok()) - co_return std::optional>(loaded.status()); - co_return std::optional>( - EncodeDiskValue(std::move(*loaded))); + + LAVIK_FAULT_INJECT( + if (LAVIK_FAULT_MATCHES("LAVIK_FAIL_VALUE_READ_KEY", key)) { + result.ready_.emplace( + absl::InternalError("injected value payload read failure")); + return; + }); + const auto [file_id, block_offset] = FileOffset(location.block_id()); + const auto absolute_offset = block_offset + location.record_offset(); + const auto mask = static_cast(direct_io_alignment_ - 1); + const auto aligned_offset = absolute_offset & ~mask; + result.record_headroom_ = absolute_offset - aligned_offset; + result.read_bytes_ = (result.record_headroom_ + location.total_disk_bytes() + + direct_io_alignment_ - 1) & + ~mask; + + // Admission is only an optimization. If the local bounded pool is busy, + // let the owner path wait for its buffer rather than adding a child task or + // retaining a remote pin in a second asynchronous state machine. + auto acquire = local.buffers_.AcquireReadBuffer(result.read_bytes_); + if (!acquire.await_ready()) return; + auto acquired = acquire.await_resume(); + if (!acquired.ok()) { + result.ready_.emplace(acquired.status()); + return; + } + result.lease_ = std::move(*acquired); + auto buffer = result.lease_.io_buffer(); + assert(result.read_bytes_ <= buffer.size_); + buffer.size_ = result.read_bytes_; + result.io_.emplace(*local.worker_, local.files_[file_id], buffer, + aligned_offset, + result.lease_.registered() + ? bycorf::SizeIoAwaitable::Operation::kReadFixed + : bycorf::SizeIoAwaitable::Operation::kRead); +} + +StorageEngine::RemoteGetAwaitable::Pin::~Pin() { + if (engine_ != nullptr) { + engine_->UnpinRemoteBlock(engine_->BlockStateAt(block_id_), owner_, + block_id_); + } +} + +bool StorageEngine::RemoteGetAwaitable::Current( + const Impl& engine) const noexcept { + return engine.stores_[pin_.owner_]->remote_population_generation_.load( + std::memory_order_acquire) == generation_ && + engine.DbEpoch(db_id_) == db_epoch_ && + engine.remote_replication_epochs_[slot_].load( + std::memory_order_acquire) == replication_epoch_ && + !engine.replica_loading_.load(std::memory_order_acquire) && + !engine.replica_recovery_fenced_.load(std::memory_order_acquire) && + !engine.request_serving_fenced_until_restart_.load( + std::memory_order_acquire); +} + +StorageEngine::RemoteGetAwaitable::Result +StorageEngine::RemoteGetAwaitable::await_resume() { + if (!io_) { + if (pin_.engine_ != nullptr && !Current(*pin_.engine_)) return std::nullopt; + return std::move(ready_); + } + auto read = io_->await_resume(); + auto& engine = *pin_.engine_; + if (!read.ok() || *read != read_bytes_) { + if (!Current(engine) || (!read.ok() && absl::IsAborted(read.status()))) + return std::nullopt; + return read.ok() ? absl::InternalError("short compact record read") + : read.status(); + } + auto loaded = + engine.DecodeRemoteValue(std::move(lease_), db_id_, key_, location_, + replication_epoch_, record_headroom_, db_epoch_); + if (!Current(engine) || (!loaded.ok() && absl::IsAborted(loaded.status()))) + return std::nullopt; + if (!loaded.ok()) return loaded.status(); + return engine.EncodeDiskValue(std::move(*loaded)); } // Keep optimistic-read admission out of the already large command dispatcher. @@ -1424,10 +1492,9 @@ StorageEngine::Impl::LoadValueAwaitable StorageEngine::Impl::LoadValue( StorageEngine::Impl::LoadValueAwaitable StorageEngine::Impl::LoadValueLocal( WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, - ReadLatencyTrace* trace, std::optional expected_db_epoch, - bool remotely_pinned) { + ReadLatencyTrace* trace, std::optional expected_db_epoch) { return LoadValueAwaitable(*this, store, db_id, key, location, - replication_epoch, trace, expected_db_epoch, remotely_pinned); + replication_epoch, trace, expected_db_epoch); } StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( @@ -1464,8 +1531,7 @@ StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( Impl& engine, WorkerStore& store, std::uint8_t db_id, std::string_view key, RecordLocation location, std::uint64_t replication_epoch, - ReadLatencyTrace* trace, std::optional expected_db_epoch, - bool remotely_pinned) + ReadLatencyTrace* trace, std::optional expected_db_epoch) : engine_(engine), store_(store), key_(key), @@ -1473,7 +1539,6 @@ StorageEngine::Impl::LoadValueAwaitable::LoadValueAwaitable( trace_(trace), replication_epoch_(replication_epoch), expected_db_epoch_(expected_db_epoch), - remotely_pinned_(remotely_pinned), db_id_(db_id) { PrepareLocal(); } @@ -1527,13 +1592,13 @@ void StorageEngine::Impl::LoadValueAwaitable::PrepareLocal() { // Resolve staging memory only after acquiring the buffer: waiting can let // flush/defrag retire it. The staged copy itself never suspends, so only // disk I/O needs a physical pin. - auto* state = remotely_pinned_ ? nullptr : engine_.FindBlockState(store_, location_.block_id()); - if (!remotely_pinned_ && (!state || !state->allocated_ || state->freeing_ || - state->allocation_epoch_ != location_.allocation_epoch())) { + auto* state = engine_.FindBlockState(store_, location_.block_id()); + if (!state || !state->allocated_ || state->freeing_ || + state->allocation_epoch_ != location_.allocation_epoch()) { ready_.emplace(absl::AbortedError("stale index block epoch")); return; } - if (!remotely_pinned_ && location_.in_memory() && state->in_memory_) [[unlikely]] { + if (location_.in_memory() && state->in_memory_) [[unlikely]] { auto buffer = engine_.StagingBufferFor(store_, *state); if (!buffer.data_ || location_.record_offset() + location_.total_disk_bytes() > @@ -1558,7 +1623,7 @@ void StorageEngine::Impl::LoadValueAwaitable::PrepareLocal() { "record exceeds registered read buffer capacity")); return; } - if (state) state->PinLocal(); + state->PinLocal(); pinned_ = state; if (trace_) trace_->disk_read_ = true; } @@ -1738,4 +1803,59 @@ StorageEngine::Impl::LoadValueAwaitable::FinishRecord(const std::byte* data, return LoadedValue{std::move(lease_), offset, payload.bytes_, payload.txid_}; } +absl::StatusOr +StorageEngine::Impl::DecodeRemoteValue(ReadBufferLease lease, + std::uint8_t db_id, std::string_view key, + RecordLocation location, + std::uint64_t replication_epoch, + std::size_t record_headroom, + std::uint64_t expected_db_epoch) { + const FixedBuffer io = lease.io_buffer(); + RecordHeader record{}; + std::string_view disk_key; + const std::byte* record_data = io.data_ + record_headroom; + std::span record_bytes(record_data, + location.total_disk_bytes()); + if (!DecodeRecordHeader(record_bytes, &record, &disk_key) || + record.db_id_ != db_id || (!record.key_indirect_ && disk_key != key) || + record.kind_ != RecordKind::kValue || + record.db_epoch_ != expected_db_epoch || + record.mutation_sequence_ != location.mutation_sequence_ || + record.replication_epoch_ != replication_epoch || + record.allocation_epoch_ != location.allocation_epoch() || + record.expire_at_ms_ != location.expire_at_ms_ || + record.value_type_ != location.value_type() || + record.external_ != location.external() || + record.key_indirect_ != location.key_indirect() || + record.logical_size_ != location.logical_size_ || + record.total_disk_bytes_ != location.total_disk_bytes()) { + return absl::Status(absl::StatusCode::kAborted, + "record does not match in-memory location"); + } + const std::byte* payload_data = record_data + record.header_bytes_; + if (record.key_indirect_ && record.key_bytes_ != key.size()) { + return absl::Status(absl::StatusCode::kAborted, + "record key does not match location"); + } + const std::size_t value_bytes = record.payload_bytes_; + if (record.value_type_ == ValueType::kString && !record.grouped_ && + !record.auxiliary_group_ && value_bytes != record.logical_size_) { + return absl::Status(absl::StatusCode::kInternal, + "inline value length does not match metadata"); + } + if (Crc32c(std::span(payload_data, record.payload_bytes_)) != + record.payload_checksum_) { + return absl::Status(absl::StatusCode::kInternal, + "record value checksum mismatch"); + } + const std::byte* framed_value = payload_data; + if (value_bytes > DirectGetValueLimit()) { + std::memmove(io.data_, payload_data, value_bytes); + framed_value = io.data_; + } + const std::size_t value_offset = + static_cast(framed_value - lease.bytes().data()); + return LoadedValue{std::move(lease), value_offset, value_bytes, record.txid_}; +} + } // namespace lavik::storage diff --git a/tests/load_value_awaitable_test.cpp b/tests/load_value_awaitable_test.cpp index d5e1a4302..23b594273 100644 --- a/tests/load_value_awaitable_test.cpp +++ b/tests/load_value_awaitable_test.cpp @@ -33,6 +33,7 @@ class LoadValueAwaitableTestPeer : public testing::Test { engine.stores_.push_back( std::make_unique()); auto& store = *engine.stores_[0]; + store.engine_ = &engine; store.worker_ = &worker_; store.partitions_.resize(1); store.partition_indexes_.resize(kLogicalDatabaseCount); @@ -271,7 +272,7 @@ TEST_F(LoadValueAwaitableTestPeer, BufferWaitersPreserveFifoHandoff) { Start(first_task, true); Start(second_task); EXPECT_TRUE(BufferWaitIsBackground(first)); - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); held->Reset(); // Handoff schedules I/O; it must not resume inside the release stack. EXPECT_FALSE(first_task.done()); @@ -298,7 +299,7 @@ TEST_F(LoadValueAwaitableTestPeer, PopulationChangeDuringBufferWaitSkipsIo) { ASSERT_NO_FATAL_FAILURE(Drain(task)); EXPECT_TRUE(absl::IsNotFound(std::move(task).TakeResult().status())); EXPECT_FALSE(HasHelper(read)); - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); } @@ -308,7 +309,7 @@ TEST_F(LoadValueAwaitableTestPeer, RelocationDuringBufferWaitUsesNewLocation) { auto read = Read(); auto task = AwaitExisting(read); Start(task, true); - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); // No physical pointer was retained while waiting: recycle before handoff. block_.Reset(0, 8); block_.allocated_ = true; @@ -320,7 +321,7 @@ TEST_F(LoadValueAwaitableTestPeer, RelocationDuringBufferWaitUsesNewLocation) { ASSERT_NO_FATAL_FAILURE(Drain(task)); ExpectValue(std::move(task).TakeResult()); EXPECT_TRUE(HasHelper(read)); - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); } @@ -332,7 +333,7 @@ TEST_F(LoadValueAwaitableTestPeer, ExtentWaitRetainsPinAndReturnsSlice) { { auto read = ExtentSlice(ExtentReference(), 2, output); EXPECT_FALSE(read.await_ready()); - EXPECT_EQ(block_.pins_, 1); + EXPECT_EQ(block_.pin_count(), 1); auto task = AwaitExtent(read); Start(task, true); held->Reset(); @@ -344,7 +345,7 @@ TEST_F(LoadValueAwaitableTestPeer, ExtentWaitRetainsPinAndReturnsSlice) { output.size()), std::string_view(value_).substr(2, output.size())); } - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); } @@ -362,7 +363,7 @@ TEST_F(LoadValueAwaitableTestPeer, ExtentSliceChecksWholePayloadBeforeCopying) { EXPECT_EQ(output[0], std::byte{0x55}); EXPECT_EQ(output[1], std::byte{0x55}); } - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); } @@ -375,7 +376,7 @@ TEST_F(LoadValueAwaitableTestPeer, CompletionTransfersLeaseAndReleasesPin) { result = read.await_resume(); ExpectValue(*result); } - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 0); result.reset(); EXPECT_EQ(Pool().available_read_buffers(), 1); @@ -397,7 +398,7 @@ TEST_F(LoadValueAwaitableTestPeer, PopulationChangesDiscardCompletedRead) { EXPECT_FALSE(HasHelper(read)); EXPECT_TRUE(absl::IsNotFound(read.await_resume().status())); } - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); if (database) ++header_.db_epoch_; @@ -425,7 +426,7 @@ TEST_F(LoadValueAwaitableTestPeer, ErrorsReleaseResources) { header_.payload_checksum_ ^= 1; EXPECT_FALSE(read.await_resume().ok()); } - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); } } @@ -447,7 +448,7 @@ TEST_F(LoadValueAwaitableTestPeer, ASSERT_NO_FATAL_FAILURE(Complete(read)); EXPECT_FALSE(task.done()); EXPECT_TRUE(HelperIsBackground(read)); - EXPECT_EQ(block_.pins_, 0); + EXPECT_EQ(block_.pin_count(), 0); EXPECT_EQ(Pool().available_read_buffers(), 1); --header_.allocation_epoch_; // The new physical record has the same logical mutation and lives in staging. diff --git a/tests/remote_get_awaitable_test.cpp b/tests/remote_get_awaitable_test.cpp new file mode 100644 index 000000000..a9b93b070 --- /dev/null +++ b/tests/remote_get_awaitable_test.cpp @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "../src/storage/engine/impl.h" +#include "gtest/gtest.h" + +namespace lavik::storage { + +// Build a tiny published index and deliver deterministic CQEs without issuing +// disk I/O. The network e2e covers real submission; this fixture can invalidate +// admission exactly between snapshot and completion, and inspect every lease. +class RemoteGetTestPeer : public testing::Test { + protected: + using Awaitable = StorageEngine::RemoteGetAwaitable; + + void SetUp() override { + saved_context_ = std::move(bycorf::MutableThisWorker()); + wake_fd_ = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + ASSERT_GE(wake_fd_, 0); + cross_core_.mailbox(0).wake_fd_ = wake_fd_; + worker_.BindCrossCore(0, &cross_core_); + owner_.BindCrossCore(1, &cross_core_); + bycorf::SetThisWorker(0, &cross_core_, &worker_); + auto status = worker_.Init({.recv_buffer_count_ = 0}); + ASSERT_TRUE(status.ok()) << status; + auto& engine = *storage_.impl_; + engine.worker_count_ = 2; + engine.stores_.push_back( + std::make_unique()); + engine.stores_.push_back( + std::make_unique()); + engine.stores_[0]->engine_ = &engine; + engine.stores_[0]->worker_ = &worker_; + auto& source = *engine.stores_[1]; + source.engine_ = &engine; + source.worker_ = &owner_; + source.partitions_.resize(1); + source.partitions_[0].id_ = 1; + source.partitions_[0].indexes_ = std::span(&index_, 1); + source.remote_reads_ready_.store(true); + source.remote_population_generation_.store(2); + engine.db_epochs_[0].store(1); + engine.remote_replication_epochs_[1].store(1); + engine.devices_.push_back(StorageDevice{.capacity_blocks_ = 2}); + engine.block_state_lookup_.push_back({&block_, 1, 2}); + engine.stores_[0]->files_.push_back(bycorf::FixedFile{}); + block_.Reset(1, 7); + block_.allocated_ = true; + + RegisteredBufferPoolOptions options; + options.storage_write_buffer_count_ = 1; + options.registered_bytes_ = + options.write_buffer_bytes_ + options.read_slot_bytes_; + ASSERT_TRUE(Pool().Init(worker_, options).ok()); + ASSERT_EQ(Pool().available_read_buffers(), 1); + + for (unsigned i = 0;; ++i) { + key_ = "remote-" + std::to_string(i); + if (RedisSlot(key_) == 1) break; + } + auto arena = std::make_shared(63); + ASSERT_TRUE(arena->PreallocateDirectory()); + index_ = RecordIndex(std::move(arena)); + header_.header_bytes_ = RecordHeaderBytes(key_.size()); + header_.key_bytes_ = key_.size(); + header_.value_type_ = ValueType::kString; + header_.logical_size_ = value_.size(); + header_.payload_bytes_ = value_.size(); + header_.total_disk_bytes_ = + AlignRecord(header_.header_bytes_ + value_.size()); + header_.allocation_epoch_ = 7; + header_.mutation_sequence_ = 9; + header_.payload_checksum_ = Crc32c(std::as_bytes(std::span(value_))); + const auto metadata = RecordLocation::PackedMetadata::Encode( + kBlockHeaderBytes + 8, header_.total_disk_bytes_, 1, false, false, + false, false, false, false, RecordKind::kValue, ValueType::kString); + ASSERT_NE( + index_.InsertNew(ComputeDigest(key_), key_, + RecordLocation(1, 9, 7, 0, value_.size(), metadata)), + nullptr); + index_.EnableConcurrentReads(); + } + + void TearDown() override { + worker_.Shutdown(); + storage_.impl_.reset(); + if (wake_fd_ >= 0) close(wake_fd_); + bycorf::MutableThisWorker() = std::move(saved_context_); + } + + RegisteredBufferPool& Pool() { return storage_.impl_->stores_[0]->buffers_; } + auto Read() { return storage_.TryRemoteGet(0, key_, 1); } + + void Complete(Awaitable& read, int bytes = -1) { + ASSERT_TRUE(read.io_.has_value()); + const auto io = read.lease_.io_buffer(); + auto record = + std::span(io.data_ + read.record_headroom_, header_.total_disk_bytes_); + header_.db_epoch_ = read.db_epoch_; + header_.replication_epoch_ = read.replication_epoch_; + ASSERT_TRUE( + EncodeRecordHeader(header_, key_, record.first(header_.header_bytes_))); + std::memcpy(record.data() + header_.header_bytes_, value_.data(), + value_.size()); + // No awaiting handle was registered: Complete stores the result, and the + // worker ignores the empty continuation. await_resume is otherwise real. + read.io_->Complete(worker_, bytes == -1 ? read.read_bytes_ : bytes, 0); + } + + void Invalidate(unsigned which) { + auto& engine = *storage_.impl_; + switch (which) { + case 0: + engine.stores_[1]->remote_population_generation_.fetch_add(2); + break; + case 1: + engine.db_epochs_[0].fetch_add(1); + break; + case 2: + engine.remote_replication_epochs_[1].fetch_add(1); + break; + case 3: + engine.replica_loading_.store(true); + break; + case 4: + engine.replica_recovery_fenced_.store(true); + break; + case 5: + engine.request_serving_fenced_until_restart_.store(true); + break; + } + } + + void ResetFences() { + auto& engine = *storage_.impl_; + engine.replica_loading_.store(false); + engine.replica_recovery_fenced_.store(false); + engine.request_serving_fenced_until_restart_.store(false); + } + + bycorf::CurrentWorker saved_context_; + bycorf::CrossCore cross_core_{2}; + int wake_fd_ = -1; + bycorf::Worker worker_; + bycorf::Worker owner_; + RecordIndex index_; + BlockState block_; + StorageEngine storage_{StorageEngineOptions{}}; + std::string key_; + std::string value_ = "remote value"; + RecordHeader header_; +}; + +TEST_F(RemoteGetTestPeer, SynchronousMissAndClosedIndexFallback) { + { + auto read = storage_.TryRemoteGet(0, "absent", 1); + ASSERT_TRUE(read.await_ready()); + auto result = read.await_resume(); + ASSERT_TRUE(result.has_value()); + EXPECT_TRUE(absl::IsNotFound(result->status())); + } + auto writer = index_.CloseAllReaders(); + auto read = Read(); + ASSERT_TRUE(read.await_ready()); + EXPECT_FALSE(read.await_resume().has_value()); + EXPECT_EQ(block_.pin_count(), 0); +} + +TEST_F(RemoteGetTestPeer, UnawaitedReadReleasesPinAndBuffer) { + { + auto read = Read(); + EXPECT_FALSE(read.await_ready()); + EXPECT_EQ(block_.pin_count(), 1); + EXPECT_EQ(Pool().available_read_buffers(), 0); + } + EXPECT_EQ(block_.pin_count(), 0); + EXPECT_EQ(Pool().available_read_buffers(), 1); +} + +TEST_F(RemoteGetTestPeer, ExhaustedPoolFallsBackWithoutEnqueuingWaiter) { + { + auto held = Pool().AcquireReadBuffer().await_resume(); + ASSERT_TRUE(held.ok()); + auto read = Read(); + ASSERT_TRUE(read.await_ready()); + EXPECT_FALSE(read.await_resume().has_value()); + } + EXPECT_EQ(block_.pin_count(), 0); + EXPECT_EQ(Pool().available_read_buffers(), 1); +} + +TEST_F(RemoteGetTestPeer, CompletionTransfersReplyLeaseAndReleasesPin) { + Awaitable::Result result; + { + auto read = Read(); + ASSERT_FALSE(read.await_ready()); + ASSERT_NO_FATAL_FAILURE(Complete(read)); + result = read.await_resume(); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result->ok()) << result->status(); + auto bytes = result->value().value_bytes(); + EXPECT_EQ(std::string_view(reinterpret_cast(bytes.data()), + bytes.size()), + value_); + } + EXPECT_EQ(block_.pin_count(), 0); + EXPECT_EQ(Pool().available_read_buffers(), 0); + result.reset(); + EXPECT_EQ(Pool().available_read_buffers(), 1); +} + +TEST_F(RemoteGetTestPeer, PopulationChangesDiscardCompletedRead) { + // Reset only test admission so each independent invalidation starts with a + // fresh snapshot; production serving fences remain irreversible. + for (unsigned which = 0; which < 6; ++which) { + { + auto read = Read(); + ASSERT_FALSE(read.await_ready()) << which; + ASSERT_NO_FATAL_FAILURE(Complete(read)); + Invalidate(which); + EXPECT_FALSE(read.await_resume().has_value()) << which; + } + EXPECT_EQ(block_.pin_count(), 0); + EXPECT_EQ(Pool().available_read_buffers(), 1); + // Epoch changes persist and are snapshotted by the next read. + ResetFences(); + } +} + +TEST_F(RemoteGetTestPeer, CorruptShortAndFailedReadsReleaseResources) { + for (int completion : {0, -EIO, 4096}) { + { + auto read = Read(); + ASSERT_FALSE(read.await_ready()); + header_.payload_checksum_ ^= 1; + ASSERT_NO_FATAL_FAILURE(Complete(read, completion)); + header_.payload_checksum_ ^= 1; + auto result = read.await_resume(); + ASSERT_TRUE(result.has_value()); + EXPECT_FALSE(result->ok()); + } + EXPECT_EQ(block_.pin_count(), 0); + EXPECT_EQ(Pool().available_read_buffers(), 1); + } +} + +} // namespace lavik::storage From 1de8cf97cde65b6b4cbe999bf401321acdf3af26 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 06:04:01 +0000 Subject: [PATCH 11/15] Fix remote read flush wakeups and GET latency traces Close remote pin admission before testing whether a sealed staging buffer can be released. Both flush completion paths use the same helper so the last remote unpin either precedes the zero-count check or notifies the owner; write buffers cannot be stranded behind release_pending with no wakeup. Initialize GET tracing before remote admission and attach it to fast hits, misses, and errors. Preserve the original request start across fallback while resetting per-path stages. Trace-only I/O completion records the actual CQE time; trace-disabled builds retain the existing awaitable layout and I/O path. Validation: 856 Debug unit tests, 20 trace-enabled awaitable tests, remote GET and defrag payload end-to-end tests. Live trace logs contain all 1,025 requests in each of fast miss, durable hit, and indirect-key owner fallback workloads, with the expected remote/hit/disk ratios. --- docs/architecture/04-storage-and-recovery.md | 12 +-- include/lavik/storage/engine.h | 41 +++++++++- src/redis/command.cpp | 40 ++++++++-- src/storage/engine.cpp | 14 +++- src/storage/engine/flush.cpp | 31 ++++---- src/storage/engine/impl.h | 3 + src/storage/engine/read.cpp | 82 +++++++++++++++++++- tests/remote_get_awaitable_test.cpp | 80 ++++++++++++++++++- 8 files changed, 268 insertions(+), 35 deletions(-) diff --git a/docs/architecture/04-storage-and-recovery.md b/docs/architecture/04-storage-and-recovery.md index d431b6ec6..c9865b89d 100644 --- a/docs/architecture/04-storage-and-recovery.md +++ b/docs/architecture/04-storage-and-recovery.md @@ -684,11 +684,13 @@ The index snapshot orders the GET before a subsequently admitted writer. The block pin survives I/O and cancellation cleanup, so writes can append and retire old live-byte accounting while the read finishes. Defrag closes remote pin admission before inspecting the count; the last pin on a pending block asks its -owner to reconsider reclamation. Local fallback reads retain their existing pin -rules. Database/replication epoch and population-generation changes invalidate -an in-flight fast read. Worker shutdown drains I/O and destroys suspended frames -on all workers before destroying any remotely referenced index, arena or block -state. +owner to reconsider reclamation. Flush uses the same close-before-count order +when retiring a sealed staging buffer: either it releases the buffer immediately +or the last pin notifies the owner to finish the release. Local fallback reads +retain their existing pin rules. Database/replication epoch and population-generation +changes invalidate an in-flight fast read. Worker shutdown drains I/O and destroys +suspended frames on all workers before destroying any remotely referenced index, +arena or block state. An owner-local ordinary single-key GET may resolve its complete inline-key index entry without registering a shared transaction lock when that logical database's diff --git a/include/lavik/storage/engine.h b/include/lavik/storage/engine.h index b2a77dd8e..7def8131c 100644 --- a/include/lavik/storage/engine.h +++ b/include/lavik/storage/engine.h @@ -1612,8 +1612,15 @@ class StorageEngine { // Admission runs synchronously; immediately co_await the result on this // worker, keeping key and request admission alive through completion. The // fast path submits I/O without a child Task; buffer contention falls back. + // Trace builds optionally borrow caller-owned timing state through + // completion. RemoteGetAwaitable TryRemoteGet(std::uint8_t db_id, std::string_view key, - std::uint16_t slot); + std::uint16_t slot +#if LAVIK_ENABLE_TRACE + , + ReadLatencyTrace* trace = nullptr +#endif + ); // These operations must execute on OwnerForKey(key), normally through // SubmitTaskTo. Only digest/location metadata is retained after completion. @@ -1995,7 +2002,12 @@ class StorageEngine::RemoteGetAwaitable { }; RemoteGetAwaitable(Impl& engine, std::uint8_t db_id, std::string_view key, - std::uint16_t slot); + std::uint16_t slot +#if LAVIK_ENABLE_TRACE + , + ReadLatencyTrace* trace +#endif + ); bool Current(const Impl& engine) const noexcept; Result ready_; @@ -2012,7 +2024,32 @@ class StorageEngine::RemoteGetAwaitable { // including when an admitted awaitable is destroyed without being awaited. Pin pin_; ReadBufferLease lease_; +#if LAVIK_ENABLE_TRACE + // Timestamp the actual CQE before the connection is scheduled again. This + // trace-only completion leaves the production I/O awaitable and frame intact. + class TracedReadIo : public bycorf::OneShotIoAwaitable { + public: + TracedReadIo(bycorf::Worker& worker, bycorf::FixedFile file, + bycorf::FixedBuffer buffer, std::uint64_t offset, + bycorf::SizeIoAwaitable::Operation operation, + ReadLatencyTrace* trace); + bool await_suspend(std::coroutine_handle<> awaiting); + absl::StatusOr await_resume(); + void Complete(bycorf::Worker& worker, int result, unsigned flags) override; + + private: + bycorf::Worker& worker_; + bycorf::FixedFile file_; + bycorf::FixedBuffer buffer_; + std::uint64_t offset_; + bool fixed_; + ReadLatencyTrace* trace_; + }; + ReadLatencyTrace* trace_; + std::optional io_; +#else std::optional io_; +#endif }; } // namespace lavik::storage diff --git a/src/redis/command.cpp b/src/redis/command.cpp index c3914874e..58427fd95 100644 --- a/src/redis/command.cpp +++ b/src/redis/command.cpp @@ -13116,31 +13116,57 @@ Task ExecuteCommandBody( const std::uint16_t slot = routed ? request.RoutedPartitionId() : storage::RedisSlot(args[1]); const unsigned target = slot % g_storage->worker_count(); +#if LAVIK_ENABLE_TRACE + ReadLatencyTrace trace; + if (request.kind_ == CommandKind::kGet) { + trace.request_start_ns_ = ReadTraceNowNanos(); + // Remote describes key ownership, even when this worker issues I/O. + trace.remote_ = target != ThisWorker().id_; + } +#endif if (request.kind_ == CommandKind::kGet && args.size() == 2 && target != ThisWorker().id_) { // Reuse the route hash for admission on the connection worker. +#if LAVIK_ENABLE_TRACE + trace.owner_start_ns_ = trace.request_start_ns_; +#endif auto fast = - co_await g_storage->TryRemoteGet(request.db_id_, args[1], slot); + co_await g_storage->TryRemoteGet(request.db_id_, args[1], slot +#if LAVIK_ENABLE_TRACE + , + &trace +#endif + ); if (fast) { + CommandReply reply; if (const char* error = CommandServingGenerationError(request); error) - co_return BuiltReply(reply_builder.AppendError(error)); - CommandReply reply; - if (fast->ok()) + reply.encoded_ = reply_builder.AppendError(error); + else if (fast->ok()) reply.disk_value_ = std::move(**fast); else if (absl::IsNotFound(fast->status())) reply.encoded_ = reply_builder.AppendNull(); else reply.encoded_ = AppendStorageError(reply_builder, fast->status()); +#if LAVIK_ENABLE_TRACE + trace.owner_done_ns_ = ReadTraceNowNanos(); + trace.origin_resume_ns_ = trace.owner_done_ns_; + reply.read_trace_ = trace; +#endif co_return reply; } +#if LAVIK_ENABLE_TRACE + // Keep attempted fast-read time in total/route-out latency, but let + // the owner result determine hit/disk flags and its stage timings. + const auto started = trace.request_start_ns_; + trace = {}; + trace.request_start_ns_ = started; + trace.remote_ = true; +#endif } #if LAVIK_ENABLE_TRACE if (request.kind_ == CommandKind::kGet) { - ReadLatencyTrace trace; - trace.request_start_ns_ = ReadTraceNowNanos(); - trace.remote_ = target != ThisWorker().id_; CommandReply reply; if (trace.remote_) { reply = co_await SubmitTaskTo( diff --git a/src/storage/engine.cpp b/src/storage/engine.cpp index 6e6c031e9..14690a1fa 100644 --- a/src/storage/engine.cpp +++ b/src/storage/engine.cpp @@ -635,8 +635,18 @@ void StorageEngine::SetReplicaLoading(bool loading) noexcept { } StorageEngine::RemoteGetAwaitable StorageEngine::TryRemoteGet( - std::uint8_t db_id, std::string_view key, std::uint16_t slot) { - return RemoteGetAwaitable(*impl_, db_id, key, slot); + std::uint8_t db_id, std::string_view key, std::uint16_t slot +#if LAVIK_ENABLE_TRACE + , + ReadLatencyTrace* trace +#endif +) { + return RemoteGetAwaitable(*impl_, db_id, key, slot +#if LAVIK_ENABLE_TRACE + , + trace +#endif + ); } Task> StorageEngine::Get( diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index 65bef19d3..dea58e537 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -284,13 +284,7 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { if (state->kind_ == BlockKind::kTransaction) { tx_cleaner_dirty_.store(true, std::memory_order_release); } - if (state->pin_count() > 0) { - state->release_pending_ = true; - state->CloseRemotePins(); - } else { - ReleaseStagingBuffer(*store, *state); - MaybeQueueDefrag(*store, block_id); - } + ReleaseFlushedStagingBuffer(*store, *state, block_id); } continue; } @@ -601,14 +595,23 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { if (state->kind_ == BlockKind::kTransaction) { tx_cleaner_dirty_.store(true, std::memory_order_release); } - if (state->pin_count() > 0) { - state->release_pending_ = true; - state->CloseRemotePins(); - continue; - } - ReleaseStagingBuffer(*store, *state); - MaybeQueueDefrag(*store, pending->block_id_); + ReleaseFlushedStagingBuffer(*store, *state, pending->block_id_); + } +} + +void StorageEngine::Impl::ReleaseFlushedStagingBuffer(WorkerStore& store, + BlockState& state, + std::uint64_t block_id) { + // Close before checking the count: a last remote unpin either precedes this + // observation (release here) or sees the closed bit and notifies the owner. + // Checking first can strand release_pending_ with zero pins and no wakeup. + state.CloseRemotePins(); + if (state.pin_count() != 0) { + state.release_pending_ = true; + return; } + ReleaseStagingBuffer(store, state); + MaybeQueueDefrag(store, block_id); } bool StorageEngine::Impl::IsActiveBlock(const WorkerStore& store, diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 402c88638..01a79f3be 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -3416,6 +3416,9 @@ class StorageEngine::Impl { const BlockState& state) const; static void ReleaseStagingBuffer(WorkerStore& store, BlockState& state); + // Retires a sealed, durable staging buffer after draining remote pins. + void ReleaseFlushedStagingBuffer(WorkerStore& store, BlockState& state, + std::uint64_t block_id); struct LoadedValue { ReadBufferLease lease_; diff --git a/src/storage/engine/read.cpp b/src/storage/engine/read.cpp index 6827a524f..a267d0dd7 100644 --- a/src/storage/engine/read.cpp +++ b/src/storage/engine/read.cpp @@ -229,8 +229,20 @@ StorageEngine::Impl::RandomKeyLocal(std::uint8_t db_id) { StorageEngine::RemoteGetAwaitable::RemoteGetAwaitable(Impl& engine, std::uint8_t db_id, std::string_view key, - std::uint16_t slot) - : key_(key), slot_(slot), db_id_(db_id) { + std::uint16_t slot +#if LAVIK_ENABLE_TRACE + , + ReadLatencyTrace* trace +#endif + ) + : key_(key), + slot_(slot), + db_id_(db_id) +#if LAVIK_ENABLE_TRACE + , + trace_(trace) +#endif +{ engine.PrepareRemoteGet(*this); } @@ -263,6 +275,9 @@ void StorageEngine::Impl::PrepareRemoteGet(RemoteGetAwaitable& result) { auto reader = index.TryRead(digest); if (!reader) return; const auto* entry = std::as_const(index).Find(digest, key); +#if LAVIK_ENABLE_TRACE + if (result.trace_) result.trace_->lookup_done_ns_ = ReadTraceNowNanos(); +#endif if (!result.Current(*this)) return; if (entry && !entry->key_complete()) return; if (entry == nullptr || entry->value_.kind() == RecordKind::kTombstone) { @@ -316,6 +331,12 @@ void StorageEngine::Impl::PrepareRemoteGet(RemoteGetAwaitable& result) { // Admission is only an optimization. If the local bounded pool is busy, // let the owner path wait for its buffer rather than adding a child task or // retaining a remote pin in a second asynchronous state machine. +#if LAVIK_ENABLE_TRACE + if (result.trace_) { + result.trace_->hit_ = true; + result.trace_->buffer_acquire_start_ns_ = ReadTraceNowNanos(); + } +#endif auto acquire = local.buffers_.AcquireReadBuffer(result.read_bytes_); if (!acquire.await_ready()) return; auto acquired = acquire.await_resume(); @@ -324,6 +345,12 @@ void StorageEngine::Impl::PrepareRemoteGet(RemoteGetAwaitable& result) { return; } result.lease_ = std::move(*acquired); +#if LAVIK_ENABLE_TRACE + if (result.trace_) { + result.trace_->buffer_acquired_ns_ = ReadTraceNowNanos(); + result.trace_->heap_read_buffer_ = result.lease_.buffer_id() == 0; + } +#endif auto buffer = result.lease_.io_buffer(); assert(result.read_bytes_ <= buffer.size_); buffer.size_ = result.read_bytes_; @@ -331,9 +358,55 @@ void StorageEngine::Impl::PrepareRemoteGet(RemoteGetAwaitable& result) { aligned_offset, result.lease_.registered() ? bycorf::SizeIoAwaitable::Operation::kReadFixed - : bycorf::SizeIoAwaitable::Operation::kRead); + : bycorf::SizeIoAwaitable::Operation::kRead +#if LAVIK_ENABLE_TRACE + , + result.trace_ +#endif + ); +} + +#if LAVIK_ENABLE_TRACE +StorageEngine::RemoteGetAwaitable::TracedReadIo::TracedReadIo( + bycorf::Worker& worker, bycorf::FixedFile file, bycorf::FixedBuffer buffer, + std::uint64_t offset, bycorf::SizeIoAwaitable::Operation operation, + ReadLatencyTrace* trace) + : worker_(worker), + file_(file), + buffer_(buffer), + offset_(offset), + fixed_(operation == bycorf::SizeIoAwaitable::Operation::kReadFixed), + trace_(trace) {} + +bool StorageEngine::RemoteGetAwaitable::TracedReadIo::await_suspend( + std::coroutine_handle<> awaiting) { + if (trace_) { + trace_->disk_read_ = true; + trace_->io_submit_ns_ = ReadTraceNowNanos(); + } + auto status = fixed_ ? worker_.SubmitReadFixed(file_, buffer_, offset_, this) + : worker_.SubmitRead( + file_, std::span(buffer_.data_, buffer_.size_), + offset_, this); + if (!status.ok() && trace_) trace_->io_complete_ns_ = ReadTraceNowNanos(); + return Suspend(awaiting, std::move(status)); } +void StorageEngine::RemoteGetAwaitable::TracedReadIo::Complete( + bycorf::Worker& worker, int result, unsigned flags) { + if (trace_) trace_->io_complete_ns_ = ReadTraceNowNanos(); + OneShotIoAwaitable::Complete(worker, result, flags); +} + +absl::StatusOr +StorageEngine::RemoteGetAwaitable::TracedReadIo::await_resume() { + auto result = Resume(fixed_ ? "fixed-buffer read failed" : "read failed"); + if (!result.ok()) return result.status(); + worker_.RecordStorageReadCompletion(*result); + return static_cast(*result); +} +#endif + StorageEngine::RemoteGetAwaitable::Pin::~Pin() { if (engine_ != nullptr) { engine_->UnpinRemoteBlock(engine_->BlockStateAt(block_id_), owner_, @@ -371,6 +444,9 @@ StorageEngine::RemoteGetAwaitable::await_resume() { auto loaded = engine.DecodeRemoteValue(std::move(lease_), db_id_, key_, location_, replication_epoch_, record_headroom_, db_epoch_); +#if LAVIK_ENABLE_TRACE + if (trace_) trace_->decode_done_ns_ = ReadTraceNowNanos(); +#endif if (!Current(engine) || (!loaded.ok() && absl::IsAborted(loaded.status()))) return std::nullopt; if (!loaded.ok()) return loaded.status(); diff --git a/tests/remote_get_awaitable_test.cpp b/tests/remote_get_awaitable_test.cpp index a9b93b070..0709b4257 100644 --- a/tests/remote_get_awaitable_test.cpp +++ b/tests/remote_get_awaitable_test.cpp @@ -93,7 +93,54 @@ class RemoteGetTestPeer : public testing::Test { } RegisteredBufferPool& Pool() { return storage_.impl_->stores_[0]->buffers_; } - auto Read() { return storage_.TryRemoteGet(0, key_, 1); } + auto Read() { + return storage_.TryRemoteGet(0, key_, 1 +#if LAVIK_ENABLE_TRACE + , + &trace_ +#endif + ); + } + + void CheckFlushedBufferRelease(bool unpin_first) { + auto& engine = *storage_.impl_; + auto& store = *engine.stores_[0]; + block_.Reset(0, 7); + block_.allocated_ = true; + block_.committed_bytes_ = kBlockHeaderBytes + 4096; + block_.live_bytes_ = + 4096; // No defrag: only the flush may return this slot. + block_.staging_slot_ = store.staging_slots_.size(); + store.staging_slots_.emplace_back(); + ASSERT_TRUE(Pool().TryAcquireWriteBuffer( + &store.staging_slots_.back().write_buffer_id_)); + ASSERT_EQ(Pool().available_write_buffers(), 0); + ASSERT_TRUE(block_.TryRemotePin()); + auto unpin = [&] { + bycorf::SetThisWorker(1, &cross_core_, &owner_); + engine.UnpinRemoteBlock(block_, 0, 1); + bycorf::SetThisWorker(0, &cross_core_, &worker_); + }; + if (unpin_first) unpin(); + engine.ReleaseFlushedStagingBuffer(store, block_, 1); + if (!unpin_first) { + EXPECT_TRUE(block_.release_pending_); + EXPECT_FALSE(block_.TryRemotePin()); + EXPECT_EQ(Pool().available_write_buffers(), 0); + unpin(); + // A one-way notification alone must finish the release; no later local + // read/unpin or maintenance pass is allowed to rescue a lost wakeup. + const auto until = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (block_.release_pending_ && + std::chrono::steady_clock::now() < until) + worker_.RunOnce(false); + } + EXPECT_EQ(block_.pin_count(), 0); + EXPECT_FALSE(block_.release_pending_); + EXPECT_EQ(block_.staging_slot_, 0); + EXPECT_EQ(Pool().available_write_buffers(), 1); + } void Complete(Awaitable& read, int bytes = -1) { ASSERT_TRUE(read.io_.has_value()); @@ -153,15 +200,28 @@ class RemoteGetTestPeer : public testing::Test { std::string key_; std::string value_ = "remote value"; RecordHeader header_; +#if LAVIK_ENABLE_TRACE + ReadLatencyTrace trace_; +#endif }; TEST_F(RemoteGetTestPeer, SynchronousMissAndClosedIndexFallback) { { - auto read = storage_.TryRemoteGet(0, "absent", 1); + auto read = storage_.TryRemoteGet(0, "absent", 1 +#if LAVIK_ENABLE_TRACE + , + &trace_ +#endif + ); ASSERT_TRUE(read.await_ready()); auto result = read.await_resume(); ASSERT_TRUE(result.has_value()); EXPECT_TRUE(absl::IsNotFound(result->status())); +#if LAVIK_ENABLE_TRACE + EXPECT_NE(trace_.lookup_done_ns_, 0); + EXPECT_FALSE(trace_.hit_); + EXPECT_FALSE(trace_.disk_read_); +#endif } auto writer = index_.CloseAllReaders(); auto read = Read(); @@ -199,7 +259,18 @@ TEST_F(RemoteGetTestPeer, CompletionTransfersReplyLeaseAndReleasesPin) { auto read = Read(); ASSERT_FALSE(read.await_ready()); ASSERT_NO_FATAL_FAILURE(Complete(read)); +#if LAVIK_ENABLE_TRACE + EXPECT_NE(trace_.lookup_done_ns_, 0); + EXPECT_TRUE(trace_.hit_); + EXPECT_NE(trace_.buffer_acquired_ns_, 0); + const auto completed = trace_.io_complete_ns_; + EXPECT_NE(completed, 0); +#endif result = read.await_resume(); +#if LAVIK_ENABLE_TRACE + EXPECT_EQ(trace_.io_complete_ns_, completed); + EXPECT_GE(trace_.decode_done_ns_, completed); +#endif ASSERT_TRUE(result.has_value()); ASSERT_TRUE(result->ok()) << result->status(); auto bytes = result->value().value_bytes(); @@ -248,4 +319,9 @@ TEST_F(RemoteGetTestPeer, CorruptShortAndFailedReadsReleaseResources) { } } +TEST_F(RemoteGetTestPeer, FlushReturnsStagingBufferAcrossLastRemoteUnpin) { + ASSERT_NO_FATAL_FAILURE(CheckFlushedBufferRelease(true)); + ASSERT_NO_FATAL_FAILURE(CheckFlushedBufferRelease(false)); +} + } // namespace lavik::storage From 38ff92447143e0a8611303622fbd7ee77672b54b Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 06:21:25 +0000 Subject: [PATCH 12/15] test: cover last remote unpin during flush release decision --- src/storage/engine/flush.cpp | 7 +++++- src/storage/engine/impl.h | 5 ++++ tests/remote_get_awaitable_test.cpp | 37 ++++++++++++++++++++++++----- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index dea58e537..e6b93a711 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -606,7 +606,12 @@ void StorageEngine::Impl::ReleaseFlushedStagingBuffer(WorkerStore& store, // observation (release here) or sees the closed bit and notifies the owner. // Checking first can strand release_pending_ with zero pins and no wakeup. state.CloseRemotePins(); - if (state.pin_count() != 0) { + const auto pins = state.pin_count(); + // Tests complete the last remote read after the count snapshot, before the + // release/defer decision. Keep this boundary tied to the real observation. + LAVIK_FAULT_INJECT(if (staging_release_count_test_hook_) + staging_release_count_test_hook_();); + if (pins != 0) { state.release_pending_ = true; return; } diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 01a79f3be..8955ff25f 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -4857,6 +4857,11 @@ class StorageEngine::Impl { std::array replica_db_epoch_mutexes_; std::atomic active_defrags_{0}; std::atomic pending_defrags_{0}; +#if LAVIK_FAULTS_ENABLED + // Synchronous owner-side injection after a staging-release pin snapshot. + // Models a remote unpin without OS scheduling; absent in production builds. + std::function staging_release_count_test_hook_; +#endif std::atomic active_flushes_{0}; // Extent reclaims in flight (from the moment they are spawned): the block // allocator must not report the device full while one may still free space. diff --git a/tests/remote_get_awaitable_test.cpp b/tests/remote_get_awaitable_test.cpp index 0709b4257..c5577ee77 100644 --- a/tests/remote_get_awaitable_test.cpp +++ b/tests/remote_get_awaitable_test.cpp @@ -102,7 +102,9 @@ class RemoteGetTestPeer : public testing::Test { ); } - void CheckFlushedBufferRelease(bool unpin_first) { + enum class UnpinPoint { kBeforeRelease, kAfterCount, kAfterRelease }; + + void CheckFlushedBufferRelease(UnpinPoint point) { auto& engine = *storage_.impl_; auto& store = *engine.stores_[0]; block_.Reset(0, 7); @@ -121,13 +123,29 @@ class RemoteGetTestPeer : public testing::Test { engine.UnpinRemoteBlock(block_, 0, 1); bycorf::SetThisWorker(0, &cross_core_, &worker_); }; - if (unpin_first) unpin(); + if (point == UnpinPoint::kBeforeRelease) unpin(); +#if LAVIK_FAULTS_ENABLED + bool injected = false; + if (point == UnpinPoint::kAfterCount) { + engine.staging_release_count_test_hook_ = [&] { + injected = true; + unpin(); + }; + } +#endif engine.ReleaseFlushedStagingBuffer(store, block_, 1); - if (!unpin_first) { +#if LAVIK_FAULTS_ENABLED + engine.staging_release_count_test_hook_ = {}; + if (point == UnpinPoint::kAfterCount) { + ASSERT_TRUE(injected); + EXPECT_EQ(block_.pin_count(), 0); + } +#endif + if (point != UnpinPoint::kBeforeRelease) { EXPECT_TRUE(block_.release_pending_); EXPECT_FALSE(block_.TryRemotePin()); EXPECT_EQ(Pool().available_write_buffers(), 0); - unpin(); + if (point == UnpinPoint::kAfterRelease) unpin(); // A one-way notification alone must finish the release; no later local // read/unpin or maintenance pass is allowed to rescue a lost wakeup. const auto until = @@ -320,8 +338,15 @@ TEST_F(RemoteGetTestPeer, CorruptShortAndFailedReadsReleaseResources) { } TEST_F(RemoteGetTestPeer, FlushReturnsStagingBufferAcrossLastRemoteUnpin) { - ASSERT_NO_FATAL_FAILURE(CheckFlushedBufferRelease(true)); - ASSERT_NO_FATAL_FAILURE(CheckFlushedBufferRelease(false)); + ASSERT_NO_FATAL_FAILURE( + CheckFlushedBufferRelease(UnpinPoint::kBeforeRelease)); + ASSERT_NO_FATAL_FAILURE(CheckFlushedBufferRelease(UnpinPoint::kAfterRelease)); +#if LAVIK_FAULTS_ENABLED + // The owner has observed one pin, but the reader exits before it can mark + // release_pending_. Moving CloseRemotePins after the count strands the slot + // here: the reader saw an open gate and did not post a notification. + ASSERT_NO_FATAL_FAILURE(CheckFlushedBufferRelease(UnpinPoint::kAfterCount)); +#endif } } // namespace lavik::storage From 9feb35847cb7218dac1649fd1b12d0b12be47482 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 06:37:01 +0000 Subject: [PATCH 13/15] refactor(storage): share read validation and reader guard waits --- include/lavik/storage/scan_hash_map.h | 5 +- src/storage/engine/expire.cpp | 2 +- src/storage/engine/flush.cpp | 3 +- src/storage/engine/impl.h | 49 +++++----- src/storage/engine/init.cpp | 3 +- src/storage/engine/read.cpp | 131 +++++++++++--------------- src/storage/engine/replication.cpp | 15 +-- src/storage/engine/tomb_raider.cpp | 16 ++-- src/storage/engine/write.cpp | 7 +- tests/defrag_payload_e2e.py | 3 + tests/device_affinity_test.cpp | 29 +----- tests/load_value_awaitable_test.cpp | 3 + tests/remote_get_e2e.py | 43 ++------- 13 files changed, 129 insertions(+), 180 deletions(-) diff --git a/include/lavik/storage/scan_hash_map.h b/include/lavik/storage/scan_hash_map.h index e170791b9..bb51bfc83 100644 --- a/include/lavik/storage/scan_hash_map.h +++ b/include/lavik/storage/scan_hash_map.h @@ -1373,12 +1373,11 @@ class ScanHashMap { // Otherwise that final step could reopen unrelated readers before a later // insertion grows the shared overflow directory. if (Rehashing() || resize_pending_ || !tables_[0].buckets_) return true; - if (!Rehashing() && - tables_[0].used_ + 1 > + if (tables_[0].used_ + 1 > BucketCount(tables_[0]) * kTargetEntriesPerBucket && tables_[0].exponent_ < MaxBucketExponent) return true; - const Table& table = Rehashing() ? tables_[1] : tables_[0]; + const Table& table = tables_[0]; const Bucket* bucket = &table.buckets_[Hash(digest) & BucketMask(table)]; while (bucket != nullptr) { for (auto handle : bucket->entries_) diff --git a/src/storage/engine/expire.cpp b/src/storage/engine/expire.cpp index fe0fae576..2809f7356 100644 --- a/src/storage/engine/expire.cpp +++ b/src/storage/engine/expire.cpp @@ -386,7 +386,7 @@ Task StorageEngine::Impl::ExpireCandidate( // transaction dependencies. The key hold excludes logical writes, but GC // may replace the index entry and the group's physical coordinates. auto index_write = co_await AwaitIndexWrite( - *store.worker_, partition.indexes_[candidate.db_id_], candidate.digest_); + partition.indexes_[candidate.db_id_], candidate.digest_); resolved = co_await FindVerifiedEntry(store, partition.indexes_[candidate.db_id_], candidate.digest_, candidate.key_); diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index e6b93a711..c6bb46a57 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -518,8 +518,7 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { if (!index_write.ready()) [[unlikely]] { // Keep admission closed across the slow wait. Most flush updates // have no registered reader and need no waiting coroutine. - index_write = co_await AwaitIndexValueWrite(*store->worker_, index, - identity.entry_hash_); + index_write = co_await AwaitIndexReaders(std::move(index_write)); } // Detach can run while the reader drain yields. Recheck the population // before resolving the saved integer address into this index. diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index 8955ff25f..e22395dd8 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -113,8 +113,8 @@ using ExtentManifest = std::shared_ptr>; // These waits yield only the owner coroutine. Remote readers never suspend // while registered, so draining does not depend on store-state/key locks held // by the caller. Call before resolving entries, and revalidate after any wait. -inline Task AwaitIndexWrite( - [[maybe_unused]] Worker& worker, RecordIndex& index, const Digest& digest) { +inline Task AwaitIndexWrite(RecordIndex& index, + const Digest& digest) { auto guard = index.CloseForWrite(digest); bool waited = false; while (!guard.ready()) { @@ -133,16 +133,10 @@ inline Task AwaitIndexWrite( co_return std::move(guard); } -inline Task AwaitIndexValueWrite( - [[maybe_unused]] Worker& worker, RecordIndex& index, std::uint64_t hash) { - auto guard = index.CloseForValueUpdate(hash); - while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); - co_return std::move(guard); -} - -inline Task AwaitAllIndexReaders( - [[maybe_unused]] Worker& worker, RecordIndex& index) { - auto guard = index.CloseAllReaders(); +// The caller checks ready() before entering this slow path, so uncontended +// mutations keep their existing guard without allocating a waiting coroutine. +inline Task AwaitIndexReaders( + RecordIndex::WriteGuard guard) { while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); co_return std::move(guard); } @@ -3440,6 +3434,24 @@ class StorageEngine::Impl { } }; + struct RecordPayload { + std::uint64_t txid_; + std::uint32_t offset_; + std::uint32_t bytes_; + std::uint32_t checksum_; + }; + + // Metadata-only validation may run on the I/O completion poller. CRC and + // payload movement must stay in FinishValueRecord on the caller's task. + static absl::StatusOr ParseValueRecord( + const std::byte* data, std::uint8_t db_id, std::string_view key, + const RecordLocation& location, std::uint64_t db_epoch, + std::uint64_t replication_epoch); + absl::StatusOr FinishValueRecord(ReadBufferLease& lease, + const std::byte* data, + RecordPayload payload, + bool staged); + // The awaitable lives in the caller's frame. Ordinary local reads suspend // that caller directly, including through pool backpressure. Only multi-step // logical reads need a lazy helper Task. Local mode returns stale-location @@ -3468,19 +3480,14 @@ class StorageEngine::Impl { private: friend class LoadValueAwaitableTestPeer; - struct Payload { - std::uint64_t txid_; - std::uint32_t offset_; - std::uint32_t bytes_; - std::uint32_t checksum_; - }; void PrepareLocal(); void FinishBufferWait(); static void BufferReady(bycorf::RemoteWork* work); std::coroutine_handle<> SuspendRead(std::coroutine_handle<> awaiting); - absl::StatusOr ParseRecord(const std::byte* data) const; - Result FinishRecord(const std::byte* data, Payload payload, bool staged); + absl::StatusOr ParseRecord(const std::byte* data) const; + Result FinishRecord(const std::byte* data, RecordPayload payload, + bool staged); bool PopulationChanged() const; void Unpin(); void Retry(absl::Status failure); @@ -3500,7 +3507,7 @@ class StorageEngine::Impl { std::size_t read_bytes_ = 0; std::uint8_t db_id_; BlockState* pinned_ = nullptr; - std::optional payload_; + std::optional payload_; std::optional ready_; // Runtime shutdown drains I/O before destroying suspended frames. The // waiter retains its assigned slot until the scheduled continuation runs. diff --git a/src/storage/engine/init.cpp b/src/storage/engine/init.cpp index 3488e7e58..3da3a1c0d 100644 --- a/src/storage/engine/init.cpp +++ b/src/storage/engine/init.cpp @@ -1993,8 +1993,7 @@ Task StorageEngine::Impl::InitializeWorker(Worker& worker) { { UnlockGuard unlock(&store.store_state_mutex_, store.worker_); auto index_write = co_await AwaitIndexWrite( - *store.worker_, partition.indexes_[expired.db_id_], - expired.digest_); + partition.indexes_[expired.db_id_], expired.digest_); auto resolved = co_await FindVerifiedEntry( store, partition.indexes_[expired.db_id_], expired.digest_, expired.key_); diff --git a/src/storage/engine/read.cpp b/src/storage/engine/read.cpp index a267d0dd7..2758e7fa2 100644 --- a/src/storage/engine/read.cpp +++ b/src/storage/engine/read.cpp @@ -286,8 +286,8 @@ void StorageEngine::Impl::PrepareRemoteGet(RemoteGetAwaitable& result) { } // Cold representations and expiration retain the complete owner state // machine (including expiry scheduling and indirect-key collision checks). - if (!entry->key_complete() || entry->value_.grouped() || - entry->value_.external() || entry->value_.in_memory() || + if (entry->value_.grouped() || entry->value_.external() || + entry->value_.in_memory() || entry->value_.value_type() != ValueType::kString || IsExpiredNow(*entry)) return; const RecordLocation location = MaterializeIndexLocation(*entry); @@ -1832,51 +1832,69 @@ StorageEngine::Impl::LoadValueAwaitable::await_resume() { return FinishRecord(data, *payload_, false); } -absl::StatusOr +absl::StatusOr StorageEngine::Impl::LoadValueAwaitable::ParseRecord( const std::byte* data) const { + return ParseValueRecord( + data, db_id_, key_, location_, + expected_db_epoch_ ? *expected_db_epoch_ : engine_.DbEpoch(db_id_), + replication_epoch_); +} + +StorageEngine::Impl::LoadValueAwaitable::Result +StorageEngine::Impl::LoadValueAwaitable::FinishRecord(const std::byte* data, + RecordPayload payload, + bool staged) { + auto loaded = engine_.FinishValueRecord(lease_, data, payload, staged); + if (loaded.ok() && trace_) trace_->decode_done_ns_ = ReadTraceNowNanos(); + return loaded; +} + +absl::StatusOr +StorageEngine::Impl::ParseValueRecord(const std::byte* data, std::uint8_t db_id, + std::string_view key, + const RecordLocation& location, + std::uint64_t db_epoch, + std::uint64_t replication_epoch) { RecordHeader record{}; std::string_view disk_key; - if (!DecodeRecordHeader(std::span(data, location_.total_disk_bytes()), - &record, &disk_key) || - record.db_id_ != db_id_ || (!record.key_indirect_ && disk_key != key_) || - record.kind_ != RecordKind::kValue || - record.db_epoch_ != (expected_db_epoch_ ? *expected_db_epoch_ - : engine_.DbEpoch(db_id_)) || - record.mutation_sequence_ != location_.mutation_sequence_ || - record.replication_epoch_ != replication_epoch_ || - record.allocation_epoch_ != location_.allocation_epoch() || - record.expire_at_ms_ != location_.expire_at_ms_ || - record.value_type_ != location_.value_type() || - record.external_ != location_.external() || - record.key_indirect_ != location_.key_indirect() || - record.logical_size_ != location_.logical_size_ || - record.total_disk_bytes_ != location_.total_disk_bytes()) { + if (!DecodeRecordHeader(std::span(data, location.total_disk_bytes()), &record, + &disk_key) || + record.db_id_ != db_id || (!record.key_indirect_ && disk_key != key) || + record.kind_ != RecordKind::kValue || record.db_epoch_ != db_epoch || + record.mutation_sequence_ != location.mutation_sequence_ || + record.replication_epoch_ != replication_epoch || + record.allocation_epoch_ != location.allocation_epoch() || + record.expire_at_ms_ != location.expire_at_ms_ || + record.value_type_ != location.value_type() || + record.external_ != location.external() || + record.key_indirect_ != location.key_indirect() || + record.logical_size_ != location.logical_size_ || + record.total_disk_bytes_ != location.total_disk_bytes()) { return absl::AbortedError("record does not match in-memory location"); } - if (record.key_indirect_ && record.key_bytes_ != key_.size()) + if (record.key_indirect_ && record.key_bytes_ != key.size()) return absl::AbortedError("record key does not match location"); if (record.value_type_ == ValueType::kString && !record.grouped_ && !record.auxiliary_group_ && record.payload_bytes_ != record.logical_size_) return absl::InternalError("inline value length does not match metadata"); - return Payload{record.txid_, record.header_bytes_, record.payload_bytes_, - record.payload_checksum_}; + return RecordPayload{record.txid_, record.header_bytes_, + record.payload_bytes_, record.payload_checksum_}; } -StorageEngine::Impl::LoadValueAwaitable::Result -StorageEngine::Impl::LoadValueAwaitable::FinishRecord(const std::byte* data, - Payload payload, - bool staged) { +absl::StatusOr +StorageEngine::Impl::FinishValueRecord(ReadBufferLease& lease, + const std::byte* data, + RecordPayload payload, bool staged) { const auto* value = data + payload.offset_; if (Crc32c(std::span(value, payload.bytes_)) != payload.checksum_) return absl::InternalError("record value checksum mismatch"); - if (staged || payload.bytes_ > engine_.DirectGetValueLimit()) { - std::memmove(lease_.io_buffer().data_, value, payload.bytes_); - value = lease_.io_buffer().data_; + if (staged || payload.bytes_ > DirectGetValueLimit()) { + std::memmove(lease.io_buffer().data_, value, payload.bytes_); + value = lease.io_buffer().data_; } - const auto offset = static_cast(value - lease_.bytes().data()); - if (trace_) trace_->decode_done_ns_ = ReadTraceNowNanos(); - return LoadedValue{std::move(lease_), offset, payload.bytes_, payload.txid_}; + const auto offset = static_cast(value - lease.bytes().data()); + return LoadedValue{std::move(lease), offset, payload.bytes_, payload.txid_}; } absl::StatusOr @@ -1886,52 +1904,11 @@ StorageEngine::Impl::DecodeRemoteValue(ReadBufferLease lease, std::uint64_t replication_epoch, std::size_t record_headroom, std::uint64_t expected_db_epoch) { - const FixedBuffer io = lease.io_buffer(); - RecordHeader record{}; - std::string_view disk_key; - const std::byte* record_data = io.data_ + record_headroom; - std::span record_bytes(record_data, - location.total_disk_bytes()); - if (!DecodeRecordHeader(record_bytes, &record, &disk_key) || - record.db_id_ != db_id || (!record.key_indirect_ && disk_key != key) || - record.kind_ != RecordKind::kValue || - record.db_epoch_ != expected_db_epoch || - record.mutation_sequence_ != location.mutation_sequence_ || - record.replication_epoch_ != replication_epoch || - record.allocation_epoch_ != location.allocation_epoch() || - record.expire_at_ms_ != location.expire_at_ms_ || - record.value_type_ != location.value_type() || - record.external_ != location.external() || - record.key_indirect_ != location.key_indirect() || - record.logical_size_ != location.logical_size_ || - record.total_disk_bytes_ != location.total_disk_bytes()) { - return absl::Status(absl::StatusCode::kAborted, - "record does not match in-memory location"); - } - const std::byte* payload_data = record_data + record.header_bytes_; - if (record.key_indirect_ && record.key_bytes_ != key.size()) { - return absl::Status(absl::StatusCode::kAborted, - "record key does not match location"); - } - const std::size_t value_bytes = record.payload_bytes_; - if (record.value_type_ == ValueType::kString && !record.grouped_ && - !record.auxiliary_group_ && value_bytes != record.logical_size_) { - return absl::Status(absl::StatusCode::kInternal, - "inline value length does not match metadata"); - } - if (Crc32c(std::span(payload_data, record.payload_bytes_)) != - record.payload_checksum_) { - return absl::Status(absl::StatusCode::kInternal, - "record value checksum mismatch"); - } - const std::byte* framed_value = payload_data; - if (value_bytes > DirectGetValueLimit()) { - std::memmove(io.data_, payload_data, value_bytes); - framed_value = io.data_; - } - const std::size_t value_offset = - static_cast(framed_value - lease.bytes().data()); - return LoadedValue{std::move(lease), value_offset, value_bytes, record.txid_}; + const auto* data = lease.io_buffer().data_ + record_headroom; + auto payload = ParseValueRecord(data, db_id, key, location, expected_db_epoch, + replication_epoch); + if (!payload.ok()) return payload.status(); + return FinishValueRecord(lease, data, *payload, false); } } // namespace lavik::storage diff --git a/src/storage/engine/replication.cpp b/src/storage/engine/replication.cpp index 296abdd77..1d7b4002f 100644 --- a/src/storage/engine/replication.cpp +++ b/src/storage/engine/replication.cpp @@ -1577,8 +1577,9 @@ StorageEngine::Impl::ResetReplicaPartitions( sync->local_db_epochs_ = local_db_epochs; for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; - auto index_write = co_await AwaitAllIndexReaders( - *store.worker_, partition.indexes_[db_id]); + auto index_write = partition.indexes_[db_id].CloseAllReaders(); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; @@ -1689,8 +1690,9 @@ Task StorageEngine::Impl::ResetPartitionsDetachLocal( auto& partition = PartitionFor(store, partition_ids[i]); for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; - auto index_write = co_await AwaitAllIndexReaders( - *store.worker_, partition.indexes_[db_id]); + auto index_write = partition.indexes_[db_id].CloseAllReaders(); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; @@ -2415,8 +2417,9 @@ Task StorageEngine::Impl::AbortReplicaRoot( for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; - auto index_write = co_await AwaitAllIndexReaders( - *store.worker_, partition.indexes_[db_id]); + auto index_write = partition.indexes_[db_id].CloseAllReaders(); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; diff --git a/src/storage/engine/tomb_raider.cpp b/src/storage/engine/tomb_raider.cpp index 8fb318ea3..1fdf0a3cf 100644 --- a/src/storage/engine/tomb_raider.cpp +++ b/src/storage/engine/tomb_raider.cpp @@ -441,7 +441,9 @@ Task StorageEngine::Impl::TombMarkLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) { co_return absl::OkStatus(); // forfeit the round } - auto index_write = co_await AwaitAllIndexReaders(*store.worker_, index); + auto index_write = index.CloseAllReaders(); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); cursor = index.Scan(cursor, [](RecordIndex::Entry& entry) { if (entry.value_.kind() == RecordKind::kTombstone || @@ -469,9 +471,10 @@ Task StorageEngine::Impl::TombClaimLocal( } auto& partition = PartitionForKey(store, claim.key_); if (partition.replication_epoch_ == claim.replication_epoch_) { - auto index_write = co_await AwaitIndexValueWrite( - *store.worker_, partition.indexes_[claim.db_id_], + auto index_write = partition.indexes_[claim.db_id_].CloseForValueUpdate( claim.digest_.value_); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto resolved = co_await FindVerifiedEntry( store, partition.indexes_[claim.db_id_], claim.digest_, claim.key_); @@ -878,7 +881,9 @@ Task StorageEngine::Impl::TombReapLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) { co_return absl::OkStatus(); } - auto index_write = co_await AwaitAllIndexReaders(*store.worker_, index); + auto index_write = index.CloseAllReaders(); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); cursor = index.Scan(cursor, [&](RecordIndex::Entry& entry) { if (!entry.value_.unclaimed()) { @@ -964,8 +969,7 @@ Task StorageEngine::Impl::TombReapLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto& partition = PartitionForKey(store, candidate.key_); auto index_write = co_await AwaitIndexWrite( - *store.worker_, partition.indexes_[candidate.db_id_], - candidate.digest_); + partition.indexes_[candidate.db_id_], candidate.digest_); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto resolved = co_await FindVerifiedEntry(store, partition.indexes_[candidate.db_id_], diff --git a/src/storage/engine/write.cpp b/src/storage/engine/write.cpp index 3fb867351..89e511cd8 100644 --- a/src/storage/engine/write.cpp +++ b/src/storage/engine/write.cpp @@ -2936,8 +2936,7 @@ Task StorageEngine::Impl::WriteRecordLocked( if (!index_write.ready()) [[unlikely]] { // Retain the outer closure while the slow path drains readers and // rechecks capacity. Ordinary writes need no extra coroutine frame. - index_write = - co_await AwaitIndexWrite(*store.worker_, *index_ptr, request.digest_); + index_write = co_await AwaitIndexWrite(*index_ptr, request.digest_); } } RecordIndex::Entry* previous_entry = nullptr; @@ -2946,7 +2945,9 @@ Task StorageEngine::Impl::WriteRecordLocked( if (previous_entry != nullptr && !previous_entry->key_complete()) [[unlikely]] { index_write.Reset(); - index_write = co_await AwaitAllIndexReaders(*store.worker_, *index_ptr); + index_write = index_ptr->CloseAllReaders(); + if (!index_write.ready()) [[unlikely]] + index_write = co_await AwaitIndexReaders(std::move(index_write)); auto resolved = co_await FindVerifiedEntry(store, *index_ptr, request.digest_, request.key_); if (!resolved.ok()) { diff --git a/tests/defrag_payload_e2e.py b/tests/defrag_payload_e2e.py index 0060f3998..7a5f73e5d 100644 --- a/tests/defrag_payload_e2e.py +++ b/tests/defrag_payload_e2e.py @@ -48,6 +48,9 @@ def __init__(self, p): def response(self): head = self.stream.readline() assert head.endswith(b"\r\n"), head + if head[:1] == b"*": + size = int(head[1:-2]) + return None if size == -1 else [self.response() for _ in range(size)] if head[:1] == b"$": size = int(head[1:-2]) if size == -1: diff --git a/tests/device_affinity_test.cpp b/tests/device_affinity_test.cpp index 4fd080d9c..93806b87a 100644 --- a/tests/device_affinity_test.cpp +++ b/tests/device_affinity_test.cpp @@ -55,6 +55,11 @@ TEST(DeviceAffinityTest, AllocatesWorkersByCapacityWeightAndQpairCap) { EXPECT_EQ(owned.size(), 1); } + ASSERT_EQ(plan->controller_qpair_workers_.size(), controllers.size()); + for (const auto& workers : plan->controller_qpair_workers_) { + EXPECT_EQ(workers, (std::vector{0, 1, 2, 3})); + } + const std::vector capped{ {.id_ = "0000:01:00.0", .foreground_weight_ = 100, .io_qpair_count_ = 1}, {.id_ = "0000:02:00.0", .foreground_weight_ = 1, .io_qpair_count_ = 3}, @@ -76,30 +81,6 @@ TEST(DeviceAffinityTest, AssignsEveryControllerWhenControllersExceedWorkers) { EXPECT_EQ(plan->worker_controllers_[1], (std::vector{1, 2})); } -TEST(DeviceAffinityTest, SixControllersGiveEveryWorkerDirectAccess) { - std::vector controllers; - for (unsigned controller = 0; controller < 6; ++controller) { - controllers.push_back({.id_ = std::to_string(controller), - .foreground_weight_ = 10, - .io_qpair_count_ = 16}); - } - auto plan = PlanControllerAffinity(controllers, 16); - ASSERT_TRUE(plan.ok()) << plan.status(); - // More read access must preserve the existing weighted write distribution. - EXPECT_EQ(plan->controller_quotas_, - (std::vector{3, 3, 3, 3, 2, 2})); - for (const auto& home : plan->worker_controllers_) { - EXPECT_EQ(home.size(), 1); - } - ASSERT_EQ(plan->controller_qpair_workers_.size(), 6); - for (const auto& workers : plan->controller_qpair_workers_) { - ASSERT_EQ(workers.size(), 16); - for (unsigned worker = 0; worker < 16; ++worker) { - EXPECT_EQ(workers[worker], worker); - } - } -} - TEST(DeviceAffinityTest, SpareQpairsRespectCapacityAndRetainHomeAccess) { const std::vector controllers{ {.id_ = "0", .foreground_weight_ = 10, .io_qpair_count_ = 2}, diff --git a/tests/load_value_awaitable_test.cpp b/tests/load_value_awaitable_test.cpp index 23b594273..845412973 100644 --- a/tests/load_value_awaitable_test.cpp +++ b/tests/load_value_awaitable_test.cpp @@ -424,6 +424,9 @@ TEST_F(LoadValueAwaitableTestPeer, ErrorsReleaseResources) { header_.payload_checksum_ ^= 1; ASSERT_NO_FATAL_FAILURE(Complete(read, completion)); header_.payload_checksum_ ^= 1; + // A full CQE parses metadata only; CRC failure is discovered when the + // caller resumes, not while the I/O poller is processing completions. + if (completion == 4096) EXPECT_FALSE(read.await_ready()); EXPECT_FALSE(read.await_resume().ok()); } EXPECT_EQ(block_.pin_count(), 0); diff --git a/tests/remote_get_e2e.py b/tests/remote_get_e2e.py index f3855921c..e4f9e315f 100644 --- a/tests/remote_get_e2e.py +++ b/tests/remote_get_e2e.py @@ -14,23 +14,6 @@ import defrag_payload_e2e as support -class Client(support.Client): - def response(self): - head = self.stream.readline() - assert head.endswith(b"\r\n"), head - assert not head.startswith(b"-"), head - if head.startswith(b"*"): - return [self.response() for _ in range(int(head[1:-2]))] - if head.startswith(b"$"): - size = int(head[1:-2]) - if size < 0: - return None - data = self.stream.read(size) - assert self.stream.read(2) == b"\r\n" - return data - return head[1:-2] - - def stats(client): return dict( line.split(b":", 1) @@ -75,19 +58,16 @@ def check(): missing = [b"{" + key + b"}:missing" for key in keys[:100]] assert admin.batch([[b"GET", key] for key in missing]) == [None] * len(missing) - def read(): - client = Client(port) + def read(count, allowed): + client = support.Client(port) try: - for i in range(6000): - assert client.batch([[b"GET", keys[i % 1000]]])[0] in ( - before, - after, - ) + for i in range(count): + assert client.batch([[b"GET", keys[i % 1000]]])[0] in allowed finally: client.close() def write(): - client = Client(port) + client = support.Client(port) try: # Retire enough of the first sealed blocks to overlap reads # with the ordinary defrag admission and last-unpin retry. @@ -101,22 +81,15 @@ def write(): client.close() with ThreadPoolExecutor(max_workers=5) as pool: - tasks = [pool.submit(read) for _ in range(4)] + [pool.submit(write)] + tasks = [pool.submit(read, 6000, (before, after)) for _ in range(4)] + tasks.append(pool.submit(write)) for task in tasks: task.result() # Clear while requests are arriving. Either generation is valid for # an overlapping GET, but stale bytes must never reappear after FLUSHDB. - def read_during_flush(): - client = Client(port) - try: - for i in range(1000): - assert client.batch([[b"GET", keys[i]]])[0] in (None, before, after) - finally: - client.close() - with ThreadPoolExecutor(max_workers=2) as pool: - task = pool.submit(read_during_flush) + task = pool.submit(read, 1000, (None, before, after)) assert admin.batch([[b"FLUSHDB"]]) == [b"OK"] task.result() for batch in support.chunks(keys[:1000]): From 571067cbc0c61162015025f1d9d2f412d5bb2618 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 06:50:24 +0000 Subject: [PATCH 14/15] Restore independent read validation and reader guard waits --- src/storage/engine/flush.cpp | 3 +- src/storage/engine/impl.h | 44 ++++------ src/storage/engine/read.cpp | 127 +++++++++++++++++------------ src/storage/engine/replication.cpp | 15 ++-- src/storage/engine/tomb_raider.cpp | 14 +--- src/storage/engine/write.cpp | 4 +- 6 files changed, 106 insertions(+), 101 deletions(-) diff --git a/src/storage/engine/flush.cpp b/src/storage/engine/flush.cpp index 355c98cdf..930fa8afb 100644 --- a/src/storage/engine/flush.cpp +++ b/src/storage/engine/flush.cpp @@ -540,7 +540,8 @@ Task StorageEngine::Impl::FlushPendingBlocks(WorkerStore* store) { if (!index_write.ready()) [[unlikely]] { // Keep admission closed across the slow wait. Most flush updates // have no registered reader and need no waiting coroutine. - index_write = co_await AwaitIndexReaders(std::move(index_write)); + index_write = + co_await AwaitIndexValueWrite(index, identity.entry_hash_); } // Detach can run while the reader drain yields. Recheck the population // before resolving the saved integer address into this index. diff --git a/src/storage/engine/impl.h b/src/storage/engine/impl.h index e22395dd8..f5e4bb9e8 100644 --- a/src/storage/engine/impl.h +++ b/src/storage/engine/impl.h @@ -133,10 +133,15 @@ inline Task AwaitIndexWrite(RecordIndex& index, co_return std::move(guard); } -// The caller checks ready() before entering this slow path, so uncontended -// mutations keep their existing guard without allocating a waiting coroutine. -inline Task AwaitIndexReaders( - RecordIndex::WriteGuard guard) { +inline Task AwaitIndexValueWrite(RecordIndex& index, + std::uint64_t hash) { + auto guard = index.CloseForValueUpdate(hash); + while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); + co_return std::move(guard); +} + +inline Task AwaitAllIndexReaders(RecordIndex& index) { + auto guard = index.CloseAllReaders(); while (!guard.ready()) co_await tx::CurrentTxShard().WaitForReaders(); co_return std::move(guard); } @@ -3434,24 +3439,6 @@ class StorageEngine::Impl { } }; - struct RecordPayload { - std::uint64_t txid_; - std::uint32_t offset_; - std::uint32_t bytes_; - std::uint32_t checksum_; - }; - - // Metadata-only validation may run on the I/O completion poller. CRC and - // payload movement must stay in FinishValueRecord on the caller's task. - static absl::StatusOr ParseValueRecord( - const std::byte* data, std::uint8_t db_id, std::string_view key, - const RecordLocation& location, std::uint64_t db_epoch, - std::uint64_t replication_epoch); - absl::StatusOr FinishValueRecord(ReadBufferLease& lease, - const std::byte* data, - RecordPayload payload, - bool staged); - // The awaitable lives in the caller's frame. Ordinary local reads suspend // that caller directly, including through pool backpressure. Only multi-step // logical reads need a lazy helper Task. Local mode returns stale-location @@ -3480,14 +3467,19 @@ class StorageEngine::Impl { private: friend class LoadValueAwaitableTestPeer; + struct Payload { + std::uint64_t txid_; + std::uint32_t offset_; + std::uint32_t bytes_; + std::uint32_t checksum_; + }; void PrepareLocal(); void FinishBufferWait(); static void BufferReady(bycorf::RemoteWork* work); std::coroutine_handle<> SuspendRead(std::coroutine_handle<> awaiting); - absl::StatusOr ParseRecord(const std::byte* data) const; - Result FinishRecord(const std::byte* data, RecordPayload payload, - bool staged); + absl::StatusOr ParseRecord(const std::byte* data) const; + Result FinishRecord(const std::byte* data, Payload payload, bool staged); bool PopulationChanged() const; void Unpin(); void Retry(absl::Status failure); @@ -3507,7 +3499,7 @@ class StorageEngine::Impl { std::size_t read_bytes_ = 0; std::uint8_t db_id_; BlockState* pinned_ = nullptr; - std::optional payload_; + std::optional payload_; std::optional ready_; // Runtime shutdown drains I/O before destroying suspended frames. The // waiter retains its assigned slot until the scheduled continuation runs. diff --git a/src/storage/engine/read.cpp b/src/storage/engine/read.cpp index 2758e7fa2..05f45b1a6 100644 --- a/src/storage/engine/read.cpp +++ b/src/storage/engine/read.cpp @@ -1832,69 +1832,51 @@ StorageEngine::Impl::LoadValueAwaitable::await_resume() { return FinishRecord(data, *payload_, false); } -absl::StatusOr +absl::StatusOr StorageEngine::Impl::LoadValueAwaitable::ParseRecord( const std::byte* data) const { - return ParseValueRecord( - data, db_id_, key_, location_, - expected_db_epoch_ ? *expected_db_epoch_ : engine_.DbEpoch(db_id_), - replication_epoch_); -} - -StorageEngine::Impl::LoadValueAwaitable::Result -StorageEngine::Impl::LoadValueAwaitable::FinishRecord(const std::byte* data, - RecordPayload payload, - bool staged) { - auto loaded = engine_.FinishValueRecord(lease_, data, payload, staged); - if (loaded.ok() && trace_) trace_->decode_done_ns_ = ReadTraceNowNanos(); - return loaded; -} - -absl::StatusOr -StorageEngine::Impl::ParseValueRecord(const std::byte* data, std::uint8_t db_id, - std::string_view key, - const RecordLocation& location, - std::uint64_t db_epoch, - std::uint64_t replication_epoch) { RecordHeader record{}; std::string_view disk_key; - if (!DecodeRecordHeader(std::span(data, location.total_disk_bytes()), &record, - &disk_key) || - record.db_id_ != db_id || (!record.key_indirect_ && disk_key != key) || - record.kind_ != RecordKind::kValue || record.db_epoch_ != db_epoch || - record.mutation_sequence_ != location.mutation_sequence_ || - record.replication_epoch_ != replication_epoch || - record.allocation_epoch_ != location.allocation_epoch() || - record.expire_at_ms_ != location.expire_at_ms_ || - record.value_type_ != location.value_type() || - record.external_ != location.external() || - record.key_indirect_ != location.key_indirect() || - record.logical_size_ != location.logical_size_ || - record.total_disk_bytes_ != location.total_disk_bytes()) { + if (!DecodeRecordHeader(std::span(data, location_.total_disk_bytes()), + &record, &disk_key) || + record.db_id_ != db_id_ || (!record.key_indirect_ && disk_key != key_) || + record.kind_ != RecordKind::kValue || + record.db_epoch_ != (expected_db_epoch_ ? *expected_db_epoch_ + : engine_.DbEpoch(db_id_)) || + record.mutation_sequence_ != location_.mutation_sequence_ || + record.replication_epoch_ != replication_epoch_ || + record.allocation_epoch_ != location_.allocation_epoch() || + record.expire_at_ms_ != location_.expire_at_ms_ || + record.value_type_ != location_.value_type() || + record.external_ != location_.external() || + record.key_indirect_ != location_.key_indirect() || + record.logical_size_ != location_.logical_size_ || + record.total_disk_bytes_ != location_.total_disk_bytes()) { return absl::AbortedError("record does not match in-memory location"); } - if (record.key_indirect_ && record.key_bytes_ != key.size()) + if (record.key_indirect_ && record.key_bytes_ != key_.size()) return absl::AbortedError("record key does not match location"); if (record.value_type_ == ValueType::kString && !record.grouped_ && !record.auxiliary_group_ && record.payload_bytes_ != record.logical_size_) return absl::InternalError("inline value length does not match metadata"); - return RecordPayload{record.txid_, record.header_bytes_, - record.payload_bytes_, record.payload_checksum_}; + return Payload{record.txid_, record.header_bytes_, record.payload_bytes_, + record.payload_checksum_}; } -absl::StatusOr -StorageEngine::Impl::FinishValueRecord(ReadBufferLease& lease, - const std::byte* data, - RecordPayload payload, bool staged) { +StorageEngine::Impl::LoadValueAwaitable::Result +StorageEngine::Impl::LoadValueAwaitable::FinishRecord(const std::byte* data, + Payload payload, + bool staged) { const auto* value = data + payload.offset_; if (Crc32c(std::span(value, payload.bytes_)) != payload.checksum_) return absl::InternalError("record value checksum mismatch"); - if (staged || payload.bytes_ > DirectGetValueLimit()) { - std::memmove(lease.io_buffer().data_, value, payload.bytes_); - value = lease.io_buffer().data_; + if (staged || payload.bytes_ > engine_.DirectGetValueLimit()) { + std::memmove(lease_.io_buffer().data_, value, payload.bytes_); + value = lease_.io_buffer().data_; } - const auto offset = static_cast(value - lease.bytes().data()); - return LoadedValue{std::move(lease), offset, payload.bytes_, payload.txid_}; + const auto offset = static_cast(value - lease_.bytes().data()); + if (trace_) trace_->decode_done_ns_ = ReadTraceNowNanos(); + return LoadedValue{std::move(lease_), offset, payload.bytes_, payload.txid_}; } absl::StatusOr @@ -1904,11 +1886,52 @@ StorageEngine::Impl::DecodeRemoteValue(ReadBufferLease lease, std::uint64_t replication_epoch, std::size_t record_headroom, std::uint64_t expected_db_epoch) { - const auto* data = lease.io_buffer().data_ + record_headroom; - auto payload = ParseValueRecord(data, db_id, key, location, expected_db_epoch, - replication_epoch); - if (!payload.ok()) return payload.status(); - return FinishValueRecord(lease, data, *payload, false); + const FixedBuffer io = lease.io_buffer(); + RecordHeader record{}; + std::string_view disk_key; + const std::byte* record_data = io.data_ + record_headroom; + std::span record_bytes(record_data, + location.total_disk_bytes()); + if (!DecodeRecordHeader(record_bytes, &record, &disk_key) || + record.db_id_ != db_id || (!record.key_indirect_ && disk_key != key) || + record.kind_ != RecordKind::kValue || + record.db_epoch_ != expected_db_epoch || + record.mutation_sequence_ != location.mutation_sequence_ || + record.replication_epoch_ != replication_epoch || + record.allocation_epoch_ != location.allocation_epoch() || + record.expire_at_ms_ != location.expire_at_ms_ || + record.value_type_ != location.value_type() || + record.external_ != location.external() || + record.key_indirect_ != location.key_indirect() || + record.logical_size_ != location.logical_size_ || + record.total_disk_bytes_ != location.total_disk_bytes()) { + return absl::Status(absl::StatusCode::kAborted, + "record does not match in-memory location"); + } + const std::byte* payload_data = record_data + record.header_bytes_; + if (record.key_indirect_ && record.key_bytes_ != key.size()) { + return absl::Status(absl::StatusCode::kAborted, + "record key does not match location"); + } + const std::size_t value_bytes = record.payload_bytes_; + if (record.value_type_ == ValueType::kString && !record.grouped_ && + !record.auxiliary_group_ && value_bytes != record.logical_size_) { + return absl::Status(absl::StatusCode::kInternal, + "inline value length does not match metadata"); + } + if (Crc32c(std::span(payload_data, record.payload_bytes_)) != + record.payload_checksum_) { + return absl::Status(absl::StatusCode::kInternal, + "record value checksum mismatch"); + } + const std::byte* framed_value = payload_data; + if (value_bytes > DirectGetValueLimit()) { + std::memmove(io.data_, payload_data, value_bytes); + framed_value = io.data_; + } + const std::size_t value_offset = + static_cast(framed_value - lease.bytes().data()); + return LoadedValue{std::move(lease), value_offset, value_bytes, record.txid_}; } } // namespace lavik::storage diff --git a/src/storage/engine/replication.cpp b/src/storage/engine/replication.cpp index 1d7b4002f..ac027bbb3 100644 --- a/src/storage/engine/replication.cpp +++ b/src/storage/engine/replication.cpp @@ -1577,9 +1577,8 @@ StorageEngine::Impl::ResetReplicaPartitions( sync->local_db_epochs_ = local_db_epochs; for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; - auto index_write = partition.indexes_[db_id].CloseAllReaders(); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + auto index_write = + co_await AwaitAllIndexReaders(partition.indexes_[db_id]); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; @@ -1690,9 +1689,8 @@ Task StorageEngine::Impl::ResetPartitionsDetachLocal( auto& partition = PartitionFor(store, partition_ids[i]); for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; - auto index_write = partition.indexes_[db_id].CloseAllReaders(); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + auto index_write = + co_await AwaitAllIndexReaders(partition.indexes_[db_id]); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; @@ -2417,9 +2415,8 @@ Task StorageEngine::Impl::AbortReplicaRoot( for (std::uint8_t db_id = 0; db_id < options_.database_count_; ++db_id) { ++partition.grouped_generations_[db_id]; - auto index_write = partition.indexes_[db_id].CloseAllReaders(); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + auto index_write = + co_await AwaitAllIndexReaders(partition.indexes_[db_id]); QueueDetachedIndex(store, partition.indexes_[db_id], db_id, &partition.grouped_objects_[db_id]); partition.fullsync_coverage_bytes_[db_id] = 0; diff --git a/src/storage/engine/tomb_raider.cpp b/src/storage/engine/tomb_raider.cpp index 1fdf0a3cf..6523e4a89 100644 --- a/src/storage/engine/tomb_raider.cpp +++ b/src/storage/engine/tomb_raider.cpp @@ -441,9 +441,7 @@ Task StorageEngine::Impl::TombMarkLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) { co_return absl::OkStatus(); // forfeit the round } - auto index_write = index.CloseAllReaders(); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + auto index_write = co_await AwaitAllIndexReaders(index); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); cursor = index.Scan(cursor, [](RecordIndex::Entry& entry) { if (entry.value_.kind() == RecordKind::kTombstone || @@ -471,10 +469,8 @@ Task StorageEngine::Impl::TombClaimLocal( } auto& partition = PartitionForKey(store, claim.key_); if (partition.replication_epoch_ == claim.replication_epoch_) { - auto index_write = partition.indexes_[claim.db_id_].CloseForValueUpdate( - claim.digest_.value_); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + auto index_write = co_await AwaitIndexValueWrite( + partition.indexes_[claim.db_id_], claim.digest_.value_); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); auto resolved = co_await FindVerifiedEntry( store, partition.indexes_[claim.db_id_], claim.digest_, claim.key_); @@ -881,9 +877,7 @@ Task StorageEngine::Impl::TombReapLocal(WorkerStore& store, if (!TombRaiderRoundValid(round, &store)) { co_return absl::OkStatus(); } - auto index_write = index.CloseAllReaders(); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + auto index_write = co_await AwaitAllIndexReaders(index); if (!TombRaiderRoundValid(round, &store)) co_return absl::OkStatus(); cursor = index.Scan(cursor, [&](RecordIndex::Entry& entry) { if (!entry.value_.unclaimed()) { diff --git a/src/storage/engine/write.cpp b/src/storage/engine/write.cpp index 89e511cd8..d4aa9dd73 100644 --- a/src/storage/engine/write.cpp +++ b/src/storage/engine/write.cpp @@ -2945,9 +2945,7 @@ Task StorageEngine::Impl::WriteRecordLocked( if (previous_entry != nullptr && !previous_entry->key_complete()) [[unlikely]] { index_write.Reset(); - index_write = index_ptr->CloseAllReaders(); - if (!index_write.ready()) [[unlikely]] - index_write = co_await AwaitIndexReaders(std::move(index_write)); + index_write = co_await AwaitAllIndexReaders(*index_ptr); auto resolved = co_await FindVerifiedEntry(store, *index_ptr, request.digest_, request.key_); if (!resolved.ok()) { From a2ba846c241a8a51e30b9e2d4661cfa94acf0864 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 08:58:55 +0000 Subject: [PATCH 15/15] Fix CI memory budgets and late blocking-wait cancellation --- CMakeLists.txt | 8 +++-- docs/architecture/02-request-serving.md | 4 ++- include/lavik/command.h | 2 ++ include/lavik/session.h | 4 +-- src/redis/command.cpp | 10 ++++++- src/redis/server.cpp | 4 +-- tests/client_retirement_test.cpp | 37 ++++++++++++++++++++++++ tests/grouped/ordered_write_e2e_test.cpp | 7 +++-- tests/grouped/rdb_stream_e2e_test.cpp | 3 +- tests/grouped/transfer_e2e_test.cpp | 7 +++-- 10 files changed, 71 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e42de64a3..91ddbf6f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1685,10 +1685,12 @@ if (BUILD_TESTING) NAME cluster_integration.replication_manager_control_api COMMAND lavik_cluster_replication_manager_integration_test ) + # Allow repeated server recovery across the full suite on Debug CI runners. + # Individual operation waits remain bounded inside each test. set_tests_properties( cluster_integration.replication_manager_control_api PROPERTIES LABELS cluster-integration - TIMEOUT 90 + TIMEOUT 180 RUN_SERIAL TRUE ) @@ -2244,8 +2246,10 @@ endforeach() NAME lavik_grouped_ordered_write_e2e COMMAND lavik_grouped_ordered_write_e2e_test $ ) + # This aggregate suite repeatedly starts and recovers storage, including + # large RDB and crash-recovery cases on Debug CI runners. set_tests_properties(lavik_grouped_ordered_write_e2e PROPERTIES - TIMEOUT 1200 RUN_SERIAL TRUE + TIMEOUT 1800 RUN_SERIAL TRUE ) add_executable(lavik_list_e2e_test diff --git a/docs/architecture/02-request-serving.md b/docs/architecture/02-request-serving.md index bcdebc6fb..0665060fa 100644 --- a/docs/architecture/02-request-serving.md +++ b/docs/architecture/02-request-serving.md @@ -51,7 +51,9 @@ worker-local client record separately tracks data exposed by `CLIENT`, including RESP version, client library name/version, subscription counts, blocking state, and replication-session identity. All disconnect paths return through one cleanup point which unregisters client, monitor, Pub/Sub, and -WATCH state. +WATCH state. Peer EOF remains latched across asynchronous command work; +registering a blocking waiter after EOF immediately cancels it, including +waiters created by commands already buffered when the peer disconnected. Meta-managed serving boundaries retire ordinary client connections, including idle, blocking and Pub/Sub sessions. After authority admission closes, each diff --git a/include/lavik/command.h b/include/lavik/command.h index e6938d393..6ad2a305c 100644 --- a/include/lavik/command.h +++ b/include/lavik/command.h @@ -665,6 +665,8 @@ void SetClientName(std::uint64_t id, std::string name) noexcept; void SetClientRespVersion(std::uint64_t id, RespVersion version) noexcept; void SetClientPubSubCounts(std::uint64_t id, std::size_t subscriptions, std::size_t pattern_subscriptions) noexcept; +// Updates CLIENT state. After registering a waiter, setting blocked also +// cancels it if EOF or connection retirement already occurred. void SetClientBlocked(std::uint64_t id, bool blocked) noexcept; void UnregisterClientConnection(std::uint64_t id) noexcept; diff --git a/include/lavik/session.h b/include/lavik/session.h index 76ac57ec8..89b7e2b56 100644 --- a/include/lavik/session.h +++ b/include/lavik/session.h @@ -43,8 +43,8 @@ struct ConnectionContext { ReplyBuilder&); // Worker-local shutdown marker: buffered commands stop once cleanup starts. bool closing_ = false; - // Peer EOF can leave complete commands buffered. Only WAIT needs to remember - // it across suspension before its blocking waiter is registered. + // Peer EOF can leave complete commands buffered. Remember it across + // suspension so a blocking waiter registered later is cancelled too. bool wait_peer_disconnected_ = false; std::uint8_t selected_db_ = 0; bool authenticated_ = true; diff --git a/src/redis/command.cpp b/src/redis/command.cpp index 58427fd95..0fe72fd7e 100644 --- a/src/redis/command.cpp +++ b/src/redis/command.cpp @@ -11640,7 +11640,15 @@ void SetClientBlocked(std::uint64_t id, bool blocked) noexcept { const auto found = std::find_if( g_worker_clients[worker].begin(), g_worker_clients[worker].end(), [id](const auto& client) { return client.id_ == id; }); - if (found != g_worker_clients[worker].end()) found->blocked_ = blocked; + if (found == g_worker_clients[worker].end()) return; + found->blocked_ = blocked; + // EOF or retirement may precede registration while a buffered command is + // suspended on another worker. The waiter is now in the local registry, so + // replay cancellation before it can sleep indefinitely. + if (blocked && found->context_ != nullptr && + (found->context_->closing_ || found->context_->wait_peer_disconnected_)) { + (void)CancelBlockedClientOnCurrentWorker(id); + } } void UnregisterClientConnection(std::uint64_t id) noexcept { diff --git a/src/redis/server.cpp b/src/redis/server.cpp index 72fb45c16..68d09a765 100644 --- a/src/redis/server.cpp +++ b/src/redis/server.cpp @@ -1305,8 +1305,8 @@ Task RedisService::Serve(TcpStream stream) { [](void* context) noexcept { auto* connection = static_cast(context); // A command may be suspended before registering its blocking waiter. - // Preserve peer EOF for WAIT without retiring buffered commands or - // changing Pub/Sub's existing connection lifecycle. + // Preserve peer EOF for late waiters without retiring buffered commands + // or changing Pub/Sub's existing connection lifecycle. connection->wait_peer_disconnected_ = true; (void)CancelBlockedClientOnCurrentWorker(connection->conn_id_); }, diff --git a/tests/client_retirement_test.cpp b/tests/client_retirement_test.cpp index fdd64115c..b64fd55db 100644 --- a/tests/client_retirement_test.cpp +++ b/tests/client_retirement_test.cpp @@ -25,18 +25,55 @@ #include #include +#include "../src/redis/blocking_wait.h" #include "absl/cleanup/cleanup.h" #include "bycorf/runtime/foreign_executor.h" #include "bycorf/runtime/runtime.h" #include "bycorf/runtime/worker.h" #include "gtest/gtest.h" #include "lavik/command.h" +#include "lavik/metrics.h" #include "lavik/session.h" +#include "lavik/tx/tx_shard.h" namespace lavik { namespace { using namespace std::chrono_literals; +TEST(ClientRetirementTest, LateBlockingRegistrationObservesDisconnect) { + auto saved = std::move(bycorf::MutableThisWorker()); + bycorf::CrossCore cross_core(1); + bycorf::Worker worker; + bycorf::SetThisWorker(0, &cross_core, &worker); + absl::Cleanup restore = [&] { + bycorf::MutableThisWorker() = std::move(saved); + }; + SetServerInfo("127.0.0.1", 0, 1, ""); + InitWorkerMetrics(1); + if (tx::TxRuntime::Get() == nullptr) tx::TxRuntime::Create(1); + for (unsigned state = 0; state < 3; ++state) { + ConnectionContext context; + context.wait_peer_disconnected_ = state == 1; + context.closing_ = state == 2; + RegisterClientConnection(1, -1, "late-waiter", false, false, 0, &context); + absl::Cleanup unregister = [] { UnregisterClientConnection(1); }; + // Force cancellation before registration, when the disconnect callback + // cannot find a waiter. Both keyed and keyless waits share this registry. + EXPECT_FALSE(CancelBlockedClientOnCurrentWorker(1)); + auto task = RegisterClientBlockingWait(1); + std::move(task) + .operator co_await() + .await_suspend(std::noop_coroutine()) + .resume(); + ASSERT_TRUE(task.done()); + auto result = std::move(task).TakeResult(); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_EQ(BlockingWaitState(**result), + state == 0 ? BlockingWakeReason::kWaiting + : BlockingWakeReason::kCancelled); + } +} + TEST(ClientRetirementTest, DelayedNotificationClosesReconnectsButPreservesInternalConnections) { std::array, 6> sockets; diff --git a/tests/grouped/ordered_write_e2e_test.cpp b/tests/grouped/ordered_write_e2e_test.cpp index 445f803d0..018a84f14 100644 --- a/tests/grouped/ordered_write_e2e_test.cpp +++ b/tests/grouped/ordered_write_e2e_test.cpp @@ -543,13 +543,14 @@ TEST(IndirectKeyE2e, RemoteGroupedReadsRespectParentKeyOwnership) { ASSERT_EQ(server.Wait(true), 0) << server.Log(); } { - // Each logical owner retains two 16 MiB side-index keys. Its 48 MiB - // retained-memory quota fits the pages, but not another complete key copy. + // Each logical owner retains two 16 MiB side-index keys plus its fixed + // 16 MiB arena directory. Its 57.6 MiB steady-state quota fits the pages, + // but not another complete key copy. // Hash remote reads still own a key copy and must reject that allocation. // Ordered remote reads borrow the caller's key across the awaited hop, so // they must succeed with only page scratch. Client request buffers have a // separate budget so they cannot mask this storage admission check. - Server limited(disk, 3, {}, {}, false, 2, "160M", {}, "128M"); + Server limited(disk, 3, {}, {}, false, 2, "192M", {}, "128M"); limited.PreserveOnFailure(); Client client(limited.port()); unsigned rejected_hash = 0; diff --git a/tests/grouped/rdb_stream_e2e_test.cpp b/tests/grouped/rdb_stream_e2e_test.cpp index 201508b7d..3d3187ccd 100644 --- a/tests/grouped/rdb_stream_e2e_test.cpp +++ b/tests/grouped/rdb_stream_e2e_test.cpp @@ -503,7 +503,8 @@ TEST(GroupedRdbStreamE2e, ZsetRestoreBatchesPreserveOrderAndRollback) { }; PrivateDisk disk; { - Server server(disk, 2, {}, {}, false, 2, "128M"); + // Include the fixed 16 MiB arena directory per worker in the budget. + Server server(disk, 2, {}, {}, false, 2, "176M"); Client client(server.port()); for (unsigned mode = 0; mode < 4; ++mode) { SCOPED_TRACE(mode); diff --git a/tests/grouped/transfer_e2e_test.cpp b/tests/grouped/transfer_e2e_test.cpp index eb5fa3aaa..d9c730c63 100644 --- a/tests/grouped/transfer_e2e_test.cpp +++ b/tests/grouped/transfer_e2e_test.cpp @@ -176,10 +176,11 @@ TEST(GroupedTransferE2e, FullImageScratchOomPreservesValuesAndMetadataReads) { ASSERT_EQ(server.Wait(true), 0) << server.Log(); } { - // Each worker has less than the 9 MiB reply itself. Reject the full image - // even when a read retains just one copy, rather than relying on mutation + // After its fixed 16 MiB arena directory, each worker has less than the + // 9 MiB reply left in its 21.6 MiB steady-state quota. Reject the full + // image even when a read retains just one copy, without relying on mutation // scratch multipliers to make an otherwise affordable reply fail. - Server server(disk, 4, {}, {}, false, 2, "32M"); + Server server(disk, 4, {}, {}, false, 2, "96M"); Client client(server.port()); for (const std::vector command : {std::vector{"HGETALL", "hash"},