From 09dba739610b2b7916af5d58d4864935f76ac53f Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 12 Oct 2026 15:20:17 +0000 Subject: [PATCH 1/3] perf: compact coroutine and I/O state member layouts --- include/bycorf/io/storage.h | 5 +++-- include/bycorf/net/connection.h | 2 +- include/bycorf/runtime/cross_core.h | 8 +++++--- include/bycorf/runtime/task.h | 5 +++-- src/io/storage.cpp | 8 ++++---- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/include/bycorf/io/storage.h b/include/bycorf/io/storage.h index ba01143..1060e61 100644 --- a/include/bycorf/io/storage.h +++ b/include/bycorf/io/storage.h @@ -96,9 +96,9 @@ class SizeIoAwaitable final : public OneShotIoAwaitable { private: Worker* worker_ = nullptr; FixedFile file_{}; + Operation operation_ = Operation::kRead; FixedBuffer buffer_{}; std::uint64_t offset_ = 0; - Operation operation_ = Operation::kRead; }; class OpenFixedFileAwaitable final : public OneShotIoAwaitable { @@ -111,9 +111,10 @@ class OpenFixedFileAwaitable final : public OneShotIoAwaitable { absl::Status await_resume(); private: + // A leading int can occupy OneShotIoAwaitable's tail padding. + int flags_ = 0; Worker* worker_ = nullptr; std::string path_; - int flags_ = 0; mode_t mode_ = 0; FixedFile file_{}; }; diff --git a/include/bycorf/net/connection.h b/include/bycorf/net/connection.h index 8d99179..ba4bcab 100644 --- a/include/bycorf/net/connection.h +++ b/include/bycorf/net/connection.h @@ -72,6 +72,7 @@ struct Connection { std::uint64_t id_ = 0; RegisteredFile file_; ConnectionState state_ = ConnectionState::kActive; + RecvMode recv_mode_ = RecvMode::kMultishot; std::uint64_t generation_ = 0; std::int64_t last_active_ms_ = 0; @@ -106,7 +107,6 @@ struct Connection { bool peer_disconnect_poll_cancel_requested_ = false; PeerDisconnectCallback peer_disconnect_callback_ = nullptr; void* peer_disconnect_context_ = nullptr; - RecvMode recv_mode_ = RecvMode::kMultishot; std::shared_ptr tls_state_; void* protocol_context_ = nullptr; }; diff --git a/include/bycorf/runtime/cross_core.h b/include/bycorf/runtime/cross_core.h index cf6db97..7894055 100644 --- a/include/bycorf/runtime/cross_core.h +++ b/include/bycorf/runtime/cross_core.h @@ -60,15 +60,17 @@ using WorkerId = std::uint16_t; // pointer. run_fn is the one unavoidable indirect call: it invokes the // type-erased user closure, and only on the request leg. struct RemoteWork { - WorkerId origin_ = 0; + // Keep aligned fields first and small bookkeeping last. Derived awaiters + // can then reuse the base's tail padding for their target worker ID. std::coroutine_handle<> waiter_{}; - TaskClass task_class_ = TaskClass::kForeground; void (*run_fn_)(RemoteWork*) = nullptr; // runs the user fn, stores result - bool reply_deferred_ = false; #if BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE std::uint64_t request_post_ns_ = 0; std::uint64_t reply_post_ns_ = 0; #endif + WorkerId origin_ = 0; + TaskClass task_class_ = TaskClass::kForeground; + bool reply_deferred_ = false; }; // Small one-way control message. It is copied into the target mailbox, so the diff --git a/include/bycorf/runtime/task.h b/include/bycorf/runtime/task.h index 9fa672b..7828876 100644 --- a/include/bycorf/runtime/task.h +++ b/include/bycorf/runtime/task.h @@ -66,12 +66,10 @@ class Task { // Task has exactly one successful co_return and unhandled exceptions // terminate. Keep the result directly in the promise so return_value does // not pay optional::emplace's reset/engagement branch on every completion. - // The flag is still required for destroying a frame before it completes. union { char empty_; T value_; }; - bool value_constructed_ = false; std::coroutine_handle<> continuation_{}; void* completion_context_ = nullptr; completion_fn completion_ = nullptr; @@ -79,6 +77,9 @@ class Task { // or kNotDetached while the task is owned by a Task object/awaiter. static constexpr std::uint32_t kNotDetached = 0xffffffffu; std::uint32_t detached_index_ = kNotDetached; + // Keep the flag beside the index to avoid a separate alignment gap. It is + // still required when a frame is destroyed before constructing its result. + bool value_constructed_ = false; static void* operator new(std::size_t size) { return detail::AllocateCoroutineFrame(size); diff --git a/src/io/storage.cpp b/src/io/storage.cpp index 9d0fe2c..055a755 100644 --- a/src/io/storage.cpp +++ b/src/io/storage.cpp @@ -127,9 +127,9 @@ SizeIoAwaitable::SizeIoAwaitable(Worker& worker, FixedFile file, Operation operation) noexcept : worker_(&worker), file_(file), + operation_(operation), buffer_(buffer), - offset_(offset), - operation_(operation) {} + offset_(offset) {} bool SizeIoAwaitable::await_suspend(std::coroutine_handle<> awaiting) { absl::Status status; @@ -191,9 +191,9 @@ absl::StatusOr SizeIoAwaitable::await_resume() { OpenFixedFileAwaitable::OpenFixedFileAwaitable(Worker& worker, std::string path, int flags, mode_t mode, FixedFile file) - : worker_(&worker), + : flags_(flags), + worker_(&worker), path_(std::move(path)), - flags_(flags), mode_(mode), file_(file) {} From 58197d1cafc94353e06f40ed4bb9ca130bb11ccd Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 19 Oct 2026 10:05:07 +0800 Subject: [PATCH 2/3] perf: group Connection receive control fields --- include/bycorf/net/connection.h | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/include/bycorf/net/connection.h b/include/bycorf/net/connection.h index ba4bcab..92c6862 100644 --- a/include/bycorf/net/connection.h +++ b/include/bycorf/net/connection.h @@ -68,25 +68,33 @@ struct ReceivedBuffer { struct Connection { using PeerDisconnectCallback = void (*)(void*) noexcept; + // Group owner-local read readiness and completion bookkeeping: these fields + // are checked together when a reader suspends and a receive completes. Worker* worker_ = nullptr; - std::uint64_t id_ = 0; + std::coroutine_handle<> read_waiter_{}; + absl::Status last_error_ = absl::OkStatus(); + std::int64_t last_active_ms_ = 0; RegisteredFile file_; + std::uint32_t inflight_ops_ = 0; ConnectionState state_ = ConnectionState::kActive; RecvMode recv_mode_ = RecvMode::kMultishot; - std::uint64_t generation_ = 0; - std::int64_t last_active_ms_ = 0; - bool closing_ = false; bool closed_ = true; - bool retired_ = false; bool read_inflight_ = false; bool write_inflight_ = false; // Live connection handoff pauses recv before the fd is duplicated onto a // different worker. In particular, an armed multishot recv must not keep // consuming bytes from the shared socket after the handoff. bool recv_paused_ = false; + bool recv_armed_ = false; + bool needs_recv_rearm_ = + false; // re-arm deferred out of the completion handler + bool recv_eof_ = false; + bool retired_ = false; + bool peer_disconnect_poll_armed_ = false; + bool peer_disconnect_poll_cancel_requested_ = false; - std::uint32_t inflight_ops_ = 0; + std::vector read_buffer_; // Session coroutines borrow their Connection storage for their whole // lifetime: BeginClose/RetireConnection still close the fd and wake waiters // immediately, but ReclaimConnections skips storage while it is borrowed, so @@ -94,17 +102,10 @@ struct Connection { // reclaimed memory. Borrow and release run on the owning worker thread (or // before the first await that could observe retirement). std::uint32_t storage_borrows_ = 0; - absl::Status last_error_ = absl::OkStatus(); - - std::vector read_buffer_; std::deque received_buffers_; - std::coroutine_handle<> read_waiter_{}; - bool recv_armed_ = false; - bool needs_recv_rearm_ = - false; // re-arm deferred out of the completion handler - bool recv_eof_ = false; - bool peer_disconnect_poll_armed_ = false; - bool peer_disconnect_poll_cancel_requested_ = false; + + std::uint64_t id_ = 0; + std::uint64_t generation_ = 0; PeerDisconnectCallback peer_disconnect_callback_ = nullptr; void* peer_disconnect_context_ = nullptr; std::shared_ptr tls_state_; From 7c42d3bfffc14b797ab74087b9e1e18b6ec9a96f Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 19 Oct 2026 10:24:39 +0800 Subject: [PATCH 3/3] perf: retain only Connection member layout changes --- include/bycorf/io/storage.h | 5 ++--- include/bycorf/net/connection.h | 11 +++++++++-- include/bycorf/runtime/cross_core.h | 8 +++----- include/bycorf/runtime/task.h | 5 ++--- src/io/storage.cpp | 8 ++++---- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/include/bycorf/io/storage.h b/include/bycorf/io/storage.h index 1060e61..ba01143 100644 --- a/include/bycorf/io/storage.h +++ b/include/bycorf/io/storage.h @@ -96,9 +96,9 @@ class SizeIoAwaitable final : public OneShotIoAwaitable { private: Worker* worker_ = nullptr; FixedFile file_{}; - Operation operation_ = Operation::kRead; FixedBuffer buffer_{}; std::uint64_t offset_ = 0; + Operation operation_ = Operation::kRead; }; class OpenFixedFileAwaitable final : public OneShotIoAwaitable { @@ -111,10 +111,9 @@ class OpenFixedFileAwaitable final : public OneShotIoAwaitable { absl::Status await_resume(); private: - // A leading int can occupy OneShotIoAwaitable's tail padding. - int flags_ = 0; Worker* worker_ = nullptr; std::string path_; + int flags_ = 0; mode_t mode_ = 0; FixedFile file_{}; }; diff --git a/include/bycorf/net/connection.h b/include/bycorf/net/connection.h index 92c6862..596a2db 100644 --- a/include/bycorf/net/connection.h +++ b/include/bycorf/net/connection.h @@ -68,8 +68,15 @@ struct ReceivedBuffer { struct Connection { using PeerDisconnectCallback = void (*)(void*) noexcept; - // Group owner-local read readiness and completion bookkeeping: these fields - // are checked together when a reader suspends and a receive completes. + // Keep read-suspension writes together: read_waiter_ and read_inflight_ + // moved from offsets 176/51 to 8/52 in the audited GCC 13 x86-64 layout. + // With 64-byte-aligned allocations, these writes touch one cache line + // instead of two. This alignment depends on the allocator, not alignof; + // recheck actual addresses and generated stores when changing the layout. + // Normal receive completion still writes the deque tail on another line, + // so this does not establish fewer lines for the whole receive/GET path. + // Smaller sizeof or fewer touched lines alone does not establish a speedup: + // require correctness and paired throughput/tail-latency regression tests. Worker* worker_ = nullptr; std::coroutine_handle<> read_waiter_{}; absl::Status last_error_ = absl::OkStatus(); diff --git a/include/bycorf/runtime/cross_core.h b/include/bycorf/runtime/cross_core.h index 7894055..cf6db97 100644 --- a/include/bycorf/runtime/cross_core.h +++ b/include/bycorf/runtime/cross_core.h @@ -60,17 +60,15 @@ using WorkerId = std::uint16_t; // pointer. run_fn is the one unavoidable indirect call: it invokes the // type-erased user closure, and only on the request leg. struct RemoteWork { - // Keep aligned fields first and small bookkeeping last. Derived awaiters - // can then reuse the base's tail padding for their target worker ID. + WorkerId origin_ = 0; std::coroutine_handle<> waiter_{}; + TaskClass task_class_ = TaskClass::kForeground; void (*run_fn_)(RemoteWork*) = nullptr; // runs the user fn, stores result + bool reply_deferred_ = false; #if BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE std::uint64_t request_post_ns_ = 0; std::uint64_t reply_post_ns_ = 0; #endif - WorkerId origin_ = 0; - TaskClass task_class_ = TaskClass::kForeground; - bool reply_deferred_ = false; }; // Small one-way control message. It is copied into the target mailbox, so the diff --git a/include/bycorf/runtime/task.h b/include/bycorf/runtime/task.h index 7828876..9fa672b 100644 --- a/include/bycorf/runtime/task.h +++ b/include/bycorf/runtime/task.h @@ -66,10 +66,12 @@ class Task { // Task has exactly one successful co_return and unhandled exceptions // terminate. Keep the result directly in the promise so return_value does // not pay optional::emplace's reset/engagement branch on every completion. + // The flag is still required for destroying a frame before it completes. union { char empty_; T value_; }; + bool value_constructed_ = false; std::coroutine_handle<> continuation_{}; void* completion_context_ = nullptr; completion_fn completion_ = nullptr; @@ -77,9 +79,6 @@ class Task { // or kNotDetached while the task is owned by a Task object/awaiter. static constexpr std::uint32_t kNotDetached = 0xffffffffu; std::uint32_t detached_index_ = kNotDetached; - // Keep the flag beside the index to avoid a separate alignment gap. It is - // still required when a frame is destroyed before constructing its result. - bool value_constructed_ = false; static void* operator new(std::size_t size) { return detail::AllocateCoroutineFrame(size); diff --git a/src/io/storage.cpp b/src/io/storage.cpp index 055a755..9d0fe2c 100644 --- a/src/io/storage.cpp +++ b/src/io/storage.cpp @@ -127,9 +127,9 @@ SizeIoAwaitable::SizeIoAwaitable(Worker& worker, FixedFile file, Operation operation) noexcept : worker_(&worker), file_(file), - operation_(operation), buffer_(buffer), - offset_(offset) {} + offset_(offset), + operation_(operation) {} bool SizeIoAwaitable::await_suspend(std::coroutine_handle<> awaiting) { absl::Status status; @@ -191,9 +191,9 @@ absl::StatusOr SizeIoAwaitable::await_resume() { OpenFixedFileAwaitable::OpenFixedFileAwaitable(Worker& worker, std::string path, int flags, mode_t mode, FixedFile file) - : flags_(flags), - worker_(&worker), + : worker_(&worker), path_(std::move(path)), + flags_(flags), mode_(mode), file_(file) {}