Skip to content

Flush deferred SQEs before parking #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vadimskipin merged 2 commits into from
Jul 14, 2026
Merged

Flush deferred SQEs before parking #102

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2503aaf
Add sqReady / cqReady helpers for the ring counter reads
vadimskipin Jul 14, 2026
2dd997e
Flush deferred SQEs before parking
vadimskipin Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 33 additions & 8 deletions src/fibers/fiber.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,8 @@ struct FiberScheduler::ProcessorState
void wakeThread() noexcept;
void parkThread(uint64_t waitNs, CpuTimer * timer) noexcept;
bool hasWork() const noexcept;
uint32_t sqReady() const noexcept;
uint32_t cqReady() const noexcept;

void enqueueDoorbell() noexcept;
void postWakeup(ProcessorState * target) noexcept;
Expand Down Expand Up @@ -770,6 +772,13 @@ void FiberScheduler::ProcessorState::wakeThread() noexcept

void FiberScheduler::ProcessorState::parkThread(uint64_t waitNs, CpuTimer * timer) noexcept
{
// Flush deferred SQEs: the idle path has no other submit, and parking
// passes to_submit=0, so a deferred doorbell rearm, MSG_RING wakeup, or
// remote cancel would sit queued until unrelated activity lands on this
// ring. If the submit defers again: on EBUSY hasWork() sees the full CQ
// and skips the park; on EAGAIN the timed park is the retry backoff.
submitIo(true);

// Announce that we are about to park, then a seq_cst fence pairing with the one in wakeThread:
// release alone is not a StoreLoad barrier, so without this the store could reorder past the
// hasWork() re-check below while a concurrent wakeThread reads sleeping=false - both miss, and
Expand Down Expand Up @@ -822,13 +831,30 @@ bool FiberScheduler::ProcessorState::hasWork() const noexcept
{
return true;
}
if (cqReady())
{
return true;
}
return false;
}

// Ignore ring->cq.khead atomic/non-atomic mismatch.
// The kernel and other submitters update the SQ/CQ ring counters with plain
// stores; a racy read is benign - a stale count only defers or repeats work -
// so hide it from TSan here rather than annotating every call site.
uint32_t FiberScheduler::ProcessorState::sqReady() const noexcept
{
TSAN_IGNORE_BEGIN();
uint32_t count = ::io_uring_cq_ready(&ring);
uint32_t count = ::io_uring_sq_ready(&ring);
TSAN_IGNORE_END();
return count;
}

return count > 0;
uint32_t FiberScheduler::ProcessorState::cqReady() const noexcept
{
TSAN_IGNORE_BEGIN();
uint32_t count = ::io_uring_cq_ready(&ring);
TSAN_IGNORE_END();
return count;
}

void FiberScheduler::ProcessorState::enqueueDoorbell() noexcept
Expand Down Expand Up @@ -943,9 +969,7 @@ bool FiberScheduler::ProcessorState::submitIo(bool flush) noexcept
// the submission lock when there's nothing to submit or the count/staleness
// thresholds haven't been met. Kept small so it inlines into runFiber;
// the rest lives in submitIoSlow.
TSAN_IGNORE_BEGIN();
uint32_t count = ::io_uring_sq_ready(&ring);
TSAN_IGNORE_END();
uint32_t count = sqReady();
if (count == 0)
{
return false;
Expand All @@ -970,7 +994,7 @@ __attribute__((noinline)) bool FiberScheduler::ProcessorState::submitIoSlow(uint
{
std::lock_guard lock(submissionLock);

uint32_t count = ::io_uring_sq_ready(&ring);
uint32_t count = sqReady();
if (count == 0)
{
return false;
Expand Down Expand Up @@ -1970,7 +1994,8 @@ bool FiberScheduler::handleReadyQueue(ProcessorState * processor, CpuTimer * tim
bool FiberScheduler::handleCompletionQueue(ProcessorState * processor) noexcept
{
// Fast path: CQ ring is empty, nothing to drain.
if (::io_uring_cq_ready(&processor->ring) == 0)
uint32_t count = processor->cqReady();
if (count == 0)
{
return false;
}
Expand Down