diff --git a/src/fibers/fiber.cpp b/src/fibers/fiber.cpp index 8863872..aa7dab9 100644 --- a/src/fibers/fiber.cpp +++ b/src/fibers/fiber.cpp @@ -586,6 +586,8 @@ struct FiberScheduler::ProcessorState void wakeThread() noexcept; void parkThread(uint64_t waitNs, CpuTimer * timer) noexcept; bool hasWork() const noexcept; + uint32_t sqReady() const noexcept; + uint32_t cqReady() const noexcept; void enqueueDoorbell() noexcept; void postWakeup(ProcessorState * target) noexcept; @@ -770,6 +772,13 @@ void FiberScheduler::ProcessorState::wakeThread() noexcept void FiberScheduler::ProcessorState::parkThread(uint64_t waitNs, CpuTimer * timer) noexcept { + // Flush deferred SQEs: the idle path has no other submit, and parking + // passes to_submit=0, so a deferred doorbell rearm, MSG_RING wakeup, or + // remote cancel would sit queued until unrelated activity lands on this + // ring. If the submit defers again: on EBUSY hasWork() sees the full CQ + // and skips the park; on EAGAIN the timed park is the retry backoff. + submitIo(true); + // Announce that we are about to park, then a seq_cst fence pairing with the one in wakeThread: // release alone is not a StoreLoad barrier, so without this the store could reorder past the // hasWork() re-check below while a concurrent wakeThread reads sleeping=false - both miss, and @@ -822,13 +831,30 @@ bool FiberScheduler::ProcessorState::hasWork() const noexcept { return true; } + if (cqReady()) + { + return true; + } + return false; +} - // Ignore ring->cq.khead atomic/non-atomic mismatch. +// The kernel and other submitters update the SQ/CQ ring counters with plain +// stores; a racy read is benign - a stale count only defers or repeats work - +// so hide it from TSan here rather than annotating every call site. +uint32_t FiberScheduler::ProcessorState::sqReady() const noexcept +{ TSAN_IGNORE_BEGIN(); - uint32_t count = ::io_uring_cq_ready(&ring); + uint32_t count = ::io_uring_sq_ready(&ring); TSAN_IGNORE_END(); + return count; +} - return count > 0; +uint32_t FiberScheduler::ProcessorState::cqReady() const noexcept +{ + TSAN_IGNORE_BEGIN(); + uint32_t count = ::io_uring_cq_ready(&ring); + TSAN_IGNORE_END(); + return count; } void FiberScheduler::ProcessorState::enqueueDoorbell() noexcept @@ -943,9 +969,7 @@ bool FiberScheduler::ProcessorState::submitIo(bool flush) noexcept // the submission lock when there's nothing to submit or the count/staleness // thresholds haven't been met. Kept small so it inlines into runFiber; // the rest lives in submitIoSlow. - TSAN_IGNORE_BEGIN(); - uint32_t count = ::io_uring_sq_ready(&ring); - TSAN_IGNORE_END(); + uint32_t count = sqReady(); if (count == 0) { return false; @@ -970,7 +994,7 @@ __attribute__((noinline)) bool FiberScheduler::ProcessorState::submitIoSlow(uint { std::lock_guard lock(submissionLock); - uint32_t count = ::io_uring_sq_ready(&ring); + uint32_t count = sqReady(); if (count == 0) { return false; @@ -1970,7 +1994,8 @@ bool FiberScheduler::handleReadyQueue(ProcessorState * processor, CpuTimer * tim bool FiberScheduler::handleCompletionQueue(ProcessorState * processor) noexcept { // Fast path: CQ ring is empty, nothing to drain. - if (::io_uring_cq_ready(&processor->ring) == 0) + uint32_t count = processor->cqReady(); + if (count == 0) { return false; }