diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 80198866..ba11caa9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -21,5 +21,28 @@ RUN apt-get -y update && \ bundler install && \ npm install -g @anthropic-ai/claude-code +# SPDK, built from source with shared libraries so libevpl can link it via +# pkg-config (EVPL_CORE_MECH_SPDK guest mode + spdk_mem_register framework). +# nvme-cuse is disabled to avoid a libfuse3 dependency; tests run without +# hugepages via spdk_env_opts.no_huge. The sed repoints the stale DPDK +# build-tree -L path in the generated .pc files at the install prefix, since +# the source tree is deleted after install. +ARG SPDK_VERSION=v25.09 +RUN apt-get -y --no-install-recommends install \ + python3 python3-pip python3-pyelftools python3-setuptools \ + meson pkg-config nasm patchelf autoconf automake libtool help2man && \ + git clone --depth 1 --branch ${SPDK_VERSION} --recurse-submodules \ + --shallow-submodules https://github.com/spdk/spdk.git /tmp/spdk && \ + cd /tmp/spdk && \ + ./configure --prefix=/usr/local --with-shared \ + --disable-tests --disable-unit-tests --disable-examples \ + --disable-apps --without-nvme-cuse && \ + make -j$(nproc) && \ + make install && \ + sed -i "s|-L/tmp/spdk/dpdk/build/lib|-L/usr/local/lib|g" \ + /usr/local/lib/pkgconfig/*.pc && \ + ldconfig && \ + cd / && rm -rf /tmp/spdk + ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 diff --git a/CMakeLists.txt b/CMakeLists.txt index d700e970..4f77e252 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ option(IO_URING_ENABLED "Enable io_uring support" ON) option(IO_URING_NVME_ENABLED "Enable the io_uring NVMe uring_cmd block backend (auto-detected when ON; set OFF to force-disable, e.g. under the clang static analyzer)" ON) option(REQUIRE_NETNS_TESTS "Fail configure if network namespace tests cannot be enabled" OFF) set(LIBAIO_ENABLED "" CACHE STRING "Enable libaio support (empty=autodetect, YES=required, NO=disabled)") +set(SPDK_ENABLED "" CACHE STRING "Enable SPDK support (empty=autodetect, YES=required, NO=disabled)") option(EVPL_IOVEC_PROFILE "Enable retained-iovec stack profiling" OFF) project(libevpl LANGUAGES C) @@ -220,6 +221,55 @@ else() message(STATUS "xlio library not found.") endif() +# SPDK guest-mode integration (EVPL_CORE_MECH_SPDK + spdk_mem_register memory +# framework). Detected via pkg-config against SPDK shared libraries +# (./configure --with-shared); static SPDK is not supported. SPDK is Linux +# only and the mechanism composes with epoll, so both are required when found. +if(NOT SPDK_ENABLED STREQUAL "NO" AND NOT SPDK_ENABLED STREQUAL "OFF") + find_package(PkgConfig) + if(PkgConfig_FOUND) + pkg_check_modules(SPDK IMPORTED_TARGET GLOBAL + spdk_bdev spdk_sock spdk_sock_posix + spdk_thread spdk_env_dpdk spdk_util spdk_log) + endif() + if(SPDK_FOUND) + message(STATUS "SPDK found via pkg-config") + add_definitions(-DHAVE_SPDK) + set(HAVE_SPDK 1) + # --no-as-needed: DPDK driver libraries (librte_mempool_ring et al.) + # and SPDK module libraries (sock impls, bdev modules) register + # themselves via constructors and are referenced by no symbol, so the + # default --as-needed link drops them and their functionality is + # silently absent at runtime. + set(SPDK_LINK_TARGETS "-Wl,--no-as-needed" PkgConfig::SPDK) + + # SPDK's shared libspdk_util leaves ISA-L symbols undefined (SPDK + # normally links ISA-L statically into applications), so link ISA-L + # explicitly when its pkg-config -- installed by SPDK's own build -- + # is present. + pkg_check_modules(ISAL IMPORTED_TARGET GLOBAL libisal libisal_crypto) + if(ISAL_FOUND) + list(APPEND SPDK_LINK_TARGETS PkgConfig::ISAL) + endif() + + # Test-only modules: subsystem JSON bring-up (spdk_init) and the bdev + # subsystem registration constructors (spdk_event_bdev), used by the + # SPDK bdev tests to create a malloc bdev without the app framework. + pkg_check_modules(SPDK_INIT IMPORTED_TARGET GLOBAL + spdk_init spdk_event_bdev) + if(SPDK_INIT_FOUND) + set(SPDK_TEST_LINK_TARGETS PkgConfig::SPDK_INIT) + endif() + else() + if(SPDK_ENABLED STREQUAL "YES" OR SPDK_ENABLED STREQUAL "ON") + message(FATAL_ERROR "SPDK required but not found.") + endif() + message(STATUS "SPDK not found.") + endif() +else() + message(STATUS "SPDK disabled") +endif() + # OpenSSL backs the TLS protocol. macOS ships no OpenSSL/LibreSSL headers or # linkable stubs in the SDK, so this resolves to a Homebrew openssl@3 (found # via the prefixes added above) -- find_package is used rather than a bare @@ -274,6 +324,14 @@ if(NOT EVPL_MECHANISMS) message(FATAL_ERROR "No supported event core mechanism found on ${CMAKE_SYSTEM_NAME}") endif() +# The SPDK mechanism delegates its fd handling to the epoll backend. SPDK is +# deliberately NOT appended to EVPL_MECHANISMS: that list replicates generic +# tests which pump evpl from a plain pthread and cannot run in guest mode; +# SPDK coverage comes from dedicated tests under src/core/spdk/tests. +if(HAVE_SPDK AND NOT EVPL_HAVE_EPOLL) + message(FATAL_ERROR "SPDK support requires the epoll core mechanism") +endif() + message(STATUS "Event core mechanisms: ${EVPL_MECHANISMS} (default: ${EVPL_MECH})") add_definitions(-g -Wall -Werror -Wno-unused-function) diff --git a/docs/api/binds.md b/docs/api/binds.md index a8922046..ffba84e2 100644 --- a/docs/api/binds.md +++ b/docs/api/binds.md @@ -75,6 +75,8 @@ Look up a protocol ID by name. - `"STREAM_SOCKET_TCP"` → `EVPL_STREAM_SOCKET_TCP` - `"DATAGRAM_SOCKET_UDP"` → `EVPL_DATAGRAM_SOCKET_UDP` - `"STREAM_XLIO_TCP"` → `EVPL_STREAM_XLIO_TCP` +- `"STREAM_SPDK_TCP"` → `EVPL_STREAM_SPDK_TCP` (spdk_sock; requires the SPDK + core mechanism inside an SPDK host application) - `"STREAM_RDMACM_RC"` → `EVPL_STREAM_RDMACM_RC` - `"DATAGRAM_RDMACM_RC"` → `EVPL_DATAGRAM_RDMACM_RC` - `"DATAGRAM_RDMACM_UD"` → `EVPL_DATAGRAM_RDMACM_UD` diff --git a/docs/api/block.md b/docs/api/block.md index d9429b4f..92a1a116 100644 --- a/docs/api/block.md +++ b/docs/api/block.md @@ -52,6 +52,26 @@ libevpl's Block I/O subsystem offers: **Use cases:** Ultra-low latency storage, maximum IOPS, dedicated storage devices +### SPDK bdev + +**Description:** Access to any SPDK block device (bdev) of a host SPDK application + +**Availability:** Builds with SPDK installed (`SPDK_ENABLED`); at runtime +requires `EVPL_CORE_MECH_SPDK` — the opening event loop must be an evpl thread +running as an spdk_thread inside an SPDK application that owns the bdev layer. + +**Characteristics:** +- Full access to the host application's bdev stack (NVMe, malloc, RAID, + logical volumes, crypto, ...) +- Userspace polled completions delivered by the bdev I/O channel on the + owning spdk_thread +- evpl slab buffers are pre-registered with `spdk_mem_register`, so I/O is + zero-copy DMA-safe +- The URI is the bdev name registered in the host application + +**Use cases:** libevpl workloads embedded in an existing SPDK application that +need to share its storage stack + ## Types ### `struct evpl_block_device` @@ -70,6 +90,9 @@ Identifies block device backend: |----------|-------------| | `EVPL_BLOCK_PROTOCOL_IO_URING` | Linux io_uring | | `EVPL_BLOCK_PROTOCOL_VFIO` | VFIO-NVMe direct access | +| `EVPL_BLOCK_PROTOCOL_LIBAIO` | Linux libaio | +| `EVPL_BLOCK_PROTOCOL_IO_URING_NVME` | io_uring NVMe passthrough (uring_cmd) | +| `EVPL_BLOCK_PROTOCOL_SPDK_BDEV` | SPDK bdev (requires the SPDK core mechanism) | ### `evpl_block_callback_t` @@ -84,50 +107,102 @@ Callback invoked when a block operation completes. **Parameters:** - `evpl` - Event loop -- `status` - 0 on success, negative error code on failure +- `status` - 0 on success, positive errno on failure +- `private_data` - User-provided context + +### `evpl_block_open_callback_t` + +```c +typedef void (*evpl_block_open_callback_t)( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *private_data); +``` + +Callback invoked when an asynchronous device open completes. + +**Parameters:** +- `evpl` - Event loop +- `blockdev` - Opened device handle, or `NULL` on failure +- `status` - 0 on success, positive errno on failure - `private_data` - User-provided context ## Functions ### Device Management +Device open and close are asynchronous operations that run in the context of +an event loop, like every other libevpl operation. The completion callback +always fires from a later iteration of the opening loop — never inline from +the call itself — including for failures. + +The opening event loop owns the device for lifecycle purposes: + +- it must outlive the device, +- backend device events (such as hot-remove) are handled on its thread, and +- `evpl_block_close_device` must be called with this same event loop. + +Queues may still be opened against the device from any event loop / thread. + #### `evpl_block_open_device` ```c -struct evpl_block_device *evpl_block_open_device( +void evpl_block_open_device( + struct evpl *evpl, enum evpl_block_protocol_id protocol, - const char *uri); + const char *uri, + evpl_block_open_callback_t callback, + void *private_data); ``` -Open a block device. Each device should be opened once globally for the whole process. +Open a block device asynchronously. Each device should be opened once +globally for the whole process. **Parameters:** +- `evpl` - Event loop that will own the device - `protocol` - Backend protocol to use - `uri` - Device identifier (protocol-specific) - -**Returns:** Block device handle, or `NULL` on failure +- `callback` - Completion callback; receives the device handle or `NULL` +- `private_data` - User context **URI Formats:** -**io_uring:** +**io_uring / libaio:** - Device path: `/dev/nvme0n1` - File path: `/tmp/testfile` +**io_uring NVMe passthrough:** +- NVMe namespace block device: `/dev/nvme0n1` + **VFIO-NVMe:** - PCI address: `0000:01:00.0` +**SPDK bdev:** +- bdev name in the host SPDK application: `Malloc0`, `Nvme0n1` + --- #### `evpl_block_close_device` ```c -void evpl_block_close_device(struct evpl_block_device *blockdev); +void evpl_block_close_device( + struct evpl *evpl, + struct evpl_block_device *blockdev, + evpl_block_callback_t callback, + void *private_data); ``` -Close a block device. All queues must be closed first. +Close a block device asynchronously. All queues must be closed first, and the +call must be made on the event loop that opened the device. The device handle +is invalid as soon as this is called; the callback (which may be `NULL`) fires +from a later loop iteration once the backend has released the device. **Parameters:** +- `evpl` - Event loop that opened the device - `blockdev` - Device to close +- `callback` - Completion callback, or `NULL` +- `private_data` - User context --- diff --git a/include/evpl/evpl_block.h b/include/evpl/evpl_block.h index dc50936e..6450f851 100644 --- a/include/evpl/evpl_block.h +++ b/include/evpl/evpl_block.h @@ -13,13 +13,49 @@ struct evpl_block_device; struct evpl_block_queue; -struct evpl_block_device * -evpl_block_open_device( +typedef void (*evpl_block_callback_t)( + struct evpl *evpl, + int status, + void *private_data); + +/* + * Completion for evpl_block_open_device. blockdev is the opened device, or + * NULL on failure; status is 0 on success or a positive errno. + */ +typedef void (*evpl_block_open_callback_t)( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *private_data); + +/* + * Open a block device asynchronously. Like all libevpl operations this runs + * in the context of an event loop: the callback fires from a later iteration + * of `evpl`'s loop (never inline from this call), including for failures. + * + * The opening evpl owns the device for lifecycle purposes: it must outlive + * the device, backend device events (e.g. hot-remove) are handled on its + * thread, and evpl_block_close_device must be called with this same evpl. + * Queues may still be opened against the device from any evpl thread. + */ +void evpl_block_open_device( + struct evpl *evpl, enum evpl_block_protocol_id protocol, - const char *uri); + const char *uri, + evpl_block_open_callback_t callback, + void *private_data); +/* + * Close a block device asynchronously. Must be called on the evpl that + * opened the device, after all of its queues have been closed. The callback + * (which may be NULL) fires from a later loop iteration once the backend has + * released the device; blockdev is invalid as soon as this is called. + */ void evpl_block_close_device( - struct evpl_block_device *blockdev); + struct evpl *evpl, + struct evpl_block_device *blockdev, + evpl_block_callback_t callback, + void *private_data); uint64_t evpl_block_size( struct evpl_block_device *blockdev); @@ -36,11 +72,6 @@ void evpl_block_close_queue( struct evpl *evpl, struct evpl_block_queue *queue); -typedef void (*evpl_block_callback_t)( - struct evpl *evpl, - int status, - void *private_data); - void evpl_block_read( struct evpl *evpl, struct evpl_block_queue *queue, diff --git a/include/evpl/evpl_config.h b/include/evpl/evpl_config.h index f1c08ddc..08d479e1 100644 --- a/include/evpl/evpl_config.h +++ b/include/evpl/evpl_config.h @@ -28,6 +28,11 @@ enum evpl_core_mech { EVPL_CORE_MECH_EPOLL = 1, EVPL_CORE_MECH_KQUEUE = 2, EVPL_CORE_MECH_SELECT = 3, + /* Guest mode inside an SPDK application: each evpl is pumped by an + * spdk_poller on the spdk_thread that created it. Never the platform + * default; requires the host to have initialized the SPDK env and thread + * library, and evpl_create() must run on an spdk_thread. */ + EVPL_CORE_MECH_SPDK = 4, }; struct evpl_global_config * @@ -122,6 +127,20 @@ void evpl_thread_config_set_wait_ms( struct evpl_thread_config *config, int wait_ms); +/* Thread name, used e.g. to name the spdk_thread created for an evpl_thread + * under EVPL_CORE_MECH_SPDK. Truncated to the config field size. */ +void evpl_thread_config_set_name( + struct evpl_thread_config *config, + const char *name); + +/* SPDK cpumask string (as accepted by spdk_cpuset_parse, e.g. "0x3" or + * "[0,1]") constraining where the host scheduler may place the spdk_thread + * created for an evpl_thread. Empty (default) lets the host decide. Only + * meaningful under EVPL_CORE_MECH_SPDK. */ +void evpl_thread_config_set_spdk_cpumask( + struct evpl_thread_config *config, + const char *cpumask); + void evpl_global_config_set_slab_size( struct evpl_global_config *config, uint64_t size); @@ -210,6 +229,16 @@ void evpl_global_config_set_libaio_enabled( struct evpl_global_config *config, int enabled); +void evpl_global_config_set_spdk_enabled( + struct evpl_global_config *config, + int enabled); + +/* spdk_sock implementation for STREAM_SPDK_TCP ("posix", "uring", ...); + * NULL (default) selects SPDK's default implementation. */ +void evpl_global_config_set_spdk_sock_impl( + struct evpl_global_config *config, + const char *impl_name); + void evpl_global_config_set_libaio_max_pending( struct evpl_global_config *config, unsigned int max_pending); diff --git a/include/evpl/evpl_core.h b/include/evpl/evpl_core.h index 7b0ffb7c..b1124479 100644 --- a/include/evpl/evpl_core.h +++ b/include/evpl/evpl_core.h @@ -18,7 +18,8 @@ enum evpl_framework_id { EVPL_FRAMEWORK_TLS = 4, EVPL_FRAMEWORK_TCP_RDMA = 5, EVPL_FRAMEWORK_LIBAIO = 6, - EVPL_NUM_FRAMEWORK = 7 + EVPL_FRAMEWORK_SPDK = 7, + EVPL_NUM_FRAMEWORK = 8 }; enum evpl_protocol_id { @@ -31,7 +32,8 @@ enum evpl_protocol_id { EVPL_STREAM_RDMACM_RC = 6, EVPL_STREAM_SOCKET_TLS = 7, EVPL_DATAGRAM_TCP_RDMA = 8, - EVPL_NUM_PROTO = 9 + EVPL_STREAM_SPDK_TCP = 9, + EVPL_NUM_PROTO = 10 }; enum evpl_block_protocol_id { @@ -39,7 +41,8 @@ enum evpl_block_protocol_id { EVPL_BLOCK_PROTOCOL_VFIO = 1, EVPL_BLOCK_PROTOCOL_LIBAIO = 2, EVPL_BLOCK_PROTOCOL_IO_URING_NVME = 3, - EVPL_NUM_BLOCK_PROTOCOL = 4 + EVPL_BLOCK_PROTOCOL_SPDK_BDEV = 4, + EVPL_NUM_BLOCK_PROTOCOL = 5 }; struct evpl; @@ -69,12 +72,29 @@ evpl_get_hf_monotonic_time( void evpl_destroy( struct evpl *evpl); -void evpl_continue( +/* + * Run exactly one iteration of the event loop. Returns an approximate count + * of work items handled this pass (timers fired, events dispatched, deferrals + * run, poll-callback activity); 0 means the pass was idle. External loops + * embedding evpl (e.g. an SPDK reactor poller) use the return value to report + * busy/idle to their own scheduler. + */ +int evpl_continue( struct evpl *evpl); void evpl_run( struct evpl *evpl); +/* + * Wake this evpl so its next pump re-evaluates pending work. Required when + * code sharing the thread outside of an evpl callback (e.g. another SPDK + * poller on the same spdk_thread) mutates evpl state such as queuing a send; + * without it an external host loop may sleep without knowing the evpl has + * work. Safe from any thread; idempotent. + */ +void evpl_kick( + struct evpl *evpl); + typedef void (*evpl_loop_callback_t)( struct evpl *evpl, void *private_data); diff --git a/include/evpl/evpl_doorbell.h b/include/evpl/evpl_doorbell.h index e040b70b..efbb9aae 100644 --- a/include/evpl/evpl_doorbell.h +++ b/include/evpl/evpl_doorbell.h @@ -20,6 +20,24 @@ typedef void (*evpl_doorbell_callback_t)( struct evpl *evpl, struct evpl_doorbell *doorbell); +/* + * Open a doorbell's wakeup fd without registering it on an evpl. Callable + * from any thread; a later evpl_add_doorbell_opened() on the owning thread + * attaches it to the loop. Rings issued between open and add are retained + * and delivered on the first dispatch after add. Use this pair (instead of + * evpl_add_doorbell, which opens and registers in one step on the owning + * thread) when the doorbell must be ringable before its owning thread is up. + */ +void +evpl_doorbell_open( + struct evpl_doorbell *doorbell); + +void +evpl_add_doorbell_opened( + struct evpl *evpl, + struct evpl_doorbell *doorbell, + evpl_doorbell_callback_t callback); + void evpl_add_doorbell( struct evpl *evpl, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 503a9cab..8c78f701 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,13 @@ macro(unit_test module test_name) target_link_libraries(${module}_${test_name} evpl pthread) + # test_common.h embeds the SPDK mini-reactor harness when SPDK support is + # compiled in (for EVPL_TEST_CORE_MECH=spdk), so test binaries need the + # SPDK libraries on their own link line. + if (HAVE_SPDK) + target_link_libraries(${module}_${test_name} ${SPDK_LINK_TARGETS}) + endif() + list(GET sources 0 first_source) if(LIBEVPL_NETNS_TESTING) @@ -44,6 +51,10 @@ macro(unit_test_all_mechs module test_name) target_link_libraries(${module}_${test_name} evpl pthread) + if (HAVE_SPDK) + target_link_libraries(${module}_${test_name} ${SPDK_LINK_TARGETS}) + endif() + list(GET sources 0 first_source) foreach(mech ${EVPL_MECHANISMS}) @@ -91,6 +102,63 @@ macro(unit_test_bin_all_mechs module test_name binary_name) endforeach() endmacro() +# Like unit_test_all_mechs, but additionally registers an +# EVPL_TEST_CORE_MECH=spdk instance. Only for tests that are fully +# evpl_thread/callback driven (no evpl_run or manual pumping from the main +# thread): under spdk the harness boots a mini reactor pool and every evpl +# thread runs as an spdk_thread in guest mode. LeakSanitizer is disabled for +# the spdk instance (DPDK EAL keeps process-lifetime allocations). +macro(unit_test_all_mechs_spdk module test_name) + + unit_test_all_mechs(${module} ${test_name} ${ARGN}) + + if (HAVE_SPDK) + set(sources ${ARGN}) + list(GET sources 0 first_source) + set(_test libevpl/${module}/${test_name}_spdk) + + if(LIBEVPL_NETNS_TESTING) + add_test(NAME ${_test} COMMAND ${LIBEVPL_SOURCE_DIR}/scripts/netns_test_wrapper.sh ${CMAKE_CURRENT_BINARY_DIR}/${module}_${test_name}) + + set_tests_properties(${_test} PROPERTIES + ENVIRONMENT "TEST_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${first_source};EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0") + else() + add_test(NAME ${_test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${module}_${test_name}) + + set_tests_properties(${_test} PROPERTIES + RESOURCE_LOCK evpl_net + ENVIRONMENT "TEST_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${first_source};EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0") + endif() + endif() + +endmacro() + +# Prebuilt-binary counterpart of unit_test_all_mechs_spdk. +macro(unit_test_bin_all_mechs_spdk module test_name binary_name) + + unit_test_bin_all_mechs(${module} ${test_name} ${binary_name} ${ARGN}) + + if (HAVE_SPDK) + set(_test libevpl/${module}/${test_name}_spdk) + + get_target_property(test_file ${binary_name} TEST_FILE) + + if(LIBEVPL_NETNS_TESTING) + add_test(NAME ${_test} COMMAND ${LIBEVPL_SOURCE_DIR}/scripts/netns_test_wrapper.sh ${TEST_BIN}/${binary_name} ${ARGN}) + + set_tests_properties(${_test} PROPERTIES + ENVIRONMENT "TEST_FILE=${test_file};EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0") + else() + add_test(NAME ${_test} COMMAND ${TEST_BIN}/${binary_name} ${ARGN}) + + set_tests_properties(${_test} PROPERTIES + RESOURCE_LOCK evpl_net + ENVIRONMENT "TEST_FILE=${test_file};EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0") + endif() + endif() + +endmacro() + macro(unit_test_bin module test_name binary_name) if(LIBEVPL_NETNS_TESTING) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index dc4523fe..be0c5c23 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -86,6 +86,18 @@ if (HAVE_XLIO) add_subdirectory(xlio) endif() +if (HAVE_SPDK) + set(CORE_SRC ${CORE_SRC} + spdk/spdk_core.c spdk/evpl_spdk.h + spdk/spdk_framework.c + spdk/spdk_sock_common.h + spdk/spdk_block.c + spdk/tcp.c + thread/thread_spdk.c) + add_subdirectory(spdk) + set(BACKEND_LIBDEPS ${BACKEND_LIBDEPS} ${SPDK_LINK_TARGETS}) +endif() + set(CORE_SRC ${CORE_SRC} thread/thread.c) add_subdirectory(thread) diff --git a/src/core/allocator.c b/src/core/allocator.c index 70079791..291b508c 100644 --- a/src/core/allocator.c +++ b/src/core/allocator.c @@ -295,7 +295,7 @@ evpl_allocator_build_slab( } else { slab->data = evpl_valloc(evpl_shared->config->slab_size, - evpl_shared->config->page_size); + evpl_shared->config->slab_alignment); } diff --git a/src/core/block.c b/src/core/block.c index bc05f568..27bf63aa 100644 --- a/src/core/block.c +++ b/src/core/block.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "core/evpl_shared.h" #include "core/macros.h" @@ -94,72 +95,192 @@ evpl_block_complete( callback(evpl, status, callback_private); } /* evpl_block_complete */ -SYMBOL_EXPORT struct evpl_block_device * -evpl_block_open_device( - enum evpl_block_protocol_id protocol_id, - const char *uri) -{ +/* Transient state for an asynchronous device open. The deferral normalizes + * delivery: the user callback always fires from a loop iteration after + * evpl_block_open_device returned, whether the backend completed inline + * (io_uring/libaio/vfio) or later, and for failures too. + */ +struct evpl_block_open_request { struct evpl_block_protocol *protocol; + char *uri; + evpl_block_open_callback_t callback; + void *private_data; struct evpl_block_device *blockdev; - void *protocol_private_data; + int status; + struct evpl_deferral deferral; +}; + +static void +evpl_block_open_deliver( + struct evpl *evpl, + void *private_data) +{ + struct evpl_block_open_request *request = private_data; + evpl_block_open_callback_t callback = request->callback; + void *callback_private = request->private_data; + struct evpl_block_device *blockdev = request->blockdev; + int status = request->status; + + if (request->uri) { + evpl_free(request->uri); + } - __evpl_init(); + evpl_free(request); - if (protocol_id >= EVPL_NUM_BLOCK_PROTOCOL) { - return NULL; + callback(evpl, blockdev, status, callback_private); +} /* evpl_block_open_deliver */ + +static void +evpl_block_open_complete( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *ctx) +{ + struct evpl_block_open_request *request = ctx; + struct evpl_block_protocol *protocol = request->protocol; + + request->blockdev = blockdev; + request->status = status; + + if (blockdev) { + + blockdev->protocol = protocol; + + /* Per-device metric series, labelled with the device URI and the + * protocol type so callers can aggregate across all devices or + * break out by device/type. The histograms additionally carry an + * "op" label, so one series is created per operation class. + */ + for (int k = 0; k < EVPL_BLOCK_NUM_OP_KIND; k++) { + blockdev->m_latency[k] = prometheus_histogram_create_series( + evpl_shared->block_latency, + (const char *[]) { "device", "type", "op" }, + (const char *[]) { request->uri, protocol->name, + evpl_block_op_names[k] }, 3); + + blockdev->m_request_size[k] = prometheus_histogram_create_series( + evpl_shared->block_request_size, + (const char *[]) { "device", "type", "op" }, + (const char *[]) { request->uri, protocol->name, + evpl_block_op_names[k] }, 3); + } + + blockdev->m_queue_depth = prometheus_gauge_create_series( + evpl_shared->block_queue_depth, + (const char *[]) { "device", "type" }, + (const char *[]) { request->uri, protocol->name }, 2); } + evpl_defer(evpl, &request->deferral); +} /* evpl_block_open_complete */ + +SYMBOL_EXPORT void +evpl_block_open_device( + struct evpl *evpl, + enum evpl_block_protocol_id protocol_id, + const char *uri, + evpl_block_open_callback_t callback, + void *private_data) +{ + struct evpl_block_open_request *request; + struct evpl_block_protocol *protocol = NULL; + void *protocol_private_data; + size_t urilen; + + request = evpl_zalloc(sizeof(*request)); + + request->callback = callback; + request->private_data = private_data; + + evpl_deferral_init(&request->deferral, evpl_block_open_deliver, request); - protocol = evpl_shared->block_protocol[protocol_id]; + if (protocol_id < EVPL_NUM_BLOCK_PROTOCOL) { + protocol = evpl_shared->block_protocol[protocol_id]; + } if (!protocol) { - /* In-range id but the backend was not registered (e.g. gated out at - * build time, such as the NVMe uring_cmd backend on older liburing). - * Report failure rather than dereferencing a NULL protocol. */ - return NULL; + /* Out-of-range id, or an in-range backend that was not registered + * (gated out at build time or disabled at runtime). Uniform + * asynchronous failure rather than dereferencing a NULL protocol. */ + request->status = ENOTSUP; + evpl_defer(evpl, &request->deferral); + return; } + request->protocol = protocol; + + urilen = strlen(uri) + 1; + request->uri = evpl_zalloc(urilen); + memcpy(request->uri, uri, urilen); + evpl_attach_framework_shared(protocol->framework->id); protocol_private_data = evpl_shared->framework_private[protocol->framework-> id]; - blockdev = protocol->open_device(uri, protocol_private_data); + protocol->open_device(evpl, request->uri, protocol_private_data, + evpl_block_open_complete, request); +} /* evpl_block_open_device */ - if (!blockdev) { - return NULL; - } +/* Transient state for an asynchronous device close; same deferral + * normalization as open. The user callback may be NULL. */ +struct evpl_block_close_request { + evpl_block_callback_t callback; + void *private_data; + int status; + struct evpl_deferral deferral; +}; - blockdev->protocol = protocol; +static void +evpl_block_close_deliver( + struct evpl *evpl, + void *private_data) +{ + struct evpl_block_close_request *request = private_data; + evpl_block_callback_t callback = request->callback; + void *callback_private = request->private_data; + int status = request->status; - /* Per-device metric series, labelled with the device URI and the - * protocol type so callers can aggregate across all devices or - * break out by device/type. The histograms additionally carry an - * "op" label, so one series is created per operation class. - */ - for (int k = 0; k < EVPL_BLOCK_NUM_OP_KIND; k++) { - blockdev->m_latency[k] = prometheus_histogram_create_series( - evpl_shared->block_latency, - (const char *[]) { "device", "type", "op" }, - (const char *[]) { uri, protocol->name, evpl_block_op_names[k] }, 3); - - blockdev->m_request_size[k] = prometheus_histogram_create_series( - evpl_shared->block_request_size, - (const char *[]) { "device", "type", "op" }, - (const char *[]) { uri, protocol->name, evpl_block_op_names[k] }, 3); + evpl_free(request); + + if (callback) { + callback(evpl, status, callback_private); } +} /* evpl_block_close_deliver */ - blockdev->m_queue_depth = prometheus_gauge_create_series( - evpl_shared->block_queue_depth, - (const char *[]) { "device", "type" }, - (const char *[]) { uri, protocol->name }, 2); +static void +evpl_block_close_complete( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *ctx) +{ + struct evpl_block_close_request *request = ctx; - return blockdev; -} /* evpl_block_open_device */ + request->status = status; + + evpl_defer(evpl, &request->deferral); +} /* evpl_block_close_complete */ SYMBOL_EXPORT void -evpl_block_close_device(struct evpl_block_device *bdev) +evpl_block_close_device( + struct evpl *evpl, + struct evpl_block_device *bdev, + evpl_block_callback_t callback, + void *private_data) { + struct evpl_block_close_request *request; + + request = evpl_zalloc(sizeof(*request)); + + request->callback = callback; + request->private_data = private_data; + + evpl_deferral_init(&request->deferral, evpl_block_close_deliver, request); + + /* The metric series hang off the device struct, which the backend frees; + * destroy them before handing the device over. */ for (int k = 0; k < EVPL_BLOCK_NUM_OP_KIND; k++) { prometheus_histogram_destroy_series(evpl_shared->block_latency, bdev->m_latency[k]); @@ -169,7 +290,7 @@ evpl_block_close_device(struct evpl_block_device *bdev) prometheus_gauge_destroy_series(evpl_shared->block_queue_depth, bdev->m_queue_depth); - bdev->close_device(bdev); + bdev->close_device(evpl, bdev, evpl_block_close_complete, request); } /* evpl_block_close_device */ SYMBOL_EXPORT uint64_t diff --git a/src/core/config.c b/src/core/config.c index 4ff52cd3..dc8ebda3 100644 --- a/src/core/config.c +++ b/src/core/config.c @@ -72,6 +72,9 @@ evpl_global_config_init(void) config->libaio_enabled = 1; config->libaio_max_pending = 256; + config->spdk_enabled = 1; + config->slab_alignment = config->page_size; + config->preallocate_slabs = 0; config->preallocate_threads = 0; @@ -103,6 +106,10 @@ evpl_global_config_free(struct evpl_global_config *config) evpl_free(config->tls_cipher_list); } + if (config->spdk_sock_impl) { + evpl_free(config->spdk_sock_impl); + } + evpl_free(config); } /* evpl_global_config_free */ @@ -332,6 +339,22 @@ evpl_thread_config_set_wait_ms( config->wait_ms = wait_ms; } /* evpl_thread_config_set_wait_ms */ +SYMBOL_EXPORT void +evpl_thread_config_set_name( + struct evpl_thread_config *config, + const char *name) +{ + snprintf(config->name, sizeof(config->name), "%s", name); +} /* evpl_thread_config_set_name */ + +SYMBOL_EXPORT void +evpl_thread_config_set_spdk_cpumask( + struct evpl_thread_config *config, + const char *cpumask) +{ + snprintf(config->spdk_cpumask, sizeof(config->spdk_cpumask), "%s", cpumask); +} /* evpl_thread_config_set_spdk_cpumask */ + SYMBOL_EXPORT void evpl_global_config_set_slab_size( struct evpl_global_config *config, @@ -516,6 +539,26 @@ evpl_global_config_set_libaio_max_pending( config->libaio_max_pending = max_pending; } /* evpl_global_config_set_libaio_max_pending */ +SYMBOL_EXPORT void +evpl_global_config_set_spdk_enabled( + struct evpl_global_config *config, + int enabled) +{ + config->spdk_enabled = enabled; +} /* evpl_global_config_set_spdk_enabled */ + +SYMBOL_EXPORT void +evpl_global_config_set_spdk_sock_impl( + struct evpl_global_config *config, + const char *impl_name) +{ + if (config->spdk_sock_impl) { + evpl_free(config->spdk_sock_impl); + } + + config->spdk_sock_impl = impl_name ? strdup(impl_name) : NULL; +} /* evpl_global_config_set_spdk_sock_impl */ + SYMBOL_EXPORT void evpl_global_config_set_hf_time_mode( struct evpl_global_config *config, diff --git a/src/core/core.c b/src/core/core.c index 11efe805..1b92912b 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -45,6 +45,10 @@ evpl_core_ops_lookup(unsigned int mech) case EVPL_CORE_MECH_SELECT: return &evpl_core_select_ops; #endif /* ifdef EVPL_HAVE_SELECT */ +#ifdef HAVE_SPDK + case EVPL_CORE_MECH_SPDK: + return &evpl_core_spdk_ops; +#endif /* ifdef HAVE_SPDK */ default: return NULL; } /* switch */ @@ -62,6 +66,8 @@ evpl_core_mech_name(unsigned int mech) return "kqueue"; case EVPL_CORE_MECH_SELECT: return "select"; + case EVPL_CORE_MECH_SPDK: + return "spdk"; default: return "unknown"; } /* switch */ diff --git a/src/core/core.h b/src/core/core.h index cd235b37..889a2ac9 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -21,20 +21,26 @@ struct evpl_core; struct evpl_event; +/* The mechanism is pumped by an external loop (e.g. an SPDK reactor): the + * mechanism's own machinery invokes evpl_continue(), so evpl_run()/evpl_stop() + * must not be used and the wait op never blocks. */ +#define EVPL_CORE_OPS_EXTERNAL_LOOP 0x1 + struct evpl_core_ops { - const char *name; - int (*init)( + const char *name; + unsigned int flags; + int (*init)( struct evpl_core *evc, int max_events); - void (*destroy)( + void (*destroy)( struct evpl_core *evc); - void (*add)( + void (*add)( struct evpl_core *evc, struct evpl_event *event); - void (*remove)( + void (*remove)( struct evpl_core *evc, struct evpl_event *event); - int (*wait)( + int (*wait)( struct evpl_core *evc, int max_msecs); }; @@ -51,6 +57,10 @@ struct evpl_core_ops { #include "core/select.h" #endif /* ifdef EVPL_HAVE_SELECT */ +#ifdef HAVE_SPDK +#include "core/spdk/evpl_spdk.h" +#endif /* ifdef HAVE_SPDK */ + struct evpl_core { const struct evpl_core_ops *ops; union { @@ -63,6 +73,9 @@ struct evpl_core { #ifdef EVPL_HAVE_SELECT struct evpl_core_select select; #endif /* ifdef EVPL_HAVE_SELECT */ +#ifdef HAVE_SPDK + struct evpl_core_spdk spdk; +#endif /* ifdef HAVE_SPDK */ } u; }; diff --git a/src/core/doorbell.c b/src/core/doorbell.c index 27563a8c..994ff37d 100644 --- a/src/core/doorbell.c +++ b/src/core/doorbell.c @@ -30,17 +30,32 @@ evpl_event_user_callback( } /* evpl_event_user_callback */ +/* + * Open a doorbell's wakeup fd without registering it on an evpl. Callable + * from any thread; a later evpl_add_doorbell_opened() on the owning thread + * attaches it to the loop. Rings issued between open and add are retained + * (eventfd counter / pipe buffer) and delivered on the first dispatch after + * add. + */ SYMBOL_EXPORT void -evpl_add_doorbell( +evpl_doorbell_open(struct evpl_doorbell *doorbell) +{ + evpl_core_abort_if(evpl_wakeup_open(&doorbell->wakeup) < 0, + "evpl_doorbell_open: wakeup open failed"); +} /* evpl_doorbell_open */ + +/* + * Register a doorbell whose wakeup was already opened with + * evpl_doorbell_open(). Must run on the evpl's own thread. + */ +SYMBOL_EXPORT void +evpl_add_doorbell_opened( struct evpl *evpl, struct evpl_doorbell *doorbell, evpl_doorbell_callback_t callback) { struct evpl_event *event = &doorbell->event; - evpl_core_abort_if(evpl_wakeup_open(&doorbell->wakeup) < 0, - "evpl_add_doorbell: wakeup open failed"); - evpl_add_event(evpl, event, doorbell->wakeup.rfd, evpl_event_user_callback, NULL, NULL); @@ -48,7 +63,18 @@ evpl_add_doorbell( doorbell->callback = callback; -} /* evpl_add_event_user */ +} /* evpl_add_doorbell_opened */ + +SYMBOL_EXPORT void +evpl_add_doorbell( + struct evpl *evpl, + struct evpl_doorbell *doorbell, + evpl_doorbell_callback_t callback) +{ + evpl_doorbell_open(doorbell); + + evpl_add_doorbell_opened(evpl, doorbell, callback); +} /* evpl_add_doorbell */ SYMBOL_EXPORT void evpl_remove_doorbell( diff --git a/src/core/evpl.c b/src/core/evpl.c index 3731d292..a74a2e3c 100644 --- a/src/core/evpl.c +++ b/src/core/evpl.c @@ -108,6 +108,17 @@ evpl_shared_init(struct evpl_global_config *config) evpl_shared->config = config; +#ifdef HAVE_SPDK + /* Raised before the allocator exists so every slab ever built satisfies + * spdk_mem_register: 2 MiB covers all SPDK releases (v25.09 itself only + * needs 4 KiB). Costs a few KiB of padding when SPDK goes unused. */ + if (config->spdk_enabled) { + config->slab_alignment = 2 * 1024 * 1024; + config->slab_size = (config->slab_size + (2 * 1024 * 1024 - 1)) & + ~(uint64_t) (2 * 1024 * 1024 - 1); + } +#endif /* ifdef HAVE_SPDK */ + if (evpl_shared->config->hf_time_mode == 2) { /* Deetect if nonstop_tsc is supported, enable iff so */ @@ -257,6 +268,22 @@ evpl_shared_init(struct evpl_global_config *config) #endif /* ifdef HAVE_XLIO */ +#ifdef HAVE_SPDK + if (config->spdk_enabled) { + evpl_framework_init(evpl_shared, EVPL_FRAMEWORK_SPDK, + &evpl_framework_spdk); + + /* Registered whenever SPDK support is enabled; both fail loudly at + * open/connect/listen when the evpl is not running under + * EVPL_CORE_MECH_SPDK, rather than being invisible here. */ + evpl_protocol_init(evpl_shared, EVPL_STREAM_SPDK_TCP, + &evpl_spdk_tcp); + + evpl_block_protocol_init(evpl_shared, EVPL_BLOCK_PROTOCOL_SPDK_BDEV, + &evpl_block_protocol_spdk_bdev); + } +#endif /* ifdef HAVE_SPDK */ + } /* evpl_shared_init */ void @@ -441,6 +468,17 @@ evpl_create(struct evpl_thread_config *config) * compares it without converting on every iteration. */ evpl->spin_ticks = evpl_ns_to_ticks(evpl->config.spin_ns); +#ifdef HAVE_SPDK + /* Lazy attach: the first SPDK-mode evpl_create is the earliest point the + * guest-mode contract guarantees a live host env. Attaching registers + * every existing slab with spdk_mem_register (and future slabs register + * as they are built), making evpl buffers DMA-safe for host SPDK I/O. */ + if (evpl_shared->config->core_mech == EVPL_CORE_MECH_SPDK && + evpl_shared->framework[EVPL_FRAMEWORK_SPDK]) { + evpl_attach_framework_shared(EVPL_FRAMEWORK_SPDK); + } +#endif /* ifdef HAVE_SPDK */ + evpl_core_init(&evpl->core, 64); evpl->running = 1; @@ -456,7 +494,7 @@ evpl_create(struct evpl_thread_config *config) return evpl; } /* evpl_init */ -SYMBOL_EXPORT FORCE_INLINE void +SYMBOL_EXPORT FORCE_INLINE int evpl_continue(struct evpl *evpl) { struct evpl_event *event; @@ -466,12 +504,16 @@ evpl_continue(struct evpl *evpl) struct evpl_timer *timer; int i, n; int msecs = evpl->config.wait_ms; + int work = 0; uint64_t elapsed; int64_t remain; uint64_t now_ticks; + uint64_t poll_activity; if (evpl->poll_mode && evpl->poll_iterations < evpl->config.poll_iterations) { + poll_activity = evpl->activity; + for (i = 0; i < evpl->num_poll; ++i) { poll = &evpl->poll[i]; poll->callback(evpl, poll->private_data); @@ -479,6 +521,8 @@ evpl_continue(struct evpl *evpl) evpl->poll_iterations++; + work += (int) (evpl->activity - poll_activity); + } else { now_ticks = evpl_now_ticks(); @@ -512,6 +556,8 @@ evpl_continue(struct evpl *evpl) evpl_timer_insert(evpl, timer); } + work++; + } while (evpl->num_timers); } @@ -570,6 +616,10 @@ evpl_continue(struct evpl *evpl) evpl->loop_hooks.post_wait(evpl, evpl->loop_hooks.private_data); } + if (n > 0) { + work += n; + } + if (evpl->pending_close_binds && n == 0) { struct evpl_bind *next; @@ -596,15 +646,18 @@ evpl_continue(struct evpl *evpl) if ((event->flags & EVPL_READ_READY) == EVPL_READ_READY) { event->read_callback(evpl, event); + work++; } if ((event->flags & EVPL_WRITE_READY) == EVPL_WRITE_READY) { event->write_callback(evpl, event); + work++; } if ((event->flags & EVPL_ERROR) == EVPL_ERROR) { event->error_callback(evpl, event); + work++; } if ((event->flags & EVPL_READ_READY) != EVPL_READ_READY && @@ -633,12 +686,15 @@ evpl_continue(struct evpl *evpl) deferral->armed = 0; deferral->callback(evpl, deferral->private_data); + + work++; } if (evpl->loop_hooks.iteration_end) { evpl->loop_hooks.iteration_end(evpl, evpl->loop_hooks.private_data); } + return work; } /* evpl_continue */ SYMBOL_EXPORT void @@ -677,6 +733,11 @@ evpl_get_hf_monotonic_time( SYMBOL_EXPORT void evpl_run(struct evpl *evpl) { + evpl_core_abort_if(evpl->core.ops->flags & EVPL_CORE_OPS_EXTERNAL_LOOP, + "evpl_run: core mechanism '%s' is pumped by an external " + "loop; evpl_run() must not be called", + evpl->core.ops->name); + while (evpl->running) { evpl_continue(evpl); } @@ -700,6 +761,11 @@ evpl_stop(struct evpl *evpl) ssize_t len; int err; + evpl_core_abort_if(evpl->core.ops->flags & EVPL_CORE_OPS_EXTERNAL_LOOP, + "evpl_stop: core mechanism '%s' is pumped by an external " + "loop; there is no evpl_run() loop to stop", + evpl->core.ops->name); + evpl_core_assert(evpl->running); evpl->running = 0; @@ -715,9 +781,31 @@ evpl_stop(struct evpl *evpl) evpl->run_wakeup.wfd, len, err, strerror(err)); } /* evpl_stop */ +/* + * Wake this evpl so its next pump re-evaluates pending work. Required when + * code sharing the thread outside of an evpl callback (e.g. another SPDK + * poller on the same spdk_thread) mutates evpl state such as queuing a send: + * under an external-loop mechanism the host loop may otherwise sleep without + * knowing the evpl has work. Safe from any thread; idempotent. + */ +SYMBOL_EXPORT void +evpl_kick(struct evpl *evpl) +{ + ssize_t len; + int err; + + len = evpl_wakeup_signal(&evpl->run_wakeup); + + err = errno; + + evpl_core_abort_if(len != sizeof(uint64_t), + "evpl_kick: wakeup signal (fd %d) failed: len=%zd errno=%d (%s)", + evpl->run_wakeup.wfd, len, err, strerror(err)); +} /* evpl_kick */ + void -evpl_destroy_close_bind(struct evpl *evpl) +evpl_close_all_binds(struct evpl *evpl) { struct evpl_bind *bind; @@ -726,9 +814,21 @@ evpl_destroy_close_bind(struct evpl *evpl) { evpl_close(evpl, bind); } +} /* evpl_close_all_binds */ + +int +evpl_has_pending_binds(struct evpl *evpl) +{ + return evpl->binds != NULL || evpl->pending_close_binds != NULL; +} /* evpl_has_pending_binds */ + +void +evpl_destroy_close_bind(struct evpl *evpl) +{ + evpl_close_all_binds(evpl); /* Pump events until we have no pending close binds */ - while (evpl->binds || evpl->pending_close_binds) { + while (evpl_has_pending_binds(evpl)) { evpl_continue(evpl); } diff --git a/src/core/evpl.h b/src/core/evpl.h index 9e5353a5..63fd657f 100644 --- a/src/core/evpl.h +++ b/src/core/evpl.h @@ -21,7 +21,8 @@ struct evpl_thread_config { int poll_iterations; unsigned int spin_ns; int wait_ms; - + char name[32]; + char spdk_cpumask[40]; }; struct evpl_global_config { @@ -71,6 +72,17 @@ struct evpl_global_config { unsigned int libaio_enabled; unsigned int libaio_max_pending; + unsigned int spdk_enabled; + + /* spdk_sock implementation name for STREAM_SPDK_TCP ("posix", "uring", + * ...); NULL selects SPDK's default implementation. */ + char *spdk_sock_impl; + + /* Alignment for slab allocations; page_size by default, raised to 2 MiB + * when the SPDK framework is registered so slabs satisfy + * spdk_mem_register()'s alignment requirement. */ + unsigned int slab_alignment; + unsigned int preallocate_slabs; unsigned int preallocate_threads; @@ -187,7 +199,6 @@ struct evpl_connect_request { struct evpl_listener { struct evpl_thread *thread; - int running; struct evpl_doorbell doorbell; struct evpl_bind **binds; int num_binds; @@ -225,6 +236,16 @@ void evpl_destroy_close_bind( struct evpl *evpl); +/* Push every open bind into pending-close state without pumping the loop. */ +void +evpl_close_all_binds( + struct evpl *evpl); + +/* True while any bind is still open or draining its close. */ +int +evpl_has_pending_binds( + struct evpl *evpl); + /* Exported (defined in poll.c); also declared in the public evpl/evpl_poll.h so * out-of-tree consumers can use them. See evpl_poll.h for semantics. */ void diff --git a/src/core/io_uring/io_uring_block.c b/src/core/io_uring/io_uring_block.c index 2a2cc3f5..6c4fdc2a 100644 --- a/src/core/io_uring/io_uring_block.c +++ b/src/core/io_uring/io_uring_block.c @@ -217,23 +217,33 @@ evpl_io_uring_open_queue( } /* evpl_io_uring_open_queue */ static void -evpl_io_uring_close_device(struct evpl_block_device *bdev) +evpl_io_uring_close_device( + struct evpl *evpl, + struct evpl_block_device *bdev, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_io_uring_device *dev = bdev->private_data; close(dev->fd); evpl_free(dev); evpl_free(bdev); + + complete(evpl, NULL, 0, ctx); } /* evpl_io_uring_close_device */ -static struct evpl_block_device * +static void evpl_io_uring_open_device( - const char *uri, - void *private_data) + struct evpl *evpl, + const char *uri, + void *private_data, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_block_device *bdev; struct evpl_io_uring_device *dev; struct stat st; + int err; bdev = evpl_zalloc(sizeof(*bdev)); dev = evpl_zalloc(sizeof(*dev)); @@ -241,15 +251,20 @@ evpl_io_uring_open_device( dev->fd = open(uri, O_RDWR | O_DIRECT); if (dev->fd < 0) { + err = errno; evpl_free(dev); - return NULL; + evpl_free(bdev); + complete(evpl, NULL, err, ctx); + return; } if (fstat(dev->fd, &st) < 0) { + err = errno; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } bdev->private_data = dev; @@ -259,10 +274,12 @@ evpl_io_uring_open_device( if (S_ISBLK(st.st_mode)) { uint64_t bytes; if (ioctl(dev->fd, BLKGETSIZE64, &bytes) < 0) { + err = errno; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } bdev->size = bytes; } else { @@ -271,7 +288,7 @@ evpl_io_uring_open_device( bdev->max_request_size = 4 * 1024 * 1024; - return bdev; + complete(evpl, bdev, 0, ctx); } /* evpl_io_uring_open_device */ struct evpl_block_protocol evpl_block_protocol_io_uring = { diff --git a/src/core/io_uring/io_uring_nvme_block.c b/src/core/io_uring/io_uring_nvme_block.c index d7e8b4e2..3863e080 100644 --- a/src/core/io_uring/io_uring_nvme_block.c +++ b/src/core/io_uring/io_uring_nvme_block.c @@ -228,19 +228,28 @@ evpl_io_uring_nvme_open_queue( } /* evpl_io_uring_nvme_open_queue */ static void -evpl_io_uring_nvme_close_device(struct evpl_block_device *bdev) +evpl_io_uring_nvme_close_device( + struct evpl *evpl, + struct evpl_block_device *bdev, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_io_uring_device *dev = bdev->private_data; close(dev->fd); evpl_free(dev); evpl_free(bdev); + + complete(evpl, NULL, 0, ctx); } /* evpl_io_uring_nvme_close_device */ -static struct evpl_block_device * +static void evpl_io_uring_nvme_open_device( - const char *uri, - void *private_data) + struct evpl *evpl, + const char *uri, + void *private_data, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_block_device *bdev; struct evpl_io_uring_device *dev; @@ -248,6 +257,7 @@ evpl_io_uring_nvme_open_device( uint64_t bytes; int sector_size; int nsid; + int err; bdev = evpl_zalloc(sizeof(*bdev)); dev = evpl_zalloc(sizeof(*dev)); @@ -255,33 +265,41 @@ evpl_io_uring_nvme_open_device( dev->fd = open(uri, O_RDWR | O_DIRECT); if (dev->fd < 0) { + err = errno; evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } if (fstat(dev->fd, &st) < 0 || !S_ISBLK(st.st_mode)) { + err = errno ? errno : ENOTBLK; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } nsid = ioctl(dev->fd, NVME_IOCTL_ID); if (nsid < 0) { + err = errno; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } if (ioctl(dev->fd, BLKGETSIZE64, &bytes) < 0 || ioctl(dev->fd, BLKSSZGET, §or_size) < 0) { + err = errno; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } dev->nsid = nsid; @@ -293,7 +311,7 @@ evpl_io_uring_nvme_open_device( bdev->size = bytes; bdev->max_request_size = 4 * 1024 * 1024; - return bdev; + complete(evpl, bdev, 0, ctx); } /* evpl_io_uring_nvme_open_device */ struct evpl_block_protocol evpl_block_protocol_io_uring_nvme = { diff --git a/src/core/io_uring/tests/basic.c b/src/core/io_uring/tests/basic.c index 271d6041..9ef1aeb1 100644 --- a/src/core/io_uring/tests/basic.c +++ b/src/core/io_uring/tests/basic.c @@ -9,35 +9,71 @@ #include "evpl/evpl.h" +struct test_state { + struct evpl_block_device *bdev; + int open_status; + int opened; + int closed; + int pending; +}; + static void -read_callback( +open_callback( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + state->bdev = blockdev; + state->open_status = status; + state->opened = 1; +} /* open_callback */ + +static void +close_callback( struct evpl *evpl, int status, void *private_data) { - int *pending = private_data; + struct test_state *state = private_data; if (status) { exit(1); } - (*pending)--; + state->closed = 1; +} /* close_callback */ -} /* read_callback */ +static void +io_callback( + struct evpl *evpl, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + if (status) { + exit(1); + } + + state->pending--; + +} /* io_callback */ int main( int argc, char *argv[]) { - struct evpl *evpl; - struct evpl_block_device *bdev; - int fd; - int rc; - struct evpl_block_queue *bqueue; - int pending = 0; - struct evpl_iovec iov; - int niov; + struct evpl *evpl; + int fd; + int rc; + struct evpl_block_queue *bqueue; + struct test_state state = { 0 }; + struct evpl_iovec iov; + int niov; fd = open("test.img", O_RDWR | O_CREAT, 0666); rc = ftruncate(fd, 1024 * 1024 * 1024); @@ -51,23 +87,33 @@ main( evpl = evpl_create(NULL); - bdev = evpl_block_open_device(EVPL_BLOCK_PROTOCOL_IO_URING, "test.img"); + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_IO_URING, "test.img", + open_callback, &state); - bqueue = evpl_block_open_queue(evpl, bdev); + while (!state.opened) { + evpl_continue(evpl); + } + + if (state.open_status || !state.bdev) { + fprintf(stderr, "open failed: %d\n", state.open_status); + exit(1); + } + + bqueue = evpl_block_open_queue(evpl, state.bdev); niov = evpl_iovec_alloc(evpl, 4096, 4096, 1, 0, &iov); - pending++; - evpl_block_write(evpl, bqueue, &iov, niov, 0, 0, read_callback, &pending); + state.pending++; + evpl_block_write(evpl, bqueue, &iov, niov, 0, 0, io_callback, &state); - pending++; - evpl_block_read(evpl, bqueue, &iov, niov, 0, read_callback, &pending); + state.pending++; + evpl_block_read(evpl, bqueue, &iov, niov, 0, io_callback, &state); - pending++; - evpl_block_flush(evpl, bqueue, read_callback, &pending); + state.pending++; + evpl_block_flush(evpl, bqueue, io_callback, &state); - while (pending) { + while (state.pending) { evpl_continue(evpl); } @@ -75,9 +121,13 @@ main( evpl_block_close_queue(evpl, bqueue); - evpl_block_close_device(bdev); + evpl_block_close_device(evpl, state.bdev, close_callback, &state); + + while (!state.closed) { + evpl_continue(evpl); + } evpl_destroy(evpl); return 0; -} /* main */ \ No newline at end of file +} /* main */ diff --git a/src/core/io_uring/tests/nvme_open_fail.c b/src/core/io_uring/tests/nvme_open_fail.c index fae986a2..97cab192 100644 --- a/src/core/io_uring/tests/nvme_open_fail.c +++ b/src/core/io_uring/tests/nvme_open_fail.c @@ -9,13 +9,34 @@ #include "evpl/evpl.h" +struct test_state { + struct evpl_block_device *bdev; + int open_status; + int opened; +}; + +static void +open_callback( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + state->bdev = blockdev; + state->open_status = status; + state->opened = 1; +} /* open_callback */ + int main( int argc, char *argv[]) { - struct evpl_block_device *bdev; - int fd; + struct evpl *evpl; + struct test_state state = { 0 }; + int fd; fd = open("test.img", O_RDWR | O_CREAT, 0666); @@ -26,12 +47,23 @@ main( close(fd); - bdev = evpl_block_open_device(EVPL_BLOCK_PROTOCOL_IO_URING_NVME, "test.img"); + evpl = evpl_create(NULL); + + /* A plain file is not an NVMe namespace: the open must fail, delivered + * asynchronously with a nonzero status and a NULL device. */ + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_IO_URING_NVME, "test.img", + open_callback, &state); - if (bdev) { - evpl_block_close_device(bdev); + while (!state.opened) { + evpl_continue(evpl); + } + + if (state.open_status == 0 || state.bdev) { + fprintf(stderr, "open unexpectedly succeeded\n"); exit(1); } + evpl_destroy(evpl); + return 0; } /* main */ diff --git a/src/core/libaio/libaio_block.c b/src/core/libaio/libaio_block.c index b11c2b36..73c169c5 100644 --- a/src/core/libaio/libaio_block.c +++ b/src/core/libaio/libaio_block.c @@ -192,23 +192,33 @@ evpl_libaio_open_queue( } /* evpl_libaio_open_queue */ static void -evpl_libaio_close_device(struct evpl_block_device *bdev) +evpl_libaio_close_device( + struct evpl *evpl, + struct evpl_block_device *bdev, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_libaio_device *dev = bdev->private_data; close(dev->fd); evpl_free(dev); evpl_free(bdev); + + complete(evpl, NULL, 0, ctx); } /* evpl_libaio_close_device */ -static struct evpl_block_device * +static void evpl_libaio_open_device( - const char *uri, - void *private_data) + struct evpl *evpl, + const char *uri, + void *private_data, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_block_device *bdev; struct evpl_libaio_device *dev; struct stat st; + int err; bdev = evpl_zalloc(sizeof(*bdev)); dev = evpl_zalloc(sizeof(*dev)); @@ -216,15 +226,20 @@ evpl_libaio_open_device( dev->fd = open(uri, O_RDWR | O_DIRECT); if (dev->fd < 0) { + err = errno; evpl_free(dev); - return NULL; + evpl_free(bdev); + complete(evpl, NULL, err, ctx); + return; } if (fstat(dev->fd, &st) < 0) { + err = errno; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } bdev->private_data = dev; @@ -234,10 +249,12 @@ evpl_libaio_open_device( if (S_ISBLK(st.st_mode)) { uint64_t bytes; if (ioctl(dev->fd, BLKGETSIZE64, &bytes) < 0) { + err = errno; close(dev->fd); evpl_free(dev); evpl_free(bdev); - return NULL; + complete(evpl, NULL, err, ctx); + return; } bdev->size = bytes; } else { @@ -246,7 +263,7 @@ evpl_libaio_open_device( bdev->max_request_size = 4 * 1024 * 1024; - return bdev; + complete(evpl, bdev, 0, ctx); } /* evpl_libaio_open_device */ struct evpl_block_protocol evpl_block_protocol_libaio = { diff --git a/src/core/libaio/tests/basic.c b/src/core/libaio/tests/basic.c index 937cb3a7..e36bc702 100644 --- a/src/core/libaio/tests/basic.c +++ b/src/core/libaio/tests/basic.c @@ -9,35 +9,71 @@ #include "evpl/evpl.h" +struct test_state { + struct evpl_block_device *bdev; + int open_status; + int opened; + int closed; + int pending; +}; + static void -read_callback( +open_callback( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + state->bdev = blockdev; + state->open_status = status; + state->opened = 1; +} /* open_callback */ + +static void +close_callback( struct evpl *evpl, int status, void *private_data) { - int *pending = private_data; + struct test_state *state = private_data; if (status) { exit(1); } - (*pending)--; + state->closed = 1; +} /* close_callback */ -} /* read_callback */ +static void +io_callback( + struct evpl *evpl, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + if (status) { + exit(1); + } + + state->pending--; + +} /* io_callback */ int main( int argc, char *argv[]) { - struct evpl *evpl; - struct evpl_block_device *bdev; - int fd; - int rc; - struct evpl_block_queue *bqueue; - int pending = 0; - struct evpl_iovec iov; - int niov; + struct evpl *evpl; + int fd; + int rc; + struct evpl_block_queue *bqueue; + struct test_state state = { 0 }; + struct evpl_iovec iov; + int niov; fd = open("test.img", O_RDWR | O_CREAT, 0666); rc = ftruncate(fd, 1024 * 1024 * 1024); @@ -51,23 +87,33 @@ main( evpl = evpl_create(NULL); - bdev = evpl_block_open_device(EVPL_BLOCK_PROTOCOL_LIBAIO, "test.img"); + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_LIBAIO, "test.img", + open_callback, &state); - bqueue = evpl_block_open_queue(evpl, bdev); + while (!state.opened) { + evpl_continue(evpl); + } + + if (state.open_status || !state.bdev) { + fprintf(stderr, "open failed: %d\n", state.open_status); + exit(1); + } + + bqueue = evpl_block_open_queue(evpl, state.bdev); niov = evpl_iovec_alloc(evpl, 4096, 4096, 1, 0, &iov); - pending++; - evpl_block_write(evpl, bqueue, &iov, niov, 0, 0, read_callback, &pending); + state.pending++; + evpl_block_write(evpl, bqueue, &iov, niov, 0, 0, io_callback, &state); - pending++; - evpl_block_read(evpl, bqueue, &iov, niov, 0, read_callback, &pending); + state.pending++; + evpl_block_read(evpl, bqueue, &iov, niov, 0, io_callback, &state); - pending++; - evpl_block_flush(evpl, bqueue, read_callback, &pending); + state.pending++; + evpl_block_flush(evpl, bqueue, io_callback, &state); - while (pending) { + while (state.pending) { evpl_continue(evpl); } @@ -75,7 +121,11 @@ main( evpl_block_close_queue(evpl, bqueue); - evpl_block_close_device(bdev); + evpl_block_close_device(evpl, state.bdev, close_callback, &state); + + while (!state.closed) { + evpl_continue(evpl); + } evpl_destroy(evpl); diff --git a/src/core/listen.c b/src/core/listen.c index 3e961ca7..d67b92ea 100644 --- a/src/core/listen.c +++ b/src/core/listen.c @@ -130,11 +130,10 @@ evpl_listener_init( { struct evpl_listener *listener = private_data; - evpl_add_doorbell(evpl, &listener->doorbell, evpl_listener_callback); - - __sync_synchronize(); - - listener->running = 1; + /* The doorbell wakeup was opened by evpl_listener_create before the + * thread existed, so rings issued before this registration are retained + * and dispatched on the first loop pass. */ + evpl_add_doorbell_opened(evpl, &listener->doorbell, evpl_listener_callback); return listener; @@ -149,6 +148,13 @@ evpl_listener_create(void) listener = evpl_zalloc(sizeof(*listener)); + /* Open the doorbell before the worker exists so evpl_listen can ring it + * immediately; the worker registers it on its own evpl in + * evpl_listener_init and picks up any retained rings. No readiness wait + * is needed, which also keeps this callable in SPDK guest mode where the + * worker may share a reactor with the caller. */ + evpl_doorbell_open(&listener->doorbell); + listener->thread = evpl_thread_create(NULL, evpl_listener_init, NULL, listener); listener->max_binds = 64; @@ -157,10 +163,6 @@ evpl_listener_create(void) listener->max_attached = 64; listener->attached = evpl_calloc(listener->max_attached, sizeof(struct evpl_listener_binding *)); - while (!listener->running) { - __sync_synchronize(); - } - return listener; } /* evpl_listener_create */ diff --git a/src/core/protocol.h b/src/core/protocol.h index 56e437af..ff218042 100644 --- a/src/core/protocol.h +++ b/src/core/protocol.h @@ -172,6 +172,19 @@ enum evpl_block_op_kind { EVPL_BLOCK_NUM_OP_KIND }; +/* + * Backend completion for asynchronous device open/close. Backends may invoke + * it inline from open_device/close_device or later from their own machinery; + * the core normalizes delivery to the user through a deferral either way. + * blockdev is the opened device (NULL on open failure, and always NULL for + * close); status is 0 or a positive errno. + */ +typedef void (*evpl_block_device_complete_t)( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *ctx); + struct evpl_block_device { /* Private data owned by the protocol */ void *private_data; @@ -200,8 +213,14 @@ struct evpl_block_device { struct evpl *evpl, struct evpl_block_device *blockdev); + /* Close the device. Runs on the evpl that opened the device; the + * backend frees blockdev and invokes complete (with a NULL blockdev) + * when teardown is finished. */ void (*close_device)( - struct evpl_block_device *blockdev); + struct evpl *evpl, + struct evpl_block_device *blockdev, + evpl_block_device_complete_t complete, + void *ctx); }; struct evpl_block_queue { @@ -281,18 +300,23 @@ struct evpl_block_queue { struct evpl_block_protocol { /* unique ID number for each protocol */ - unsigned int id; + unsigned int id; /* human readable name for protocol, no spaces */ - const char *name; + const char *name; /* pointer to associated framework, or NULL if no framework */ - struct evpl_framework *framework; + struct evpl_framework *framework; - /* Open a block device */ - struct evpl_block_device * (*open_device)( - const char *uri, - void *private_data); + /* Open a block device. Runs on the opening evpl's thread; the backend + * allocates the evpl_block_device and invokes complete with it (or with + * NULL and a positive errno on failure), inline or asynchronously. */ + void (*open_device)( + struct evpl *evpl, + const char *uri, + void *private_data, + evpl_block_device_complete_t complete, + void *ctx); }; void diff --git a/src/core/socket/tests/CMakeLists.txt b/src/core/socket/tests/CMakeLists.txt index 7dedfb61..de0a80df 100644 --- a/src/core/socket/tests/CMakeLists.txt +++ b/src/core/socket/tests/CMakeLists.txt @@ -9,11 +9,11 @@ unit_test_bin_all_mechs(socket hello_world_msg_udp hello_world_msg -r DATAGRAM_S unit_test_bin_all_mechs(socket hello_world_stream_tcp hello_world_stream -r STREAM_SOCKET_TCP) unit_test_bin_all_mechs(socket hello_world_connected_msg_tcp hello_world_connected_msg -r STREAM_SOCKET_TCP) -unit_test_bin_all_mechs(socket ping_pong_msg_udp ping_pong_msg -r DATAGRAM_SOCKET_UDP) +unit_test_bin_all_mechs_spdk(socket ping_pong_msg_udp ping_pong_msg -r DATAGRAM_SOCKET_UDP) unit_test_bin_all_mechs(socket ping_pong_stream_tcp ping_pong_stream -r STREAM_SOCKET_TCP) -unit_test_bin_all_mechs(socket ping_pong_msg_tcp ping_pong_connected_msg -r STREAM_SOCKET_TCP) +unit_test_bin_all_mechs_spdk(socket ping_pong_msg_tcp ping_pong_connected_msg -r STREAM_SOCKET_TCP) -unit_test_bin_all_mechs(socket bulk_msg_udp bulk_msg -r DATAGRAM_SOCKET_UDP) +unit_test_bin_all_mechs_spdk(socket bulk_msg_udp bulk_msg -r DATAGRAM_SOCKET_UDP) unit_test_bin_all_mechs(socket bulk_msg_tcp bulk_connected_msg -r STREAM_SOCKET_TCP) unit_test_bin_all_mechs(socket bulk_stream_tcp bulk_stream -r STREAM_SOCKET_TCP) @@ -21,4 +21,4 @@ unit_test_bin_all_mechs(socket rand_full_duplex_msg_udp rand_full_duplex_msg -r unit_test_bin_all_mechs(socket rand_full_duplex_stream_tcp rand_full_duplex_stream -r STREAM_SOCKET_TCP) # TCP_RDMA tests -unit_test_bin_all_mechs(socket rdma_ops_tcp_rdma rdma_ops -r DATAGRAM_TCP_RDMA) +unit_test_bin_all_mechs_spdk(socket rdma_ops_tcp_rdma rdma_ops -r DATAGRAM_TCP_RDMA) diff --git a/src/core/spdk/CMakeLists.txt b/src/core/spdk/CMakeLists.txt new file mode 100644 index 00000000..20581100 --- /dev/null +++ b/src/core/spdk/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Ben Jarvis +# +# SPDX-License-Identifier: LGPL-2.1-only + +if (NOT DISABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/core/spdk/evpl_spdk.h b/src/core/spdk/evpl_spdk.h new file mode 100644 index 00000000..9c6f644d --- /dev/null +++ b/src/core/spdk/evpl_spdk.h @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#pragma once + +/* + * EVPL_CORE_MECH_SPDK: guest-mode integration with an SPDK application. + * + * The evpl is pumped by a period-0 spdk_poller on the spdk_thread that called + * evpl_create(); libevpl never blocks and never owns a loop (evpl_run aborts). + * All of libevpl's fds live in an internal epoll composed from the epoll + * backend, so interrupt-mode reactors need exactly one fd registered with the + * thread's fd_group, plus a timerfd that translates evpl timer deadlines and + * pending-work kicks into fd_group wakeups. This mirrors how SPDK's own + * POSIX sock groups integrate (an internal epoll swept from a poller). + * + * This header is included by core/core.h in every translation unit, so it + * must not pull in SPDK headers; SPDK types appear only as opaque pointers. + */ + +#include + +#include "core/epoll.h" +#include "core/event.h" + +#ifndef EVPL_HAVE_EPOLL +#error "EVPL_CORE_MECH_SPDK requires the epoll core mechanism" +#endif /* ifndef EVPL_HAVE_EPOLL */ + +struct spdk_thread; +struct spdk_poller; +struct spdk_interrupt; + +struct evpl_core_spdk { + /* Must be first: the spdk mechanism delegates fd handling to the epoll + * backend, whose ops locate their state via evc->u.epoll. Backend state + * lives in a union inside struct evpl_core, so placing the epoll state + * first makes evc->u.epoll and evc->u.spdk.epoll the same object. */ + struct evpl_core_epoll epoll; + + struct spdk_thread *thread; /* owning spdk_thread */ + struct spdk_poller *poller; /* period-0 pump poller */ + struct spdk_interrupt *intr; /* epoll fd in the fd_group */ + int timer_fd; /* one-shot CLOCK_MONOTONIC */ + struct evpl_event timer_event; /* drain-only; wakes the fd_group */ + unsigned int interrupt_mode; /* reactor phase, via mode cb */ + unsigned int in_pump; /* inside evpl_continue() */ + unsigned int last_wait_full; /* inner wait returned max_events */ + uint64_t armed_ns; /* current timerfd programming */ +}; + +extern const struct evpl_core_ops evpl_core_spdk_ops; + +struct evpl_framework; +struct evpl_protocol; +struct evpl_block_protocol; + +extern struct evpl_framework evpl_framework_spdk; +extern struct evpl_protocol evpl_spdk_tcp; +extern struct evpl_block_protocol evpl_block_protocol_spdk_bdev; diff --git a/src/core/spdk/spdk_block.c b/src/core/spdk/spdk_block.c new file mode 100644 index 00000000..b861e5d4 --- /dev/null +++ b/src/core/spdk/spdk_block.c @@ -0,0 +1,791 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * EVPL_BLOCK_PROTOCOL_SPDK_BDEV: block protocol backed by SPDK bdevs. + * + * Guest-mode contract: the host application owns the SPDK env and the bdev + * layer; the uri is a bdev name registered there. Device open/close run on + * the calling evpl's thread -- an spdk_thread under EVPL_CORE_MECH_SPDK -- + * which becomes the device's owner: bdev events (hot-remove, resize) are + * delivered to it, and close must run on it (enforced). + * + * Queues wrap a per-thread spdk_io_channel and need no event-loop hookup of + * their own: completions are delivered by the channel's bdev pollers on the + * owning spdk_thread, outside evpl_continue -- the completion path bumps + * evpl_activity() and kicks the loop if callbacks armed deferrals, so a + * sleeping interrupt-mode reactor cannot strand work. + * + * evpl slab buffers are pre-registered with spdk_mem_register (see + * spdk_framework.c), so evpl iovec addresses are directly DMA-usable; a + * bounce through a registered slab buffer covers the rare bdev with an + * alignment requirement the caller's iovecs do not meet. + */ + +#include +#include +#include +#include + +#include +#include + +#include "core/evpl.h" +#include "evpl/evpl.h" +#include "core/protocol.h" +#include "core/evpl_shared.h" +#include "core/spdk/evpl_spdk.h" + +#define evpl_spdk_bdev_debug(...) evpl_debug("spdk_bdev", __FILE__, \ + __LINE__, __VA_ARGS__) +#define evpl_spdk_bdev_info(...) evpl_info("spdk_bdev", __FILE__, \ + __LINE__, __VA_ARGS__) +#define evpl_spdk_bdev_error(...) evpl_error("spdk_bdev", __FILE__, \ + __LINE__, __VA_ARGS__) +#define evpl_spdk_bdev_abort_if(cond, ...) \ + evpl_abort_if(cond, "spdk_bdev", __FILE__, __LINE__, __VA_ARGS__) + +#define EVPL_SPDK_BDEV_MAX_IOV 64 + +struct evpl_spdk_bdev_device { + struct evpl *owner; /* evpl that opened the device */ + struct spdk_thread *owner_thread; + struct spdk_bdev_desc *desc; + struct spdk_bdev *bdev; + struct evpl_block_device *bdev_pub; /* backpointer for RESIZE */ + uint32_t block_size; + uint64_t num_blocks; + size_t buf_align; + unsigned int supports_unmap : 1; + unsigned int supports_write_zeroes : 1; + unsigned int supports_flush : 1; + unsigned int has_write_cache : 1; + unsigned int dead : 1; +}; + +struct evpl_spdk_bdev_queue; + +struct evpl_spdk_bdev_request { + struct evpl_spdk_bdev_queue *queue; + enum evpl_block_op_kind kind; + evpl_block_callback_t callback; /* core evpl_block_complete */ + void *private_data; + struct iovec iov[EVPL_SPDK_BDEV_MAX_IOV]; + int niov; + uint64_t offset; /* bytes */ + uint64_t length; /* bytes */ + unsigned int need_flush : 1; /* sync write: chain flush */ + unsigned int in_flush : 1; /* chained flush in flight */ + unsigned int bounced : 1; + unsigned int debounce : 1; /* read: scatter back */ + struct evpl_iovec bounce; + struct spdk_bdev_io_wait_entry wait; + struct evpl_spdk_bdev_request *next; +}; + +struct evpl_spdk_bdev_queue { + struct evpl *evpl; + struct evpl_spdk_bdev_device *dev; + struct evpl_block_queue *bq; + struct spdk_io_channel *ch; + uint64_t outstanding; + unsigned int closing; + struct evpl_spdk_bdev_request *free_requests; +}; + +static void evpl_spdk_bdev_submit( + struct evpl_spdk_bdev_queue *queue, + struct evpl_spdk_bdev_request *request); + +static void evpl_spdk_bdev_io_done( + struct spdk_bdev_io *bdev_io, + bool success, + void *cb_arg); + +/* + * Completion epilogue shared by real completions and synthetic failures. + * Runs on the queue's own spdk_thread but OUTSIDE evpl_continue: bump + * activity for honest busy accounting, and kick the loop if the callback + * chain armed deferrals so a sleeping interrupt-mode reactor wakes to run + * them. + */ +static void +evpl_spdk_bdev_request_complete( + struct evpl_spdk_bdev_request *request, + int status) +{ + struct evpl_spdk_bdev_queue *queue = request->queue; + struct evpl *evpl = queue->evpl; + evpl_block_callback_t callback; + void *callback_private; + int deferrals_before; + + if (request->bounced) { + if (request->debounce && status == 0) { + const char *src = request->bounce.data; + int i; + + for (i = 0; i < request->niov; i++) { + memcpy(request->iov[i].iov_base, src, + request->iov[i].iov_len); + src += request->iov[i].iov_len; + } + } + + evpl_iovec_release_internal(evpl, &request->bounce); + } + + /* Recycle before the callback: it may submit again (reusing this + * request) or close the queue. */ + callback = request->callback; + callback_private = request->private_data; + + request->next = queue->free_requests; + queue->free_requests = request; + + evpl_activity(evpl); + + deferrals_before = evpl->num_active_deferrals; + + callback(evpl, status, callback_private); + + if (--queue->outstanding == 0 && queue->closing) { + struct evpl_spdk_bdev_request *req; + + spdk_put_io_channel(queue->ch); + + while ((req = queue->free_requests)) { + queue->free_requests = req->next; + evpl_free(req); + } + + evpl_free(queue->bq); + evpl_free(queue); + } + + if (evpl->num_active_deferrals > deferrals_before) { + evpl_kick(evpl); + } +} /* evpl_spdk_bdev_request_complete */ + +static void +evpl_spdk_bdev_io_done( + struct spdk_bdev_io *bdev_io, + bool success, + void *cb_arg) +{ + struct evpl_spdk_bdev_request *request = cb_arg; + struct evpl_spdk_bdev_queue *queue = request->queue; + + if (bdev_io) { + spdk_bdev_free_io(bdev_io); + } + + if (success && request->kind == EVPL_BLOCK_OP_WRITE && + request->need_flush && !request->in_flush) { + /* Durable write on a write-back device: chain a flush before + * completing the operation. */ + request->in_flush = 1; + evpl_spdk_bdev_submit(queue, request); + return; + } + + evpl_spdk_bdev_request_complete(request, success ? 0 : EIO); +} /* evpl_spdk_bdev_io_done */ + +static void +evpl_spdk_bdev_io_wait_retry(void *cb_arg) +{ + struct evpl_spdk_bdev_request *request = cb_arg; + + evpl_spdk_bdev_submit(request->queue, request); +} /* evpl_spdk_bdev_io_wait_retry */ + +static void +evpl_spdk_bdev_submit( + struct evpl_spdk_bdev_queue *queue, + struct evpl_spdk_bdev_request *request) +{ + struct evpl_spdk_bdev_device *dev = queue->dev; + int rc; + + if (request->in_flush) { + rc = spdk_bdev_flush(dev->desc, queue->ch, + 0, dev->num_blocks * dev->block_size, + evpl_spdk_bdev_io_done, request); + } else { + switch (request->kind) { + case EVPL_BLOCK_OP_READ: + rc = spdk_bdev_readv(dev->desc, queue->ch, + request->iov, request->niov, + request->offset, request->length, + evpl_spdk_bdev_io_done, request); + break; + case EVPL_BLOCK_OP_WRITE: + rc = spdk_bdev_writev(dev->desc, queue->ch, + request->iov, request->niov, + request->offset, request->length, + evpl_spdk_bdev_io_done, request); + break; + case EVPL_BLOCK_OP_FLUSH: + rc = spdk_bdev_flush(dev->desc, queue->ch, + 0, dev->num_blocks * dev->block_size, + evpl_spdk_bdev_io_done, request); + break; + case EVPL_BLOCK_OP_DISCARD: + if (request->length == 0) { + rc = 0; + evpl_spdk_bdev_io_done(NULL, true, request); + break; + } + rc = spdk_bdev_unmap(dev->desc, queue->ch, + request->offset, request->length, + evpl_spdk_bdev_io_done, request); + break; + default: + /* write_zeroes rides EVPL_BLOCK_OP_WRITE accounting but is + * distinguished by niov == 0. */ + rc = spdk_bdev_write_zeroes(dev->desc, queue->ch, + request->offset, request->length, + evpl_spdk_bdev_io_done, request); + break; + } /* switch */ + } + + if (rc == 0) { + return; + } + + if (rc == -ENOMEM) { + /* bdev_io pool exhausted: park until the channel frees one. The + * request stays counted in outstanding, so queue close cannot + * finalize under it. */ + request->wait.bdev = dev->bdev; + request->wait.cb_fn = evpl_spdk_bdev_io_wait_retry; + request->wait.cb_arg = request; + + rc = spdk_bdev_queue_io_wait(dev->bdev, queue->ch, &request->wait); + + if (rc == 0) { + return; + } + } + + evpl_spdk_bdev_request_complete(request, -rc); +} /* evpl_spdk_bdev_submit */ + +static struct evpl_spdk_bdev_request * +evpl_spdk_bdev_request_alloc(struct evpl_spdk_bdev_queue *queue) +{ + struct evpl_spdk_bdev_request *request = queue->free_requests; + + if (request) { + queue->free_requests = request->next; + memset(request, 0, sizeof(*request)); + } else { + request = evpl_zalloc(sizeof(*request)); + } + + request->queue = queue; + + return request; +} /* evpl_spdk_bdev_request_alloc */ + +/* + * Validate and stage a data op. Returns 0 and completes the op with an + * error itself when the request cannot be issued. + */ +static int +evpl_spdk_bdev_prepare( + struct evpl_spdk_bdev_queue *queue, + struct evpl_spdk_bdev_request *request, + const struct evpl_iovec *iov, + int niov, + uint64_t offset, + int is_write) +{ + struct evpl_spdk_bdev_device *dev = queue->dev; + uint64_t length = 0; + int misaligned = 0; + int i; + + for (i = 0; i < niov; i++) { + length += iov[i].length; + + if ((uintptr_t) iov[i].data & (dev->buf_align - 1)) { + misaligned = 1; + } + + if (i + 1 < niov && (iov[i].length & (dev->buf_align - 1))) { + misaligned = 1; + } + } + + request->offset = offset; + request->length = length; + + if (dev->dead) { + evpl_spdk_bdev_request_complete(request, ENODEV); + return -1; + } + + if (length == 0 || + (offset & (dev->block_size - 1)) || + (length & (dev->block_size - 1)) || + offset + length > dev->num_blocks * (uint64_t) dev->block_size) { + evpl_spdk_bdev_request_complete(request, EINVAL); + return -1; + } + + if (!misaligned && niov <= EVPL_SPDK_BDEV_MAX_IOV) { + for (i = 0; i < niov; i++) { + request->iov[i].iov_base = iov[i].data; + request->iov[i].iov_len = iov[i].length; + } + request->niov = niov; + return 0; + } + + /* Bounce through a single registered slab buffer (a plain heap buffer + * would not be spdk_mem_register'ed and could not DMA). */ + if (niov > EVPL_SPDK_BDEV_MAX_IOV || length > evpl_shared->config->buffer_size) { + evpl_spdk_bdev_error( + "request (%d iov, %lu bytes) cannot be bounced for bdev " + "alignment %zu", niov, (unsigned long) length, dev->buf_align); + evpl_spdk_bdev_request_complete(request, EINVAL); + return -1; + } + + if (evpl_iovec_alloc(queue->evpl, length, + dev->buf_align < 4096 ? 4096 : dev->buf_align, + 1, 0, &request->bounce) < 1) { + evpl_spdk_bdev_request_complete(request, ENOMEM); + return -1; + } + + request->bounced = 1; + + if (is_write) { + char *dst = request->bounce.data; + + for (i = 0; i < niov; i++) { + memcpy(dst, iov[i].data, iov[i].length); + dst += iov[i].length; + } + } else { + /* Keep the caller's layout for the completion-side scatter. */ + for (i = 0; i < niov; i++) { + request->iov[i].iov_base = iov[i].data; + request->iov[i].iov_len = iov[i].length; + } + request->niov = niov; + request->debounce = 1; + } + + if (is_write) { + request->iov[0].iov_base = request->bounce.data; + request->iov[0].iov_len = length; + request->niov = 1; + } + + return 0; +} /* evpl_spdk_bdev_prepare */ + +static void +evpl_spdk_bdev_read( + struct evpl *evpl, + struct evpl_block_queue *bq, + struct evpl_iovec *iov, + int niov, + uint64_t offset, + evpl_block_callback_t callback, + void *private_data) +{ + struct evpl_spdk_bdev_queue *queue = bq->private_data; + struct evpl_spdk_bdev_request *request; + + evpl_activity(evpl); + + request = evpl_spdk_bdev_request_alloc(queue); + + request->kind = EVPL_BLOCK_OP_READ; + request->callback = callback; + request->private_data = private_data; + + queue->outstanding++; + + if (evpl_spdk_bdev_prepare(queue, request, iov, niov, offset, 0)) { + return; + } + + /* A bounced read submits into the bounce buffer, not the caller iovs. */ + if (request->bounced) { + struct iovec bounce_iov = { + .iov_base = request->bounce.data, + .iov_len = request->length, + }; + struct evpl_spdk_bdev_device *dev = queue->dev; + int rc; + + rc = spdk_bdev_readv(dev->desc, queue->ch, &bounce_iov, 1, + request->offset, request->length, + evpl_spdk_bdev_io_done, request); + + if (rc == -ENOMEM) { + request->wait.bdev = dev->bdev; + request->wait.cb_fn = evpl_spdk_bdev_io_wait_retry; + request->wait.cb_arg = request; + rc = spdk_bdev_queue_io_wait(dev->bdev, queue->ch, &request->wait); + } + + if (rc) { + evpl_spdk_bdev_request_complete(request, -rc); + } + return; + } + + evpl_spdk_bdev_submit(queue, request); +} /* evpl_spdk_bdev_read */ + +static void +evpl_spdk_bdev_write( + struct evpl *evpl, + struct evpl_block_queue *bq, + const struct evpl_iovec *iov, + int niov, + uint64_t offset, + int sync, + evpl_block_callback_t callback, + void *private_data) +{ + struct evpl_spdk_bdev_queue *queue = bq->private_data; + struct evpl_spdk_bdev_device *dev = queue->dev; + struct evpl_spdk_bdev_request *request; + + evpl_activity(evpl); + + request = evpl_spdk_bdev_request_alloc(queue); + + request->kind = EVPL_BLOCK_OP_WRITE; + request->callback = callback; + request->private_data = private_data; + + if (sync && dev->has_write_cache) { + if (dev->supports_flush) { + request->need_flush = 1; + } else { + static int warned; + if (!warned) { + warned = 1; + evpl_spdk_bdev_error( + "bdev has a write cache but no flush support; sync " + "writes cannot be made durable"); + } + } + } + + queue->outstanding++; + + if (evpl_spdk_bdev_prepare(queue, request, iov, niov, offset, 1)) { + return; + } + + evpl_spdk_bdev_submit(queue, request); +} /* evpl_spdk_bdev_write */ + +static void +evpl_spdk_bdev_flush( + struct evpl *evpl, + struct evpl_block_queue *bq, + evpl_block_callback_t callback, + void *private_data) +{ + struct evpl_spdk_bdev_queue *queue = bq->private_data; + struct evpl_spdk_bdev_request *request; + + evpl_activity(evpl); + + request = evpl_spdk_bdev_request_alloc(queue); + + request->kind = EVPL_BLOCK_OP_FLUSH; + request->callback = callback; + request->private_data = private_data; + + queue->outstanding++; + + if (queue->dev->dead) { + evpl_spdk_bdev_request_complete(request, ENODEV); + return; + } + + if (!queue->dev->supports_flush) { + /* Nothing to flush (e.g. a RAM-backed bdev): succeed as a no-op. */ + evpl_spdk_bdev_io_done(NULL, true, request); + return; + } + + evpl_spdk_bdev_submit(queue, request); +} /* evpl_spdk_bdev_flush */ + +static void +evpl_spdk_bdev_range_op( + struct evpl_block_queue *bq, + enum evpl_block_op_kind kind, + uint64_t offset, + uint64_t length, + evpl_block_callback_t callback, + void *private_data) +{ + struct evpl_spdk_bdev_queue *queue = bq->private_data; + struct evpl_spdk_bdev_device *dev = queue->dev; + struct evpl_spdk_bdev_request *request; + + evpl_activity(queue->evpl); + + request = evpl_spdk_bdev_request_alloc(queue); + + request->kind = kind; + request->callback = callback; + request->private_data = private_data; + request->offset = offset; + request->length = length; + + queue->outstanding++; + + if (dev->dead) { + evpl_spdk_bdev_request_complete(request, ENODEV); + return; + } + + if ((offset & (dev->block_size - 1)) || + (length & (dev->block_size - 1)) || + offset + length > dev->num_blocks * (uint64_t) dev->block_size) { + evpl_spdk_bdev_request_complete(request, EINVAL); + return; + } + + evpl_spdk_bdev_submit(queue, request); +} /* evpl_spdk_bdev_range_op */ + +static void +evpl_spdk_bdev_discard( + struct evpl *evpl, + struct evpl_block_queue *bq, + uint64_t offset, + uint64_t length, + evpl_block_callback_t callback, + void *private_data) +{ + evpl_spdk_bdev_range_op(bq, EVPL_BLOCK_OP_DISCARD, offset, length, + callback, private_data); +} /* evpl_spdk_bdev_discard */ + +static void +evpl_spdk_bdev_write_zeroes( + struct evpl *evpl, + struct evpl_block_queue *bq, + uint64_t offset, + uint64_t length, + evpl_block_callback_t callback, + void *private_data) +{ + /* niov == 0 distinguishes write_zeroes in the submit switch; the core + * accounts it as EVPL_BLOCK_OP_WRITE. */ + evpl_spdk_bdev_range_op(bq, EVPL_BLOCK_NUM_OP_KIND, offset, length, + callback, private_data); +} /* evpl_spdk_bdev_write_zeroes */ + +static void +evpl_spdk_bdev_close_queue( + struct evpl *evpl, + struct evpl_block_queue *bq) +{ + struct evpl_spdk_bdev_queue *queue = bq->private_data; + + queue->closing = 1; + + if (queue->outstanding) { + /* The last completion finalizes: put the channel and free. */ + return; + } + + struct evpl_spdk_bdev_request *request; + + spdk_put_io_channel(queue->ch); + + while ((request = queue->free_requests)) { + queue->free_requests = request->next; + evpl_free(request); + } + + evpl_free(queue); + evpl_free(bq); +} /* evpl_spdk_bdev_close_queue */ + +static struct evpl_block_queue * +evpl_spdk_bdev_open_queue( + struct evpl *evpl, + struct evpl_block_device *bdev_pub) +{ + struct evpl_spdk_bdev_device *dev = bdev_pub->private_data; + struct evpl_spdk_bdev_queue *queue; + struct evpl_block_queue *bq; + + evpl_spdk_bdev_abort_if(spdk_get_thread() == NULL, + "SPDK bdev queues require the evpl thread to be " + "an spdk_thread (EVPL_CORE_MECH_SPDK)"); + + evpl_spdk_bdev_abort_if(dev->dead, + "cannot open a queue on hot-removed bdev"); + + bq = evpl_zalloc(sizeof(*bq)); + queue = evpl_zalloc(sizeof(*queue)); + + queue->evpl = evpl; + queue->dev = dev; + queue->bq = bq; + queue->ch = spdk_bdev_get_io_channel(dev->desc); + + evpl_spdk_bdev_abort_if(!queue->ch, + "spdk_bdev_get_io_channel failed"); + + bq->private_data = queue; + bq->close_queue = evpl_spdk_bdev_close_queue; + bq->read = evpl_spdk_bdev_read; + bq->write = evpl_spdk_bdev_write; + bq->flush = evpl_spdk_bdev_flush; + + if (dev->supports_unmap) { + bq->discard = evpl_spdk_bdev_discard; + } + + if (dev->supports_write_zeroes) { + bq->write_zeroes = evpl_spdk_bdev_write_zeroes; + } + + return bq; +} /* evpl_spdk_bdev_open_queue */ + +static void +evpl_spdk_bdev_event_cb( + enum spdk_bdev_event_type type, + struct spdk_bdev *bdev, + void *event_ctx) +{ + struct evpl_spdk_bdev_device *dev = event_ctx; + + switch (type) { + case SPDK_BDEV_EVENT_REMOVE: + evpl_spdk_bdev_error( + "bdev hot-removed; device marked dead, in-flight and future " + "I/O will fail; close its queues and the device to release " + "it"); + dev->dead = 1; + break; + case SPDK_BDEV_EVENT_RESIZE: + dev->num_blocks = spdk_bdev_get_num_blocks(dev->bdev); + dev->bdev_pub->size = dev->num_blocks * + (uint64_t) dev->block_size; + evpl_spdk_bdev_info("bdev resized to %lu bytes", + (unsigned long) dev->bdev_pub->size); + break; + default: + evpl_spdk_bdev_info("ignoring bdev event %d", type); + break; + } /* switch */ +} /* evpl_spdk_bdev_event_cb */ + +static void +evpl_spdk_bdev_close_device( + struct evpl *evpl, + struct evpl_block_device *bdev_pub, + evpl_block_device_complete_t complete, + void *ctx) +{ + struct evpl_spdk_bdev_device *dev = bdev_pub->private_data; + + evpl_spdk_bdev_abort_if(evpl != dev->owner || + spdk_get_thread() != dev->owner_thread, + "evpl_block_close_device must be called on the " + "evpl that opened the SPDK bdev device"); + + spdk_bdev_close(dev->desc); + + evpl_free(dev); + evpl_free(bdev_pub); + + complete(evpl, NULL, 0, ctx); +} /* evpl_spdk_bdev_close_device */ + +static void +evpl_spdk_bdev_open_device( + struct evpl *evpl, + const char *uri, + void *private_data, + evpl_block_device_complete_t complete, + void *ctx) +{ + struct evpl_spdk_bdev_device *dev; + struct evpl_block_device *bdev_pub; + int rc; + + if (evpl_shared->config->core_mech != EVPL_CORE_MECH_SPDK || + spdk_get_thread() == NULL) { + evpl_spdk_bdev_error( + "EVPL_BLOCK_PROTOCOL_SPDK_BDEV requires EVPL_CORE_MECH_SPDK; " + "the opening evpl must be running on an spdk_thread"); + complete(evpl, NULL, ENOTSUP, ctx); + return; + } + + bdev_pub = evpl_zalloc(sizeof(*bdev_pub)); + dev = evpl_zalloc(sizeof(*dev)); + + rc = spdk_bdev_open_ext(uri, true, evpl_spdk_bdev_event_cb, dev, + &dev->desc); + + if (rc) { + evpl_spdk_bdev_error("spdk_bdev_open_ext(%s) failed: %s", + uri, strerror(-rc)); + evpl_free(dev); + evpl_free(bdev_pub); + complete(evpl, NULL, -rc, ctx); + return; + } + + dev->owner = evpl; + dev->owner_thread = spdk_get_thread(); + dev->bdev = spdk_bdev_desc_get_bdev(dev->desc); + dev->bdev_pub = bdev_pub; + dev->block_size = spdk_bdev_get_block_size(dev->bdev); + dev->num_blocks = spdk_bdev_get_num_blocks(dev->bdev); + dev->buf_align = spdk_bdev_get_buf_align(dev->bdev); + + if (dev->buf_align < 1) { + dev->buf_align = 1; + } + + dev->supports_unmap = + spdk_bdev_io_type_supported(dev->bdev, SPDK_BDEV_IO_TYPE_UNMAP); + dev->supports_write_zeroes = + spdk_bdev_io_type_supported(dev->bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); + dev->supports_flush = + spdk_bdev_io_type_supported(dev->bdev, SPDK_BDEV_IO_TYPE_FLUSH); + dev->has_write_cache = spdk_bdev_has_write_cache(dev->bdev); + + bdev_pub->private_data = dev; + bdev_pub->size = dev->num_blocks * (uint64_t) dev->block_size; + bdev_pub->max_request_size = 4 * 1024 * 1024; + bdev_pub->open_queue = evpl_spdk_bdev_open_queue; + bdev_pub->close_device = evpl_spdk_bdev_close_device; + + complete(evpl, bdev_pub, 0, ctx); +} /* evpl_spdk_bdev_open_device */ + +struct evpl_block_protocol evpl_block_protocol_spdk_bdev = { + .id = EVPL_BLOCK_PROTOCOL_SPDK_BDEV, + .name = "spdk_bdev", + .framework = &evpl_framework_spdk, + .open_device = evpl_spdk_bdev_open_device, +}; diff --git a/src/core/spdk/spdk_core.c b/src/core/spdk/spdk_core.c new file mode 100644 index 00000000..be1f0a35 --- /dev/null +++ b/src/core/spdk/spdk_core.c @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "core/spdk/evpl_spdk.h" +#include "core/core.h" +#include "core/event_fn.h" +#include "core/evpl.h" +#include "core/evpl_shared.h" + +_Static_assert(offsetof(struct evpl_core_spdk, epoll) == 0, + "epoll state must be first so evc->u.epoll aliases evc->u.spdk.epoll"); + +/* + * Program the one-shot timerfd to fire in ns nanoseconds (0 disarms). The + * syscall is skipped when the requested value matches the last programming; + * armed_ns is cleared by the drain callback when the timer fires, so a skip + * can never leave a fired (and therefore unarmed) timerfd believed armed. + * A fired-but-not-yet-drained timerfd is still readable, which keeps the + * fd_group hot, so skipping in that window is also safe. + */ +static void +evpl_core_spdk_arm( + struct evpl_core_spdk *s, + uint64_t ns) +{ + struct itimerspec its; + int rc; + + if (ns == s->armed_ns) { + return; + } + + memset(&its, 0, sizeof(its)); + + its.it_value.tv_sec = ns / 1000000000ULL; + its.it_value.tv_nsec = ns % 1000000000ULL; + + rc = timerfd_settime(s->timer_fd, 0, &its, NULL); + + evpl_core_abort_if(rc, "evpl_core_spdk_arm: timerfd_settime failed: %s", + strerror(errno)); + + s->armed_ns = ns; +} /* evpl_core_spdk_arm */ + +/* + * Read handler for the timerfd. The timerfd exists only to pop the thread's + * fd_group when an evpl timer deadline or pending-work kick comes due while + * an interrupt-mode reactor sleeps; actual timer dispatch is evpl_continue's + * heap logic. + */ +static void +evpl_core_spdk_timer_drain( + struct evpl *evpl, + struct evpl_event *event) +{ + struct evpl_core_spdk *s = &evpl->core.u.spdk; + uint64_t expirations; + ssize_t rc; + + for (;;) { + rc = read(event->fd, &expirations, sizeof(expirations)); + + if (rc != sizeof(expirations)) { + break; + } + + /* The one-shot fired, so the kernel has disarmed it. */ + s->armed_ns = 0; + } + + evpl_event_mark_unreadable(evpl, event); +} /* evpl_core_spdk_timer_drain */ + +/* + * Interrupt-mode sleep gate, run after every pump from fresh state (event + * callbacks and deferrals run after the wait inside evpl_continue, so any + * decision made earlier would be stale). While work is pending the timerfd + * is held at 1ns so the reactor re-enters the pump instead of sleeping; + * otherwise it is armed to the nearest evpl timer deadline (capped by a + * positive wait_ms) or disarmed entirely. + */ +static void +evpl_core_spdk_rearm(struct evpl_core *evc) +{ + struct evpl_core_spdk *s = &evc->u.spdk; + struct evpl *evpl = evpl_from_core(evc); + uint64_t ns; + uint64_t cap; + int64_t remain; + int pending; + + pending = evpl->poll_mode || + evpl->num_active_events || + evpl->num_active_deferrals || + evpl->pending_close_binds != NULL || + s->last_wait_full; + + if (pending) { + evpl_core_spdk_arm(s, 1); + return; + } + + ns = 0; + + if (evpl->num_timers) { + remain = (int64_t) (evpl->timers[0]->deadline - evpl_now_ticks()); + + ns = remain > 0 ? evpl_ticks_to_ns((uint64_t) remain) : 1; + + if (ns == 0) { + ns = 1; + } + } + + if (evpl->config.wait_ms > 0) { + cap = (uint64_t) evpl->config.wait_ms * 1000000ULL; + + if (ns == 0 || cap < ns) { + ns = cap; + } + } + + evpl_core_spdk_arm(s, ns); +} /* evpl_core_spdk_rearm */ + +static int +evpl_core_spdk_pump(struct evpl_core *evc) +{ + struct evpl_core_spdk *s = &evc->u.spdk; + struct evpl *evpl = evpl_from_core(evc); + int work; + + s->last_wait_full = 0; + + s->in_pump = 1; + + work = evpl_continue(evpl); + + s->in_pump = 0; + + if (s->interrupt_mode) { + evpl_core_spdk_rearm(evc); + } + + return work; +} /* evpl_core_spdk_pump */ + +static int +evpl_core_spdk_poller_fn(void *ctx) +{ + return evpl_core_spdk_pump(ctx) ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; +} /* evpl_core_spdk_poller_fn */ + +/* fd_group handler for the internal epoll fd: >0 events processed, 0 none. */ +static int +evpl_core_spdk_interrupt_fn(void *ctx) +{ + return evpl_core_spdk_pump(ctx); +} /* evpl_core_spdk_interrupt_fn */ + +/* + * Reactor phase transitions. Registering this callback also marks the pump + * poller interrupt-capable so SPDK suspends it during interrupt phases (the + * fd_group handler pumps instead). On entry to interrupt mode a 1ns priming + * kick closes the window where work created while polling would otherwise be + * stranded with nothing armed. + */ +static void +evpl_core_spdk_set_mode_cb( + struct spdk_poller *poller, + void *ctx, + bool interrupt_mode) +{ + struct evpl_core *evc = ctx; + struct evpl_core_spdk *s = &evc->u.spdk; + + (void) poller; + + s->interrupt_mode = interrupt_mode; + + if (interrupt_mode) { + evpl_core_spdk_arm(s, 1); + } +} /* evpl_core_spdk_set_mode_cb */ + +static int +evpl_core_spdk_init( + struct evpl_core *evc, + int max_events) +{ + struct evpl_core_spdk *s = &evc->u.spdk; + struct evpl *evpl = evpl_from_core(evc); + int rc; + + s->thread = spdk_get_thread(); + + evpl_core_abort_if(!s->thread, + "EVPL_CORE_MECH_SPDK requires evpl_create() to run on an " + "spdk_thread; the host application must initialize the SPDK " + "env and thread library first"); + + rc = evpl_core_epoll_ops.init(evc, max_events); + + evpl_core_abort_if(rc, "evpl_core_spdk_init: epoll init failed: %s", + strerror(rc)); + + /* Each fast-path pass is a separate poller callback on a shared reactor, + * so bound the pure-poll iterations between timer/fd sweeps to keep + * socket and timer latency within a few reactor iterations. */ + if (evpl->config.poll_iterations > 16) { + evpl->config.poll_iterations = 16; + } + + s->timer_fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC); + + evpl_core_abort_if(s->timer_fd < 0, + "evpl_core_spdk_init: timerfd_create failed: %s", + strerror(errno)); + + evpl_add_event(evpl, &s->timer_event, s->timer_fd, + evpl_core_spdk_timer_drain, NULL, NULL); + + evpl_event_read_interest(evpl, &s->timer_event); + + s->poller = spdk_poller_register(evpl_core_spdk_poller_fn, evc, 0); + + evpl_core_abort_if(!s->poller, + "evpl_core_spdk_init: spdk_poller_register failed"); + + if (spdk_interrupt_mode_is_enabled()) { + /* May invoke the mode callback immediately if the thread is already + * in its interrupt phase. */ + spdk_poller_register_interrupt(s->poller, evpl_core_spdk_set_mode_cb, + evc); + + s->intr = spdk_interrupt_register(s->epoll.fd, + evpl_core_spdk_interrupt_fn, + evc, "evpl"); + + evpl_core_abort_if(!s->intr, + "evpl_core_spdk_init: spdk_interrupt_register failed"); + + /* Priming kick: timers or binds created by thread-init callbacks run + * before the first pump, and a sleeping reactor must not miss them. */ + evpl_core_spdk_arm(s, 1); + } + + return 0; +} /* evpl_core_spdk_init */ + +static void +evpl_core_spdk_destroy(struct evpl_core *evc) +{ + struct evpl_core_spdk *s = &evc->u.spdk; + + evpl_core_abort_if(spdk_get_thread() != s->thread, + "evpl_destroy must run on the owning spdk_thread under " + "EVPL_CORE_MECH_SPDK"); + + evpl_core_abort_if(s->in_pump, + "evpl_destroy must not be called from inside an evpl " + "callback under EVPL_CORE_MECH_SPDK; destroy from an spdk " + "message or poller instead"); + + /* Detach from SPDK before tearing down any fds so the pump can never run + * against a dead epoll. */ + if (s->intr) { + spdk_interrupt_unregister(&s->intr); + } + + spdk_poller_unregister(&s->poller); + + /* No evpl_remove_event for timer_event: core destroy runs after event + * dispatch is finished (mirroring how run_event is handled), and closing + * the fd removes it from the epoll interest set. */ + close(s->timer_fd); + + evpl_core_epoll_ops.destroy(evc); +} /* evpl_core_spdk_destroy */ + +static void +evpl_core_spdk_add( + struct evpl_core *evc, + struct evpl_event *event) +{ + evpl_core_epoll_ops.add(evc, event); +} /* evpl_core_spdk_add */ + +static void +evpl_core_spdk_remove( + struct evpl_core *evc, + struct evpl_event *event) +{ + evpl_core_epoll_ops.remove(evc, event); +} /* evpl_core_spdk_remove */ + +static int +evpl_core_spdk_wait( + struct evpl_core *evc, + int max_msecs) +{ + struct evpl_core_spdk *s = &evc->u.spdk; + int n; + + /* Never block the reactor: sleeping is the host's job. max_msecs is + * deliberately ignored rather than used to arm the timerfd -- callbacks + * and deferrals run after this wait within the same evpl_continue, so + * any arming decision made here would be stale; evpl_core_spdk_rearm + * runs post-pump from fresh state instead. */ + (void) max_msecs; + + n = evpl_core_epoll_ops.wait(evc, 0); + + /* A full batch may leave undelivered events latched in the inner epoll + * with no new edge to pop the fd_group; the rearm kick covers it. */ + s->last_wait_full = (n == s->epoll.max_events); + + return n; +} /* evpl_core_spdk_wait */ + +const struct evpl_core_ops evpl_core_spdk_ops = { + .name = "spdk", + .flags = EVPL_CORE_OPS_EXTERNAL_LOOP, + .init = evpl_core_spdk_init, + .destroy = evpl_core_spdk_destroy, + .add = evpl_core_spdk_add, + .remove = evpl_core_spdk_remove, + .wait = evpl_core_spdk_wait, +}; diff --git a/src/core/spdk/spdk_framework.c b/src/core/spdk/spdk_framework.c new file mode 100644 index 00000000..fac7d2fc --- /dev/null +++ b/src/core/spdk/spdk_framework.c @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * EVPL_FRAMEWORK_SPDK: registers libevpl's slab buffers with SPDK's memory + * translation (spdk_mem_register) so evpl iovecs are DMA-safe for SPDK + * NVMe/bdev I/O in the host application. + * + * Guest-mode contract: the host owns the SPDK env; this framework never + * initializes or finalizes it. The framework is attached lazily by the + * first evpl_create() under EVPL_CORE_MECH_SPDK -- the one point where a + * live host env is guaranteed -- and evpl_allocator_reregister then walks + * every existing slab through register_memory. + * + * Slabs are 2 MiB-aligned when SPDK support is compiled in (see + * evpl_shared_init), which satisfies every SPDK release; v25.09 itself only + * requires 4 KiB alignment, so spdk_mem_register's return code is treated as + * authoritative rather than pre-asserting an alignment here. + */ + +#include +#include +#include + +#include + +#include "core/evpl.h" +#include "evpl/evpl.h" +#include "core/protocol.h" +#include "core/evpl_shared.h" +#include "core/spdk/spdk_sock_common.h" + +#define evpl_spdk_error(...) evpl_error("spdk", __FILE__, __LINE__, \ + __VA_ARGS__) +#define evpl_spdk_abort_if(cond, ...) \ + evpl_abort_if(cond, "spdk", __FILE__, __LINE__, __VA_ARGS__) + +struct evpl_spdk_shared { + int attached; +}; + +struct evpl_spdk_memreg { + void *vaddr; + size_t len; +}; + +static void * +evpl_spdk_init(void) +{ + struct evpl_spdk_shared *shared; + + /* Only reached when something attaches the framework (first SPDK-mode + * evpl_create), so a loud failure here always indicates a host that did + * not bring up the SPDK env. */ + evpl_spdk_abort_if(spdk_env_get_core_count() == 0, + "SPDK framework attach requires the host application to " + "have initialized the SPDK env"); + + shared = evpl_zalloc(sizeof(*shared)); + + shared->attached = 1; + + return shared; +} /* evpl_spdk_init */ + +static void +evpl_spdk_cleanup(void *private_data) +{ + /* Never spdk_env_fini/spdk_thread_lib_fini: the host owns them. */ + evpl_free(private_data); +} /* evpl_spdk_cleanup */ + +static void * +evpl_spdk_create( + struct evpl *evpl, + void *private_data) +{ + struct evpl_spdk_thread *ctx; + + ctx = evpl_zalloc(sizeof(*ctx)); + + ctx->shared = private_data; + ctx->evpl = evpl; + ctx->max_active = 256; + ctx->active = evpl_zalloc(ctx->max_active * sizeof(*ctx->active)); + + /* The sock group, poll and interrupt registration are created lazily by + * spdk/tcp.c on first socket use. */ + return ctx; +} /* evpl_spdk_create */ + +static void +evpl_spdk_destroy( + struct evpl *evpl, + void *private_data) +{ + struct evpl_spdk_thread *ctx = private_data; + + evpl_spdk_sock_thread_destroy(evpl, ctx); + + evpl_free(ctx->active); + evpl_free(ctx); +} /* evpl_spdk_destroy */ + +static void * +evpl_spdk_register_memory( + void *buffer, + int size, + void *buffer_private, + void *thread_private) +{ + struct evpl_spdk_memreg *reg; + int rc; + + /* evpl_allocator_reregister re-walks every slab on each late framework + * attach; an already-registered slab keeps its token. */ + if (buffer_private) { + return buffer_private; + } + + rc = spdk_mem_register(buffer, size); + + evpl_spdk_abort_if(rc, + "spdk_mem_register(%p, %d) failed: %s " + "(slabs must be page-aligned; older SPDK releases " + "require 2 MiB alignment)", + buffer, size, strerror(-rc)); + + reg = evpl_zalloc(sizeof(*reg)); + + reg->vaddr = buffer; + reg->len = (size_t) size; + + return reg; +} /* evpl_spdk_register_memory */ + +static void +evpl_spdk_unregister_memory( + void *buffer_private, + void *thread_private) +{ + struct evpl_spdk_memreg *reg = buffer_private; + int rc; + + if (!reg) { + return; + } + + /* Runs from evpl_cleanup at atexit, which can be after the host has + * finalized its SPDK env; failure is logged, not fatal, and process + * exit reclaims any remaining DMA mappings. */ + rc = spdk_mem_unregister(reg->vaddr, reg->len); + + if (rc) { + evpl_spdk_error("spdk_mem_unregister(%p, %zu) failed: %s", + reg->vaddr, reg->len, strerror(-rc)); + } + + evpl_free(reg); +} /* evpl_spdk_unregister_memory */ + +struct evpl_framework evpl_framework_spdk = { + .id = EVPL_FRAMEWORK_SPDK, + .name = "spdk", + .init = evpl_spdk_init, + .cleanup = evpl_spdk_cleanup, + .create = evpl_spdk_create, + .destroy = evpl_spdk_destroy, + .register_memory = evpl_spdk_register_memory, + .unregister_memory = evpl_spdk_unregister_memory, +}; diff --git a/src/core/spdk/spdk_sock_common.h b/src/core/spdk/spdk_sock_common.h new file mode 100644 index 00000000..9c9bbe60 --- /dev/null +++ b/src/core/spdk/spdk_sock_common.h @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#pragma once + +/* + * Shared internal state for the spdk_sock TCP backend (STREAM_SPDK_TCP). + * Included only by spdk_framework.c and spdk/tcp.c -- unlike evpl_spdk.h this + * header includes SPDK headers. + */ + +#include +#include +#include + +#include +#include + +#include "core/evpl.h" +#include "evpl/evpl.h" + +#define EVPL_SPDK_SOCK_REQ_MAX_IOV 64 /* iovecs per spdk_sock_request */ +#define EVPL_SPDK_SOCK_MAX_INFLIGHT 16 /* outstanding write reqs per sock */ +#define EVPL_SPDK_SOCK_PROGRESS_US 1000 /* interrupt-mode progress period */ + +struct evpl_spdk_sock; +struct evpl_spdk_thread; + +struct evpl_spdk_sock_request { + /* iov[] must sit immediately after base: spdk_sock_writev_async locates + * the iovec array via SPDK_SOCK_REQUEST_IOV, which assumes exactly this + * layout. */ + struct spdk_sock_request base; + struct iovec iov[EVPL_SPDK_SOCK_REQ_MAX_IOV]; + struct evpl_spdk_sock *sock; + uint32_t niov; + uint32_t total; + struct evpl_spdk_sock_request *next; +}; + +/* Per-connection state, stored in evpl_bind_private (4096-byte budget). */ +struct evpl_spdk_sock { + struct evpl *evpl; + struct spdk_sock *sock; + unsigned int listen : 1; + unsigned int connected : 1; + unsigned int connecting : 1; + unsigned int connect_failed : 1; + unsigned int in_group : 1; + unsigned int writable : 1; + unsigned int write_interest : 1; + unsigned int active : 1; + unsigned int closed : 1; + uint32_t outstanding; /* submitted, uncompleted writes */ + struct evpl_iovec recv1; /* whole-buffer recv staging */ + struct evpl_iovec recv2; + /* Keeps spdk_sock_opts.src_addr alive for the whole async connect: the + * posix impl shallow-copies the opts and dereferences the string across + * connect-poller retries. */ + char src_addr[INET6_ADDRSTRLEN]; +}; + +/* Per-thread sock state; lives in the SPDK framework's per-thread private + * (evpl_framework_private(evpl, EVPL_FRAMEWORK_SPDK)). The group, poll and + * interrupt registration are created lazily on first socket use, so SPDK + * threads that never touch STREAM_SPDK_TCP pay only this allocation. */ +struct evpl_spdk_thread { + void *shared; /* framework global private */ + struct evpl *evpl; + struct spdk_sock_group *group; + struct evpl_poll *poll; + int intr_registered; + struct evpl_spdk_sock **active; /* write-interest sweep array */ + int num_active; + int max_active; + int num_socks; + /* Interrupt-mode progress: the group epoll only reports EPOLLIN, so + * in-flight connects and sndbuf-stalled writes need a timer to advance + * while the reactor sleeps. Counts (connecting socks + socks with + * outstanding writes); the timer is armed while nonzero. */ + int progress; + int progress_timer_armed; + struct evpl_timer progress_timer; + struct evpl_spdk_sock_request *free_requests; +}; + +/* Implemented in spdk/tcp.c; called from the framework's per-thread destroy + * so group teardown stays with the sock code. */ +void +evpl_spdk_sock_thread_destroy( + struct evpl *evpl, + struct evpl_spdk_thread *ctx); diff --git a/src/core/spdk/tcp.c b/src/core/spdk/tcp.c new file mode 100644 index 00000000..e9440144 --- /dev/null +++ b/src/core/spdk/tcp.c @@ -0,0 +1,987 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * STREAM_SPDK_TCP: TCP stream protocol over spdk_sock (pluggable posix/uring + * implementations, selected with evpl_global_config_set_spdk_sock_impl). + * + * Requires EVPL_CORE_MECH_SPDK: each evpl thread keeps an spdk_sock_group in + * its SPDK framework state (created lazily on first socket use) and pumps it + * with a shared sweep -- spdk_sock_group_poll (which fires per-sock recv and + * accept callbacks, flushes queued write requests, and advances asynchronous + * connects) followed by an XLIO-style active-socket write pass. The sweep + * runs from three places: + * + * - an evpl_add_poll callback, every loop pass while in poll mode. When + * interrupt mode is not enabled process-wide, force_poll_mode pins the + * loop so the group is always pumped (XLIO precedent); + * - the group's interrupt registration when interrupt mode is enabled, so + * sock-only threads can sleep. This runs OUTSIDE evpl_continue, so it + * ends with evpl_kick() whenever work was done or deferrals/closes are + * pending, waking the mechanism's pump before the reactor sleeps; + * - a 1ms progress timer, armed in interrupt mode only while connects are + * in flight or write requests are outstanding: the group epoll reports + * only EPOLLIN, and SPDK expects the caller to retry partially-flushed + * writes ("caller in interrupt mode should retry"), so both need a + * time-based nudge while the reactor sleeps. + * + * Sends are zero-copy spdk_sock_writev_async requests using the rdmacm waist + * model: iovecs stay in bind->iovec_send holding their reference for the + * whole flight; submit advances the ring's waist, and the completion (FIFO + * for the non-zcopy posix path -- zcopy is deliberately disabled) releases + * from the tail. Cancelled requests (-ECANCELED from group removal or sock + * close) leave the ring alone; evpl_bind_destroy's ring clear reclaims them. + * + * Interrupt-mode wiring is code-complete but not covered by the mini-reactor + * test harness (which never enables interrupt mode); it shares this sweep + * with the tested poll path. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "core/evpl.h" +#include "evpl/evpl.h" +#include "core/macros.h" +#include "core/protocol.h" +#include "core/evpl_shared.h" +#include "core/bind.h" +#include "core/address.h" +#include "core/iovec.h" +#include "core/spdk/evpl_spdk.h" +#include "core/spdk/spdk_sock_common.h" + +extern struct evpl_shared *evpl_shared; + +#define evpl_spdk_sock_debug(...) evpl_debug("spdk_sock", __FILE__, \ + __LINE__, __VA_ARGS__) +#define evpl_spdk_sock_info(...) evpl_info("spdk_sock", __FILE__, \ + __LINE__, __VA_ARGS__) +#define evpl_spdk_sock_error(...) evpl_error("spdk_sock", __FILE__, \ + __LINE__, __VA_ARGS__) +#define evpl_spdk_sock_abort_if(cond, ...) \ + evpl_abort_if(cond, "spdk_sock", __FILE__, __LINE__, __VA_ARGS__) + +_Static_assert(offsetof(struct evpl_spdk_sock_request, iov) == + sizeof(struct spdk_sock_request), + "iov[] must directly follow spdk_sock_request " + "(SPDK_SOCK_REQUEST_IOV contract)"); + +struct evpl_spdk_accepted_sock { + struct spdk_sock *sock; +}; + +static void evpl_spdk_sock_progress_timer_cb( + struct evpl *evpl, + struct evpl_timer *timer); + +static int evpl_spdk_sock_sweep( + struct evpl *evpl, + struct evpl_spdk_thread *ctx); + +static int evpl_spdk_sock_submit( + struct evpl *evpl, + struct evpl_spdk_thread *ctx, + struct evpl_spdk_sock *s, + struct evpl_bind *bind); + +static int evpl_spdk_sock_intr_fn( + void *arg); + +static void +evpl_spdk_sock_poll_cb( + struct evpl *evpl, + void *arg) +{ + evpl_spdk_sock_sweep(evpl, arg); +} /* evpl_spdk_sock_poll_cb */ + +/* Interrupt-mode progress accounting: (connecting socks + socks with + * outstanding write requests); the 1ms timer runs while nonzero. */ +static void +evpl_spdk_sock_progress_inc(struct evpl_spdk_thread *ctx) +{ + if (ctx->progress++ == 0 && ctx->intr_registered && + !ctx->progress_timer_armed) { + evpl_add_timer(ctx->evpl, &ctx->progress_timer, + evpl_spdk_sock_progress_timer_cb, + EVPL_SPDK_SOCK_PROGRESS_US); + ctx->progress_timer_armed = 1; + } +} /* evpl_spdk_sock_progress_inc */ + +static void +evpl_spdk_sock_progress_dec(struct evpl_spdk_thread *ctx) +{ + if (--ctx->progress == 0 && ctx->progress_timer_armed) { + evpl_remove_timer(ctx->evpl, &ctx->progress_timer); + ctx->progress_timer_armed = 0; + } +} /* evpl_spdk_sock_progress_dec */ + +static void +evpl_spdk_sock_check_active( + struct evpl_spdk_thread *ctx, + struct evpl_spdk_sock *s) +{ + if (s->active) { + return; + } + + if (ctx->num_active >= ctx->max_active) { + ctx->max_active *= 2; + ctx->active = evpl_realloc(ctx->active, + ctx->max_active * + sizeof(*ctx->active)); + } + + ctx->active[ctx->num_active++] = s; + + s->active = 1; +} /* evpl_spdk_sock_check_active */ + +/* Extract ip string + port from an evpl address. */ +static void +evpl_spdk_sock_endpoint_parts( + struct evpl_address *address, + char *ip, + size_t iplen, + uint16_t *port) +{ + if (address->addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *) address->addr; + inet_ntop(AF_INET, &sin->sin_addr, ip, iplen); + *port = ntohs(sin->sin_port); + } else { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) address->addr; + inet_ntop(AF_INET6, &sin6->sin6_addr, ip, iplen); + *port = ntohs(sin6->sin6_port); + } +} /* evpl_spdk_sock_endpoint_parts */ + +/* Fill an evpl address from an ip string + port. */ +static void +evpl_spdk_sock_address_fill( + struct evpl_address *address, + const char *ip, + uint16_t port) +{ + struct sockaddr_in *sin = (struct sockaddr_in *) address->addr; + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) address->addr; + + if (inet_pton(AF_INET, ip, &sin->sin_addr) == 1) { + sin->sin_family = AF_INET; + sin->sin_port = htons(port); + address->addrlen = sizeof(*sin); + } else if (inet_pton(AF_INET6, ip, &sin6->sin6_addr) == 1) { + sin6->sin6_family = AF_INET6; + sin6->sin6_port = htons(port); + address->addrlen = sizeof(*sin6); + } else { + address->addrlen = 0; + } +} /* evpl_spdk_sock_address_fill */ + +/* + * Lazy per-thread group bring-up on first socket use. The framework's + * per-thread ctx (struct evpl_spdk_thread) was created by bind_prepare's + * evpl_attach_framework before any protocol op runs. + */ +static struct evpl_spdk_thread * +evpl_spdk_sock_thread(struct evpl *evpl) +{ + struct evpl_spdk_thread *ctx; + int rc; + + ctx = evpl_framework_private(evpl, EVPL_FRAMEWORK_SPDK); + + if (ctx->group) { + return ctx; + } + + ctx->group = spdk_sock_group_create(NULL); + + evpl_spdk_sock_abort_if(!ctx->group, "spdk_sock_group_create failed"); + + ctx->poll = evpl_add_poll(evpl, NULL, NULL, evpl_spdk_sock_poll_cb, ctx); + + if (spdk_interrupt_mode_is_enabled()) { + rc = spdk_sock_group_register_interrupt(ctx->group, + SPDK_INTERRUPT_EVENT_IN, + evpl_spdk_sock_intr_fn, + ctx, "evpl_sock"); + + evpl_spdk_sock_abort_if(rc, + "spdk_sock_group_register_interrupt failed: %d (sock impl without interrupt support?)", + rc); + + ctx->intr_registered = 1; + } else { + /* No interrupt path: the loop must stay in poll mode so the group + * is pumped every pass (requires config poll_mode, the default). */ + evpl->force_poll_mode = 1; + } + + return ctx; +} /* evpl_spdk_sock_thread */ + +static void +evpl_spdk_sock_write_complete( + void *cb_arg, + int err) +{ + struct evpl_spdk_sock_request *request = cb_arg; + struct evpl_spdk_sock *s = request->sock; + struct evpl *evpl = s->evpl; + struct evpl_bind *bind = evpl_private2bind(s); + struct evpl_spdk_thread *ctx; + struct evpl_notify notify; + struct evpl_iovec *iovec; + int i, niov_sent, msg_sent = 0; + + ctx = evpl_framework_private(evpl, EVPL_FRAMEWORK_SPDK); + + if (--s->outstanding == 0) { + evpl_spdk_sock_progress_dec(ctx); + } + + if (err == 0) { + + /* FIFO completion: this request's iovecs are exactly at the ring + * tail. Release their flight references and retire them. */ + for (i = 0; i < request->niov; i++) { + iovec = evpl_iovec_ring_tail(&bind->iovec_send); + evpl_iovec_release_internal(evpl, iovec); + evpl_iovec_ring_remove(&bind->iovec_send); + } + + niov_sent = request->niov; + + if (bind->segment_callback) { + while (niov_sent) { + struct evpl_dgram *dgram = + evpl_dgram_ring_tail(&bind->dgram_send); + + if (!dgram) { + break; + } + + if (dgram->niov > niov_sent) { + dgram->niov -= niov_sent; + break; + } + + niov_sent -= dgram->niov; + msg_sent++; + evpl_dgram_ring_remove(&bind->dgram_send); + } + } + + if (bind->flags & EVPL_BIND_SENT_NOTIFY) { + notify.notify_type = EVPL_NOTIFY_SENT; + notify.notify_status = 0; + notify.sent.bytes = request->total; + notify.sent.msgs = msg_sent; + bind->notify_callback(evpl, bind, ¬ify, bind->private_data); + } + + if (evpl_iovec_ring_is_empty(&bind->iovec_send) && + (bind->flags & EVPL_BIND_FINISH)) { + evpl_close(evpl, bind); + } + + } else { + /* -ECANCELED (group removal / sock close) or a transport error. Do + * NOT touch the send ring: releases beyond the completed prefix are + * evpl_bind_destroy's job via its ring clear. Once one request + * errors the sock is dying, so no success completion can follow. */ + if (err != -ECANCELED && !s->closed) { + evpl_spdk_sock_debug("spdk_sock write failed: %d", err); + } + + s->closed = 1; + + evpl_close(evpl, bind); + } + + request->next = ctx->free_requests; + ctx->free_requests = request; +} /* evpl_spdk_sock_write_complete */ + +/* + * Waist-based zero-copy submission from the send ring. Reentrancy note: + * spdk_sock_writev_async and spdk_sock_flush may invoke the completion + * callback synchronously (immediate kernel send, or -EBADF on a dead sock); + * the callback touches only tail/outstanding while this loop reads + * waist/head, so the interleaving is safe. + */ +static int +evpl_spdk_sock_submit( + struct evpl *evpl, + struct evpl_spdk_thread *ctx, + struct evpl_spdk_sock *s, + struct evpl_bind *bind) +{ + struct evpl_iovec_ring *ring = &bind->iovec_send; + struct evpl_spdk_sock_request *request; + struct evpl_iovec *cur; + int work = 0; + + while (ring->waist != ring->head && !s->closed && + s->outstanding < EVPL_SPDK_SOCK_MAX_INFLIGHT) { + + request = ctx->free_requests; + + if (request) { + ctx->free_requests = request->next; + } else { + request = evpl_zalloc(sizeof(*request)); + } + + memset(&request->base, 0, sizeof(request->base)); + + request->sock = s; + request->niov = 0; + request->total = 0; + + while (ring->waist != ring->head && + request->niov < EVPL_SPDK_SOCK_REQ_MAX_IOV) { + cur = &ring->iovec[ring->waist]; + + request->iov[request->niov].iov_base = cur->data; + request->iov[request->niov].iov_len = cur->length; + + request->total += cur->length; + request->niov++; + + ring->waist = (ring->waist + 1) & ring->mask; + } + + request->base.iovcnt = request->niov; + request->base.cb_fn = evpl_spdk_sock_write_complete; + request->base.cb_arg = request; + + if (s->outstanding++ == 0) { + evpl_spdk_sock_progress_inc(ctx); + } + + spdk_sock_writev_async(s->sock, &request->base); + + work++; + } + + /* Prod the impl to hit the kernel now rather than on the next group + * poll; a partial flush (EAGAIN) is retried by the sweep. */ + if (work && s->sock && !s->closed) { + if (spdk_sock_flush(s->sock) < 0 && + errno != EAGAIN && errno != EWOULDBLOCK && errno != EBADF) { + evpl_close(evpl, bind); + } + } + + s->write_interest = (ring->waist != ring->head); + + if (s->write_interest && !s->closed) { + evpl_spdk_sock_check_active(ctx, s); + } + + return work; +} /* evpl_spdk_sock_submit */ + +/* Per-sock group callback: data available to read (socket/tcp.c model). */ +static void +evpl_spdk_sock_recv_cb( + void *arg, + struct spdk_sock_group *group, + struct spdk_sock *sock) +{ + struct evpl_spdk_sock *s = arg; + struct evpl *evpl = s->evpl; + struct evpl_bind *bind = evpl_private2bind(s); + struct evpl_iovec *iovec; + struct evpl_notify notify; + struct iovec iov[2]; + ssize_t res, total, remain; + int length, niov, iter; + uint64_t got = 0; + + if (s->closed || !s->sock) { + return; + } + + for (iter = 0; iter < 8; iter++) { + + if (s->recv1.length == 0) { + if (s->recv2.length) { + evpl_iovec_move(&s->recv1, &s->recv2); + s->recv2.length = 0; + } else { + evpl_iovec_alloc_whole(evpl, &s->recv1); + } + } + + if (s->recv2.length == 0) { + evpl_iovec_alloc_whole(evpl, &s->recv2); + } + + iov[0].iov_base = s->recv1.data; + iov[0].iov_len = s->recv1.length; + iov[1].iov_base = s->recv2.data; + iov[1].iov_len = s->recv2.length; + + total = iov[0].iov_len + iov[1].iov_len; + + res = spdk_sock_readv(s->sock, iov, 2); + + if (res < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + evpl_close(evpl, bind); + } + break; + } else if (res == 0) { + evpl_close(evpl, bind); + break; + } + + if (s->recv1.length >= res) { + evpl_iovec_ring_append(evpl, &bind->iovec_recv, &s->recv1, res); + } else { + remain = res - s->recv1.length; + evpl_iovec_ring_append(evpl, &bind->iovec_recv, &s->recv1, + s->recv1.length); + evpl_iovec_ring_append(evpl, &bind->iovec_recv, &s->recv2, remain); + } + + got += res; + + if (res < total) { + break; + } + } + + if (!got) { + return; + } + + if (bind->segment_callback) { + + iovec = alloca(sizeof(struct evpl_iovec) * + evpl_shared->config->max_num_iovec); + + while (1) { + + length = bind->segment_callback(evpl, bind, bind->private_data); + + if (length == 0 || + evpl_iovec_ring_bytes(&bind->iovec_recv) < length) { + break; + } + + if (unlikely(length < 0)) { + evpl_close(evpl, bind); + return; + } + + niov = evpl_iovec_ring_copyv(evpl, iovec, &bind->iovec_recv, + length); + + notify.notify_type = EVPL_NOTIFY_RECV_MSG; + notify.recv_msg.iovec = iovec; + notify.recv_msg.niov = niov; + notify.recv_msg.length = length; + notify.recv_msg.addr = bind->remote; + + bind->notify_callback(evpl, bind, ¬ify, bind->private_data); + } + + } else { + notify.notify_type = EVPL_NOTIFY_RECV_DATA; + notify.notify_status = 0; + bind->notify_callback(evpl, bind, ¬ify, bind->private_data); + } +} /* evpl_spdk_sock_recv_cb */ + +/* + * The shared pump: group poll (recv/accept dispatch, queued-write flushing, + * async-connect progress) plus the active-socket write sweep. + */ +static int +evpl_spdk_sock_sweep( + struct evpl *evpl, + struct evpl_spdk_thread *ctx) +{ + struct evpl_spdk_sock *s; + struct evpl_bind *bind; + int i, n, work = 0; + + if (!ctx->group) { + return 0; + } + + n = spdk_sock_group_poll(ctx->group); + + if (n > 0) { + work += n; + } + + for (i = 0; i < ctx->num_active;) { + s = ctx->active[i]; + bind = evpl_private2bind(s); + + if (!s->closed && s->connected && s->write_interest) { + work += evpl_spdk_sock_submit(evpl, ctx, s, bind); + } + + if (s->closed || !s->write_interest) { + s->active = 0; + ctx->active[i] = ctx->active[--ctx->num_active]; + } else { + i++; + } + } + + if (work) { + evpl_activity(evpl); + } + + return work; +} /* evpl_spdk_sock_sweep */ + +/* + * fd_group handler for the sock group in interrupt mode. Runs OUTSIDE + * evpl_continue on the owning thread: anything the dispatched callbacks + * deferred (sends from recv callbacks arm flush deferrals, closes arm close + * deferrals) would otherwise sit unprocessed while the reactor sleeps, so + * kick the mechanism's pump whenever there is any sign of follow-up work. + */ +static int +evpl_spdk_sock_intr_fn(void *arg) +{ + struct evpl_spdk_thread *ctx = arg; + struct evpl *evpl = ctx->evpl; + int work; + + work = evpl_spdk_sock_sweep(evpl, ctx); + + if (work || evpl->num_active_deferrals || evpl->pending_close_binds) { + evpl_kick(evpl); + } + + return work; +} /* evpl_spdk_sock_intr_fn */ + +static void +evpl_spdk_sock_progress_timer_cb( + struct evpl *evpl, + struct evpl_timer *timer) +{ + struct evpl_spdk_thread *ctx = + container_of(timer, struct evpl_spdk_thread, progress_timer); + + evpl_spdk_sock_sweep(evpl, ctx); +} /* evpl_spdk_sock_progress_timer_cb */ + +/* Framework per-thread teardown; runs from evpl_destroy after every bind has + * been closed, so the group is empty. */ +void +evpl_spdk_sock_thread_destroy( + struct evpl *evpl, + struct evpl_spdk_thread *ctx) +{ + struct evpl_spdk_sock_request *request; + int rc; + + if (ctx->progress_timer_armed) { + evpl_remove_timer(evpl, &ctx->progress_timer); + ctx->progress_timer_armed = 0; + } + + if (ctx->intr_registered) { + spdk_sock_group_unregister_interrupt(ctx->group); + ctx->intr_registered = 0; + } + + if (ctx->poll) { + evpl_remove_poll(evpl, ctx->poll); + ctx->poll = NULL; + evpl->force_poll_mode = 0; + } + + if (ctx->group) { + rc = spdk_sock_group_close(&ctx->group); + + if (rc) { + evpl_spdk_sock_error("spdk_sock_group_close failed: %s", + strerror(errno)); + } + } + + while ((request = ctx->free_requests)) { + ctx->free_requests = request->next; + evpl_free(request); + } +} /* evpl_spdk_sock_thread_destroy */ + +static void +evpl_spdk_sock_connect_done( + void *cb_arg, + int status) +{ + struct evpl_spdk_sock *s = cb_arg; + struct evpl *evpl = s->evpl; + struct evpl_bind *bind = evpl_private2bind(s); + struct evpl_spdk_thread *ctx; + struct evpl_notify notify; + char laddr[INET6_ADDRSTRLEN]; + char paddr[INET6_ADDRSTRLEN]; + uint16_t lport, pport; + + ctx = evpl_framework_private(evpl, EVPL_FRAMEWORK_SPDK); + + if (s->connecting) { + s->connecting = 0; + evpl_spdk_sock_progress_dec(ctx); + } + + if (status != 0) { + s->connect_failed = 1; + + if (!s->closed) { + evpl_spdk_sock_debug("spdk_sock connect failed: %d", status); + evpl_close(evpl, bind); + } + return; + } + + s->connected = 1; + s->writable = 1; + + /* Resolve the actual local endpoint; kept only when the caller did not + * bind an explicit local address. */ + if (!bind->local && + spdk_sock_getaddr(s->sock, laddr, sizeof(laddr), &lport, + paddr, sizeof(paddr), &pport) == 0) { + bind->local = evpl_address_alloc(); + evpl_spdk_sock_address_fill(bind->local, laddr, lport); + } + + notify.notify_type = EVPL_NOTIFY_CONNECTED; + notify.notify_status = 0; + bind->notify_callback(evpl, bind, ¬ify, bind->private_data); + + /* Data staged before the connect completed. */ + if (!evpl_iovec_ring_is_empty(&bind->iovec_send)) { + evpl_defer(evpl, &bind->flush_deferral); + } +} /* evpl_spdk_sock_connect_done */ + +static void +evpl_spdk_tcp_connect( + struct evpl *evpl, + struct evpl_bind *bind) +{ + struct evpl_spdk_thread *ctx; + struct evpl_spdk_sock *s = evpl_bind_private(bind); + struct spdk_sock_opts opts; + char ip[INET6_ADDRSTRLEN]; + uint16_t port; + const char *impl = evpl_shared->config->spdk_sock_impl; + int rc; + + evpl_spdk_sock_abort_if( + evpl_shared->config->core_mech != EVPL_CORE_MECH_SPDK, + "STREAM_SPDK_TCP requires EVPL_CORE_MECH_SPDK"); + + ctx = evpl_spdk_sock_thread(evpl); + + s->evpl = evpl; + + evpl_spdk_sock_endpoint_parts(bind->remote, ip, sizeof(ip), &port); + + memset(&opts, 0, sizeof(opts)); + opts.opts_size = sizeof(opts); + spdk_sock_get_default_opts(&opts); + + /* zcopy off: write completions must be FIFO for the waist/tail release + * model, which is only guaranteed for the non-zcopy path. */ + opts.zcopy = false; + + if (bind->local) { + uint16_t lport; + + evpl_spdk_sock_endpoint_parts(bind->local, s->src_addr, + sizeof(s->src_addr), &lport); + opts.src_addr = s->src_addr; + opts.src_port = lport; + } + + s->connecting = 1; + evpl_spdk_sock_progress_inc(ctx); + + s->sock = spdk_sock_connect_async(ip, port, impl, &opts, + evpl_spdk_sock_connect_done, s); + + if (!s->sock) { + /* Implementation without async connect, or an immediate failure + * (posix fires the callback on failure even when returning NULL -- + * connect_failed guards double handling). */ + if (s->connecting) { + s->connecting = 0; + evpl_spdk_sock_progress_dec(ctx); + } + + if (!s->connect_failed) { + s->sock = spdk_sock_connect_ext(ip, port, impl, &opts); + } + + if (!s->sock) { + evpl_spdk_sock_debug("spdk_sock connect to %s:%u failed: %s", + ip, port, strerror(errno)); + evpl_close(evpl, bind); + return; + } + } + + rc = spdk_sock_group_add_sock(ctx->group, s->sock, + evpl_spdk_sock_recv_cb, s); + + evpl_spdk_sock_abort_if(rc, "spdk_sock_group_add_sock failed: %s", + strerror(errno)); + + s->in_group = 1; + ctx->num_socks++; + + if (!s->connecting && !s->connected) { + /* Synchronous fallback connected inline. */ + evpl_spdk_sock_connect_done(s, 0); + } +} /* evpl_spdk_tcp_connect */ + +/* Listen-socket group callback: connections ready to accept. Runs on the + * shared listener thread; accepted socks are in no group (verified), so the + * wrapper hands bare spdk_sock ownership to the worker's attach. */ +static void +evpl_spdk_sock_accept_cb( + void *arg, + struct spdk_sock_group *group, + struct spdk_sock *sock) +{ + struct evpl_spdk_sock *ls = arg; + struct evpl *evpl = ls->evpl; + struct evpl_bind *listen_bind = evpl_private2bind(ls); + struct spdk_sock *new_sock; + struct evpl_spdk_accepted_sock *wrapper; + struct evpl_address *remote; + char laddr[INET6_ADDRSTRLEN]; + char paddr[INET6_ADDRSTRLEN]; + uint16_t lport, pport; + + while ((new_sock = spdk_sock_accept(ls->sock)) != NULL) { + + remote = evpl_address_alloc(); + + if (spdk_sock_getaddr(new_sock, laddr, sizeof(laddr), &lport, + paddr, sizeof(paddr), &pport) == 0) { + evpl_spdk_sock_address_fill(remote, paddr, pport); + } + + wrapper = evpl_zalloc(sizeof(*wrapper)); + wrapper->sock = new_sock; + + listen_bind->accept_callback(evpl, listen_bind, remote, wrapper, + listen_bind->private_data); + } +} /* evpl_spdk_sock_accept_cb */ + +static void +evpl_spdk_tcp_listen( + struct evpl *evpl, + struct evpl_bind *listen_bind) +{ + struct evpl_spdk_thread *ctx; + struct evpl_spdk_sock *s = evpl_bind_private(listen_bind); + struct spdk_sock_opts opts; + char ip[INET6_ADDRSTRLEN]; + uint16_t port; + int rc; + + evpl_spdk_sock_abort_if( + evpl_shared->config->core_mech != EVPL_CORE_MECH_SPDK, + "STREAM_SPDK_TCP requires EVPL_CORE_MECH_SPDK"); + + ctx = evpl_spdk_sock_thread(evpl); + + s->evpl = evpl; + s->listen = 1; + + evpl_spdk_sock_endpoint_parts(listen_bind->local, ip, sizeof(ip), &port); + + memset(&opts, 0, sizeof(opts)); + opts.opts_size = sizeof(opts); + spdk_sock_get_default_opts(&opts); + opts.zcopy = false; + + s->sock = spdk_sock_listen_ext(ip, port, + evpl_shared->config->spdk_sock_impl, + &opts); + + evpl_spdk_sock_abort_if(!s->sock, "Failed to listen on %s:%u: %s", + ip, port, strerror(errno)); + + rc = spdk_sock_group_add_sock(ctx->group, s->sock, + evpl_spdk_sock_accept_cb, s); + + evpl_spdk_sock_abort_if(rc, "spdk_sock_group_add_sock failed: %s", + strerror(errno)); + + s->in_group = 1; + ctx->num_socks++; +} /* evpl_spdk_tcp_listen */ + +static void +evpl_spdk_tcp_attach( + struct evpl *evpl, + struct evpl_bind *bind, + void *accepted) +{ + struct evpl_spdk_accepted_sock *wrapper = accepted; + struct evpl_spdk_thread *ctx; + struct evpl_spdk_sock *s = evpl_bind_private(bind); + struct evpl_notify notify; + char laddr[INET6_ADDRSTRLEN]; + char paddr[INET6_ADDRSTRLEN]; + uint16_t lport, pport; + int rc; + + ctx = evpl_spdk_sock_thread(evpl); + + s->evpl = evpl; + s->sock = wrapper->sock; + s->connected = 1; + s->writable = 1; + + rc = spdk_sock_group_add_sock(ctx->group, s->sock, + evpl_spdk_sock_recv_cb, s); + + evpl_spdk_sock_abort_if(rc, + "failed to add accepted sock to group: %s", + strerror(errno)); + + s->in_group = 1; + ctx->num_socks++; + + /* The connect request's local_address is deliberately NULL: resolve the + * actual local endpoint of the accepted connection here. */ + if (spdk_sock_getaddr(s->sock, laddr, sizeof(laddr), &lport, + paddr, sizeof(paddr), &pport) == 0) { + bind->local = evpl_address_alloc(); + evpl_spdk_sock_address_fill(bind->local, laddr, lport); + } + + notify.notify_type = EVPL_NOTIFY_CONNECTED; + notify.notify_status = 0; + bind->notify_callback(evpl, bind, ¬ify, bind->private_data); + + evpl_free(wrapper); +} /* evpl_spdk_tcp_attach */ + +static void +evpl_spdk_tcp_flush( + struct evpl *evpl, + struct evpl_bind *bind) +{ + struct evpl_spdk_sock *s = evpl_bind_private(bind); + struct evpl_spdk_thread *ctx; + + if (!s->sock || s->closed) { + return; + } + + if (!s->connected) { + /* connect_done re-defers the flush once established. */ + s->write_interest = 1; + return; + } + + ctx = evpl_framework_private(evpl, EVPL_FRAMEWORK_SPDK); + + evpl_spdk_sock_submit(evpl, ctx, s, bind); +} /* evpl_spdk_tcp_flush */ + +/* + * Runs from the close deferral inside evpl_continue -- never from within + * spdk_sock_group_poll, so mutating the group here is safe. Removing the + * sock from the group aborts outstanding write requests: their completions + * fire synchronously here with -ECANCELED and leave the send ring alone. + * Fully synchronous teardown; EVPL_BIND_CLOSE_DEFERRED is never needed. + */ +static void +evpl_spdk_tcp_pending_close( + struct evpl *evpl, + struct evpl_bind *bind) +{ + struct evpl_spdk_sock *s = evpl_bind_private(bind); + struct evpl_spdk_thread *ctx; + + ctx = evpl_framework_private(evpl, EVPL_FRAMEWORK_SPDK); + + if (s->connecting) { + s->connecting = 0; + evpl_spdk_sock_progress_dec(ctx); + } + + s->closed = 1; + + if (s->sock) { + if (s->in_group) { + spdk_sock_group_remove_sock(ctx->group, s->sock); + s->in_group = 0; + ctx->num_socks--; + } + + spdk_sock_close(&s->sock); + } + + if (s->outstanding) { + evpl_spdk_sock_error("sock closed with %u write requests still " + "outstanding", s->outstanding); + } +} /* evpl_spdk_tcp_pending_close */ + +static void +evpl_spdk_tcp_close( + struct evpl *evpl, + struct evpl_bind *bind) +{ + struct evpl_spdk_sock *s = evpl_bind_private(bind); + + if (s->recv1.length) { + evpl_iovec_release_internal(evpl, &s->recv1); + s->recv1.length = 0; + } + + if (s->recv2.length) { + evpl_iovec_release_internal(evpl, &s->recv2); + s->recv2.length = 0; + } +} /* evpl_spdk_tcp_close */ + +struct evpl_protocol evpl_spdk_tcp = { + .id = EVPL_STREAM_SPDK_TCP, + .connected = 1, + .stream = 1, + .name = "STREAM_SPDK_TCP", + .framework = &evpl_framework_spdk, + .connect = evpl_spdk_tcp_connect, + .pending_close = evpl_spdk_tcp_pending_close, + .close = evpl_spdk_tcp_close, + .listen = evpl_spdk_tcp_listen, + .attach = evpl_spdk_tcp_attach, + .flush = evpl_spdk_tcp_flush, +}; diff --git a/src/core/spdk/tests/CMakeLists.txt b/src/core/spdk/tests/CMakeLists.txt new file mode 100644 index 00000000..c6e10687 --- /dev/null +++ b/src/core/spdk/tests/CMakeLists.txt @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2026 Ben Jarvis +# +# SPDX-License-Identifier: LGPL-2.1-only + +# SPDK guest-mode tests run under a mini-reactor harness (spdk_test_harness.h) +# rather than the per-mechanism replication used by generic tests, which pump +# evpl from a plain pthread and cannot run in guest mode. The harness calls +# SPDK env/thread APIs directly, so the test binaries link SPDK themselves. +# +# LeakSanitizer is disabled for these tests: DPDK's EAL allocates +# process-lifetime state it never frees, which would otherwise fail every +# debug (ASAN) run. +macro(spdk_unit_test test_name) + unit_test(spdk ${test_name} spdk_${test_name}.c) + + target_link_libraries(spdk_${test_name} ${SPDK_LINK_TARGETS}) + + set_property(TEST libevpl/spdk/${test_name} APPEND PROPERTY + ENVIRONMENT "ASAN_OPTIONS=detect_leaks=0") +endmacro() + +spdk_unit_test(thread_basic) +spdk_unit_test(thread_same_reactor) +spdk_unit_test(threadpool_pingpong) +spdk_unit_test(doorbell) +spdk_unit_test(timer) +spdk_unit_test(memreg) +spdk_unit_test(sock_pingpong) +spdk_unit_test(sock_msg) + +# Protocol coverage for STREAM_SPDK_TCP via the shared, evpl_thread-driven +# stream test binary (windowed message ping-pong with segment framing). +if(LIBEVPL_NETNS_TESTING) + add_test(NAME libevpl/spdk/sock_bulk_connected_msg + COMMAND ${LIBEVPL_SOURCE_DIR}/scripts/netns_test_wrapper.sh + ${TEST_BIN}/ping_pong_connected_msg -r STREAM_SPDK_TCP) + set_tests_properties(libevpl/spdk/sock_bulk_connected_msg PROPERTIES + ENVIRONMENT "TEST_FILE=${LIBEVPL_SOURCE_DIR}/src/tests/ping_pong_connected_msg.c;EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0") +else() + add_test(NAME libevpl/spdk/sock_bulk_connected_msg + COMMAND ${TEST_BIN}/ping_pong_connected_msg -r STREAM_SPDK_TCP) + set_tests_properties(libevpl/spdk/sock_bulk_connected_msg PROPERTIES + RESOURCE_LOCK evpl_net + ENVIRONMENT "TEST_FILE=${LIBEVPL_SOURCE_DIR}/src/tests/ping_pong_connected_msg.c;EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0") +endif() + +# bdev tests bring up the bdev subsystem from JSON (spdk_init + +# spdk_event_bdev, test-only link modules). +spdk_unit_test(bdev_basic) +target_link_libraries(spdk_bdev_basic ${SPDK_TEST_LINK_TARGETS}) +spdk_unit_test(bdev_open_fail) +target_link_libraries(spdk_bdev_open_fail ${SPDK_TEST_LINK_TARGETS}) diff --git a/src/core/spdk/tests/spdk_bdev_basic.c b/src/core/spdk/tests/spdk_bdev_basic.c new file mode 100644 index 00000000..3edce85b --- /dev/null +++ b/src/core/spdk/tests/spdk_bdev_basic.c @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * SPDK bdev block backend: async open on the owning evpl thread, then a + * callback-driven chain over a malloc bdev covering sync write (flush + * chain), read-verify, flush, discard, write_zeroes, zero-verify, a + * misaligned-buffer write/read pair (bounce path when the bdev has an + * alignment requirement), async close. + */ + +#include +#include + +#include "spdk_test_harness.h" +#include "spdk_bdev_test_common.h" + +#define TEST_PATTERN_A 0xAB +#define TEST_PATTERN_B 0x5C + +struct bdev_test_state { + struct evpl *evpl; + struct evpl_block_device *bdev; + struct evpl_block_queue *queue; + struct evpl_iovec iov; /* 8 KiB scratch */ + int step; + volatile int done; +}; + +static void bdev_step( + struct evpl *evpl, + struct bdev_test_state *state); + +static void +bdev_step_done( + struct evpl *evpl, + int status, + void *private_data) +{ + struct bdev_test_state *state = private_data; + + evpl_test_abort_if(status, "step %d failed: %d", state->step, status); + + state->step++; + + bdev_step(evpl, state); +} /* bdev_step_done */ + +static void +bdev_close_done( + struct evpl *evpl, + int status, + void *private_data) +{ + struct bdev_test_state *state = private_data; + + evpl_test_abort_if(status, "device close failed: %d", status); + + state->done = 1; +} /* bdev_close_done */ + +static void +bdev_step( + struct evpl *evpl, + struct bdev_test_state *state) +{ + struct evpl_iovec view; + + switch (state->step) { + case 0: + /* Durable write: exercises the write-then-flush chain when the + * bdev reports a write cache. */ + memset(state->iov.data, TEST_PATTERN_A, 4096); + view = state->iov; + view.length = 4096; + evpl_block_write(evpl, state->queue, &view, 1, 0, 1, + bdev_step_done, state); + break; + case 1: + memset(state->iov.data, 0, 4096); + view = state->iov; + view.length = 4096; + evpl_block_read(evpl, state->queue, &view, 1, 0, + bdev_step_done, state); + break; + case 2: + for (int i = 0; i < 4096; i++) { + evpl_test_abort_if( + ((unsigned char *) state->iov.data)[i] != TEST_PATTERN_A, + "read data mismatch at %d", i); + } + evpl_block_flush(evpl, state->queue, bdev_step_done, state); + break; + case 3: + evpl_block_discard(evpl, state->queue, 0, 8 * 4096, + bdev_step_done, state); + break; + case 4: + /* First re-establish a known pattern... */ + memset(state->iov.data, TEST_PATTERN_A, 4096); + view = state->iov; + view.length = 4096; + evpl_block_write(evpl, state->queue, &view, 1, 4096, 0, + bdev_step_done, state); + break; + case 5: + /* ...then zero it. */ + evpl_block_write_zeroes(evpl, state->queue, 4096, 4096, + bdev_step_done, state); + break; + case 6: + memset(state->iov.data, TEST_PATTERN_A, 4096); + view = state->iov; + view.length = 4096; + evpl_block_read(evpl, state->queue, &view, 1, 4096, + bdev_step_done, state); + break; + case 7: + for (int i = 0; i < 4096; i++) { + evpl_test_abort_if( + ((unsigned char *) state->iov.data)[i] != 0, + "write_zeroes left nonzero data at %d", i); + } + + /* Misaligned view into the buffer: exercises the bounce path on + * bdevs with a buffer alignment requirement (correctness holds + * either way). */ + view = state->iov; + view.data = (char *) state->iov.data + 512; + view.length = 4096; + memset(view.data, TEST_PATTERN_B, 4096); + evpl_block_write(evpl, state->queue, &view, 1, 8192, 0, + bdev_step_done, state); + break; + case 8: + view = state->iov; + view.data = (char *) state->iov.data + 512; + view.length = 4096; + memset(view.data, 0, 4096); + evpl_block_read(evpl, state->queue, &view, 1, 8192, + bdev_step_done, state); + break; + case 9: + for (int i = 0; i < 4096; i++) { + evpl_test_abort_if( + ((unsigned char *) state->iov.data)[512 + i] != + TEST_PATTERN_B, + "misaligned read mismatch at %d", i); + } + + evpl_iovec_release(evpl, &state->iov); + + evpl_block_close_queue(evpl, state->queue); + + evpl_block_close_device(evpl, state->bdev, + bdev_close_done, state); + break; + default: + evpl_test_abort_if(1, "unexpected step %d", state->step); + } /* switch */ +} /* bdev_step */ + +static void +bdev_open_done( + struct evpl *evpl, + struct evpl_block_device *bdev, + int status, + void *private_data) +{ + struct bdev_test_state *state = private_data; + int niov; + + evpl_test_abort_if(status || !bdev, "device open failed: %d", status); + + state->bdev = bdev; + + evpl_test_abort_if(evpl_block_size(bdev) != 16384ULL * 4096, + "unexpected device size %lu", + (unsigned long) evpl_block_size(bdev)); + + state->queue = evpl_block_open_queue(evpl, bdev); + + niov = evpl_iovec_alloc(evpl, 8192, 4096, 1, 0, &state->iov); + + evpl_test_abort_if(niov < 1, "iovec alloc failed"); + + bdev_step(evpl, state); +} /* bdev_open_done */ + +static void * +bdev_thread_init( + struct evpl *evpl, + void *private_data) +{ + struct bdev_test_state *state = private_data; + + state->evpl = evpl; + + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_SPDK_BDEV, "Malloc0", + bdev_open_done, state); + + return private_data; +} /* bdev_thread_init */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *thread; + struct bdev_test_state state = { 0 }; + + evpl_spdk_test_init(2); + + evpl_spdk_bdev_test_up(EVPL_SPDK_BDEV_TEST_MALLOC_JSON); + + evpl_spdk_test_config(); + + thread = evpl_thread_create(NULL, bdev_thread_init, NULL, &state); + + while (!state.done) { + usleep(1000); + } + + evpl_test_info("bdev op chain completed"); + + evpl_thread_destroy(thread); + + evpl_spdk_bdev_test_down(); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_bdev_open_fail.c b/src/core/spdk/tests/spdk_bdev_open_fail.c new file mode 100644 index 00000000..1916797c --- /dev/null +++ b/src/core/spdk/tests/spdk_bdev_open_fail.c @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * Opening a nonexistent bdev fails asynchronously with a nonzero status and + * a NULL device, and a subsequent good open on the same evpl succeeds. + */ + +#include + +#include "spdk_test_harness.h" +#include "spdk_bdev_test_common.h" + +struct open_fail_state { + struct evpl_block_device *bdev; + volatile int done; +}; + +static void +close_done( + struct evpl *evpl, + int status, + void *private_data) +{ + struct open_fail_state *state = private_data; + + evpl_test_abort_if(status, "close failed: %d", status); + + state->done = 1; +} /* close_done */ + +static void +good_open_done( + struct evpl *evpl, + struct evpl_block_device *bdev, + int status, + void *private_data) +{ + struct open_fail_state *state = private_data; + + evpl_test_abort_if(status || !bdev, + "expected open of Malloc0 to succeed: %d", status); + + state->bdev = bdev; + + evpl_block_close_device(evpl, bdev, close_done, state); +} /* good_open_done */ + +static void +bad_open_done( + struct evpl *evpl, + struct evpl_block_device *bdev, + int status, + void *private_data) +{ + evpl_test_abort_if(status == 0 || bdev, + "open of nonexistent bdev unexpectedly succeeded"); + + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_SPDK_BDEV, "Malloc0", + good_open_done, private_data); +} /* bad_open_done */ + +static void * +thread_init( + struct evpl *evpl, + void *private_data) +{ + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_SPDK_BDEV, + "DoesNotExist", bad_open_done, private_data); + + return private_data; +} /* thread_init */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *thread; + struct open_fail_state state = { 0 }; + + evpl_spdk_test_init(2); + + evpl_spdk_bdev_test_up(EVPL_SPDK_BDEV_TEST_MALLOC_JSON); + + evpl_spdk_test_config(); + + thread = evpl_thread_create(NULL, thread_init, NULL, &state); + + while (!state.done) { + usleep(1000); + } + + evpl_thread_destroy(thread); + + evpl_spdk_bdev_test_down(); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_bdev_test_common.h b/src/core/spdk/tests/spdk_bdev_test_common.h new file mode 100644 index 00000000..0901abe5 --- /dev/null +++ b/src/core/spdk/tests/spdk_bdev_test_common.h @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#pragma once + +/* + * bdev subsystem bring-up for SPDK bdev tests, without the app framework: + * a bootstrap spdk_thread runs spdk_subsystem_load_config with a JSON config + * (e.g. creating a malloc bdev), the way SPDK's fio plugin does. + * + * The bootstrap thread is created before any evpl call so it is the first + * spdk_thread in the process (SPDK's "app thread", which some init paths + * are restricted to). Call evpl_spdk_bdev_test_up right after + * evpl_spdk_test_init, and evpl_spdk_bdev_test_down before returning from + * main; the bootstrap thread itself is reaped by the harness atexit fini. + */ + +#include +#include + +#include "spdk_test_harness.h" + +static struct spdk_thread *evpl_spdk_bdev_test_thread; +static volatile int evpl_spdk_bdev_test_rc; +static volatile int evpl_spdk_bdev_test_flag; + +struct evpl_spdk_bdev_test_json { + const char *json; + size_t len; +}; + +static void +evpl_spdk_bdev_test_init_done( + int rc, + void *ctx) +{ + evpl_spdk_bdev_test_rc = rc; + evpl_spdk_bdev_test_flag = 1; +} /* evpl_spdk_bdev_test_init_done */ + +static void +evpl_spdk_bdev_test_subsystems_up( + int rc, + void *ctx) +{ + if (rc == 0) { + /* The RPC state gates which config methods spdk_subsystem_load_config + * will replay; runtime methods like bdev_malloc_create are silently + * skipped unless the state is advanced (the app framework does this + * in its own init-done callback). */ + spdk_rpc_set_state(SPDK_RPC_RUNTIME); + } + + evpl_spdk_bdev_test_init_done(rc, ctx); +} /* evpl_spdk_bdev_test_subsystems_up */ + +static void +evpl_spdk_bdev_test_init_msg(void *ctx) +{ + spdk_subsystem_init(evpl_spdk_bdev_test_subsystems_up, NULL); +} /* evpl_spdk_bdev_test_init_msg */ + +static void +evpl_spdk_bdev_test_load_msg(void *ctx) +{ + struct evpl_spdk_bdev_test_json *cfg = ctx; + + /* Replays the config RPCs against the current (post-init) RPC state; + * unlike the old init_from_json_config this does NOT initialize the + * subsystems itself, hence the spdk_subsystem_init step first. */ + spdk_subsystem_load_config((void *) cfg->json, cfg->len, + evpl_spdk_bdev_test_init_done, NULL, true); +} /* evpl_spdk_bdev_test_load_msg */ + +static void +evpl_spdk_bdev_test_step( + spdk_msg_fn fn, + void *arg, + const char *what) +{ + int rc; + + evpl_spdk_bdev_test_flag = 0; + evpl_spdk_bdev_test_rc = 0; + + rc = spdk_thread_send_msg(evpl_spdk_bdev_test_thread, fn, arg); + + evpl_test_abort_if(rc, "spdk_thread_send_msg failed: %d", rc); + + while (!evpl_spdk_bdev_test_flag) { + usleep(1000); + } + + evpl_test_abort_if(evpl_spdk_bdev_test_rc, "%s failed: %d", + what, evpl_spdk_bdev_test_rc); +} /* evpl_spdk_bdev_test_step */ + +static void +evpl_spdk_bdev_test_up(const char *json) +{ + static struct evpl_spdk_bdev_test_json cfg; + + cfg.json = json; + cfg.len = strlen(json); + + evpl_spdk_bdev_test_thread = spdk_thread_create("bootstrap", NULL); + + evpl_test_abort_if(!evpl_spdk_bdev_test_thread, + "bootstrap spdk_thread_create failed"); + + evpl_spdk_bdev_test_step(evpl_spdk_bdev_test_init_msg, NULL, + "subsystem init"); + + evpl_spdk_bdev_test_step(evpl_spdk_bdev_test_load_msg, &cfg, + "JSON config load"); +} /* evpl_spdk_bdev_test_up */ + +static void +evpl_spdk_bdev_test_fini_done(void *ctx) +{ + evpl_spdk_bdev_test_flag = 1; +} /* evpl_spdk_bdev_test_fini_done */ + +static void +evpl_spdk_bdev_test_fini_msg(void *ctx) +{ + spdk_subsystem_fini(evpl_spdk_bdev_test_fini_done, NULL); +} /* evpl_spdk_bdev_test_fini_msg */ + +static void +evpl_spdk_bdev_test_down(void) +{ + int rc; + + evpl_spdk_bdev_test_flag = 0; + + rc = spdk_thread_send_msg(evpl_spdk_bdev_test_thread, + evpl_spdk_bdev_test_fini_msg, NULL); + + evpl_test_abort_if(rc, "spdk_thread_send_msg failed: %d", rc); + + while (!evpl_spdk_bdev_test_flag) { + usleep(1000); + } +} /* evpl_spdk_bdev_test_down */ + +#define EVPL_SPDK_BDEV_TEST_MALLOC_JSON \ + "{\"subsystems\":[{\"subsystem\":\"bdev\",\"config\":[" \ + "{\"method\":\"bdev_malloc_create\",\"params\":" \ + "{\"name\":\"Malloc0\",\"num_blocks\":16384,\"block_size\":4096" \ + "}}]}]}" diff --git a/src/core/spdk/tests/spdk_doorbell.c b/src/core/spdk/tests/spdk_doorbell.c new file mode 100644 index 00000000..c30f79ab --- /dev/null +++ b/src/core/spdk/tests/spdk_doorbell.c @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * Doorbell ping-pong between two SPDK evpl threads: exercises eventfd events + * through the mechanism's internal epoll and cross-thread wakeups under the + * poller pump. + */ + +#include + +#include "spdk_test_harness.h" + +#define DB_ITERS 100 + +static struct evpl_doorbell a_doorbell; +static struct evpl_doorbell b_doorbell; +static volatile int count; +static volatile int done; + +static void +a_callback( + struct evpl *evpl, + struct evpl_doorbell *doorbell) +{ + if (done) { + return; + } + + count++; + + if (count >= DB_ITERS) { + done = 1; + return; + } + + evpl_ring_doorbell(&b_doorbell); +} /* a_callback */ + +static void +b_callback( + struct evpl *evpl, + struct evpl_doorbell *doorbell) +{ + if (done) { + return; + } + + evpl_ring_doorbell(&a_doorbell); +} /* b_callback */ + +static void * +a_init( + struct evpl *evpl, + void *private_data) +{ + evpl_add_doorbell(evpl, &a_doorbell, a_callback); + + return private_data; +} /* a_init */ + +static void +a_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_remove_doorbell(evpl, &a_doorbell); +} /* a_shutdown */ + +static void * +b_init( + struct evpl *evpl, + void *private_data) +{ + evpl_add_doorbell(evpl, &b_doorbell, b_callback); + + return private_data; +} /* b_init */ + +static void +b_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_remove_doorbell(evpl, &b_doorbell); +} /* b_shutdown */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *thread_a; + struct evpl_thread *thread_b; + + evpl_spdk_test_init(2); + + evpl_spdk_test_config(); + + thread_a = evpl_thread_create(NULL, a_init, a_shutdown, NULL); + thread_b = evpl_thread_create(NULL, b_init, b_shutdown, NULL); + + evpl_ring_doorbell(&a_doorbell); + + while (!done) { + usleep(1000); + } + + evpl_test_info("doorbell ping-pong completed %d rounds", count); + + evpl_thread_destroy(thread_a); + evpl_thread_destroy(thread_b); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_memreg.c b/src/core/spdk/tests/spdk_memreg.c new file mode 100644 index 00000000..022d5fde --- /dev/null +++ b/src/core/spdk/tests/spdk_memreg.c @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * Validates the EVPL_FRAMEWORK_SPDK memory registration path: the first + * SPDK-mode evpl_create attaches the framework, every slab is registered + * with spdk_mem_register as it is built, and slabs are 2 MiB aligned. + * Verified by allocating an iovec (forcing a slab) on an SPDK evpl thread, + * then walking SPDK's existing registrations via the notify pass that + * spdk_mem_map_alloc performs, checking the iovec's buffer is covered. + */ + +#include +#include + +#include "spdk_test_harness.h" + +static void *iovec_addr; +static volatile int covered; +static volatile int covered_aligned; + +static void * +memreg_init( + struct evpl *evpl, + void *private_data) +{ + struct evpl_iovec iovec; + int niov; + + niov = evpl_iovec_alloc(evpl, 4096, 4096, 1, 0, &iovec); + + evpl_test_abort_if(niov < 1, "evpl_iovec_alloc failed"); + + iovec_addr = evpl_iovec_data(&iovec); + + evpl_iovec_release(evpl, &iovec); + + return private_data; +} /* memreg_init */ + +static int +memreg_notify( + void *cb_ctx, + struct spdk_mem_map *map, + enum spdk_mem_map_notify_action action, + void *vaddr, + size_t size) +{ + if (action != SPDK_MEM_MAP_NOTIFY_REGISTER) { + return 0; + } + + /* Alignment is asserted only for the region covering our slab: SPDK + * registers its own EAL heap regions too, and those need not be 2 MiB + * aligned (e.g. the no-huge heap). */ + if ((uintptr_t) iovec_addr >= (uintptr_t) vaddr && + (uintptr_t) iovec_addr < (uintptr_t) vaddr + size) { + covered = 1; + + covered_aligned = ((uintptr_t) vaddr & + (2 * 1024 * 1024 - 1)) == 0; + } + + return 0; +} /* memreg_notify */ + +int +main( + int argc, + char *argv[]) +{ + static const struct spdk_mem_map_ops ops = { + .notify_cb = memreg_notify, + .are_contiguous = NULL, + }; + struct evpl_thread *thread; + struct spdk_mem_map *map; + + evpl_spdk_test_init(2); + + evpl_spdk_test_config(); + + /* Blocks until memreg_init allocated the iovec on the worker. */ + thread = evpl_thread_create(NULL, memreg_init, NULL, NULL); + + /* Allocation walks all existing registrations through memreg_notify. */ + map = spdk_mem_map_alloc(0, &ops, NULL); + + evpl_test_abort_if(!map, "spdk_mem_map_alloc failed"); + + spdk_mem_map_free(&map); + + evpl_test_abort_if(!covered, + "slab containing the iovec was not registered with SPDK"); + + evpl_test_abort_if(!covered_aligned, + "the slab's registered region was not 2 MiB aligned"); + + evpl_test_info("iovec buffer %p is SPDK-registered and aligned", + iovec_addr); + + evpl_thread_destroy(thread); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_sock_msg.c b/src/core/spdk/tests/spdk_sock_msg.c new file mode 100644 index 00000000..2a99ef41 --- /dev/null +++ b/src/core/spdk/tests/spdk_sock_msg.c @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * Message-framed ping-pong over STREAM_SPDK_TCP: a segment callback frames + * fixed 4-byte messages, so this covers the EVPL_NOTIFY_RECV_MSG branch and + * the per-message dgram accounting on the writev_async completion path. + */ + +#include +#include + +#include "spdk_test_harness.h" + +static const char localhost[] = "127.0.0.1"; +static int port = 8000; + +struct msg_state { + volatile int run; + int sent; + int recv; + int niters; + uint32_t value; +}; + +static struct evpl_listener_binding *server_binding; +static struct evpl_listener *listener; + +static int +msg_segment_callback( + struct evpl *evpl, + struct evpl_bind *bind, + void *private_data) +{ + return sizeof(uint32_t); +} /* msg_segment_callback */ + +static void +client_callback( + struct evpl *evpl, + struct evpl_bind *bind, + struct evpl_notify *notify, + void *private_data) +{ + struct msg_state *state = private_data; + uint32_t value; + + switch (notify->notify_type) { + case EVPL_NOTIFY_RECV_MSG: + + evpl_test_abort_if(notify->recv_msg.length != sizeof(value), + "unexpected message length %u", + notify->recv_msg.length); + + memcpy(&value, notify->recv_msg.iovec[0].data, sizeof(value)); + + evpl_iovecs_release(evpl, notify->recv_msg.iovec, + notify->recv_msg.niov); + + state->recv++; + + if (state->recv < state->niters) { + evpl_send(evpl, bind, &state->value, sizeof(state->value)); + state->value++; + state->sent++; + } else { + state->run = 0; + } + break; + + case EVPL_NOTIFY_DISCONNECTED: + evpl_test_info("client disconnected"); + break; + } /* switch */ + +} /* client_callback */ + +static void +server_callback( + struct evpl *evpl, + struct evpl_bind *bind, + struct evpl_notify *notify, + void *private_data) +{ + uint32_t value; + + switch (notify->notify_type) { + case EVPL_NOTIFY_RECV_MSG: + memcpy(&value, notify->recv_msg.iovec[0].data, sizeof(value)); + evpl_iovecs_release(evpl, notify->recv_msg.iovec, + notify->recv_msg.niov); + evpl_send(evpl, bind, &value, sizeof(value)); + break; + } /* switch */ + +} /* server_callback */ + +static void +accept_callback( + struct evpl *evpl, + struct evpl_bind *accepted_bind, + evpl_notify_callback_t *notify_callback, + evpl_segment_callback_t *segment_callback, + void **conn_private_data, + void *private_data) +{ + *notify_callback = server_callback; + *segment_callback = msg_segment_callback; + *conn_private_data = private_data; +} /* accept_callback */ + +static void * +server_init( + struct evpl *evpl, + void *private_data) +{ + server_binding = evpl_listener_attach(evpl, listener, accept_callback, + private_data); + + return private_data; +} /* server_init */ + +static void +server_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_listener_detach(evpl, server_binding); +} /* server_shutdown */ + +static void * +client_init( + struct evpl *evpl, + void *private_data) +{ + struct msg_state *state = private_data; + struct evpl_endpoint *ep; + struct evpl_bind *bind; + + ep = evpl_endpoint_create(localhost, port); + + bind = evpl_connect(evpl, EVPL_STREAM_SPDK_TCP, NULL, ep, + client_callback, msg_segment_callback, state); + + evpl_send(evpl, bind, &state->value, sizeof(state->value)); + + state->value++; + state->sent++; + + return private_data; +} /* client_init */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + struct evpl_endpoint *listen_ep; + struct msg_state state = { + .run = 1, + .sent = 0, + .recv = 0, + .niters = 100, + .value = 1 + }; + + evpl_spdk_test_init(3); + + evpl_spdk_test_config(); + + listener = evpl_listener_create(); + + server_thread = evpl_thread_create(NULL, server_init, server_shutdown, + &state); + + listen_ep = evpl_endpoint_create("0.0.0.0", port); + + evpl_listen(listener, EVPL_STREAM_SPDK_TCP, listen_ep); + + client_thread = evpl_thread_create(NULL, client_init, NULL, &state); + + while (state.run) { + usleep(1000); + } + + evpl_test_info("msg ping-pong completed: sent %d recv %d", + state.sent, state.recv); + + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); + + evpl_listener_destroy(listener); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_sock_pingpong.c b/src/core/spdk/tests/spdk_sock_pingpong.c new file mode 100644 index 00000000..f2871b2c --- /dev/null +++ b/src/core/spdk/tests/spdk_sock_pingpong.c @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * spdk_sock TCP ping-pong between evpl threads: covers the listener-thread + * listener over an SPDK evpl thread, the cross-thread accept handoff + * (connect_requests + run_wakeup), and socket send/recv under the poller + * pump. All I/O logic runs in callbacks on the SPDK evpl threads; main() + * stays on a plain pthread where the blocking listener/thread APIs are safe. + */ + +#include +#include + +#include "spdk_test_harness.h" + +static const char localhost[] = "127.0.0.1"; +static int port = 8000; + +struct pingpong_state { + volatile int run; + int sent; + int recv; + int niters; + uint32_t value; +}; + +static struct evpl_listener_binding *server_binding; +static struct evpl_listener *listener; + +static void +client_callback( + struct evpl *evpl, + struct evpl_bind *bind, + struct evpl_notify *notify, + void *private_data) +{ + struct pingpong_state *state = private_data; + uint32_t value; + int length; + + switch (notify->notify_type) { + case EVPL_NOTIFY_CONNECTED: + evpl_test_info("client connected"); + break; + case EVPL_NOTIFY_RECV_DATA: + + do { + length = evpl_recv(evpl, bind, &value, sizeof(value), + EVPL_RECV_FLAG_ALL_OR_NONE); + + if (length == sizeof(value)) { + + state->recv++; + + if (state->recv < state->niters) { + + evpl_send(evpl, bind, &state->value, + sizeof(state->value)); + + state->value++; + state->sent++; + } else { + state->run = 0; + } + } + } while (length > 0); + + break; + + case EVPL_NOTIFY_DISCONNECTED: + evpl_test_info("client disconnected"); + break; + } /* switch */ + +} /* client_callback */ + +static void +server_callback( + struct evpl *evpl, + struct evpl_bind *bind, + struct evpl_notify *notify, + void *private_data) +{ + uint32_t value; + int length; + + switch (notify->notify_type) { + case EVPL_NOTIFY_CONNECTED: + evpl_test_info("server connected"); + break; + case EVPL_NOTIFY_DISCONNECTED: + evpl_test_info("server disconnected"); + break; + case EVPL_NOTIFY_RECV_DATA: + + do { + length = evpl_recv(evpl, bind, &value, sizeof(value), + EVPL_RECV_FLAG_ALL_OR_NONE); + + if (length == sizeof(value)) { + evpl_send(evpl, bind, &value, sizeof(value)); + } + } while (length > 0); + + break; + } /* switch */ + +} /* server_callback */ + +static void +accept_callback( + struct evpl *evpl, + struct evpl_bind *accepted_bind, + evpl_notify_callback_t *notify_callback, + evpl_segment_callback_t *segment_callback, + void **conn_private_data, + void *private_data) +{ + *notify_callback = server_callback; + *conn_private_data = private_data; +} /* accept_callback */ + +static void * +server_init( + struct evpl *evpl, + void *private_data) +{ + server_binding = evpl_listener_attach(evpl, listener, accept_callback, + private_data); + + return private_data; +} /* server_init */ + +static void +server_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_listener_detach(evpl, server_binding); +} /* server_shutdown */ + +static void * +client_init( + struct evpl *evpl, + void *private_data) +{ + struct pingpong_state *state = private_data; + struct evpl_endpoint *ep; + struct evpl_bind *bind; + + ep = evpl_endpoint_create(localhost, port); + + bind = evpl_connect(evpl, EVPL_STREAM_SPDK_TCP, NULL, ep, + client_callback, NULL, state); + + evpl_send(evpl, bind, &state->value, sizeof(state->value)); + + state->value++; + state->sent++; + + return private_data; +} /* client_init */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + struct evpl_endpoint *listen_ep; + struct pingpong_state state = { + .run = 1, + .sent = 0, + .recv = 0, + .niters = 100, + .value = 1 + }; + + evpl_spdk_test_init(3); + + evpl_spdk_test_config(); + + listener = evpl_listener_create(); + + /* Blocks until server_init attached the binding on the worker. */ + server_thread = evpl_thread_create(NULL, server_init, server_shutdown, + &state); + + listen_ep = evpl_endpoint_create("0.0.0.0", port); + + evpl_listen(listener, EVPL_STREAM_SPDK_TCP, listen_ep); + + client_thread = evpl_thread_create(NULL, client_init, NULL, &state); + + while (state.run) { + usleep(1000); + } + + evpl_test_info("ping-pong completed: sent %d recv %d", + state.sent, state.recv); + + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); + + evpl_listener_destroy(listener); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_test_harness.h b/src/core/spdk/tests/spdk_test_harness.h new file mode 100644 index 00000000..765e3bfd --- /dev/null +++ b/src/core/spdk/tests/spdk_test_harness.h @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#pragma once + +/* + * Mini-reactor harness for SPDK guest-mode tests. + * + * Production libevpl never pumps spdk_threads -- the host application's + * reactors do -- so the tests provide the host side themselves: an SPDK env + * (no hugepages, so it runs in CI containers), the thread library with a + * round-robin scheduler op, and N pthreads that poll their assigned + * spdk_threads and reap exited ones. Test main() stays on a plain pthread, + * where the blocking forms of evpl_thread_create/evpl_listen/ + * evpl_thread_destroy are safe. + * + * Teardown ordering: evpl_spdk_test_init() registers its atexit handler + * before the test calls evpl_init(), so evpl_cleanup (registered later, run + * first -- atexit is LIFO) unregisters slab memory while the SPDK env is + * still alive. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "core/test_log.h" +#include "evpl/evpl.h" + +#define EVPL_SPDK_TEST_MAX_REACTORS 8 +#define EVPL_SPDK_TEST_BATCH 256 + +struct evpl_spdk_test_reactor { + pthread_t pthread; + pthread_mutex_t lock; + struct spdk_thread **threads; + int num_threads; + int max_threads; + volatile int stop; +}; + +static struct evpl_spdk_test_reactor + evpl_spdk_test_reactors[EVPL_SPDK_TEST_MAX_REACTORS]; +static int evpl_spdk_test_num_reactors; +static unsigned int evpl_spdk_test_rotor; + +static int +evpl_spdk_test_new_thread(struct spdk_thread *thread) +{ + struct evpl_spdk_test_reactor *reactor; + unsigned int idx; + + idx = __sync_fetch_and_add(&evpl_spdk_test_rotor, 1); + + reactor = &evpl_spdk_test_reactors[idx % evpl_spdk_test_num_reactors]; + + pthread_mutex_lock(&reactor->lock); + + if (reactor->num_threads >= reactor->max_threads) { + reactor->max_threads = reactor->max_threads ? + reactor->max_threads * 2 : 8; + reactor->threads = realloc(reactor->threads, + reactor->max_threads * + sizeof(*reactor->threads)); + } + + reactor->threads[reactor->num_threads++] = thread; + + pthread_mutex_unlock(&reactor->lock); + + return 0; +} /* evpl_spdk_test_new_thread */ + +static void * +evpl_spdk_test_reactor_fn(void *arg) +{ + struct evpl_spdk_test_reactor *reactor = arg; + struct spdk_thread *threads[EVPL_SPDK_TEST_BATCH]; + struct spdk_thread *thread; + int i, j, n, done, busy; + + for (;;) { + pthread_mutex_lock(&reactor->lock); + + n = reactor->num_threads; + + if (n > EVPL_SPDK_TEST_BATCH) { + n = EVPL_SPDK_TEST_BATCH; + } + + for (i = 0; i < n; i++) { + threads[i] = reactor->threads[i]; + } + + done = reactor->stop && reactor->num_threads == 0; + + pthread_mutex_unlock(&reactor->lock); + + if (done) { + break; + } + + busy = 0; + + for (i = 0; i < n; i++) { + thread = threads[i]; + + /* Poll without holding the lock: pollers and messages may create + * new spdk_threads, which re-enters the scheduler op. */ + busy |= spdk_thread_poll(thread, 0, 0); + + if (spdk_thread_is_exited(thread)) { + pthread_mutex_lock(&reactor->lock); + + for (j = 0; j < reactor->num_threads; j++) { + if (reactor->threads[j] == thread) { + reactor->threads[j] = + reactor->threads[--reactor->num_threads]; + break; + } + } + + pthread_mutex_unlock(&reactor->lock); + + spdk_thread_destroy(thread); + } + } + + if (!busy) { + usleep(100); + } + } + + return NULL; +} /* evpl_spdk_test_reactor_fn */ + +static void +evpl_spdk_test_thread_exit_msg(void *ctx) +{ + spdk_thread_exit(spdk_get_thread()); +} /* evpl_spdk_test_thread_exit_msg */ + +static void +evpl_spdk_test_fini(void) +{ + struct evpl_spdk_test_reactor *reactor; + int i, j; + + /* Ask any threads the test left behind (e.g. bootstrap threads) to exit; + * evpl worker threads exit themselves during evpl_thread_destroy. */ + for (i = 0; i < evpl_spdk_test_num_reactors; i++) { + reactor = &evpl_spdk_test_reactors[i]; + + pthread_mutex_lock(&reactor->lock); + + for (j = 0; j < reactor->num_threads; j++) { + if (!spdk_thread_is_exited(reactor->threads[j])) { + spdk_thread_send_msg(reactor->threads[j], + evpl_spdk_test_thread_exit_msg, NULL); + } + } + + pthread_mutex_unlock(&reactor->lock); + + reactor->stop = 1; + } + + for (i = 0; i < evpl_spdk_test_num_reactors; i++) { + pthread_join(evpl_spdk_test_reactors[i].pthread, NULL); + free(evpl_spdk_test_reactors[i].threads); + } + + spdk_thread_lib_fini(); + spdk_env_fini(); +} /* evpl_spdk_test_fini */ + +static void +evpl_spdk_test_init(int nreactors) +{ + struct spdk_env_opts opts; + int rc, i; + + evpl_test_abort_if(nreactors < 1 || + nreactors > EVPL_SPDK_TEST_MAX_REACTORS, + "invalid reactor count %d", nreactors); + + memset(&opts, 0, sizeof(opts)); + + opts.opts_size = sizeof(opts); + + spdk_env_opts_init(&opts); + + opts.name = "evpl_spdk_test"; + + /* No hugepages so the suite runs in unprivileged CI containers; evpl + * slabs are external memory, so DPDK only backs SPDK internals. 512MB + * leaves room for the iobuf subsystem's default pools (~200MB), which + * the bdev tests initialize. */ + opts.no_huge = true; + opts.mem_size = 512; + + rc = spdk_env_init(&opts); + + evpl_test_abort_if(rc, "spdk_env_init failed: %d", rc); + + rc = spdk_thread_lib_init(evpl_spdk_test_new_thread, 0); + + evpl_test_abort_if(rc, "spdk_thread_lib_init failed: %d", rc); + + evpl_spdk_test_num_reactors = nreactors; + + for (i = 0; i < nreactors; i++) { + pthread_mutex_init(&evpl_spdk_test_reactors[i].lock, NULL); + + rc = pthread_create(&evpl_spdk_test_reactors[i].pthread, NULL, + evpl_spdk_test_reactor_fn, + &evpl_spdk_test_reactors[i]); + + evpl_test_abort_if(rc, "reactor pthread_create failed: %d", rc); + } + + /* Before any evpl call, so evpl_cleanup's atexit runs first (LIFO). */ + atexit(evpl_spdk_test_fini); +} /* evpl_spdk_test_init */ + +static void +evpl_spdk_test_config(void) +{ + struct evpl_global_config *config = evpl_global_config_init(); + + evpl_global_config_set_core_mech(config, EVPL_CORE_MECH_SPDK); + + evpl_init(config); +} /* evpl_spdk_test_config */ diff --git a/src/core/spdk/tests/spdk_thread_basic.c b/src/core/spdk/tests/spdk_thread_basic.c new file mode 100644 index 00000000..4cef2548 --- /dev/null +++ b/src/core/spdk/tests/spdk_thread_basic.c @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#include + +#include "spdk_test_harness.h" + +static void * +thread_init( + struct evpl *evpl, + void *private_data) +{ + int *number = private_data; + + evpl_test_info("thread_init ran with number=%d", *number); + + evpl_test_abort_if(*number != 42, + "got wrong argument in thread init function"); + + return private_data; +} /* thread_init */ + +static void +thread_shutdown( + struct evpl *evpl, + void *private_data) +{ + int *number = private_data; + + evpl_test_info("thread_shutdown ran with number=%d", *number); +} /* thread_shutdown */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *thread; + int number = 42; + + evpl_spdk_test_init(3); + + evpl_spdk_test_config(); + + /* From a plain pthread the create blocks until init ran on the worker + * spdk_thread and destroy blocks until teardown completed. */ + thread = evpl_thread_create(NULL, thread_init, thread_shutdown, &number); + + evpl_test_info("thread created, now destroying"); + + evpl_thread_destroy(thread); + + evpl_test_info("thread destroyed"); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_thread_same_reactor.c b/src/core/spdk/tests/spdk_thread_same_reactor.c new file mode 100644 index 00000000..ac066b50 --- /dev/null +++ b/src/core/spdk/tests/spdk_thread_same_reactor.c @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * Deadlock regression test: with a single reactor, an spdk_thread that calls + * evpl_thread_create/evpl_thread_destroy shares its reactor with the new + * worker, so any blocking wait inside those calls can never complete -- the + * reactor is executing the caller. The SPDK paths must return immediately + * for spdk_thread callers; this test hangs (and trips the ctest timeout) if + * anyone reintroduces a wait. + */ + +#include + +#include "spdk_test_harness.h" + +static volatile int bootstrap_done; +static volatile int init_ran; +static volatile int shutdown_ran; + +static void * +inner_init( + struct evpl *evpl, + void *private_data) +{ + init_ran = 1; + + return private_data; +} /* inner_init */ + +static void +inner_shutdown( + struct evpl *evpl, + void *private_data) +{ + shutdown_ran = 1; +} /* inner_shutdown */ + +static void +bootstrap_fn(void *ctx) +{ + struct evpl_thread *thread; + + /* On an spdk_thread: must return without waiting for readiness. */ + thread = evpl_thread_create(NULL, inner_init, inner_shutdown, NULL); + + /* Must detach rather than wait; the worker frees the handle itself. */ + evpl_thread_destroy(thread); + + bootstrap_done = 1; + + spdk_thread_exit(spdk_get_thread()); +} /* bootstrap_fn */ + +int +main( + int argc, + char *argv[]) +{ + struct spdk_thread *bootstrap; + int rc; + + evpl_spdk_test_init(1); + + evpl_spdk_test_config(); + + bootstrap = spdk_thread_create("bootstrap", NULL); + + evpl_test_abort_if(!bootstrap, "bootstrap spdk_thread_create failed"); + + rc = spdk_thread_send_msg(bootstrap, bootstrap_fn, NULL); + + evpl_test_abort_if(rc, "spdk_thread_send_msg failed: %d", rc); + + while (!bootstrap_done || !init_ran || !shutdown_ran) { + usleep(1000); + } + + evpl_test_info("bootstrap completed; worker init and shutdown ran"); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_threadpool_pingpong.c b/src/core/spdk/tests/spdk_threadpool_pingpong.c new file mode 100644 index 00000000..6bb5c546 --- /dev/null +++ b/src/core/spdk/tests/spdk_threadpool_pingpong.c @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * TCP ping-pong between evpl threads running as spdk_threads: covers the + * listener over an SPDK evpl thread, the cross-thread accept handoff + * (connect_requests + run_wakeup), and socket send/recv under the poller + * pump. All I/O logic runs in callbacks on the SPDK evpl threads; main() + * stays on a plain pthread where the blocking listener/thread APIs are safe. + */ + +#include +#include + +#include "spdk_test_harness.h" + +static const char localhost[] = "127.0.0.1"; +static int port = 8000; + +struct pingpong_state { + volatile int run; + int sent; + int recv; + int niters; + uint32_t value; +}; + +static struct evpl_listener_binding *server_binding; +static struct evpl_listener *listener; + +static void +client_callback( + struct evpl *evpl, + struct evpl_bind *bind, + struct evpl_notify *notify, + void *private_data) +{ + struct pingpong_state *state = private_data; + uint32_t value; + int length; + + switch (notify->notify_type) { + case EVPL_NOTIFY_CONNECTED: + evpl_test_info("client connected"); + break; + case EVPL_NOTIFY_RECV_DATA: + + do { + length = evpl_recv(evpl, bind, &value, sizeof(value), + EVPL_RECV_FLAG_ALL_OR_NONE); + + if (length == sizeof(value)) { + + state->recv++; + + if (state->recv < state->niters) { + + evpl_send(evpl, bind, &state->value, + sizeof(state->value)); + + state->value++; + state->sent++; + } else { + state->run = 0; + } + } + } while (length > 0); + + break; + + case EVPL_NOTIFY_DISCONNECTED: + evpl_test_info("client disconnected"); + break; + } /* switch */ + +} /* client_callback */ + +static void +server_callback( + struct evpl *evpl, + struct evpl_bind *bind, + struct evpl_notify *notify, + void *private_data) +{ + uint32_t value; + int length; + + switch (notify->notify_type) { + case EVPL_NOTIFY_CONNECTED: + evpl_test_info("server connected"); + break; + case EVPL_NOTIFY_DISCONNECTED: + evpl_test_info("server disconnected"); + break; + case EVPL_NOTIFY_RECV_DATA: + + do { + length = evpl_recv(evpl, bind, &value, sizeof(value), + EVPL_RECV_FLAG_ALL_OR_NONE); + + if (length == sizeof(value)) { + evpl_send(evpl, bind, &value, sizeof(value)); + } + } while (length > 0); + + break; + } /* switch */ + +} /* server_callback */ + +static void +accept_callback( + struct evpl *evpl, + struct evpl_bind *accepted_bind, + evpl_notify_callback_t *notify_callback, + evpl_segment_callback_t *segment_callback, + void **conn_private_data, + void *private_data) +{ + *notify_callback = server_callback; + *conn_private_data = private_data; +} /* accept_callback */ + +static void * +server_init( + struct evpl *evpl, + void *private_data) +{ + server_binding = evpl_listener_attach(evpl, listener, accept_callback, + private_data); + + return private_data; +} /* server_init */ + +static void +server_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_listener_detach(evpl, server_binding); +} /* server_shutdown */ + +static void * +client_init( + struct evpl *evpl, + void *private_data) +{ + struct pingpong_state *state = private_data; + struct evpl_endpoint *ep; + struct evpl_bind *bind; + + ep = evpl_endpoint_create(localhost, port); + + bind = evpl_connect(evpl, EVPL_STREAM_SOCKET_TCP, NULL, ep, + client_callback, NULL, state); + + evpl_send(evpl, bind, &state->value, sizeof(state->value)); + + state->value++; + state->sent++; + + return private_data; +} /* client_init */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + struct evpl_endpoint *listen_ep; + struct pingpong_state state = { + .run = 1, + .sent = 0, + .recv = 0, + .niters = 100, + .value = 1 + }; + + evpl_spdk_test_init(3); + + evpl_spdk_test_config(); + + listener = evpl_listener_create(); + + /* Blocks until server_init attached the binding on the worker. */ + server_thread = evpl_thread_create(NULL, server_init, server_shutdown, + &state); + + listen_ep = evpl_endpoint_create("0.0.0.0", port); + + evpl_listen(listener, EVPL_STREAM_SOCKET_TCP, listen_ep); + + client_thread = evpl_thread_create(NULL, client_init, NULL, &state); + + while (state.run) { + usleep(1000); + } + + evpl_test_info("ping-pong completed: sent %d recv %d", + state.sent, state.recv); + + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); + + evpl_listener_destroy(listener); + + return 0; +} /* main */ diff --git a/src/core/spdk/tests/spdk_timer.c b/src/core/spdk/tests/spdk_timer.c new file mode 100644 index 00000000..58148968 --- /dev/null +++ b/src/core/spdk/tests/spdk_timer.c @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * Timer coverage under the poller pump: one-shot and periodic evpl timers on + * an SPDK evpl thread, serviced by evpl_continue's slow path as the reactor + * pumps. + */ + +#include + +#include "spdk_test_harness.h" + +static struct evpl_timer oneshot_timer; +static struct evpl_timer periodic_timer; +static volatile int oneshot_fired; +static volatile int periodic_count; + +static void +oneshot_callback( + struct evpl *evpl, + struct evpl_timer *timer) +{ + oneshot_fired = 1; +} /* oneshot_callback */ + +static void +periodic_callback( + struct evpl *evpl, + struct evpl_timer *timer) +{ + periodic_count++; +} /* periodic_callback */ + +static void * +timer_init( + struct evpl *evpl, + void *private_data) +{ + evpl_add_oneshot_timer(evpl, &oneshot_timer, oneshot_callback, 10000); + + evpl_add_timer(evpl, &periodic_timer, periodic_callback, 5000); + + return private_data; +} /* timer_init */ + +static void +timer_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_remove_timer(evpl, &periodic_timer); +} /* timer_shutdown */ + +int +main( + int argc, + char *argv[]) +{ + struct evpl_thread *thread; + + evpl_spdk_test_init(2); + + evpl_spdk_test_config(); + + thread = evpl_thread_create(NULL, timer_init, timer_shutdown, NULL); + + while (!oneshot_fired || periodic_count < 5) { + usleep(1000); + } + + evpl_test_info("oneshot fired, periodic fired %d times", periodic_count); + + evpl_thread_destroy(thread); + + return 0; +} /* main */ diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index 0aa9b217..db519bdb 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -18,4 +18,4 @@ unit_test(core init_auto_no_config init_auto_no_config.c) unit_test(core init_with_clean_config init_with_clean_config.c) unit_test(core unused_config unused_config.c) unit_test(core tls_cipher_config tls_cipher_config.c) -unit_test_all_mechs(core timer_oneshot timer_oneshot.c) +unit_test_all_mechs_spdk(core timer_oneshot timer_oneshot.c) diff --git a/src/core/tests/timer_oneshot.c b/src/core/tests/timer_oneshot.c index c2a7a2e3..7f26cad2 100644 --- a/src/core/tests/timer_oneshot.c +++ b/src/core/tests/timer_oneshot.c @@ -3,10 +3,12 @@ // SPDX-License-Identifier: LGPL-2.1-only #include +#include #include "evpl/evpl.h" #include "tests/test_common.h" +static volatile int test_done; static int oneshot_fired; static int rearm_fired; static int periodic_fired; @@ -36,8 +38,8 @@ rearm_cb( } } /* rearm_cb */ -/* Drives the loop: fires every 1ms and stops the loop once enough time has - * passed for the one-shots to have fired and to prove they do not refire. */ +/* Fires every 1ms and signals completion once enough time has passed for the + * one-shots to have fired and to prove they do not refire. */ static void periodic_cb( struct evpl *evpl, @@ -45,29 +47,51 @@ periodic_cb( { periodic_fired++; if (periodic_fired >= 10) { - evpl_stop(evpl); + test_done = 1; } } /* periodic_cb */ +static void * +timer_thread_init( + struct evpl *evpl, + void *private_data) +{ + evpl_add_oneshot_timer(evpl, &oneshot, oneshot_cb, 1000); /* 1 ms */ + evpl_add_oneshot_timer(evpl, &rearm_timer, rearm_cb, 1000); + evpl_add_timer(evpl, &periodic, periodic_cb, 1000); /* 1 ms periodic */ + + return private_data; +} /* timer_thread_init */ + +static void +timer_thread_shutdown( + struct evpl *evpl, + void *private_data) +{ + /* Removing an already-fired one-shot must be a harmless no-op. */ + evpl_remove_timer(evpl, &oneshot); + evpl_remove_timer(evpl, &rearm_timer); + evpl_remove_timer(evpl, &periodic); +} /* timer_thread_shutdown */ + int main( int argc, char *argv[]) { - struct evpl *evpl; + struct evpl_thread *thread; /* Selects the core mechanism from EVPL_TEST_CORE_MECH so ctest can run * this against every mechanism compiled in -- the timer deadline drives * the core wait timeout, which each mechanism converts differently. */ test_evpl_config(); - evpl = evpl_create(NULL); - - evpl_add_oneshot_timer(evpl, &oneshot, oneshot_cb, 1000); /* 1 ms */ - evpl_add_oneshot_timer(evpl, &rearm_timer, rearm_cb, 1000); - evpl_add_timer(evpl, &periodic, periodic_cb, 1000); /* 1 ms periodic */ + thread = evpl_thread_create(NULL, timer_thread_init, + timer_thread_shutdown, NULL); - evpl_run(evpl); + while (!test_done) { + usleep(1000); + } /* By the time the periodic timer has fired 10 times (~10ms), the plain * one-shot must have fired exactly once and the re-armed one-shot exactly @@ -87,12 +111,7 @@ main( return 1; } - /* Removing an already-fired one-shot must be a harmless no-op. */ - evpl_remove_timer(evpl, &oneshot); - evpl_remove_timer(evpl, &rearm_timer); - evpl_remove_timer(evpl, &periodic); - - evpl_destroy(evpl); + evpl_thread_destroy(thread); printf("oneshot timer test passed\n"); return 0; diff --git a/src/core/thread/thread.c b/src/core/thread/thread.c index ff222e96..f512a2c6 100644 --- a/src/core/thread/thread.c +++ b/src/core/thread/thread.c @@ -14,6 +14,7 @@ #include "core/macros.h" #include "core/wakeup.h" #include "core/pthread_util.h" +#include "core/thread/thread_internal.h" extern struct evpl_shared *evpl_shared; @@ -34,31 +35,6 @@ extern struct evpl_shared *evpl_shared; #define evpl_thread_abort_if(cond, ...) \ evpl_abort_if(cond, "thread", __FILE__, __LINE__, __VA_ARGS__) -struct evpl_thread { - pthread_t thread; - pthread_mutex_t lock; - pthread_cond_t cond; - int ready; - /* Stop signal. Owned by evpl_thread (this struct outlives the worker's - * evpl, since it is freed only after pthread_join), so evpl_thread_destroy - * can stop the worker by writing this fd without ever dereferencing the - * worker's evpl -- which the worker creates, runs, and destroys entirely on - * its own thread. The event is registered on the worker's evpl and its - * handler clears running from the worker thread. */ - struct evpl_wakeup stop_wakeup; - struct evpl_event stop_event; - struct evpl_thread_config *config; - struct evpl *evpl; - evpl_thread_init_callback_t init_callback; - evpl_thread_shutdown_callback_t shutdown_callback; - void *private_data; -}; - -struct evpl_threadpool { - struct evpl_thread **threads; - int nthreads; -}; - /* * Read handler for a thread's stop wakeup, registered on the worker's own evpl. * Runs on the worker thread, so it clears running directly (the same thread @@ -133,6 +109,13 @@ evpl_thread_create( __evpl_init(); +#ifdef HAVE_SPDK + if (evpl_shared->config->core_mech == EVPL_CORE_MECH_SPDK) { + return evpl_thread_create_spdk(config, init_function, + shutdown_function, private_data); + } +#endif /* ifdef HAVE_SPDK */ + evpl_thread = evpl_zalloc(sizeof(*evpl_thread)); evpl_thread->config = config; @@ -170,6 +153,13 @@ evpl_thread_destroy(struct evpl_thread *evpl_thread) { ssize_t len; +#ifdef HAVE_SPDK + if (evpl_thread->spdk_mode) { + evpl_thread_destroy_spdk(evpl_thread); + return; + } +#endif /* ifdef HAVE_SPDK */ + /* Signal stop via our own fd (never touch the worker's evpl, which the * worker frees on its own thread); the worker's stop_event handler clears * running. Then join and only then close the fd. */ diff --git a/src/core/thread/thread_internal.h b/src/core/thread/thread_internal.h new file mode 100644 index 00000000..0f16b7c2 --- /dev/null +++ b/src/core/thread/thread_internal.h @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2025 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +#pragma once + +#include + +#include "core/evpl.h" +#include "core/wakeup.h" +#include "evpl/evpl.h" + +struct evpl_thread { + pthread_t thread; + pthread_mutex_t lock; + pthread_cond_t cond; + int ready; + /* Stop signal. Owned by evpl_thread (this struct outlives the worker's + * evpl, since it is freed only after pthread_join), so evpl_thread_destroy + * can stop the worker by writing this fd without ever dereferencing the + * worker's evpl -- which the worker creates, runs, and destroys entirely on + * its own thread. The event is registered on the worker's evpl and its + * handler clears running from the worker thread. */ + struct evpl_wakeup stop_wakeup; + struct evpl_event stop_event; + struct evpl_thread_config *config; + struct evpl *evpl; + evpl_thread_init_callback_t init_callback; + evpl_thread_shutdown_callback_t shutdown_callback; + void *private_data; + + /* SPDK guest mode (EVPL_CORE_MECH_SPDK): the worker is an spdk_thread + * scheduled by the host application, not a pthread. done/detached are + * the teardown handshake: a plain-pthread destroyer waits on done; an + * spdk_thread destroyer sets detached and the worker frees this struct + * itself. Pointers are void* so this header stays free of SPDK types. */ + unsigned int spdk_mode; + int done; + int detached; + void *spdk_thread; + void *spdk_teardown_poller; +}; + +struct evpl_threadpool { + struct evpl_thread **threads; + int nthreads; +}; + +#ifdef HAVE_SPDK +struct evpl_thread * +evpl_thread_create_spdk( + struct evpl_thread_config *config, + evpl_thread_init_callback_t init_function, + evpl_thread_shutdown_callback_t shutdown_function, + void *private_data); + +void +evpl_thread_destroy_spdk( + struct evpl_thread *evpl_thread); +#endif /* ifdef HAVE_SPDK */ diff --git a/src/core/thread/thread_spdk.c b/src/core/thread/thread_spdk.c new file mode 100644 index 00000000..b38b09c1 --- /dev/null +++ b/src/core/thread/thread_spdk.c @@ -0,0 +1,288 @@ +// SPDX-FileCopyrightText: 2026 Ben Jarvis +// +// SPDX-License-Identifier: LGPL-2.1-only + +/* + * evpl_thread over spdk_thread (EVPL_CORE_MECH_SPDK guest mode). + * + * The worker is a lightweight spdk_thread created here and placed onto a + * reactor by the host application's scheduler; libevpl never creates an OS + * thread and never pumps the spdk_thread itself. Creation and teardown are + * message-driven so no step ever blocks a reactor: + * + * create: spdk_thread_create -> send_msg(start): evpl_create (which + * registers the pump poller), register stop event, init_callback, + * signal ready. The creator blocks for readiness only when it is + * a plain pthread; an spdk_thread creator returns immediately, + * because the new thread may be co-scheduled on the creator's own + * reactor and no form of waiting could then ever complete. + * + * destroy: signal the stop eventfd. The stop handler (inside the pump) + * pushes every bind into pending close and registers a teardown + * poller. The teardown poller (outside the pump) idles until the + * binds drain -- the reactor keeps servicing other threads -- then + * runs shutdown_callback, evpl_destroy, and spdk_thread_exit. A + * plain-pthread destroyer waits for completion; an spdk_thread + * destroyer detaches and the worker frees the handle itself. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "core/evpl.h" +#include "evpl/evpl.h" +#include "core/evpl_shared.h" +#include "core/event_fn.h" +#include "core/macros.h" +#include "core/wakeup.h" +#include "core/thread/thread_internal.h" + +#define evpl_thread_abort_if(cond, ...) \ + evpl_abort_if(cond, "thread", __FILE__, __LINE__, __VA_ARGS__) + +static int +evpl_thread_spdk_teardown(void *ctx) +{ + struct evpl_thread *evpl_thread = ctx; + struct evpl *evpl = evpl_thread->evpl; + int detached; + + if (evpl_has_pending_binds(evpl)) { + /* The pump poller is still draining closes; let the reactor service + * other threads instead of spinning here. */ + return SPDK_POLLER_IDLE; + } + + evpl_remove_event(evpl, &evpl_thread->stop_event); + + if (evpl_thread->shutdown_callback) { + evpl_thread->shutdown_callback(evpl, evpl_thread->private_data); + } + + /* Unregisters the pump poller and interrupt via the mechanism's destroy; + * runs outside the pump, satisfying its re-entrancy guard. */ + evpl_destroy(evpl); + + evpl_thread->evpl = NULL; + + spdk_poller_unregister( + (struct spdk_poller **) &evpl_thread->spdk_teardown_poller); + + pthread_mutex_lock(&evpl_thread->lock); + evpl_thread->done = 1; + detached = evpl_thread->detached; + pthread_cond_signal(&evpl_thread->cond); + pthread_mutex_unlock(&evpl_thread->lock); + + if (detached) { + /* The destroyer returned without waiting; the handle is ours. */ + evpl_wakeup_close(&evpl_thread->stop_wakeup); + + if (evpl_thread->config) { + evpl_free(evpl_thread->config); + } + + evpl_free(evpl_thread); + } + + spdk_thread_exit(spdk_get_thread()); + + return SPDK_POLLER_BUSY; +} /* evpl_thread_spdk_teardown */ + +/* + * Stop-eventfd handler; runs inside evpl_continue() via the pump poller, so + * it must not destroy the evpl. It initiates the close of every bind and + * hands completion to the teardown poller. + */ +static void +evpl_thread_spdk_stop_event( + struct evpl *evpl, + struct evpl_event *event) +{ + struct evpl_thread *evpl_thread = + container_of(event, struct evpl_thread, stop_event); + + if (evpl_wakeup_drain(event->fd) < 0) { + evpl_event_mark_unreadable(evpl, event); + } + + evpl_close_all_binds(evpl); + + if (!evpl_thread->spdk_teardown_poller) { + evpl_thread->spdk_teardown_poller = + spdk_poller_register(evpl_thread_spdk_teardown, evpl_thread, 0); + + evpl_thread_abort_if(!evpl_thread->spdk_teardown_poller, + "evpl_thread: teardown poller registration failed"); + } +} /* evpl_thread_spdk_stop_event */ + +/* Runs on the new spdk_thread the first time its reactor polls it. */ +static void +evpl_thread_spdk_start(void *ctx) +{ + struct evpl_thread *evpl_thread = ctx; + struct evpl *evpl; + + evpl = evpl_create(evpl_thread->config); + + evpl_thread->evpl = evpl; + + evpl_add_event(evpl, &evpl_thread->stop_event, + evpl_thread->stop_wakeup.rfd, + evpl_thread_spdk_stop_event, NULL, NULL); + + evpl_event_read_interest(evpl, &evpl_thread->stop_event); + + if (evpl_thread->init_callback) { + evpl_thread->private_data = evpl_thread->init_callback( + evpl, + evpl_thread->private_data); + } + + pthread_mutex_lock(&evpl_thread->lock); + evpl_thread->ready = 1; + pthread_cond_signal(&evpl_thread->cond); + pthread_mutex_unlock(&evpl_thread->lock); +} /* evpl_thread_spdk_start */ + +struct evpl_thread * +evpl_thread_create_spdk( + struct evpl_thread_config *config, + evpl_thread_init_callback_t init_function, + evpl_thread_shutdown_callback_t shutdown_function, + void *private_data) +{ + static unsigned int evpl_thread_spdk_id = 0; + struct evpl_thread *evpl_thread; + struct spdk_thread *thread; + struct spdk_cpuset cpuset; + struct spdk_cpuset *cpuset_ptr = NULL; + char name[64]; + unsigned int id; + int rc; + + evpl_thread = evpl_zalloc(sizeof(*evpl_thread)); + + evpl_thread->spdk_mode = 1; + evpl_thread->init_callback = init_function; + evpl_thread->shutdown_callback = shutdown_function; + evpl_thread->private_data = private_data; + + /* Copy the config: the creator may return before the start message runs + * on the new thread, so the caller's config cannot be borrowed. */ + if (config) { + evpl_thread->config = evpl_zalloc(sizeof(*evpl_thread->config)); + *evpl_thread->config = *config; + } + + pthread_mutex_init(&evpl_thread->lock, NULL); + pthread_cond_init(&evpl_thread->cond, NULL); + + evpl_thread_abort_if(evpl_wakeup_open(&evpl_thread->stop_wakeup) < 0, + "evpl_thread_create: wakeup open failed"); + + id = __sync_fetch_and_add(&evpl_thread_spdk_id, 1); + + snprintf(name, sizeof(name), "%s-%u", + (config && config->name[0]) ? config->name : "evpl", id); + + if (config && config->spdk_cpumask[0]) { + spdk_cpuset_zero(&cpuset); + + rc = spdk_cpuset_parse(&cpuset, config->spdk_cpumask); + + evpl_thread_abort_if(rc, + "evpl_thread_create: invalid spdk cpumask '%s'", + config->spdk_cpumask); + + cpuset_ptr = &cpuset; + } + + thread = spdk_thread_create(name, cpuset_ptr); + + evpl_thread_abort_if(!thread, + "evpl_thread_create: spdk_thread_create failed; the " + "host application must initialize the SPDK env and " + "thread library before using EVPL_CORE_MECH_SPDK"); + + evpl_thread->spdk_thread = thread; + + rc = spdk_thread_send_msg(thread, evpl_thread_spdk_start, evpl_thread); + + evpl_thread_abort_if(rc, + "evpl_thread_create: spdk_thread_send_msg failed: %s", + strerror(-rc)); + + /* Block for readiness only when the creator is a plain pthread. An + * spdk_thread creator may share a reactor with the new thread, in which + * case any wait here could never complete; init_callback (which runs on + * the new thread) is the readiness hook for such callers. */ + if (spdk_get_thread() == NULL) { + pthread_mutex_lock(&evpl_thread->lock); + + while (!evpl_thread->ready) { + pthread_cond_wait(&evpl_thread->cond, &evpl_thread->lock); + } + + pthread_mutex_unlock(&evpl_thread->lock); + } + + return evpl_thread; +} /* evpl_thread_create_spdk */ + +void +evpl_thread_destroy_spdk(struct evpl_thread *evpl_thread) +{ + ssize_t len; + int free_now = 0; + + len = evpl_wakeup_signal(&evpl_thread->stop_wakeup); + + evpl_thread_abort_if(len != sizeof(uint64_t), + "evpl_thread_destroy: stop wakeup signal failed: " + "len=%zd errno=%d (%s)", len, errno, strerror(errno)); + + if (spdk_get_thread() == NULL) { + /* Plain pthread: safe to wait for the worker's teardown poller. */ + pthread_mutex_lock(&evpl_thread->lock); + + while (!evpl_thread->done) { + pthread_cond_wait(&evpl_thread->cond, &evpl_thread->lock); + } + + pthread_mutex_unlock(&evpl_thread->lock); + + free_now = 1; + } else { + /* spdk_thread caller: waiting could deadlock against a co-scheduled + * worker, so detach; the worker frees the handle when done. If the + * worker already finished, freeing falls to us. */ + pthread_mutex_lock(&evpl_thread->lock); + + if (evpl_thread->done) { + free_now = 1; + } else { + evpl_thread->detached = 1; + } + + pthread_mutex_unlock(&evpl_thread->lock); + } + + if (free_now) { + evpl_wakeup_close(&evpl_thread->stop_wakeup); + + if (evpl_thread->config) { + evpl_free(evpl_thread->config); + } + + evpl_free(evpl_thread); + } +} /* evpl_thread_destroy_spdk */ diff --git a/src/core/vfio/tests/basic.c b/src/core/vfio/tests/basic.c index c774c809..c58ff7b4 100644 --- a/src/core/vfio/tests/basic.c +++ b/src/core/vfio/tests/basic.c @@ -9,67 +9,125 @@ #include "evpl/evpl.h" +struct test_state { + struct evpl_block_device *bdev; + int open_status; + int opened; + int closed; + int pending; +}; + +static void +open_callback( + struct evpl *evpl, + struct evpl_block_device *blockdev, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + state->bdev = blockdev; + state->open_status = status; + state->opened = 1; +} /* open_callback */ + +static void +close_callback( + struct evpl *evpl, + int status, + void *private_data) +{ + struct test_state *state = private_data; + + if (status) { + exit(1); + } + + state->closed = 1; +} /* close_callback */ + static void -read_callback( - int status, - void *private_data) +io_callback( + struct evpl *evpl, + int status, + void *private_data) { - int *pending = private_data; + struct test_state *state = private_data; if (status) { exit(1); } - (*pending)--; + state->pending--; -} /* read_callback */ +} /* io_callback */ int main( int argc, char *argv[]) { - struct evpl *evpl; - struct evpl_block_device *bdev; - int fd; - struct evpl_block_queue *bqueue; - int pending = 0; - struct evpl_iovec iov; - int niov; + struct evpl *evpl; + int fd; + int rc; + struct evpl_block_queue *bqueue; + struct test_state state = { 0 }; + struct evpl_iovec iov; + int niov; fd = open("test.img", O_RDWR | O_CREAT, 0666); - ftruncate(fd, 1024 * 1024 * 1024); + rc = ftruncate(fd, 1024 * 1024 * 1024); + + if (rc < 0) { + perror("ftruncate"); + exit(1); + } + close(fd); - evpl = evpl_create(); + evpl = evpl_create(NULL); - bdev = evpl_block_open_device(EVPL_BLOCK_PROTOCOL_VFIO, "test.img"); + evpl_block_open_device(evpl, EVPL_BLOCK_PROTOCOL_VFIO, "test.img", + open_callback, &state); - bqueue = evpl_block_open_queue(evpl, bdev); + while (!state.opened) { + evpl_continue(evpl); + } + + if (state.open_status || !state.bdev) { + fprintf(stderr, "open failed: %d\n", state.open_status); + exit(1); + } + + bqueue = evpl_block_open_queue(evpl, state.bdev); niov = evpl_iovec_alloc(evpl, 4096, 4096, 1, 0, &iov); - pending++; - evpl_block_write(evpl, bqueue, &iov, niov, 0, 0, read_callback, &pending); + state.pending++; + evpl_block_write(evpl, bqueue, &iov, niov, 0, 0, io_callback, &state); - pending++; - evpl_block_read(evpl, bqueue, &iov, niov, 0, read_callback, &pending); + state.pending++; + evpl_block_read(evpl, bqueue, &iov, niov, 0, io_callback, &state); - pending++; - evpl_block_flush(evpl, bqueue, read_callback, &pending); + state.pending++; + evpl_block_flush(evpl, bqueue, io_callback, &state); - while (pending) { - evpl_wait(evpl, -1); + while (state.pending) { + evpl_continue(evpl); } evpl_iovec_release(evpl, &iov); evpl_block_close_queue(evpl, bqueue); - evpl_block_close_device(bdev); + evpl_block_close_device(evpl, state.bdev, close_callback, &state); + + while (!state.closed) { + evpl_continue(evpl); + } evpl_destroy(evpl); return 0; -} /* main */ \ No newline at end of file +} /* main */ diff --git a/src/core/vfio/vfio.c b/src/core/vfio/vfio.c index 4130c838..1dcd0365 100644 --- a/src/core/vfio/vfio.c +++ b/src/core/vfio/vfio.c @@ -1782,7 +1782,11 @@ evpl_vfio_open_queue( } /* evpl_vfio_open_queue */ static void -evpl_vfio_close_device(struct evpl_block_device *bdev) +evpl_vfio_close_device( + struct evpl *evpl, + struct evpl_block_device *bdev, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_vfio_device *dev = bdev->private_data; struct evpl_vfio_queue *queue, *tmp; @@ -1838,13 +1842,18 @@ evpl_vfio_close_device(struct evpl_block_device *bdev) evpl_free(dev); evpl_free(bdev); + + complete(evpl, NULL, 0, ctx); } /* evpl_vfio_close_device */ -static struct evpl_block_device * +static void evpl_vfio_open_device( - const char *uri, - void *private_data) + struct evpl *evpl, + const char *uri, + void *private_data, + evpl_block_device_complete_t complete, + void *ctx) { struct evpl_vfio_shared *vfio = private_data; struct evpl_block_device *bdev; @@ -1967,7 +1976,7 @@ evpl_vfio_open_device( evpl_vfio_free(dev->vfio, inquiry_mr); - return bdev; + complete(evpl, bdev, 0, ctx); } /* evpl_vfio_open_device */ struct evpl_framework evpl_framework_vfio = { diff --git a/src/rpc2/tests/CMakeLists.txt b/src/rpc2/tests/CMakeLists.txt index 81e47ea6..7546fb5a 100644 --- a/src/rpc2/tests/CMakeLists.txt +++ b/src/rpc2/tests/CMakeLists.txt @@ -30,6 +30,9 @@ macro(unit_test_rpc2 test_name xdr_file) add_executable(evpl_rpc2_${test_name} ${sources} ${XDR_C} ${XDR_H}) target_link_libraries(evpl_rpc2_${test_name} evpl_rpc2 evpl pthread) + if (HAVE_SPDK) + target_link_libraries(evpl_rpc2_${test_name} ${SPDK_LINK_TARGETS}) + endif() # Set the first source for TEST_FILE environment variable list(GET sources 0 first_source) @@ -66,6 +69,32 @@ macro(unit_test_rpc2_instance test_name protocol port) endforeach() endmacro() +# EVPL_TEST_CORE_MECH=spdk instance of an rpc2 test that has been refactored +# to be fully evpl_thread/callback driven; mirrors unit_test_rpc2_instance. +macro(unit_test_rpc2_instance_spdk test_name protocol port) + if (HAVE_SPDK) + set(_test libevpl/rpc2/${test_name}_${protocol}_spdk) + + if(LIBEVPL_NETNS_TESTING) + add_test(NAME ${_test} + COMMAND ${LIBEVPL_SOURCE_DIR}/scripts/netns_test_wrapper.sh + ${RPC2_TEST_EXE_${test_name}} -r ${protocol} -p ${port}) + + set_tests_properties(${_test} PROPERTIES + ENVIRONMENT "TEST_FILE=${RPC2_TEST_SRC_${test_name}};EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0" + TIMEOUT 10) + else() + add_test(NAME ${_test} + COMMAND ${RPC2_TEST_EXE_${test_name}} -r ${protocol} -p ${port}) + + set_tests_properties(${_test} PROPERTIES + RESOURCE_LOCK evpl_net + ENVIRONMENT "TEST_FILE=${RPC2_TEST_SRC_${test_name}};EVPL_TEST_CORE_MECH=spdk;ASAN_OPTIONS=detect_leaks=0" + TIMEOUT 10) + endif() + endif() +endmacro() + # Build RPC2 test executables unit_test_rpc2(hello_world hello_world.x hello_world.c) unit_test_rpc2(builtin_bool builtin_bool.x builtin_bool.c) @@ -78,6 +107,9 @@ unit_test_rpc2_instance(builtin_bool STREAM_SOCKET_TCP 8101) unit_test_rpc2_instance(rdma_ddp STREAM_SOCKET_TCP 8102) # multifrag is TCP-only: RDMA bypasses rpc2_segment_callback entirely. unit_test_rpc2_instance(multifrag STREAM_SOCKET_TCP 8103) +# multifrag is fully evpl_thread driven, so it also runs in SPDK guest mode +# (kernel TCP on spdk_threads). +unit_test_rpc2_instance_spdk(multifrag STREAM_SOCKET_TCP 8103) # Add test instances for DATAGRAM_TCP_RDMA unit_test_rpc2_instance(hello_world DATAGRAM_TCP_RDMA 8200) diff --git a/src/rpc2/tests/multifrag.c b/src/rpc2/tests/multifrag.c index 5a3c6fd7..e60849cd 100644 --- a/src/rpc2/tests/multifrag.c +++ b/src/rpc2/tests/multifrag.c @@ -7,7 +7,7 @@ * * libevpl's send path never fragments outbound replies, so a * libevpl-on-libevpl client cannot exercise the new code. This test - * runs a libevpl HELLO_PROGRAM server on a pump thread and drives it + * runs a libevpl HELLO_PROGRAM server on an evpl thread and drives it * from the main thread with a hand-rolled TCP client that splits one * RPC CALL across multiple record-mark fragments. */ @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -35,10 +34,11 @@ static enum evpl_protocol_id proto = EVPL_STREAM_SOCKET_TCP; static int port = 8000; struct server_ctx { - struct HELLO_V1 prog; - struct evpl *evpl; - volatile int ready; - volatile int received_count; + struct HELLO_V1 prog; + struct evpl_rpc2_program *programs[1]; + struct evpl_rpc2_server *server; + struct evpl_rpc2_thread *thread; + volatile int received_count; }; static void @@ -78,47 +78,28 @@ server_recv_greet( } /* server_recv_greet */ static void * -server_pump(void *arg) +server_thread_init( + struct evpl *evpl, + void *private_data) { - struct server_ctx *ctx = arg; - struct evpl *evpl; - struct evpl_rpc2_server *server; - struct evpl_rpc2_thread *thread; - struct evpl_endpoint *endpoint; - struct evpl_rpc2_program *programs[1]; - - evpl = evpl_create(NULL); - - HELLO_V1_init(&ctx->prog); - ctx->prog.recv_call_GREET = server_recv_greet; - programs[0] = &ctx->prog.rpc2; + struct server_ctx *ctx = private_data; - server = evpl_rpc2_server_init(programs, 1); - endpoint = evpl_endpoint_create("0.0.0.0", port); - evpl_rpc2_server_start(server, proto, endpoint); + ctx->thread = evpl_rpc2_thread_init(evpl, ctx->programs, 1, NULL, NULL); + evpl_rpc2_server_attach(ctx->thread, ctx->server, ctx); - thread = evpl_rpc2_thread_init(evpl, programs, 1, NULL, NULL); - evpl_rpc2_server_attach(thread, server, ctx); + return private_data; +} /* server_thread_init */ - /* Publish evpl so the main thread can signal evpl_stop. The - * store must be visible before ctx->ready, so use a barrier. */ - ctx->evpl = evpl; - __sync_synchronize(); - ctx->ready = 1; - - /* evpl_run blocks on epoll until evpl_stop writes the eventfd - * doorbell. A plain `while (!stop) evpl_continue(...)` deadlocks - * because epoll_wait has no wakeup when stop is set from another - * thread. */ - evpl_run(evpl); +static void +server_thread_shutdown( + struct evpl *evpl, + void *private_data) +{ + struct server_ctx *ctx = private_data; - evpl_rpc2_server_stop(server); - evpl_rpc2_server_detach(thread, server); - evpl_rpc2_thread_destroy(thread); - evpl_rpc2_server_destroy(server); - evpl_destroy(evpl); - return NULL; -} /* server_pump */ + evpl_rpc2_server_detach(ctx->thread, ctx->server); + evpl_rpc2_thread_destroy(ctx->thread); +} /* server_thread_shutdown */ /* * Hand-marshal a HELLO_PROGRAM GREET CALL into a flat buffer (no @@ -308,10 +289,11 @@ main( int argc, char *argv[]) { - struct server_ctx ctx = { 0 }; - pthread_t pump; - int opt, rc; - int i; + struct server_ctx ctx = { 0 }; + struct evpl_thread *server_thread; + struct evpl_endpoint *endpoint; + int opt, rc; + int i; test_evpl_config(); @@ -336,12 +318,20 @@ main( return 0; } - rc = pthread_create(&pump, NULL, server_pump, &ctx); - evpl_test_abort_if(rc != 0, "pthread_create: %d", rc); + HELLO_V1_init(&ctx.prog); + ctx.prog.recv_call_GREET = server_recv_greet; + ctx.programs[0] = &ctx.prog.rpc2; - while (!ctx.ready) { - usleep(1000); - } + /* Server start (which listens, a blocking operation) runs on this plain + * pthread; the rpc2 worker runs on an evpl thread, whose init callback + * attaches it to the server. evpl_thread_create returns only after the + * init callback ran, so the server is ready when it returns. */ + ctx.server = evpl_rpc2_server_init(ctx.programs, 1); + endpoint = evpl_endpoint_create("0.0.0.0", port); + evpl_rpc2_server_start(ctx.server, proto, endpoint); + + server_thread = evpl_thread_create(NULL, server_thread_init, + server_thread_shutdown, &ctx); /* Sub-test 1: 3-way fragmented CALL. */ test_fragmented_call(3, 0x11111111); @@ -363,8 +353,11 @@ main( "server processed %d calls, expected 3", ctx.received_count); - evpl_stop(ctx.evpl); - pthread_join(pump, NULL); + evpl_rpc2_server_stop(ctx.server); + + evpl_thread_destroy(server_thread); + + evpl_rpc2_server_destroy(ctx.server); printf("Test PASSED\n"); return 0; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 0e288efc..99656708 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -7,6 +7,9 @@ set(TEST_BIN ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE) macro(evpl_test name) add_executable(${name} ${name}.c) target_link_libraries(${name} evpl) + if (HAVE_SPDK) + target_link_libraries(${name} ${SPDK_LINK_TARGETS}) + endif() set_target_properties(${name} PROPERTIES TEST_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}.c") endmacro() diff --git a/src/tests/bulk_msg.c b/src/tests/bulk_msg.c index 298e9d01..7716145f 100644 --- a/src/tests/bulk_msg.c +++ b/src/tests/bulk_msg.c @@ -1,11 +1,10 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ben Jarvis +// SPDX-FileCopyrightText: 2024 - 2026 Ben Jarvis // // SPDX-License-Identifier: LGPL-2.1-only #include #include #include -#include #include #include @@ -18,17 +17,44 @@ const char localhost[] = "127.0.0.1"; const char *address = localhost; int port = 8000; - struct client_state { - int inflight; - int depth; - int sent; - int recv; - int niters; - uint32_t value; - struct evpl *server_evpl; + volatile int done; + int inflight; + int depth; + int sent; + int recv; + int niters; + uint32_t value; + struct evpl_endpoint *server_ep; + struct evpl_endpoint *client_ep; }; +/* Keep the send window full; called from the init callback and again from + * the SENT notifications as the window drains. Like the original manual + * pump, the test completes when everything has been SENT -- datagrams are + * unreliable, so completion must not depend on every echo arriving. */ +static void +client_fill_window( + struct evpl *evpl, + struct evpl_bind *bind, + struct client_state *state) +{ + while (state->inflight < state->depth && + state->sent < state->niters) { + + evpl_sendtoep(evpl, bind, state->server_ep, &state->value, + sizeof(state->value)); + + state->sent++; + state->inflight++; + + state->value++; + } + + if (state->sent == state->niters) { + state->done = 1; + } +} /* client_fill_window */ void client_callback( @@ -44,6 +70,8 @@ client_callback( evpl_test_info("sent %u msgs %u bytes", notify->sent.msgs, notify->sent.bytes); state->inflight -= notify->sent.msgs; + + client_fill_window(evpl, bind, state); break; case EVPL_NOTIFY_RECV_MSG: @@ -60,49 +88,22 @@ client_callback( } /* client_callback */ -void * -client_thread(void *arg) +static void * +client_thread_init( + struct evpl *evpl, + void *private_data) { - struct evpl *evpl; - struct evpl_endpoint *me, *server; - struct evpl_bind *bind; - struct client_state *state = arg; - - evpl = evpl_create(NULL); - - me = evpl_endpoint_create(address, port + 1); - server = evpl_endpoint_create(address, port); + struct client_state *state = private_data; + struct evpl_bind *bind; - bind = evpl_bind(evpl, proto, me, client_callback, state); + bind = evpl_bind(evpl, proto, state->client_ep, client_callback, state); evpl_bind_request_send_notifications(evpl, bind); - while (state->sent < state->niters) { - - while (state->inflight < state->depth && - state->sent < state->niters) { + client_fill_window(evpl, bind, state); - evpl_sendtoep(evpl, bind, server, &state->value, - sizeof(state->value)); - - state->sent++; - state->inflight++; - - state->value++; - - } - - evpl_continue(evpl); - } - - evpl_test_debug("client completed iterations"); - - evpl_stop(state->server_evpl); - - evpl_destroy(evpl); - - return NULL; -} /* client_thread */ + return private_data; +} /* client_thread_init */ void server_callback( @@ -128,16 +129,28 @@ server_callback( } /* server_callback */ +static void * +server_thread_init( + struct evpl *evpl, + void *private_data) +{ + struct client_state *state = private_data; + + evpl_bind(evpl, proto, state->server_ep, server_callback, + state->client_ep); + + return private_data; +} /* server_thread_init */ + int main( int argc, char *argv[]) { - pthread_t thr; - struct evpl *evpl; - struct evpl_endpoint *me, *client; - int rc, opt; - struct client_state state = { + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + int rc, opt; + struct client_state state = { .inflight = 0, .depth = 100, .sent = 0, @@ -171,23 +184,22 @@ main( } /* switch */ } + state.server_ep = evpl_endpoint_create(address, port); + state.client_ep = evpl_endpoint_create(address, port + 1); - evpl = evpl_create(NULL); - - state.server_evpl = evpl; - - me = evpl_endpoint_create(address, port); - client = evpl_endpoint_create(address, port + 1); + /* evpl_thread_create blocks until the init callback ran, so the server + * bind exists before the client's first burst. */ + server_thread = evpl_thread_create(NULL, server_thread_init, NULL, &state); + client_thread = evpl_thread_create(NULL, client_thread_init, NULL, &state); - evpl_bind(evpl, proto, me, server_callback, client); - - pthread_create(&thr, NULL, client_thread, &state); - - evpl_run(evpl); + while (!state.done) { + usleep(1000); + } - pthread_join(thr, NULL); + evpl_test_debug("client completed iterations"); - evpl_destroy(evpl); + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); return 0; } /* main */ diff --git a/src/tests/ping_pong_connected_msg.c b/src/tests/ping_pong_connected_msg.c index 994a04c5..58e65802 100644 --- a/src/tests/ping_pong_connected_msg.c +++ b/src/tests/ping_pong_connected_msg.c @@ -1,11 +1,10 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ben Jarvis +// SPDX-FileCopyrightText: 2024 - 2026 Ben Jarvis // // SPDX-License-Identifier: LGPL-2.1-only #include #include #include -#include #include #include @@ -13,14 +12,16 @@ #include "evpl/evpl.h" #include "test_common.h" -enum evpl_protocol_id proto = EVPL_DATAGRAM_SOCKET_UDP; -const char localhost[] = "127.0.0.1"; -const char *address = localhost; -int port = 8000; +enum evpl_protocol_id proto = EVPL_DATAGRAM_SOCKET_UDP; +const char localhost[] = "127.0.0.1"; +const char *address = localhost; +int port = 8000; +static struct evpl_listener *listener; +static struct evpl_listener_binding *server_binding; struct client_state { - struct evpl *server_evpl; + volatile int done; int inflight; int depth; int sent; @@ -38,6 +39,28 @@ test_segment_callback( return sizeof(uint32_t); } /* test_segment_callback */ +/* Keep the send window full; called from the init callback and again from + * the receive notifications as the window drains. */ +static void +client_fill_window( + struct evpl *evpl, + struct evpl_bind *bind, + struct client_state *state) +{ + while (state->inflight < state->depth && + state->sent < state->niters) { + + evpl_send(evpl, bind, &state->value, sizeof(state->value)); + + state->inflight++; + state->sent++; + + evpl_test_debug("client sending value %u sent %u recv %u", + state->value, state->sent, state->recv); + + state->value++; + } +} /* client_fill_window */ void client_callback( @@ -60,56 +83,35 @@ client_callback( evpl_iovecs_release(evpl, notify->recv_msg.iovec, notify->recv_msg.niov); + if (state->recv == state->niters) { + state->done = 1; + } else { + client_fill_window(evpl, bind, state); + } + break; } /* switch */ } /* client_callback */ -void * -client_thread(void *arg) +static void * +client_thread_init( + struct evpl *evpl, + void *private_data) { - struct evpl *evpl; + struct client_state *state = private_data; struct evpl_endpoint *server; struct evpl_bind *bind; - struct client_state *state = arg; - - evpl = evpl_create(NULL); server = evpl_endpoint_create(address, port); bind = evpl_connect(evpl, proto, NULL, server, client_callback, test_segment_callback, state); - while (state->recv != state->niters) { - - while (state->inflight < state->depth && - state->sent < state->niters) { - - evpl_send(evpl, bind, &state->value, sizeof(state->value)); - - state->inflight++; - state->sent++; - - evpl_test_debug("client sending value %u sent %u recv %u", - state->value, state->sent, state->recv); - - state->value++; - } - - evpl_test_debug("client continue sent %u recv %u", - state->sent, state->recv); - - evpl_continue(evpl); - } - - evpl_test_debug("client completed iterations"); - - evpl_stop(state->server_evpl); + client_fill_window(evpl, bind, state); - evpl_destroy(evpl); - - return NULL; -} /* client_thread */ + return private_data; +} /* client_thread_init */ void server_callback( @@ -150,18 +152,35 @@ accept_callback( *conn_private_data = private_data; } /* accept_callback */ +static void * +server_thread_init( + struct evpl *evpl, + void *private_data) +{ + server_binding = evpl_listener_attach(evpl, listener, accept_callback, + private_data); + + return private_data; +} /* server_thread_init */ + +static void +server_thread_shutdown( + struct evpl *evpl, + void *private_data) +{ + evpl_listener_detach(evpl, server_binding); +} /* server_thread_shutdown */ + int main( int argc, char *argv[]) { - pthread_t thr; - struct evpl *evpl; - struct evpl_endpoint *me; - struct evpl_listener *listener; - struct evpl_listener_binding *binding; - int rc, opt; - struct client_state state = { + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + struct evpl_endpoint *me; + int rc, opt; + struct client_state state = { .inflight = 0, .depth = 100, .sent = 0, @@ -195,29 +214,28 @@ main( } /* switch */ } - evpl = evpl_create(NULL); - - state.server_evpl = evpl; - me = evpl_endpoint_create("0.0.0.0", port); listener = evpl_listener_create(); - binding = evpl_listener_attach(evpl, listener, accept_callback, &state); + /* Blocks until the binding is attached on the server thread. */ + server_thread = evpl_thread_create(NULL, server_thread_init, + server_thread_shutdown, &state); evpl_listen(listener, proto, me); - pthread_create(&thr, NULL, client_thread, &state); + client_thread = evpl_thread_create(NULL, client_thread_init, NULL, &state); - evpl_run(evpl); + while (!state.done) { + usleep(1000); + } - pthread_join(thr, NULL); + evpl_test_debug("client completed iterations"); - evpl_listener_detach(evpl, binding); + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); evpl_listener_destroy(listener); - evpl_destroy(evpl); - return 0; } /* main */ diff --git a/src/tests/ping_pong_msg.c b/src/tests/ping_pong_msg.c index e0d9d6af..ec090f6b 100644 --- a/src/tests/ping_pong_msg.c +++ b/src/tests/ping_pong_msg.c @@ -1,11 +1,10 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ben Jarvis +// SPDX-FileCopyrightText: 2024 - 2026 Ben Jarvis // // SPDX-License-Identifier: LGPL-2.1-only #include #include #include -#include #include #include @@ -18,13 +17,14 @@ const char localhost[] = "127.0.0.1"; const char *address = localhost; int port = 8000; - struct client_state { - int sent; - int recv; - int niters; - uint32_t value; - struct evpl *server_evpl; + volatile int done; + int sent; + int recv; + int niters; + uint32_t value; + struct evpl_endpoint *server_ep; + struct evpl_endpoint *client_ep; }; void @@ -47,53 +47,43 @@ client_callback( evpl_iovecs_release(evpl, notify->recv_msg.iovec, notify->recv_msg.niov); + /* Lock-step: only ever one message in flight. */ + if (state->sent < state->niters) { + evpl_sendtoep(evpl, bind, state->server_ep, &state->value, + sizeof(state->value)); + + state->value++; + state->sent++; + + evpl_test_debug("client sent sent %u recv %u", + state->sent, state->recv); + } else { + state->done = 1; + } + break; } /* switch */ } /* client_callback */ -void * -client_thread(void *arg) +static void * +client_thread_init( + struct evpl *evpl, + void *private_data) { - struct evpl *evpl; - struct evpl_endpoint *me, *server; - struct evpl_bind *bind; - struct client_state *state = arg; - - evpl = evpl_create(NULL); - - me = evpl_endpoint_create(address, port + 1); - - server = evpl_endpoint_create(address, port); - - bind = evpl_bind(evpl, proto, me, client_callback, state); - - while (state->sent < state->niters) { - - if (state->sent == state->recv) { - - evpl_sendtoep(evpl, bind, server, &state->value, - sizeof(state->value)); - - state->value++; - state->sent++; - - evpl_test_debug("client sent sent %u recv %u", - state->sent, state->recv); - - } - - evpl_continue(evpl); - } + struct client_state *state = private_data; + struct evpl_bind *bind; - evpl_test_debug("client completed iterations"); + bind = evpl_bind(evpl, proto, state->client_ep, client_callback, state); - evpl_stop(state->server_evpl); + evpl_sendtoep(evpl, bind, state->server_ep, &state->value, + sizeof(state->value)); - evpl_destroy(evpl); + state->value++; + state->sent++; - return NULL; -} /* client_thread */ + return private_data; +} /* client_thread_init */ void server_callback( @@ -118,16 +108,28 @@ server_callback( } /* server_callback */ +static void * +server_thread_init( + struct evpl *evpl, + void *private_data) +{ + struct client_state *state = private_data; + + evpl_bind(evpl, proto, state->server_ep, server_callback, + state->client_ep); + + return private_data; +} /* server_thread_init */ + int main( int argc, char *argv[]) { - pthread_t thr; - struct evpl *evpl; - struct evpl_endpoint *me, *client; - int rc, opt; - struct client_state state = { + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + int rc, opt; + struct client_state state = { .sent = 0, .recv = 0, .niters = 100, @@ -159,23 +161,22 @@ main( } /* switch */ } + state.server_ep = evpl_endpoint_create(address, port); + state.client_ep = evpl_endpoint_create(address, port + 1); - evpl = evpl_create(NULL); - - state.server_evpl = evpl; - - me = evpl_endpoint_create(address, port); - client = evpl_endpoint_create(address, port + 1); - - evpl_bind(evpl, proto, me, server_callback, client); + /* evpl_thread_create blocks until the init callback ran, so the server + * bind exists before the client sends its first message. */ + server_thread = evpl_thread_create(NULL, server_thread_init, NULL, &state); + client_thread = evpl_thread_create(NULL, client_thread_init, NULL, &state); - pthread_create(&thr, NULL, client_thread, &state); - - evpl_run(evpl); + while (!state.done) { + usleep(1000); + } - pthread_join(thr, NULL); + evpl_test_debug("client completed iterations"); - evpl_destroy(evpl); + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); return 0; } /* main */ diff --git a/src/tests/rdma_ops.c b/src/tests/rdma_ops.c index 1d46d1c4..f4b9926d 100644 --- a/src/tests/rdma_ops.c +++ b/src/tests/rdma_ops.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -47,11 +46,10 @@ struct server_state { struct evpl_iovec rdma_buffer; int rdma_buffer_valid; int phase; - int complete; + volatile int complete; }; struct client_state { - struct evpl *server_evpl; struct evpl *evpl; struct evpl_bind *bind; struct evpl_iovec local_buffer; @@ -62,10 +60,13 @@ struct client_state { int phase; int read_complete; int write_complete; - int complete; + volatile int complete; int passed; }; +static struct evpl_listener *listener; +static struct evpl_listener_binding *server_binding; + int test_segment_callback( struct evpl *evpl, @@ -226,40 +227,35 @@ client_callback( } /* switch */ } /* client_callback */ -void * -client_thread(void *arg) +static void * +client_thread_init( + struct evpl *evpl, + void *private_data) { - struct evpl *evpl; + struct client_state *state = private_data; struct evpl_endpoint *server; - struct evpl_bind *bind; - struct client_state *state = arg; - evpl = evpl_create(NULL); state->evpl = evpl; server = evpl_endpoint_create(address, port); - bind = evpl_connect(evpl, proto, NULL, server, client_callback, - test_segment_callback, state); + state->bind = evpl_connect(evpl, proto, NULL, server, client_callback, + test_segment_callback, state); - state->bind = bind; - - while (!state->complete) { - evpl_continue(evpl); - } + return private_data; +} /* client_thread_init */ - evpl_test_info("Client completed"); - - evpl_stop(state->server_evpl); +static void +client_thread_shutdown( + struct evpl *evpl, + void *private_data) +{ + struct client_state *state = private_data; if (state->local_buffer_valid) { - evpl_iovec_release(state->evpl, &state->local_buffer); + evpl_iovec_release(evpl, &state->local_buffer); } - - evpl_destroy(evpl); - - return NULL; -} /* client_thread */ +} /* client_thread_shutdown */ void server_callback( @@ -356,19 +352,46 @@ accept_callback( *conn_private_data = private_data; } /* accept_callback */ +static void * +server_thread_init( + struct evpl *evpl, + void *private_data) +{ + struct server_state *state = private_data; + + state->evpl = evpl; + + server_binding = evpl_listener_attach(evpl, listener, accept_callback, + state); + + return private_data; +} /* server_thread_init */ + +static void +server_thread_shutdown( + struct evpl *evpl, + void *private_data) +{ + struct server_state *state = private_data; + + evpl_listener_detach(evpl, server_binding); + + if (state->rdma_buffer_valid) { + evpl_iovec_release(evpl, &state->rdma_buffer); + } +} /* server_thread_shutdown */ + int main( int argc, char *argv[]) { - pthread_t thr; - struct evpl *evpl; - struct evpl_endpoint *me; - struct evpl_listener *listener; - struct evpl_listener_binding *binding; - int rc, opt; - struct server_state server_state = { 0 }; - struct client_state client_state = { 0 }; + struct evpl_thread *server_thread; + struct evpl_thread *client_thread; + struct evpl_endpoint *me; + int rc, opt; + struct server_state server_state = { 0 }; + struct client_state client_state = { 0 }; test_evpl_config(); @@ -395,34 +418,31 @@ main( } /* switch */ } - evpl = evpl_create(NULL); - - server_state.evpl = evpl; - client_state.server_evpl = evpl; - me = evpl_endpoint_create("0.0.0.0", port); listener = evpl_listener_create(); - binding = evpl_listener_attach(evpl, listener, accept_callback, &server_state); + /* Blocks until the binding is attached on the server thread. */ + server_thread = evpl_thread_create(NULL, server_thread_init, + server_thread_shutdown, &server_state); evpl_listen(listener, proto, me); - pthread_create(&thr, NULL, client_thread, &client_state); + client_thread = evpl_thread_create(NULL, client_thread_init, + client_thread_shutdown, &client_state); - evpl_run(evpl); + /* Gate on the client only: on a failure path the server never receives + * MSG_TYPE_COMPLETE, and the client outcome decides the test anyway. */ + while (!client_state.complete) { + usleep(1000); + } - pthread_join(thr, NULL); + evpl_test_info("Client completed"); - evpl_listener_detach(evpl, binding); + evpl_thread_destroy(client_thread); + evpl_thread_destroy(server_thread); evpl_listener_destroy(listener); - if (server_state.rdma_buffer_valid) { - evpl_iovec_release(evpl, &server_state.rdma_buffer); - } - - evpl_destroy(evpl); - return client_state.passed ? 0 : 1; } /* main */ diff --git a/src/tests/test_common.h b/src/tests/test_common.h index 0b3a47e3..def40654 100644 --- a/src/tests/test_common.h +++ b/src/tests/test_common.h @@ -10,6 +10,10 @@ #include "evpl/evpl.h" +#ifdef HAVE_SPDK +#include "core/spdk/tests/spdk_test_harness.h" +#endif /* ifdef HAVE_SPDK */ + /* * Optional core-mechanism override for the test suite. * @@ -17,6 +21,11 @@ * the build (see src/core/socket/tests/CMakeLists.txt) and selects it through * EVPL_TEST_CORE_MECH. Keeping this in the harness rather than in the library * means no test source -- and no libevpl consumer -- needs to know about it. + * + * "spdk" is only valid for tests that are fully evpl_thread/callback driven + * (no evpl_run or manual pumping from the main thread) and registered via the + * *_spdk CMake macros: it boots the mini-reactor harness so every evpl thread + * runs as an spdk_thread in guest mode. */ static inline void test_evpl_set_core_mech(struct evpl_global_config *config) @@ -33,6 +42,13 @@ test_evpl_set_core_mech(struct evpl_global_config *config) evpl_global_config_set_core_mech(config, EVPL_CORE_MECH_KQUEUE); } else if (strcmp(mech, "select") == 0) { evpl_global_config_set_core_mech(config, EVPL_CORE_MECH_SELECT); +#ifdef HAVE_SPDK + } else if (strcmp(mech, "spdk") == 0) { + /* Must precede evpl_init so the harness's atexit teardown runs after + * evpl_cleanup (atexit is LIFO). */ + evpl_spdk_test_init(3); + evpl_global_config_set_core_mech(config, EVPL_CORE_MECH_SPDK); +#endif /* ifdef HAVE_SPDK */ } else { fprintf(stderr, "EVPL_TEST_CORE_MECH: unknown mechanism '%s'\n", mech); exit(1);