From 1e13caab5dab953e4175b8bc4430342b72972f2d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 6 Sep 2026 13:49:26 -0700 Subject: [PATCH] rpc: do not abort on shutdown when a teardown round trip fails llama-server built with GGML_RPC aborts on exit, after it has printed "cleaning up before exit", inside ggml_backend_rpc_get_device_memory called from common_memory_breakdown_print: ggml-rpc.cpp: Remote RPC server crashed or returned malformed response #3 ggml_backend_rpc_get_device_memory #4 common_memory_breakdown_print(llama_context const*) #5 llama_server(common_params&, int, char**) The memory breakdown is printed after clean_up(), which calls llama_backend_free(). The RPC device answers get_memory with a round trip to the peer, and by that point the round trip fails, so RPC_STATUS_ASSERT aborts the process. Reproduced with a peer rpc-server that is alive and healthy the whole time, so this is not about the peer going away first. Device memory is an informational property, not part of the data path, and the function already has a defined answer for an endpoint it cannot reach: report 0/0. Report the same when the query itself fails. Freeing a remote buffer is released state and is also a teardown operation, so a failure there is logged instead of aborting; if the peer is gone then so is the buffer. Every other RPC_STATUS_ASSERT, including all the data path ones, is unchanged. Confined to ggml/src/ggml-rpc/ggml-rpc.cpp, which is compiled only when GGML_RPC is on (OFF by default, ggml_add_backend(RPC) in ggml/src/CMakeLists.txt), so a build without RPC is untouched. --- ggml/src/ggml-rpc/ggml-rpc.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 69a8a08ae172..9669ada85997 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -393,8 +393,11 @@ static std::shared_ptr get_socket(const std::string & endpoint) { static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; rpc_msg_free_buffer_req request = {ctx->remote_ptr}; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_FREE_BUFFER, &request, sizeof(request), nullptr, 0); - RPC_STATUS_ASSERT(status); + // releasing a remote buffer must not abort: this runs during teardown, and if the peer is + // already gone then so is the buffer. The local context is freed either way. + if (!send_rpc_cmd(ctx->sock, RPC_CMD_FREE_BUFFER, &request, sizeof(request), nullptr, 0)) { + GGML_LOG_ERROR("%s: failed to free the remote buffer, the connection is gone\n", __func__); + } delete ctx; } @@ -823,24 +826,34 @@ bool ggml_backend_is_rpc(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rpc_guid()); } -static void get_device_memory(const std::shared_ptr & sock, uint32_t device, size_t * free, size_t * total) { +static bool get_device_memory(const std::shared_ptr & sock, uint32_t device, size_t * free, size_t * total) { rpc_msg_get_device_memory_req request; request.device = device; rpc_msg_get_device_memory_rsp response; - bool status = send_rpc_cmd(sock, RPC_CMD_GET_DEVICE_MEMORY, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + if (!send_rpc_cmd(sock, RPC_CMD_GET_DEVICE_MEMORY, &request, sizeof(request), &response, sizeof(response))) { + return false; + } *free = response.free_mem; *total = response.total_mem; + return true; } void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) { + *free = 0; + *total = 0; auto sock = get_socket(endpoint); if (sock == nullptr) { + return; + } + // this is an informational device property, not part of the data path, and it is queried during + // teardown as well (the memory breakdown printed on exit), by which time the peer rpc-server may + // already be gone. A dead connection here must not abort the process: report the memory as + // unknown, which is what an endpoint we cannot connect to at all already reports. + if (!get_device_memory(sock, device, free, total)) { + GGML_LOG_ERROR("%s: failed to query device memory of %s, reporting 0\n", __func__, endpoint); *free = 0; *total = 0; - return; } - get_device_memory(sock, device, free, total); } // RPC server-side implementation