rpc: one tensor read per decode step and asynchronous split copies

.gitignore

@@ -153,3 +153,4 @@ a.out.*
 
 AGENTS.local.md
 .pi/SYSTEM.md
+bench/

ggml/include/ggml-rpc.h

@@ -7,7 +7,7 @@ extern "C" {
 #endif
 
 #define RPC_PROTO_MAJOR_VERSION    5
-#define RPC_PROTO_MINOR_VERSION    1
+#define RPC_PROTO_MINOR_VERSION    2
 #define RPC_PROTO_PATCH_VERSION    0
 
 #ifdef  __cplusplus

ggml/src/ggml-backend.cpp

@@ -499,6 +499,31 @@ void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor
     }
 }
 
+// Asks the destination backend to take the copy, then the source backend. Only the destination
+// used to be asked, which left a backend that can accelerate reads out of itself (the RPC
+// backend, which stages through pinned memory on the peer device) on the synchronous fallback.
+//
+// The source is only asked when the two backends have DIFFERENT implementations. Two backends
+// of the same type (two CUDA devices, say) share one implementation that already saw the pair
+// and declined, so asking it again would be a second call with the same answer. That keeps
+// every single-type setup, including one GPU and multi GPU, on exactly the old path.
+static bool ggml_backend_cpy_tensor_async_impl(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
+    if (backend_dst != NULL && backend_dst->iface.cpy_tensor_async != NULL) {
+        if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
+            return true;
+        }
+    }
+
+    if (backend_src != NULL && backend_src->iface.cpy_tensor_async != NULL &&
+        (backend_dst == NULL || backend_src->iface.cpy_tensor_async != backend_dst->iface.cpy_tensor_async)) {
+        if (backend_src->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
     GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
 
@@ -507,10 +532,8 @@ void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t b
     }
 
     GGML_ASSERT(backend_dst);
-    if (backend_dst->iface.cpy_tensor_async != NULL) {
-        if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
-            return;
-        }
+    if (ggml_backend_cpy_tensor_async_impl(backend_src, backend_dst, src, dst)) {
+        return;
     }
 
     // an async copy would normally happen after all the queued operations on both backends are completed
@@ -1728,7 +1751,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
                 } else {
                     // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
                     // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
-                    if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
+                    if (!ggml_backend_cpy_tensor_async_impl(input_backend, split_backend, input, input_cpy)) {
                         ggml_backend_synchronize(input_backend);
                         if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
                             ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);