From 893effdaf5d1bdfaab00cbf2fc96f9117cf06ed1 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 22 Apr 2026 00:28:27 -0700 Subject: [PATCH 1/3] realm: initial implementation of fast many-field copies for CPUs --- src/realm/runtime_impl.cc | 2 - src/realm/runtime_impl.h | 1 - src/realm/transfer/channel.cc | 117 +++++-- src/realm/transfer/channel.h | 2 + src/realm/transfer/memcpy_channel.cc | 141 +++++++- src/realm/transfer/memcpy_channel.h | 2 + src/realm/transfer/transfer.cc | 21 +- tests/CMakeLists.txt | 23 +- tests/multifield_transfer.cc | 29 +- tests/unit_tests/memcpy_multi_field_test.cc | 341 ++++++++++++++++++++ 10 files changed, 621 insertions(+), 58 deletions(-) create mode 100644 tests/unit_tests/memcpy_multi_field_test.cc diff --git a/src/realm/runtime_impl.cc b/src/realm/runtime_impl.cc index 59282320ce4..117e19c417e 100644 --- a/src/realm/runtime_impl.cc +++ b/src/realm/runtime_impl.cc @@ -762,7 +762,6 @@ namespace Realm { config_map.insert({"report_sparsity_leaks", &report_sparsity_leaks}); config_map.insert({"barrier_broadcast_radix", &barrier_broadcast_radix}); config_map.insert({"diskmem", &disk_mem_size}); - config_map.insert({"dma_multi_field", &dma_multi_field}); resource_map.insert({"cpu", &res_num_cpus}); resource_map.insert({"sysmem", &res_sysmem_size}); @@ -817,7 +816,6 @@ namespace Realm { .add_option_int_units("-ll:ib_rsize", reg_ib_mem_size, 'm') .add_option_int_units("-ll:dsize", disk_mem_size, 'm') .add_option_int("-ll:dma", dma_worker_threads) - .add_option_int("-ll:dma_multi_field", dma_multi_field) .add_option_bool("-ll:pin_dma", pin_dma_threads) .add_option_int("-ll:dummy_rsrv_ok", dummy_reservation_ok) .add_option_bool("-ll:show_rsrv", show_reservations) diff --git a/src/realm/runtime_impl.h b/src/realm/runtime_impl.h index 8624cb433b2..72b72c75feb 100644 --- a/src/realm/runtime_impl.h +++ b/src/realm/runtime_impl.h @@ -145,7 +145,6 @@ namespace Realm { size_t reg_mem_size = 0; size_t disk_mem_size = 0; unsigned dma_worker_threads = 0; // unused - warning on application use - bool dma_multi_field = true; #ifdef EVENT_TRACING size_t event_trace_block_size = 1 << 20; double event_trace_exp_arrv_rate = 1e3; diff --git a/src/realm/transfer/channel.cc b/src/realm/transfer/channel.cc index adb7b6afb9b..008e2735a3b 100644 --- a/src/realm/transfer/channel.cc +++ b/src/realm/transfer/channel.cc @@ -2542,6 +2542,17 @@ namespace Realm { log_xd.info() << "remote write chunk: min=" << min_xfer_size << " max=" << max_bytes; + // multi-field fast path: when an IDIndexedFieldsIterator has + // attached a FieldBlock to one or both address lists, each such + // entry describes one rectangle covering N fields at offsets + // field_id * field_stride. The 1D-dst / 1D-src branch below (by + // far the common case, and the only one that can be reached when + // idindexed_fields is set - 2D dst and scatter dst are assert(0), + // 2D src and gather are #ifdef-gated) picks up per-field offset + // adjustment and - when both sides have a FieldBlock - batches + // advance() by fields_left so XferDes/iterator setup is amortized + // across all fields in the block. + while(total_bytes < max_bytes) { AddressListCursor &in_alc = in_port->addrcursor; AddressListCursor &out_alc = out_port->addrcursor; @@ -2627,34 +2638,79 @@ namespace Realm { if(src_1d_maxbytes == 0) break; - // 1D source + // 1D source - single AM per field (N fields per full-rect + // consume when both sides have a FieldBlock). The AM + // framing cost is not amortized by batching - each field + // gets its own message - but the XferDes/iterator setup is + // amortized across all fields (one XD for the whole block + // instead of N). bytes = src_1d_maxbytes; - // log_xd.info() << "remote write 1d: guid=" << guid - // << " src=" << src_buf << " dst=" << dst_buf - // << " bytes=" << bytes; - ActiveMessage amsg(dst_node, src_buf, bytes, dst_buf); - amsg->next_xd_guid = out_port->peer_guid; - amsg->next_port_idx = out_port->peer_port_idx; - amsg->span_start = out_span_start; + const FieldBlock *fblock_in = in_alc.field_block(); + const FieldBlock *fblock_out = out_alc.field_block(); + const size_t src_fstride = + fblock_in ? in_alc.addrlist->full_field_bytes() : 0; + const size_t dst_fstride = + fblock_out ? out_alc.addrlist->full_field_bytes() : 0; + const FieldID *const src_fields_arr = + fblock_in ? in_alc.fields_data() : nullptr; + const FieldID *const dst_fields_arr = + fblock_out ? out_alc.fields_data() : nullptr; + const size_t fields_left_in = fblock_in ? in_alc.remaining_fields() : 1; + const size_t fields_left_out = + fblock_out ? out_alc.remaining_fields() : 1; +#ifdef DEBUG_REALM + if(fblock_in && fblock_out) { + assert(fields_left_in == fields_left_out); + } +#endif + const size_t fields_left = std::min(fields_left_in, fields_left_out); + const bool full_rect_src = (in_dim == 1) && (bytes == icount); + const bool full_rect_dst = (out_dim == 1) && (bytes == ocount); + const size_t n = + (fblock_in && fblock_out && full_rect_src && full_rect_dst) + ? fields_left + : 1; +#ifdef DEBUG_REALM + assert(n == 1 || (full_rect_src && full_rect_dst)); +#endif - // reads aren't consumed until local completion, but - // only ask if we have a previous xd that's going to - // care - if(in_port->peer_guid != XFERDES_NO_GUID) { - // a ReadBytesUpdater holds a reference to the xd - add_reference(); - amsg.add_local_completion(ReadBytesUpdater( - this, input_control.current_io_port, in_span_start, bytes)); + for(size_t f = 0; f < n; f++) { + const uintptr_t src_fofs = + src_fields_arr ? (uintptr_t(src_fields_arr[f]) * src_fstride) : 0; + const uintptr_t dst_fofs = + dst_fields_arr ? (uintptr_t(dst_fields_arr[f]) * dst_fstride) : 0; + LocalAddress src_buf_f = src_buf; + src_buf_f.offset += src_fofs; + RemoteAddress dst_buf_f = dst_buf; + dst_buf_f.ptr += dst_fofs; + + ActiveMessage amsg(dst_node, src_buf_f, bytes, + dst_buf_f); + amsg->next_xd_guid = out_port->peer_guid; + amsg->next_port_idx = out_port->peer_port_idx; + amsg->span_start = out_span_start; + + // reads aren't consumed until local completion, but + // only ask if we have a previous xd that's going to + // care + if(in_port->peer_guid != XFERDES_NO_GUID) { + // a ReadBytesUpdater holds a reference to the xd + add_reference(); + amsg.add_local_completion(ReadBytesUpdater( + this, input_control.current_io_port, in_span_start, bytes)); + } + in_span_start += bytes; + // the write isn't complete until it's ack'd by the target + amsg.add_remote_completion(WriteBytesUpdater( + this, output_control.current_io_port, out_span_start, bytes)); + out_span_start += bytes; + + amsg.commit(); } - in_span_start += bytes; - // the write isn't complete until it's ack'd by the target - amsg.add_remote_completion(WriteBytesUpdater( - this, output_control.current_io_port, out_span_start, bytes)); - out_span_start += bytes; - amsg.commit(); - in_alc.advance(0, bytes); - out_alc.advance(0, bytes); + in_alc.advance(0, bytes, n); + out_alc.advance(0, bytes, n); + bytes *= n; } else if(src_2d_maxbytes >= src_ga_maxbytes) { // 2D source size_t bytes_per_line = icount; @@ -4207,6 +4263,19 @@ namespace Realm { RemoteWriteChannel::~RemoteWriteChannel() {} + bool RemoteWriteChannel::support_idindexed_fields(Memory src_mem, Memory dst_mem) const + { + // The multi-field branch in RemoteWriteXferDes::progress_xd reuses the + // existing per-rect 1D/1D active-message path, issuing one message per + // field of the attached FieldBlock. This is safe for any memory pair + // that RemoteWriteChannel already accepts (same preconditions as the + // single-field slow path); the real win is amortizing XferDes/iterator + // setup across all the fields in the block rather than paying it N + // times. 2D/scatter dst and 2D/gather src remain single-field (they + // are gated assert(0) or #ifdef today). + return true; + } + XferDes *RemoteWriteChannel::create_xfer_des( uintptr_t dma_op, NodeID launch_node, XferDesID guid, const std::vector &inputs_info, diff --git a/src/realm/transfer/channel.h b/src/realm/transfer/channel.h index ad886b82969..cdb183db0b5 100644 --- a/src/realm/transfer/channel.h +++ b/src/realm/transfer/channel.h @@ -1119,6 +1119,8 @@ namespace Realm { const void *fill_data, size_t fill_size, size_t fill_total); + virtual bool support_idindexed_fields(Memory src_mem, Memory dst_mem) const override; + long submit(Request **requests, long nr); }; diff --git a/src/realm/transfer/memcpy_channel.cc b/src/realm/transfer/memcpy_channel.cc index 4d6bf9a4153..c3ba67020bc 100644 --- a/src/realm/transfer/memcpy_channel.cc +++ b/src/realm/transfer/memcpy_channel.cc @@ -182,6 +182,14 @@ namespace Realm { uintptr_t out_base = reinterpret_cast(out_port->mem->get_direct_ptr(0, 0)); + // multi-field fast path: each FieldBlock-attached address list (via + // IDIndexedFieldsIterator) has entries that describe one rectangle + // covering N fields at offsets field_id * field_stride. Both ports + // have a FieldBlock in the common case (instance-to-instance copy). + // Asymmetric cases (e.g. instance <-> IB boundary XD) leave one + // side without a FieldBlock; the side that has one still needs its + // per-field offset added on every rect slab, but batching (n>1) + // requires both sides. while(total_bytes < max_bytes) { AddressListCursor &in_alc = in_port->addrcursor; AddressListCursor &out_alc = out_port->addrcursor; @@ -194,13 +202,44 @@ namespace Realm { int in_dim = in_alc.get_dim(); int out_dim = out_alc.get_dim(); + const FieldBlock *fblock_in = in_alc.field_block(); + const FieldBlock *fblock_out = out_alc.field_block(); + // multi-field: per-field rect stride (= per-field bytes in this entry) + const size_t src_fstride = + fblock_in ? in_alc.addrlist->full_field_bytes() : 0; + const size_t dst_fstride = + fblock_out ? out_alc.addrlist->full_field_bytes() : 0; + const FieldID *const src_fields = fblock_in ? in_alc.fields_data() : nullptr; + const FieldID *const dst_fields = + fblock_out ? out_alc.fields_data() : nullptr; + const size_t fields_left_in = fblock_in ? in_alc.remaining_fields() : 1; + const size_t fields_left_out = fblock_out ? out_alc.remaining_fields() : 1; +#ifdef DEBUG_REALM + if(fblock_in && fblock_out) { + assert(fields_left_in == fields_left_out); + } +#endif + const size_t fields_left = std::min(fields_left_in, fields_left_out); + size_t bytes = 0; size_t bytes_left = max_bytes - total_bytes; // memcpys don't need to be particularly big to achieve // peak efficiency, so trim to something that takes - // 10's of us to be responsive to the time limit + // 10's of us to be responsive to the time limit. This is a + // per-field cap - a multi-field iteration may issue up to + // fields_left copies of a rectangle up to this size. bytes_left = std::min(bytes_left, size_t(256 << 10)); + // geometry computed by the 1D/2D/3D selection below, then + // dispatched at the bottom - this lets multi-field vs single-field + // share a single memcpy dispatch. + int copy_kind = 0; // 1, 2, or 3 dimensions + size_t copy_contig = 0, copy_lines = 1, copy_planes = 1; + uintptr_t copy_in_lstride = 0, copy_out_lstride = 0; + uintptr_t copy_in_pstride = 0, copy_out_pstride = 0; + int adv_id = 0, adv_od = 0; + size_t adv_amt_in = 0, adv_amt_out = 0; + if(in_dim > 0) { if(out_dim > 0) { size_t icount = in_alc.remaining(0); @@ -213,10 +252,12 @@ namespace Realm { if((contig_bytes == bytes_left) || ((contig_bytes == icount) && (in_dim == 1)) || ((contig_bytes == ocount) && (out_dim == 1))) { - bytes = contig_bytes; - memcpy_1d(out_base + out_offset, in_base + in_offset, bytes); - in_alc.advance(0, bytes); - out_alc.advance(0, bytes); + copy_kind = 1; + copy_contig = contig_bytes; + adv_id = 0; + adv_amt_in = contig_bytes; + adv_od = 0; + adv_amt_out = contig_bytes; } else { // grow to a 2D copy int id; @@ -268,11 +309,15 @@ namespace Realm { if(((contig_bytes * lines) == bytes_left) || ((lines == icount) && (id == (in_dim - 1))) || ((lines == ocount) && (od == (out_dim - 1)))) { - bytes = contig_bytes * lines; - memcpy_2d(out_base + out_offset, out_lstride, in_base + in_offset, - in_lstride, contig_bytes, lines); - in_alc.advance(id, lines * iscale); - out_alc.advance(od, lines * oscale); + copy_kind = 2; + copy_contig = contig_bytes; + copy_lines = lines; + copy_in_lstride = in_lstride; + copy_out_lstride = out_lstride; + adv_id = id; + adv_amt_in = lines * iscale; + adv_od = od; + adv_amt_out = lines * oscale; } else { uintptr_t in_pstride; if(lines < icount) { @@ -308,12 +353,18 @@ namespace Realm { size_t planes = std::min(std::min(icount, ocount), (bytes_left / (contig_bytes * lines))); - bytes = contig_bytes * lines * planes; - memcpy_3d(out_base + out_offset, out_lstride, out_pstride, - in_base + in_offset, in_lstride, in_pstride, contig_bytes, - lines, planes); - in_alc.advance(id, planes * iscale); - out_alc.advance(od, planes * oscale); + copy_kind = 3; + copy_contig = contig_bytes; + copy_lines = lines; + copy_planes = planes; + copy_in_lstride = in_lstride; + copy_in_pstride = in_pstride; + copy_out_lstride = out_lstride; + copy_out_pstride = out_pstride; + adv_id = id; + adv_amt_in = planes * iscale; + adv_od = od; + adv_amt_out = planes * oscale; } } } else { @@ -330,8 +381,53 @@ namespace Realm { } } + // ---- dispatch ------------------------------------------------ + // multi-field promotes to n=fields_left only when both sides have + // a FieldBlock and the advance completes the rectangle on both + // cursors (AddressListCursor only bumps partial_fields on a full- + // rect consume - see AddressListCursor::advance in + // address_list.cc). partial-rect consumes and asymmetric cases + // always run one field at a time (n=1, f=1) so the partial_fields + // counter stays in lockstep with the data actually moved. + const bool full_rect_src = (adv_id == (in_alc.get_dim() - 1)) && + (adv_amt_in == in_alc.remaining(adv_id)); + const bool full_rect_dst = (adv_od == (out_alc.get_dim() - 1)) && + (adv_amt_out == out_alc.remaining(adv_od)); + const size_t n = (fblock_in && fblock_out && full_rect_src && full_rect_dst) + ? fields_left + : 1; +#ifdef DEBUG_REALM + // invariant: advance() with f>1 is only valid on a full-rect + // consume; see AddressListCursor::advance in address_list.cc. + assert(n == 1 || (full_rect_src && full_rect_dst)); +#endif + + for(size_t f = 0; f < n; f++) { + const uintptr_t src_fofs = + src_fields ? (uintptr_t(src_fields[f]) * src_fstride) : 0; + const uintptr_t dst_fofs = + dst_fields ? (uintptr_t(dst_fields[f]) * dst_fstride) : 0; + if(copy_kind == 1) { + memcpy_1d(out_base + out_offset + dst_fofs, + in_base + in_offset + src_fofs, copy_contig); + } else if(copy_kind == 2) { + memcpy_2d(out_base + out_offset + dst_fofs, copy_out_lstride, + in_base + in_offset + src_fofs, copy_in_lstride, copy_contig, + copy_lines); + } else { + memcpy_3d(out_base + out_offset + dst_fofs, copy_out_lstride, + copy_out_pstride, in_base + in_offset + src_fofs, + copy_in_lstride, copy_in_pstride, copy_contig, copy_lines, + copy_planes); + } + } + + bytes = copy_contig * copy_lines * copy_planes * n; + in_alc.advance(adv_id, adv_amt_in, n); + out_alc.advance(adv_od, adv_amt_out, n); + #ifdef DEBUG_REALM - assert(bytes <= bytes_left); + assert(bytes <= bytes_left * n); #endif total_bytes += bytes; @@ -474,6 +570,17 @@ namespace Realm { priority); } + bool MemcpyChannel::support_idindexed_fields(Memory src_mem, Memory dst_mem) const + { + // MemcpyChannel only accepts paths where both memories are host-direct + // addressable (local system memories plus remote-shared-memory segments + // whose get_direct_ptr resolves - see constructor's add_path calls). Any + // pair path-enumeration has already routed through this channel is safe + // to batch multi-field, because the multi-field progress_xd branch uses + // the same get_direct_ptr + memcpy_1d/2d/3d calls as the legacy branch. + return true; + } + long MemcpyChannel::submit(Request **requests, long nr) { MemcpyRequest **mem_cpy_reqs = (MemcpyRequest **)requests; diff --git a/src/realm/transfer/memcpy_channel.h b/src/realm/transfer/memcpy_channel.h index faf96d10831..5f39c89935d 100644 --- a/src/realm/transfer/memcpy_channel.h +++ b/src/realm/transfer/memcpy_channel.h @@ -76,6 +76,8 @@ namespace Realm { const void *fill_data, size_t fill_size, size_t fill_total); + virtual bool support_idindexed_fields(Memory src_mem, Memory dst_mem) const override; + virtual long submit(Request **requests, long nr); const Node *node = nullptr; diff --git a/src/realm/transfer/transfer.cc b/src/realm/transfer/transfer.cc index c999b8839cc..8ed90430761 100644 --- a/src/realm/transfer/transfer.cc +++ b/src/realm/transfer/transfer.cc @@ -1913,6 +1913,15 @@ namespace Realm { template bool serialize(S &serializer) const; + // Minimum field count for switching from the per-field iterator to the + // IDIndexedFieldsIterator fast path. Derivation: the per-path latencies + // are + // T_slow = N * C_iter + W/B_eff (one iterator per field) + // T_fast = C_iter + C_fb + W/B_eff (one iterator + FieldBlock) + // so T_slow - T_fast = (N-1)*C_iter - C_fb. With C_fb ~ one replicated- + // heap alloc + a short memcpy of the field-ID list and C_iter comparable + // in magnitude, the break-even is N >= 2 in expectation. At N=1 there + // is no batch to form, so the slow path is forced. static constexpr size_t MIN_IDINDEXED_FIELDS = 2; // protected: @@ -4387,14 +4396,12 @@ namespace Realm { xdn.scatter_control_input = -1; xdn.target_node = path_info.xd_channels[j]->node; - bool enable_multi_field = false; - RealmStatus success = - get_runtime()->get_module_config("core")->get_property( - "dma_multi_field", enable_multi_field); - assert(success == REALM_SUCCESS); - + // The fast path is selected automatically: the channel votes + // on capability (support_idindexed_fields) and TransferDesc:: + // create_iterator adds the layout/field-count gate + // (MIN_IDINDEXED_FIELDS). No user-facing knob - Realm picks + // the lower-latency path from the copy parameters. xdn.idindexed_fields = - enable_multi_field && path_info.xd_channels[j]->support_idindexed_fields(src_mem, dst_mem); xdn.channel = path_info.xd_channels[j]; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bf0adf14fec..5adc5984645 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -97,6 +97,7 @@ list( fragmented_message_test.cc incoming_message_manager_test.cc memcpy_channel_test.cc + memcpy_multi_field_test.cc comp_queue_test.cc event_test.cc path_cache_test.cc @@ -292,6 +293,16 @@ add_integration_test(coverings "${REALM_TEST_DIR}/coverings.cc") set(alltoall_ARGS -ll:csize 1024) add_integration_test(alltoall "${REALM_TEST_DIR}/alltoall.cc") add_integration_test(realm_reinit "${REALM_TEST_DIR}/realm_reinit.cc") +# multifield_transfer supports CPU and GPU memories via -memkind. Always +# build and run the CPU variant; GPU variant is registered below when CUDA +# is enabled. +set(multifield_transfer_ARGS + # max_ops=1 serializes the copy operations so verification is deterministic + # (concurrent copies on the same dst_inst race). Raise max_ops for + # throughput testing with -verify 0. + -memkind cpu -num_fields 32 -size 256 -max_ops 1 -ll:cpu 2 -ll:csize 1024 -verify 1 +) +add_integration_test(multifield_transfer "${REALM_TEST_DIR}/multifield_transfer.cc") set(sparse_construct_ARGS -verbose) add_integration_test(sparse_construct "${REALM_TEST_DIR}/sparse_construct.cc") add_integration_test(extres_alias "${REALM_TEST_DIR}/extres_alias.cc") @@ -429,10 +440,14 @@ if(TEST_USE_GPU) target_link_libraries(cuda_memcpy_test CUDA::cudart) target_link_libraries(cuda_scatter_test CUDA::cudart) target_link_libraries(test_cuhook CUDA::cudart) - set(multifield_transfer_ARGS -ll:gpu 1) - set(multifield_transfer_RESOURCE_LOCK gpu) - add_integration_test(multifield_transfer "${REALM_TEST_DIR}/multifield_transfer.cc") - target_link_libraries(multifield_transfer CUDA::cudart) + # Re-use the multifield_transfer binary (built unconditionally above) with + # GPU-mode args. The binary itself has no CUDA code. + add_test(NAME multifield_transfer_gpu + COMMAND $ + -memkind gpu -num_fields 32 -size 256 -max_ops 1 -ll:gpu 1 + -verify 1) + set_property(TEST multifield_transfer_gpu PROPERTY RESOURCE_LOCK gpu) + list(APPEND _integ_list multifield_transfer_gpu) endif() set(transpose_test_gpu_ARGS -ll:gpu 1) set(transpose_test_gpu_RESOURCE_LOCK gpu) diff --git a/tests/multifield_transfer.cc b/tests/multifield_transfer.cc index 6aed49989ae..eda8312a117 100644 --- a/tests/multifield_transfer.cc +++ b/tests/multifield_transfer.cc @@ -55,8 +55,26 @@ namespace TestConfig { size_t num_fields = 1; size_t field_size = sizeof(ElementType); size_t size = 4ULL * 256ULL; // 1024ULL * 4ULL;//1024ULL; + // Which memory to allocate instances in. Accepts "cpu" / "sysmem" for + // host system memory, and "gpu" / "fbmem" for GPU framebuffer memory. + // The caller is expected to state the intent explicitly - there is no + // hidden autoselection based on what's available. + std::string memkind = "cpu"; }; // namespace TestConfig +static Realm::Memory::Kind parse_memkind(const std::string &s) +{ + if(s == "cpu" || s == "sysmem" || s == "SYSTEM_MEM") { + return Realm::Memory::SYSTEM_MEM; + } + if(s == "gpu" || s == "fbmem" || s == "GPU_FB_MEM") { + return Realm::Memory::GPU_FB_MEM; + } + log_app.fatal() << "unrecognized -memkind " << s + << " (expected cpu/sysmem or gpu/fbmem)"; + abort(); +} + template inline void copy(RegionInstance src_inst, RegionInstance dst_inst, const std::vector &fields, IndexSpace index_space) @@ -700,9 +718,10 @@ static void update_operation_time(const void *args, size_t arglen, const void *u static void bench_timing_task(const void *args, size_t arglen, const void *userdata, size_t userlen, Processor p) { - log_app.print("=== Memory Info ==="); + const Memory::Kind selected_kind = parse_memkind(TestConfig::memkind); + log_app.print() << "=== Memory Info (kind=" << TestConfig::memkind << ") ==="; Realm::Machine::MemoryQuery mq(Realm::Machine::get_machine()); - mq.only_kind(Memory::GPU_FB_MEM).has_capacity(1); + mq.only_kind(selected_kind).has_capacity(1); std::vector memories(mq.begin(), mq.end()); for(Memory m : memories) { display_memory_info(m); @@ -712,6 +731,9 @@ static void bench_timing_task(const void *args, size_t arglen, const void *userd // mq = mq.has_capacity(pow(TestConfig::size, MAX_DIM) * sizeof(ElementType)); memories.assign(mq.begin(), mq.end()); if(memories.size() == 0) { + log_app.fatal() << "no memories of kind " << TestConfig::memkind + << " available - check runtime flags (-ll:gpu N for gpu, " + "-ll:csize M for cpu)"; abort(); } @@ -784,7 +806,8 @@ int main(int argc, char **argv) .add_option_int("-field_size", TestConfig::field_size) .add_option_int("-graphviz", TestConfig::graphviz) .add_option_int("-max_ops", TestConfig::max_ops) - .add_option_int("-verify", TestConfig::verify); + .add_option_int("-verify", TestConfig::verify) + .add_option_string("-memkind", TestConfig::memkind); ok = cp.parse_command_line(argc, (const char **)argv); assert(ok); diff --git a/tests/unit_tests/memcpy_multi_field_test.cc b/tests/unit_tests/memcpy_multi_field_test.cc new file mode 100644 index 00000000000..34ca0d9c333 --- /dev/null +++ b/tests/unit_tests/memcpy_multi_field_test.cc @@ -0,0 +1,341 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "realm/bgwork.h" +#include "realm/transfer/channel.h" +#include "realm/transfer/memcpy_channel.h" +#include "realm/transfer/transfer.h" +#include "realm/inst_layout.h" +#include "test_common.h" +#include +#include +#include +#include +#include + +using namespace Realm; + +namespace { + + struct MockHeap : public ReplicatedHeap { + void *alloc_obj(std::size_t bytes, std::size_t align = 16) override + { + void *ptr = nullptr; +#ifdef REALM_ON_WINDOWS + ptr = _aligned_malloc(bytes, align); +#else + int ret = posix_memalign(&ptr, align, bytes); + if(ret != 0) + ptr = nullptr; +#endif + assert(ptr != nullptr); + return ptr; + } + + void free_obj(void *ptr) override + { +#ifdef REALM_ON_WINDOWS + _aligned_free(ptr); +#else + free(ptr); +#endif + } + }; + + // Build an AoS-style layout where all fields share one piece list and the + // i-th field sits at offset i * field_size. This matches what + // InstanceLayoutGeneric produces when idindexed_fields is detected, which + // is the shape that IDIndexedFieldsIterator expects. + template + static InstanceLayout *make_aos_layout(Rect bounds, + const std::vector &field_ids, + size_t field_size) + { + auto *inst_layout = new InstanceLayout(); + inst_layout->bytes_used = 0; + inst_layout->space = bounds; + inst_layout->idindexed_fields = true; + + size_t stride = field_size; + for(int d = 0; d < N; d++) { + const size_t count = + static_cast(bounds.hi[d] - bounds.lo[d] + 1); + // Fields are packed first, so the next-higher spatial dim stride is + // (num_fields * field_size * prefix product). + stride *= count; + } + const size_t total_per_field = stride; + (void)total_per_field; + + inst_layout->piece_lists.resize(1); + auto *affine = new AffineLayoutPiece(); + affine->bounds = bounds; + affine->offset = 0; + size_t cur_stride = field_size; + for(int d = 0; d < N; d++) { + affine->strides[d] = cur_stride; + affine->offset -= bounds.lo[d] * cur_stride; + cur_stride *= static_cast(bounds.hi[d] - bounds.lo[d] + 1); + } + inst_layout->piece_lists[0].pieces.push_back(affine); + const size_t bytes_per_field = cur_stride; + + for(size_t i = 0; i < field_ids.size(); i++) { + InstanceLayoutGeneric::FieldLayout fl; + fl.list_idx = 0; + fl.rel_offset = static_cast(field_ids[i]) * bytes_per_field; + fl.size_in_bytes = static_cast(field_size); + inst_layout->fields[field_ids[i]] = fl; + } + // Account for all N_fields fields' storage. Pad bytes_used to cover + // the highest-ID field in the block. + size_t max_fid = 0; + for(FieldID f : field_ids) + max_fid = std::max(max_fid, static_cast(f)); + inst_layout->bytes_used = (max_fid + 1) * bytes_per_field; + return inst_layout; + } + + template + static RegionInstanceImpl *make_aos_inst(Rect bounds, + const std::vector &field_ids, + size_t field_size, + uintptr_t storage_base_offset = 0) + { + RegionInstance inst = ID::make_instance(0, 0, 0, 0).convert(); + auto *layout = make_aos_layout(bounds, field_ids, field_size); + auto *impl = new RegionInstanceImpl(nullptr, inst, nullptr); + impl->metadata.layout = layout; + impl->metadata.inst_offset = storage_base_offset; + return impl; + } + + template + static void fill_with_pattern(std::vector &buf, T base) + { + for(size_t i = 0; i < buf.size(); i++) { + buf[i] = base + static_cast(i); + } + } + + // Drive one MemcpyXferDes through progress_xd until it stops making progress + // and verify every field's byte pattern landed correctly at the right + // destination offset. + template + static void run_multi_field_case(Rect bounds, + const std::vector &src_fids, + const std::vector &dst_fids, + size_t field_size) + { + ASSERT_EQ(src_fids.size(), dst_fids.size()); + + // allocate max-field-ID + 1 slots on each side so every id in the + // copy lists is addressable. + size_t src_max_fid = 0, dst_max_fid = 0; + for(FieldID f : src_fids) + src_max_fid = std::max(src_max_fid, static_cast(f)); + for(FieldID f : dst_fids) + dst_max_fid = std::max(dst_max_fid, static_cast(f)); + + const size_t volume = bounds.volume(); + const size_t bytes_per_field = volume * field_size; + const size_t src_bytes = (src_max_fid + 1) * bytes_per_field; + const size_t dst_bytes = (dst_max_fid + 1) * bytes_per_field; + + std::vector src_storage(src_bytes, 0); + std::vector dst_storage(dst_bytes, 0xAB); + + // write a distinct byte pattern into each src field slot so we can + // verify the right field landed at the right dst offset. + for(FieldID f : src_fids) { + const uint8_t base = 0x10u + static_cast(f & 0xFFu); + uint8_t *slot = src_storage.data() + static_cast(f) * bytes_per_field; + for(size_t i = 0; i < bytes_per_field; i++) { + slot[i] = base + static_cast(i); + } + } + + auto *src_inst = make_aos_inst(bounds, src_fids, field_size); + auto *dst_inst = make_aos_inst(bounds, dst_fids, field_size); + + MockHeap heap; + std::vector dim_order(N); + std::iota(dim_order.begin(), dim_order.end(), 0); + + auto *src_it = new IDIndexedFieldsIterator( + dim_order.data(), src_fids, field_size, src_inst, bounds, &heap); + auto *dst_it = new IDIndexedFieldsIterator( + dim_order.data(), dst_fids, field_size, dst_inst, bounds, &heap); + + // -------- MemcpyXferDes wiring (mirrors memcpy_channel_test) -------- + Node node_data; + std::unordered_map remote_shared_memory_mappings; + XferDesRedopInfo redop_info; + auto bgwork = std::make_unique(); + auto channel = std::make_unique( + bgwork.get(), &node_data, remote_shared_memory_mappings); + + std::vector inputs_info, outputs_info; + std::unique_ptr xfer_desc( + dynamic_cast(channel->create_xfer_des( + 0, 0 /*owner*/, 0 /*guid*/, inputs_info, outputs_info, 0 /*priority*/, + redop_info, nullptr, 0, 0))); + + auto src_mem = std::make_unique( + nullptr, Memory::NO_MEMORY, src_storage.size(), 0, Memory::SYSTEM_MEM, + src_storage.data()); + auto dst_mem = std::make_unique( + nullptr, Memory::NO_MEMORY, dst_storage.size(), 0, Memory::SYSTEM_MEM, + dst_storage.data()); + + xfer_desc->input_ports.resize(1); + auto &input_port = xfer_desc->input_ports[0]; + input_port.mem = src_mem.get(); + input_port.peer_port_idx = 0; + input_port.iter = src_it; + input_port.addrcursor.set_addrlist(&input_port.addrlist); + + xfer_desc->output_ports.resize(1); + auto &output_port = xfer_desc->output_ports[0]; + output_port.mem = dst_mem.get(); + output_port.peer_port_idx = 0; + output_port.iter = dst_it; + output_port.addrcursor.set_addrlist(&output_port.addrlist); + + while(xfer_desc->progress_xd(channel.get(), TimeLimit::relative(10000000))) { + } + + // check: every dst_fid slot contains the pattern we wrote into the + // corresponding src_fid slot. every other byte in dst remains 0xAB. + std::vector dst_written(dst_storage.size(), false); + for(size_t k = 0; k < src_fids.size(); k++) { + const FieldID sf = src_fids[k]; + const FieldID df = dst_fids[k]; + const uint8_t base = 0x10u + static_cast(sf & 0xFFu); + const uint8_t *dst_slot = + dst_storage.data() + static_cast(df) * bytes_per_field; + for(size_t i = 0; i < bytes_per_field; i++) { + const uint8_t expected = base + static_cast(i); + EXPECT_EQ(dst_slot[i], expected) + << " src_fid=" << sf << " dst_fid=" << df << " byte=" << i; + dst_written[static_cast(df) * bytes_per_field + i] = true; + } + } + for(size_t i = 0; i < dst_storage.size(); i++) { + if(!dst_written[i]) { + EXPECT_EQ(dst_storage[i], 0xABu) << " byte=" << i << " unexpectedly modified"; + } + } + + channel->shutdown(); + } + +} // namespace + +// ---- 1D cases ---- + +TEST(MemcpyMultiField, OneDim_TwoFields_Sequential) +{ + run_multi_field_case<1, int>(Rect<1, int>(0, 63), + /*src_fids=*/{0, 1}, + /*dst_fids=*/{0, 1}, + /*field_size=*/sizeof(int)); +} + +TEST(MemcpyMultiField, OneDim_FourFields_Sequential) +{ + run_multi_field_case<1, int>(Rect<1, int>(0, 255), + /*src_fids=*/{0, 1, 2, 3}, + /*dst_fids=*/{0, 1, 2, 3}, + /*field_size=*/sizeof(int)); +} + +TEST(MemcpyMultiField, OneDim_FieldReorder) +{ + // src reads fields 0..3, dst writes into fields 4..7 (reordered). + // Confirms per-field offsets on src and dst can differ. + run_multi_field_case<1, int>(Rect<1, int>(0, 127), + /*src_fids=*/{0, 1, 2, 3}, + /*dst_fids=*/{4, 5, 6, 7}, + /*field_size=*/sizeof(int)); +} + +TEST(MemcpyMultiField, OneDim_FieldPermutation) +{ + // src fields in different order than dst - validates that the per-field + // offset pairing tracks src_fields[k] <-> dst_fields[k] rather than + // assuming both arrays are sorted. + run_multi_field_case<1, int>(Rect<1, int>(0, 63), + /*src_fids=*/{3, 1, 0, 2}, + /*dst_fids=*/{0, 2, 3, 1}, + /*field_size=*/sizeof(int)); +} + +TEST(MemcpyMultiField, OneDim_ManyFields) +{ + // 1024 fields exercises the multi-field iteration path at realistic + // Legion workload scale; the 256-KiB per-field cap still keeps each + // progress_xd iteration bounded. + std::vector fids(1024); + std::iota(fids.begin(), fids.end(), 0); + run_multi_field_case<1, int>(Rect<1, int>(0, 15), + /*src_fids=*/fids, + /*dst_fids=*/fids, + /*field_size=*/sizeof(int)); +} + +// ---- 2D cases ---- + +TEST(MemcpyMultiField, TwoDim_FourFields) +{ + run_multi_field_case<2, int>(Rect<2, int>({0, 0}, {15, 15}), + /*src_fids=*/{0, 1, 2, 3}, + /*dst_fids=*/{0, 1, 2, 3}, + /*field_size=*/sizeof(int)); +} + +TEST(MemcpyMultiField, TwoDim_FieldReorder) +{ + run_multi_field_case<2, int>(Rect<2, int>({0, 0}, {31, 7}), + /*src_fids=*/{2, 0, 1}, + /*dst_fids=*/{0, 1, 2}, + /*field_size=*/sizeof(long long)); +} + +// ---- 3D cases ---- + +TEST(MemcpyMultiField, ThreeDim_ThreeFields) +{ + run_multi_field_case<3, int>(Rect<3, int>({0, 0, 0}, {7, 7, 7}), + /*src_fids=*/{0, 1, 2}, + /*dst_fids=*/{0, 1, 2}, + /*field_size=*/sizeof(int)); +} + +// ---- single-field sanity ---- + +TEST(MemcpyMultiField, OneDim_SingleField_LegacyPath) +{ + // fields.size() == 1 still exercises the FieldBlock code path (the + // iterator attaches a FieldBlock unconditionally) but n collapses to 1 + // throughout. Confirms the fast-path code does the right thing when + // there's nothing to batch. + run_multi_field_case<1, int>(Rect<1, int>(0, 63), + /*src_fids=*/{5}, + /*dst_fids=*/{7}, + /*field_size=*/sizeof(int)); +} From c44604e5774f44701d35450182d16dfea9e107c6 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 22 Apr 2026 01:49:33 -0700 Subject: [PATCH 2/3] realm: more formatting fixes --- tests/unit_tests/memcpy_multi_field_test.cc | 108 ++++++++++---------- 1 file changed, 52 insertions(+), 56 deletions(-) diff --git a/tests/unit_tests/memcpy_multi_field_test.cc b/tests/unit_tests/memcpy_multi_field_test.cc index 34ca0d9c333..26968771fc1 100644 --- a/tests/unit_tests/memcpy_multi_field_test.cc +++ b/tests/unit_tests/memcpy_multi_field_test.cc @@ -62,8 +62,8 @@ namespace { // is the shape that IDIndexedFieldsIterator expects. template static InstanceLayout *make_aos_layout(Rect bounds, - const std::vector &field_ids, - size_t field_size) + const std::vector &field_ids, + size_t field_size) { auto *inst_layout = new InstanceLayout(); inst_layout->bytes_used = 0; @@ -72,8 +72,7 @@ namespace { size_t stride = field_size; for(int d = 0; d < N; d++) { - const size_t count = - static_cast(bounds.hi[d] - bounds.lo[d] + 1); + const size_t count = static_cast(bounds.hi[d] - bounds.lo[d] + 1); // Fields are packed first, so the next-higher spatial dim stride is // (num_fields * field_size * prefix product). stride *= count; @@ -111,10 +110,9 @@ namespace { } template - static RegionInstanceImpl *make_aos_inst(Rect bounds, - const std::vector &field_ids, - size_t field_size, - uintptr_t storage_base_offset = 0) + static RegionInstanceImpl * + make_aos_inst(Rect bounds, const std::vector &field_ids, + size_t field_size, uintptr_t storage_base_offset = 0) { RegionInstance inst = ID::make_instance(0, 0, 0, 0).convert(); auto *layout = make_aos_layout(bounds, field_ids, field_size); @@ -136,10 +134,9 @@ namespace { // and verify every field's byte pattern landed correctly at the right // destination offset. template - static void run_multi_field_case(Rect bounds, - const std::vector &src_fids, - const std::vector &dst_fids, - size_t field_size) + static void + run_multi_field_case(Rect bounds, const std::vector &src_fids, + const std::vector &dst_fids, size_t field_size) { ASSERT_EQ(src_fids.size(), dst_fids.size()); @@ -176,31 +173,30 @@ namespace { std::vector dim_order(N); std::iota(dim_order.begin(), dim_order.end(), 0); - auto *src_it = new IDIndexedFieldsIterator( - dim_order.data(), src_fids, field_size, src_inst, bounds, &heap); - auto *dst_it = new IDIndexedFieldsIterator( - dim_order.data(), dst_fids, field_size, dst_inst, bounds, &heap); + auto *src_it = new IDIndexedFieldsIterator(dim_order.data(), src_fids, + field_size, src_inst, bounds, &heap); + auto *dst_it = new IDIndexedFieldsIterator(dim_order.data(), dst_fids, + field_size, dst_inst, bounds, &heap); // -------- MemcpyXferDes wiring (mirrors memcpy_channel_test) -------- Node node_data; std::unordered_map remote_shared_memory_mappings; XferDesRedopInfo redop_info; auto bgwork = std::make_unique(); - auto channel = std::make_unique( - bgwork.get(), &node_data, remote_shared_memory_mappings); + auto channel = std::make_unique(bgwork.get(), &node_data, + remote_shared_memory_mappings); std::vector inputs_info, outputs_info; - std::unique_ptr xfer_desc( - dynamic_cast(channel->create_xfer_des( - 0, 0 /*owner*/, 0 /*guid*/, inputs_info, outputs_info, 0 /*priority*/, - redop_info, nullptr, 0, 0))); - - auto src_mem = std::make_unique( - nullptr, Memory::NO_MEMORY, src_storage.size(), 0, Memory::SYSTEM_MEM, - src_storage.data()); - auto dst_mem = std::make_unique( - nullptr, Memory::NO_MEMORY, dst_storage.size(), 0, Memory::SYSTEM_MEM, - dst_storage.data()); + std::unique_ptr xfer_desc(dynamic_cast( + channel->create_xfer_des(0, 0 /*owner*/, 0 /*guid*/, inputs_info, outputs_info, + 0 /*priority*/, redop_info, nullptr, 0, 0))); + + auto src_mem = + std::make_unique(nullptr, Memory::NO_MEMORY, src_storage.size(), + 0, Memory::SYSTEM_MEM, src_storage.data()); + auto dst_mem = + std::make_unique(nullptr, Memory::NO_MEMORY, dst_storage.size(), + 0, Memory::SYSTEM_MEM, dst_storage.data()); xfer_desc->input_ports.resize(1); auto &input_port = xfer_desc->input_ports[0]; @@ -251,17 +247,17 @@ namespace { TEST(MemcpyMultiField, OneDim_TwoFields_Sequential) { run_multi_field_case<1, int>(Rect<1, int>(0, 63), - /*src_fids=*/{0, 1}, - /*dst_fids=*/{0, 1}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{0, 1}, + /*dst_fids=*/{0, 1}, + /*field_size=*/sizeof(int)); } TEST(MemcpyMultiField, OneDim_FourFields_Sequential) { run_multi_field_case<1, int>(Rect<1, int>(0, 255), - /*src_fids=*/{0, 1, 2, 3}, - /*dst_fids=*/{0, 1, 2, 3}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{0, 1, 2, 3}, + /*dst_fids=*/{0, 1, 2, 3}, + /*field_size=*/sizeof(int)); } TEST(MemcpyMultiField, OneDim_FieldReorder) @@ -269,9 +265,9 @@ TEST(MemcpyMultiField, OneDim_FieldReorder) // src reads fields 0..3, dst writes into fields 4..7 (reordered). // Confirms per-field offsets on src and dst can differ. run_multi_field_case<1, int>(Rect<1, int>(0, 127), - /*src_fids=*/{0, 1, 2, 3}, - /*dst_fids=*/{4, 5, 6, 7}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{0, 1, 2, 3}, + /*dst_fids=*/{4, 5, 6, 7}, + /*field_size=*/sizeof(int)); } TEST(MemcpyMultiField, OneDim_FieldPermutation) @@ -280,9 +276,9 @@ TEST(MemcpyMultiField, OneDim_FieldPermutation) // offset pairing tracks src_fields[k] <-> dst_fields[k] rather than // assuming both arrays are sorted. run_multi_field_case<1, int>(Rect<1, int>(0, 63), - /*src_fids=*/{3, 1, 0, 2}, - /*dst_fids=*/{0, 2, 3, 1}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{3, 1, 0, 2}, + /*dst_fids=*/{0, 2, 3, 1}, + /*field_size=*/sizeof(int)); } TEST(MemcpyMultiField, OneDim_ManyFields) @@ -293,9 +289,9 @@ TEST(MemcpyMultiField, OneDim_ManyFields) std::vector fids(1024); std::iota(fids.begin(), fids.end(), 0); run_multi_field_case<1, int>(Rect<1, int>(0, 15), - /*src_fids=*/fids, - /*dst_fids=*/fids, - /*field_size=*/sizeof(int)); + /*src_fids=*/fids, + /*dst_fids=*/fids, + /*field_size=*/sizeof(int)); } // ---- 2D cases ---- @@ -303,17 +299,17 @@ TEST(MemcpyMultiField, OneDim_ManyFields) TEST(MemcpyMultiField, TwoDim_FourFields) { run_multi_field_case<2, int>(Rect<2, int>({0, 0}, {15, 15}), - /*src_fids=*/{0, 1, 2, 3}, - /*dst_fids=*/{0, 1, 2, 3}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{0, 1, 2, 3}, + /*dst_fids=*/{0, 1, 2, 3}, + /*field_size=*/sizeof(int)); } TEST(MemcpyMultiField, TwoDim_FieldReorder) { run_multi_field_case<2, int>(Rect<2, int>({0, 0}, {31, 7}), - /*src_fids=*/{2, 0, 1}, - /*dst_fids=*/{0, 1, 2}, - /*field_size=*/sizeof(long long)); + /*src_fids=*/{2, 0, 1}, + /*dst_fids=*/{0, 1, 2}, + /*field_size=*/sizeof(long long)); } // ---- 3D cases ---- @@ -321,9 +317,9 @@ TEST(MemcpyMultiField, TwoDim_FieldReorder) TEST(MemcpyMultiField, ThreeDim_ThreeFields) { run_multi_field_case<3, int>(Rect<3, int>({0, 0, 0}, {7, 7, 7}), - /*src_fids=*/{0, 1, 2}, - /*dst_fids=*/{0, 1, 2}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{0, 1, 2}, + /*dst_fids=*/{0, 1, 2}, + /*field_size=*/sizeof(int)); } // ---- single-field sanity ---- @@ -335,7 +331,7 @@ TEST(MemcpyMultiField, OneDim_SingleField_LegacyPath) // throughout. Confirms the fast-path code does the right thing when // there's nothing to batch. run_multi_field_case<1, int>(Rect<1, int>(0, 63), - /*src_fids=*/{5}, - /*dst_fids=*/{7}, - /*field_size=*/sizeof(int)); + /*src_fids=*/{5}, + /*dst_fids=*/{7}, + /*field_size=*/sizeof(int)); } From 427a57cd8fabeb214a0324e0d79e99abb94574af Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 22 Apr 2026 01:53:09 -0700 Subject: [PATCH 3/3] test: fix os dependence for header include --- tests/multifield_transfer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/multifield_transfer.cc b/tests/multifield_transfer.cc index eda8312a117..ab7a76674ff 100644 --- a/tests/multifield_transfer.cc +++ b/tests/multifield_transfer.cc @@ -21,9 +21,10 @@ #include #include -#include #include +#include "osdep.h" + using namespace Realm; const size_t MAX_DIM = 2;