From 226670e62f1003c0cdccbf4119f962c3eac80125 Mon Sep 17 00:00:00 2001 From: Micha Dery Date: Tue, 2 Sep 2025 23:04:21 +0300 Subject: [PATCH 1/6] Address list operations and dimension utilities Co-authored-by: apriakhin --- src/realm/idx_impl.cc | 3 +- src/realm/inst_impl.cc | 6 +- src/realm/inst_layout.h | 5 +- src/realm/inst_layout.inl | 36 +- src/realm/repl_heap.h | 4 +- src/realm/transfer/address_list.cc | 269 +++++++---- src/realm/transfer/address_list.h | 104 +++- src/realm/transfer/lowlevel_dma.cc | 4 +- src/realm/transfer/transfer.cc | 25 +- src/realm/transfer/transfer_utils.h | 28 ++ src/realm/transfer/transfer_utils.inl | 49 ++ tests/unit_tests/address_list_test.cc | 603 ++++++++++++++++++++---- tests/unit_tests/transfer_utils_test.cc | 110 +++++ 13 files changed, 1002 insertions(+), 244 deletions(-) diff --git a/src/realm/idx_impl.cc b/src/realm/idx_impl.cc index 5ddba26c499..1506aa576c6 100644 --- a/src/realm/idx_impl.cc +++ b/src/realm/idx_impl.cc @@ -212,8 +212,7 @@ namespace Realm { if(!layout) return false; // dimension mismatch - std::map::const_iterator it = - layout->fields.find(fid); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(fid); if(it == layout->fields.end()) return false; // invalid field ID diff --git a/src/realm/inst_impl.cc b/src/realm/inst_impl.cc index 383beeed47a..d3469ff3c97 100644 --- a/src/realm/inst_impl.cc +++ b/src/realm/inst_impl.cc @@ -359,7 +359,7 @@ namespace Realm { // each piece list that's used will turn into a program std::map piece_list_starts; std::map *> piece_list_plans; - for(std::map::const_iterator it = fields.begin(); + for(InstanceLayoutGeneric::FieldMap::const_iterator it = fields.begin(); it != fields.end(); ++it) { // did we already do this piece list? if(piece_list_starts.count(it->second.list_idx) > 0) @@ -417,7 +417,7 @@ namespace Realm { } // fill in per field info - for(std::map::const_iterator it = fields.begin(); + for(InstanceLayoutGeneric::FieldMap::const_iterator it = fields.begin(); it != fields.end(); ++it) { PieceLookup::CompiledProgram::PerField &pf = p.fields[it->first]; pf.start_inst = reinterpret_cast( @@ -1523,7 +1523,7 @@ namespace Realm { assert(inst_layout != 0); // look up the right field - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(field_offset); assert(it != inst_layout->fields.end()); diff --git a/src/realm/inst_layout.h b/src/realm/inst_layout.h index 04bbf9fdf49..a67b535790b 100644 --- a/src/realm/inst_layout.h +++ b/src/realm/inst_layout.h @@ -225,7 +225,8 @@ namespace Realm { int size_in_bytes; }; - std::map fields; + using FieldMap = std::map; + FieldMap fields; }; REALM_PUBLIC_API @@ -714,7 +715,7 @@ namespace Realm { // protected: // friend // std::ostream& operator<<(std::ostream& os, const AffineAccessor& a); -//#define REALM_ACCESSOR_DEBUG +// #define REALM_ACCESSOR_DEBUG #if defined(REALM_ACCESSOR_DEBUG) || defined(REALM_USE_KOKKOS) Rect bounds; #endif diff --git a/src/realm/inst_layout.inl b/src/realm/inst_layout.inl index cdc9214d6cc..8717c440bb7 100644 --- a/src/realm/inst_layout.inl +++ b/src/realm/inst_layout.inl @@ -554,8 +554,7 @@ namespace Realm { { os << "Layout(bytes=" << bytes_used << ", align=" << alignment_reqd << ", fields={"; bool first = true; - for(std::map::const_iterator it = fields.begin(); - it != fields.end(); ++it) { + for(FieldMap::const_iterator it = fields.begin(); it != fields.end(); ++it) { if(!first) os << ", "; first = false; @@ -582,8 +581,7 @@ namespace Realm { inline size_t InstanceLayout::calculate_offset(Point p, FieldID fid) const { // first look up the field to see which piece list it uses (and get offset) - std::map::const_iterator it = - fields.find(fid); + FieldMap::const_iterator it = fields.find(fid); assert(it != fields.end()); const InstanceLayoutPiece *ilp = piece_lists[it->second.list_idx].find_piece(p); @@ -769,8 +767,7 @@ namespace Realm { // find the right piece list for our field const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); this->piece_list = &layout->piece_lists[it->second.list_idx]; @@ -789,8 +786,7 @@ namespace Realm { // find the right piece list for our field const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); this->piece_list = &layout->piece_lists[it->second.list_idx]; @@ -913,8 +909,7 @@ namespace Realm { { const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -963,8 +958,7 @@ namespace Realm { { const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -1020,8 +1014,7 @@ namespace Realm { // instance's dimensionality should be const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -1088,8 +1081,7 @@ namespace Realm { // instance's dimensionality should be const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -1146,8 +1138,7 @@ namespace Realm { { const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); if(it == layout->fields.end()) return false; const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -1173,8 +1164,7 @@ namespace Realm { { const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); if(it == layout->fields.end()) return false; const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -1209,8 +1199,7 @@ namespace Realm { // instance's dimensionality should be const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); if(it == layout->fields.end()) return false; const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; @@ -1238,8 +1227,7 @@ namespace Realm { // instance's dimensionality should be const InstanceLayout *layout = checked_cast *>(inst.get_layout()); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); if(it == layout->fields.end()) return false; const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; diff --git a/src/realm/repl_heap.h b/src/realm/repl_heap.h index 7617b7f7084..3c42a3d143c 100644 --- a/src/realm/repl_heap.h +++ b/src/realm/repl_heap.h @@ -36,8 +36,8 @@ namespace Realm { void cleanup(); // objects may be allocated and freed - void *alloc_obj(size_t bytes, size_t alignment); - void free_obj(void *ptr); + virtual void *alloc_obj(size_t bytes, size_t alignment); + virtual void free_obj(void *ptr); // writes to an object must be via the address at which is was allocated, // and be followed by a call to commit_writes for the appropriate address diff --git a/src/realm/transfer/address_list.cc b/src/realm/transfer/address_list.cc index 269ecdc54a0..e989f3469a6 100644 --- a/src/realm/transfer/address_list.cc +++ b/src/realm/transfer/address_list.cc @@ -26,75 +26,137 @@ namespace Realm { + // helpers + // ----------------------------------------------------------------------------------------- + namespace detail { + inline size_t contig_bytes(const size_t *e) + { + return ((e[AddressList::SLOT_HEADER] >> AddressList::CONTIG_SHIFT)); + } + + inline int actdim(const size_t *e) + { + return int(e[AddressList::SLOT_HEADER] & AddressList::DIM_MASK); + } + + inline size_t count_index(int dim) { return AddressList::DIM_SLOTS * dim; } + + inline size_t stride_index(int dim) { return count_index(dim) + 1; } + } // namespace detail + //////////////////////////////////////////////////////////////////////// // // class AddressList // - AddressList::AddressList() - : total_bytes(0) - , write_pointer(0) - , read_pointer(0) + AddressList::AddressList(size_t _max_entries) + : max_entries(_max_entries) { - memset(data, 0, MAX_ENTRIES * sizeof(size_t)); + data.reserve(max_entries); } - size_t *AddressList::begin_nd_entry(int max_dim) + bool AddressList::append_entry( + int dims, size_t contig_bytes, size_t total_bytes, size_t base_offset, + const std::unordered_map> &count_strides, + bool wrap_around) { - size_t entries_needed = max_dim * 2; + size_t *entry = begin_entry(dims, wrap_around); - size_t new_wp = write_pointer + entries_needed; - if(new_wp > MAX_ENTRIES) { - // have to wrap around - if((read_pointer <= entries_needed) || (write_pointer < read_pointer)) - return 0; + if(entry == nullptr) { + return false; + } - // fill remaining entries with 0's so reader skips over them - while(write_pointer < MAX_ENTRIES) - data[write_pointer++] = 0; + entry[AddressList::SLOT_BASE] = base_offset; - write_pointer = 0; - } else { - // if the write pointer would cross over the read pointer, we have to wait - if((write_pointer < read_pointer) && (new_wp >= read_pointer)) - return 0; - - // special case: if the write pointer would wrap and read is at 0, that'd - // be a collision too - if((new_wp == MAX_ENTRIES) && (read_pointer == 0)) - return 0; + for(auto &[dim, count_stride] : count_strides) { + entry[detail::count_index(dim)] = count_stride.first; + entry[detail::stride_index(dim)] = count_stride.second; } - // all good - return a pointer to the first available entry - return (data + write_pointer); + entry[AddressList::SLOT_HEADER] = pack_entry_header(contig_bytes, dims); + commit_entry(dims, total_bytes); + return true; } - void AddressList::commit_nd_entry(int act_dim, size_t bytes) + size_t *AddressList::begin_entry(int max_dim, bool wrap_mode) { - size_t entries_used = act_dim * 2; + size_t entries_needed = detail::count_index(max_dim); - write_pointer += entries_used; - if(write_pointer >= MAX_ENTRIES) { - assert(write_pointer == MAX_ENTRIES); - write_pointer = 0; + if(wrap_mode) { + size_t new_wp = write_pointer + entries_needed; + if(new_wp > max_entries) { + if((read_pointer <= entries_needed) || (write_pointer < read_pointer)) + return nullptr; + + // fill remaining entries with 0's so reader skips + while(write_pointer < max_entries) + data[write_pointer++] = 0; + + write_pointer = 0; + new_wp = entries_needed; + } else { + if((write_pointer < read_pointer) && (new_wp >= read_pointer)) + return nullptr; + if((new_wp == max_entries) && (read_pointer == 0)) + return nullptr; + } + + // ensure capacity upfront for max_entries once + if(data.size() < max_entries) + data.resize(max_entries); + + return data.data() + write_pointer; + } else { + if(data.size() < write_pointer + entries_needed) + data.resize(write_pointer + entries_needed); + return data.data() + write_pointer; } + } + + void AddressList::commit_entry(int act_dim, size_t bytes) + { + size_t entries_used = detail::count_index(act_dim); + write_pointer += entries_used; + total_bytes += bytes * (field_block ? field_block->count : 1); + } - total_bytes += bytes; + void AddressList::attach_field_block(const FieldBlock *_field_block) + { + field_block = _field_block; } size_t AddressList::bytes_pending() const { return total_bytes; } + size_t AddressList::full_field_bytes() + { + const size_t *entry = read_entry(); + // decode header + const size_t contig = detail::contig_bytes(entry); + const int dims = detail::actdim(entry); + size_t bytes = contig; + for(int d = 1; d < dims; d++) { + bytes *= entry[detail::count_index(d)]; + } + return bytes; + } + + size_t AddressList::pack_entry_header(size_t contig_bytes, int dims) + { + return (contig_bytes << CONTIG_SHIFT) | (dims & DIM_MASK); + } + const size_t *AddressList::read_entry() { - assert(total_bytes > 0); - if(read_pointer >= MAX_ENTRIES) { - assert(read_pointer == MAX_ENTRIES); + // assert(total_bytes > 0); + if(read_pointer >= max_entries) { + assert(read_pointer == max_entries); read_pointer = 0; } + // skip trailing 0's if(data[read_pointer] == 0) read_pointer = 0; - return (data + read_pointer); + return (data.data() + read_pointer); } //////////////////////////////////////////////////////////////////////// @@ -102,14 +164,7 @@ namespace Realm { // class AddressListCursor // - AddressListCursor::AddressListCursor() - : addrlist(0) - , partial(false) - , partial_dim(0) - { - for(int i = 0; i < MAX_DIM; i++) - pos[i] = 0; - } + AddressListCursor::AddressListCursor() { pos.fill(0); } void AddressListCursor::set_addrlist(AddressList *_addrlist) { addrlist = _addrlist; } @@ -120,17 +175,15 @@ namespace Realm { if(partial) { return (partial_dim + 1); } else { - const size_t *entry = addrlist->read_entry(); - int act_dim = (entry[0] & 15); - return act_dim; + return detail::actdim(addrlist->read_entry()); } } uintptr_t AddressListCursor::get_offset() const { const size_t *entry = addrlist->read_entry(); - int act_dim = (entry[0] & 15); - uintptr_t ofs = entry[1]; + int act_dim = detail::actdim(entry); + uintptr_t ofs = entry[AddressList::SLOT_BASE]; if(partial) { for(int i = partial_dim; i < act_dim; i++) if(i == 0) { @@ -138,7 +191,7 @@ namespace Realm { ofs += pos[0]; } else { // rest use the strides from the address list - ofs += pos[i] * entry[1 + (2 * i)]; + ofs += pos[i] * entry[detail::stride_index(i)]; } } return ofs; @@ -147,19 +200,22 @@ namespace Realm { uintptr_t AddressListCursor::get_stride(int dim) const { const size_t *entry = addrlist->read_entry(); - int act_dim = (entry[0] & 15); + int act_dim = detail::actdim(entry); assert((dim > 0) && (dim < act_dim)); - return entry[2 * dim + 1]; + return entry[detail::stride_index(dim)]; } size_t AddressListCursor::remaining(int dim) const { const size_t *entry = addrlist->read_entry(); - int act_dim = (entry[0] & 15); + int act_dim = detail::actdim(entry); assert(dim < act_dim); - size_t r = entry[2 * dim]; - if(dim == 0) - r >>= 4; + size_t r = entry[detail::count_index(dim)]; + + if(dim == 0) { + r >>= AddressList::CONTIG_SHIFT; + } + if(partial) { if(dim > partial_dim) r = 1; @@ -171,58 +227,89 @@ namespace Realm { return r; } - void AddressListCursor::advance(int dim, size_t amount) + void AddressListCursor::advance(int dim, size_t amount, int f) { const size_t *entry = addrlist->read_entry(); - int act_dim = (entry[0] & 15); + int act_dim = detail::actdim(entry); assert(dim < act_dim); - size_t r = entry[2 * dim]; - if(dim == 0) - r >>= 4; + // size of this "slice" in dim + size_t r = entry[detail::count_index(dim)]; + if(dim == 0) { + r >>= AddressList::CONTIG_SHIFT; + } + + // compute how many bytes we're really removing size_t bytes = amount; if(dim > 0) { #ifdef DEBUG_REALM for(int i = 0; i < dim; i++) assert(pos[i] == 0); #endif - bytes *= (entry[0] >> 4); + bytes *= detail::contig_bytes(entry); for(int i = 1; i < dim; i++) - bytes *= entry[2 * i]; + bytes *= entry[detail::count_index(i)]; } + #ifdef DEBUG_REALM - assert(addrlist->total_bytes >= bytes); + assert(addrlist->total_bytes >= bytes * f); #endif - addrlist->total_bytes -= bytes; + addrlist->total_bytes -= bytes * f; - if(!partial) { - if((dim == (act_dim - 1)) && (amount == r)) { - // simple case - we consumed the whole thing - addrlist->read_pointer += 2 * act_dim; - return; + const FieldBlock *fields = field_block(); + + // ——— NEW: if this call exactly finishes *one* rect (the last dim) + if(dim == (act_dim - 1) && amount == r) { + if(fields && f > 0) { + // bump fields only on a full-rect consume + partial_fields += f; + if(partial_fields >= fields->count) { + partial_fields = 0; + addrlist->read_pointer += detail::count_index(act_dim); + } } else { - // record partial consumption - partial = true; - partial_dim = dim; - pos[partial_dim] = amount; + // no fields at all: consume entry immediately + addrlist->read_pointer += detail::count_index(act_dim); } + // reset any in-flight partial state + partial = false; + partial_dim = 0; + pos.fill(0); + return; + } + + // ——— otherwise fall back to the existing "partial" logic + if(!partial) { + partial = true; + partial_dim = dim; + pos[dim] = amount; } else { - // update a partial consumption in progress assert(dim <= partial_dim); partial_dim = dim; - pos[partial_dim] += amount; + pos[dim] += amount; } while(pos[partial_dim] == r) { pos[partial_dim++] = 0; + if(partial_dim == act_dim) { - // all done + // we have finished the rect described by this entry partial = false; - addrlist->read_pointer += 2 * act_dim; + + if(fields && (f > 0)) { + partial_fields += f; + if(partial_fields >= fields->count) { + partial_fields = 0; + addrlist->read_pointer += detail::count_index(act_dim); + } + } else { + addrlist->read_pointer += detail::count_index(act_dim); + } break; } else { - pos[partial_dim]++; // carry into next dimension - r = entry[2 * partial_dim]; // no shift because partial_dim > 0 + // carry into the next higher dimension + pos[partial_dim]++; // increment that dimension + r = entry[detail::count_index(partial_dim)]; } } } @@ -262,6 +349,24 @@ namespace Realm { } } + const FieldBlock *AddressListCursor::field_block() const + { + return addrlist->field_block; + } + + const FieldID *AddressListCursor::fields_data() const + { + return addrlist->field_block->fields + partial_fields; + } + + size_t AddressListCursor::remaining_fields() const + { + if(addrlist->field_block) { + return addrlist->field_block->count - partial_fields; + } + return 1; + } + std::ostream &operator<<(std::ostream &os, const AddressListCursor &alc) { os << alc.remaining(0); diff --git a/src/realm/transfer/address_list.h b/src/realm/transfer/address_list.h index 6a722622adf..d702ccdf936 100644 --- a/src/realm/transfer/address_list.h +++ b/src/realm/transfer/address_list.h @@ -19,54 +19,112 @@ #define ADDRESS_LIST #include "realm/realm_config.h" +#include "realm/indexspace.h" #include "realm/id.h" +#include +#include +#include + namespace Realm { + template + struct FieldBlockBase { + std::size_t count; + FieldID fields[1]; + + // allocate a FieldBlock via heap.alloc_obj and store n field IDs + template + static FieldBlockBase *create(Heap &heap, const FieldID *src, size_t n, + size_t align = 16) + { + const size_t bytes = sizeof(FieldBlockBase) + (n - 1) * sizeof(FieldID); + void *mem = heap.alloc_obj(bytes, align); + FieldBlockBase *field_block = new(mem) FieldBlockBase; + field_block->count = n; + std::copy_n(src, n, field_block->fields); + return field_block; + } + }; + + using FieldBlock = FieldBlockBase; + + // ================================================================================================= + // AddressList + // ================================================================================================= class AddressList { public: - AddressList(); + AddressList(size_t _max_entries = 1000); + + // ─── entry construction ────────────────────────────────────────────────────── + [[nodiscard]] bool + append_entry(int dims, size_t contig_bytes, size_t total_bytes, size_t base_offset, + const std::unordered_map> &count_strides, + bool wrap_around = false); - size_t *begin_nd_entry(int max_dim); - void commit_nd_entry(int act_dim, size_t bytes); + [[nodiscard]] size_t *begin_entry(int max_dim, bool wrap_around = true); + void commit_entry(int act_dim, size_t bytes); + void attach_field_block(const FieldBlock *_field_block); - size_t bytes_pending() const; + [[nodiscard]] size_t bytes_pending() const; + [[nodiscard]] size_t full_field_bytes(); + + // entry packs: + // the contiguous byte count (contig_bytes) in the upper bitsthe + // the actual dimension count (act_dim) in the lower 4 bits + [[nodiscard]] static size_t pack_entry_header(size_t contig_bytes, int dims); + + // ─── layout constants ─────────────────────────────────────────────────────── + static constexpr size_t SLOT_HEADER = 0; + static constexpr size_t SLOT_BASE = 1; + static constexpr size_t DIM_SLOTS = 2; + static constexpr size_t DIM_MASK = 0xF; + static constexpr size_t CONTIG_SHIFT = 4; protected: friend class AddressListCursor; + [[nodiscard]] const size_t *read_entry(); - const size_t *read_entry(); + const FieldBlock *field_block{nullptr}; - size_t total_bytes; - unsigned write_pointer; - unsigned read_pointer; - static const size_t MAX_ENTRIES = 1000; - size_t data[MAX_ENTRIES]; + size_t total_bytes{0}; + size_t write_pointer{0}; + size_t read_pointer{0}; + size_t max_entries{0}; + std::vector data; }; + // ================================================================================================= + // AddressListCursor + // ================================================================================================= class AddressListCursor { public: AddressListCursor(); void set_addrlist(AddressList *_addrlist); - int get_dim() const; - uintptr_t get_offset() const; - uintptr_t get_stride(int dim) const; - size_t remaining(int dim) const; - void advance(int dim, size_t amount); + // ─── layout accessors ────────────────────────────────────────────────────── + [[nodiscard]] int get_dim() const; + [[nodiscard]] uintptr_t get_offset() const; + [[nodiscard]] uintptr_t get_stride(int dim) const; + [[nodiscard]] size_t remaining(int dim) const; + // ─── progress─────────────────────────────────────────────────────────────── + void advance(int dim, size_t amount, int f = 1); void skip_bytes(size_t bytes); + // ─── field accessors ────────────────────────────────────────────────────── + [[nodiscard]] const FieldBlock *field_block() const; + [[nodiscard]] const FieldID *fields_data() const; + [[nodiscard]] size_t remaining_fields() const; + + AddressList *addrlist{nullptr}; + bool partial{false}; // inside a dimension + protected: - AddressList *addrlist; - bool partial; - // we need to be one larger than any index space realm supports, since - // we use the contiguous bytes within a field as a "dimension" in some - // cases - static const int MAX_DIM = REALM_MAX_DIM + 1; - int partial_dim; - size_t pos[MAX_DIM]; + int partial_dim{0}; // dimension index + size_t partial_fields{0}; + std::array pos{}; }; std::ostream &operator<<(std::ostream &os, const AddressListCursor &alc); diff --git a/src/realm/transfer/lowlevel_dma.cc b/src/realm/transfer/lowlevel_dma.cc index 3d9c4ad8c79..c2495f4925d 100644 --- a/src/realm/transfer/lowlevel_dma.cc +++ b/src/realm/transfer/lowlevel_dma.cc @@ -857,7 +857,7 @@ namespace Realm { // the same location twice) size_t lines = std::max((1 << 30) / size, 1); int dim = (lines > 1) ? 2 : 1; - size_t *data = addrlist.begin_nd_entry(dim); + size_t *data = addrlist.begin_entry(dim); if(!data) return true; // can't add more until some is consumed @@ -868,7 +868,7 @@ namespace Realm { data[2] = lines; data[3] = 0; // stride } - addrlist.commit_nd_entry(dim, size * lines); + addrlist.commit_entry(dim, size * lines); return false; // we can add more if asked } diff --git a/src/realm/transfer/transfer.cc b/src/realm/transfer/transfer.cc index 515983f4169..54020a174fb 100644 --- a/src/realm/transfer/transfer.cc +++ b/src/realm/transfer/transfer.cc @@ -177,7 +177,7 @@ namespace Realm { { size_t field_rel_offset; { - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(field_id); assert(it != inst_layout->fields.end()); assert((field_offset + field_size) <= size_t(it->second.size_in_bytes)); @@ -218,7 +218,7 @@ namespace Realm { checked_cast *>(inst_impl->metadata.layout); { - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(cur_field_id); assert(it != inst_layout->fields.end()); assert((cur_field_offset + cur_field_size) <= size_t(it->second.size_in_bytes)); @@ -369,7 +369,7 @@ namespace Realm { const InstanceLayoutPiece *layout_piece; // int field_rel_offset; { - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(cur_field_id); assert(it != inst_layout->fields.end()); assert((cur_field_offset == 0) && @@ -523,7 +523,7 @@ namespace Realm { // we may be able to compact dimensions, but ask for space to write a // an address record of the maximum possible dimension (i.e. N) - size_t *addr_data = addrlist.begin_nd_entry(N); + size_t *addr_data = addrlist.begin_entry(N); if(!addr_data) { return true; // out of space for now } @@ -532,7 +532,7 @@ namespace Realm { const InstanceLayoutPiece *layout_piece; size_t field_rel_offset; { - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(cur_field_id); assert(it != inst_layout->fields.end()); assert((cur_field_offset + cur_field_size) <= size_t(it->second.size_in_bytes)); @@ -572,6 +572,7 @@ namespace Realm { #endif // TODO: remove now-redundant condition here + // TODO: use compact_affine_dims if(layout_piece->layout_type == PieceLayoutTypes::AffineLayoutType) { const AffineLayoutPiece *affine = static_cast *>(layout_piece); @@ -626,6 +627,7 @@ namespace Realm { addr_data[cur_dim * 2] = total_count; addr_data[cur_dim * 2 + 1] = stride; + log_dma.debug() << "Add addr data dim=" << cur_dim << " total_count=" << total_count << " stride=" << stride; total_bytes *= total_count; @@ -635,7 +637,7 @@ namespace Realm { // now that we know the compacted dimension, we can finish the address // record addr_data[0] = (bytes << 4) + cur_dim; - addrlist.commit_nd_entry(cur_dim, total_bytes); + addrlist.commit_entry(cur_dim, total_bytes); log_dma.debug() << "Finalize addr data dim=" << cur_dim << " total_bytes" << total_bytes; } else { @@ -973,7 +975,7 @@ namespace Realm { checked_cast *>(this->inst_impl->metadata.layout); assert(inst_layout); - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(cur_field_id); assert(it != inst_layout->fields.end()); size_t pieces = inst_layout->piece_lists[it->second.list_idx].pieces.size(); @@ -1020,7 +1022,7 @@ namespace Realm { return false; } - size_t *addr_data = addrlist.begin_nd_entry(1); + size_t *addr_data = addrlist.begin_entry(1); if(!addr_data) { return true; } @@ -1029,7 +1031,7 @@ namespace Realm { size_t total_bytes = this->cur_rect.volume() * this->cur_field_size; this->have_rect = false; addr_data[0] = ((total_bytes) << 4) + cur_dim; - addrlist.commit_nd_entry(cur_dim, total_bytes); + addrlist.commit_entry(cur_dim, total_bytes); log_dma.debug() << "Finalize gather/scatter addr data dim=" << cur_dim << " total_bytes=" << total_bytes; break; @@ -1770,8 +1772,7 @@ namespace Realm { assert(impl->metadata.is_valid()); const InstanceLayout *layout = checked_cast *>(impl->metadata.layout); - std::map::const_iterator it = - layout->fields.find(field_id); + InstanceLayoutGeneric::FieldMap::const_iterator it = layout->fields.find(field_id); assert(it != layout->fields.end()); const InstancePieceList &ipl = layout->piece_lists[it->second.list_idx]; std::vector preferred; @@ -1930,7 +1931,7 @@ namespace Realm { const InstancePieceList *ipl; { - std::map::const_iterator it = + InstanceLayoutGeneric::FieldMap::const_iterator it = inst_layout->fields.find(fid); assert(it != inst_layout->fields.end()); ipl = &inst_layout->piece_lists[it->second.list_idx]; diff --git a/src/realm/transfer/transfer_utils.h b/src/realm/transfer/transfer_utils.h index 22749fe0ba0..69aaab3c2f0 100644 --- a/src/realm/transfer/transfer_utils.h +++ b/src/realm/transfer/transfer_utils.h @@ -19,6 +19,8 @@ #define REALM_TRANSFER_UTILS_H #include "realm/point.h" +#include "realm/inst_layout.h" +#include namespace Realm { // finds the largest subrectangle of 'domain' that starts with 'start', @@ -48,6 +50,32 @@ namespace Realm { Point &cur_point, Rect &target_subrect, const int *dim_order); + /** + * @brief Computes compact affine addressing information for a subrectangle. + * + * Given an affine layout piece and a target subrectangle, this function determines + * the base offset, total number of bytes, and dimension stride/count pairs for + * efficient memory access. It compactly represents contiguous dimensions and + * generates stride-based indexing information for the remaining ones. + * + * @tparam N Number of dimensions in the layout. + * @tparam T Coordinate type (e.g., int, long). + * + * @param affine Pointer to the affine layout piece being queried. + * @param subrect Subrectangle to compute addressing for. + * @param dim_order Ordering of dimensions to compact/iterate over. + * @param[out] total_bytes Total bytes represented by the subrect. + * @param[out] contig_bytes Number of contiguous bytes in the compacted dimensions. + * @param[out] count_strides Output stride/count pairs indexed by dimension. + * count_strides[i][0] = count, count_strides[i][1] = stride. + */ + template + inline int + compact_affine_dims(const AffineLayoutPiece *affine, const Rect &subrect, + const int dim_order[N], size_t field_size, size_t &total_bytes, + size_t &contig_bytes, + std::unordered_map> &count_strides); + } // namespace Realm #include "realm/transfer/transfer_utils.inl" diff --git a/src/realm/transfer/transfer_utils.inl b/src/realm/transfer/transfer_utils.inl index 0262b52ca84..b09486430b4 100644 --- a/src/realm/transfer/transfer_utils.inl +++ b/src/realm/transfer/transfer_utils.inl @@ -143,4 +143,53 @@ namespace Realm { return have_rect; } + + template + inline int + compact_affine_dims(const AffineLayoutPiece *affine, const Rect &subrect, + const int dim_order[N], size_t field_size, size_t &total_bytes, + size_t &contig_bytes, + std::unordered_map> &count_strides) + { + size_t bytes = field_size; + int cur_dim = 1; + int di = 0; + + for(; di < N; di++) { + int d = dim_order[di]; + if(subrect.lo[d] == subrect.hi[d]) + continue; + if(affine->strides[d] != bytes) + break; + bytes *= (subrect.hi[d] - subrect.lo[d] + 1); + } + + contig_bytes = bytes; + total_bytes = bytes; + + for(; di < N;) { + int d = dim_order[di]; + size_t stride = affine->strides[d]; + size_t total_count = 1; + + for(; di < N; di++) { + d = dim_order[di]; + if(subrect.lo[d] == subrect.hi[d]) + continue; + + size_t count = (subrect.hi[d] - subrect.lo[d] + 1); + if(affine->strides[d] != (stride * total_count)) + break; + + total_count *= count; + } + + count_strides[cur_dim] = {total_count, stride}; + + total_bytes *= total_count; + cur_dim++; + } + return cur_dim; + // ndims_out = cur_dim; + } } // namespace Realm diff --git a/tests/unit_tests/address_list_test.cc b/tests/unit_tests/address_list_test.cc index 8cad3ecd3c0..cc943f940b5 100644 --- a/tests/unit_tests/address_list_test.cc +++ b/tests/unit_tests/address_list_test.cc @@ -16,99 +16,518 @@ */ #include "realm/transfer/address_list.h" - -#include #include +#include using namespace Realm; -TEST(AddressListTestsWithParams, Create1DEntry) -{ - const size_t dim = 1; - const size_t stride = 8; - const size_t bytes = 1024; - assert(stride <= bytes); - - AddressList addrlist; - size_t *addr_data = addrlist.begin_nd_entry(dim); - addr_data[0] = (bytes << 4) + dim; - addrlist.commit_nd_entry(dim, bytes); - - AddressListCursor addrcursor; - addrcursor.set_addrlist(&addrlist); - - EXPECT_EQ(addrcursor.remaining(dim - 1), bytes); - EXPECT_EQ(addrcursor.get_dim(), dim); - EXPECT_EQ(addrcursor.get_offset(), 0); - - addrcursor.advance(dim - 1, stride); - EXPECT_EQ(addrcursor.remaining(dim - 1), bytes - stride); - - addrcursor.skip_bytes(stride); - EXPECT_EQ(addrcursor.remaining(dim - 1), bytes - 2 * stride); - - addrcursor.advance(dim - 1, bytes - 2 * stride); - EXPECT_EQ(addrlist.bytes_pending(), 0); -} - -TEST(AddressListTestsWithParams, Create3DEntry) -{ - AddressList addrlist; - // TODO(apryakhin): parameterize dimensions - const size_t dim = 3; - const size_t stride = 8; - const size_t bytes = 1024; - const std::vector strides{stride, stride * stride, stride * stride * stride}; - size_t *addr_data = addrlist.begin_nd_entry(dim); - - size_t cur_dim = 1; - for(size_t i = 0; i < strides.size() - 1; i++) { - addr_data[cur_dim * 2] = bytes / strides[i]; - addr_data[cur_dim * 2 + 1] = strides[i]; - cur_dim++; - } - - addr_data[0] = (strides[0] << 4) + cur_dim; - addrlist.commit_nd_entry(cur_dim, bytes); - - AddressListCursor addrcursor; - addrcursor.set_addrlist(&addrlist); - EXPECT_EQ(addrcursor.remaining(dim - 1), bytes / strides[1]); - EXPECT_EQ(addrcursor.remaining(dim - 2), bytes / strides[0]); - EXPECT_EQ(addrcursor.remaining(dim - 3), strides[0]); - - addrcursor.advance(dim - 1, 1); - EXPECT_EQ(addrlist.bytes_pending(), 0); -} - -TEST(AddressListTestsWithParams, CommitMax1DEntries) -{ - const size_t dim = 1; - const size_t stride = 8; - const size_t bytes = 1024; - const size_t max_entries = 499; - assert(stride <= bytes); - - AddressList addrlist; - for(size_t i = 0; i < max_entries; i++) { - size_t *addr_data = addrlist.begin_nd_entry(dim); - EXPECT_NE(addr_data, nullptr); - addr_data[0] = (bytes << 4) + dim; - addrlist.commit_nd_entry(dim, bytes); - } - - AddressListCursor addrcursor; - addrcursor.set_addrlist(&addrlist); - - EXPECT_EQ(addrcursor.remaining(dim - 1), bytes); - EXPECT_EQ(addrcursor.get_dim(), dim); - - EXPECT_EQ(addrcursor.get_offset(), 0); - - addrcursor.advance(dim - 1, stride); - EXPECT_EQ(addrcursor.remaining(dim - 1), bytes - stride); - - // advance rest of first entry - addrcursor.advance(dim - 1, bytes - stride); - EXPECT_EQ(addrlist.bytes_pending(), bytes * (max_entries - 1)); -} +namespace { + + constexpr size_t kStride = 8; + constexpr size_t kBytes = 1024; + + struct MockHeap { + void *alloc_obj(std::size_t bytes, std::size_t align = 16) + { + void *ptr = nullptr; +#ifdef REALM_ON_WINDOWS + ptr = _aligned_malloc(bytes, align); +#else + int ret = posix_memalign(&ptr, align, bytes); + if(ret != 0) + ptr = nullptr; +#endif + assert(ptr != nullptr); + return ptr; + } + + void free_obj(void *ptr) + { +#ifdef REALM_ON_WINDOWS + _aligned_free(ptr); +#else + free(ptr); +#endif + } + }; + + static void make_1d_entry(AddressList &alist, size_t bytes, int payload = 0) + { + size_t *e = alist.begin_entry(1); + ASSERT_NE(e, nullptr); + e[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(bytes, 1); + alist.commit_entry(1, bytes); + } + + TEST(AddressListTests, AdvanceWithFieldsBasic) + { + AddressList addrlist; + + std::vector fields = {100, 101, 102, 103}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, fields.data(), fields.size()); + addrlist.attach_field_block(fb); + + size_t *entry = addrlist.begin_entry(1); + ASSERT_NE(entry, nullptr); + entry[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(kBytes, 1); + addrlist.commit_entry(1, kBytes); + + ASSERT_NE(fb, nullptr); + ASSERT_EQ(fb->count, fields.size()); + ASSERT_NE(fb->fields, nullptr); + + AddressListCursor cursor; + cursor.set_addrlist(&addrlist); + + EXPECT_EQ(addrlist.bytes_pending(), kBytes * fields.size()); + + ASSERT_EQ(cursor.remaining_fields(), fields.size()); + cursor.advance(0, 128, 1); + EXPECT_EQ(addrlist.bytes_pending(), kBytes * fields.size() - 128); + ASSERT_EQ(cursor.remaining_fields(), fields.size()); + + cursor.advance(0, 128, 1); + EXPECT_EQ(addrlist.bytes_pending(), kBytes * fields.size() - 256); + ASSERT_EQ(cursor.remaining_fields(), fields.size()); + + cursor.advance(0, kBytes - 256, 1); + EXPECT_EQ(addrlist.bytes_pending(), kBytes * (fields.size() - 1)); + ASSERT_EQ(cursor.remaining_fields(), fields.size() - 1); + + cursor.advance(0, kBytes, fields.size() - 1); + EXPECT_EQ(addrlist.bytes_pending(), 0); + // Fields should be reset once we finished the rect entry + ASSERT_EQ(cursor.remaining_fields(), fields.size()); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, WithFields_PartialDoesNotIncrementField) + { + AddressList al; + std::vector ids = {1, 2, 3, 4}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + make_1d_entry(al, kBytes, ids.size()); + + AddressListCursor cur; + cur.set_addrlist(&al); + + // small amt < full rect: field stays at 0 + cur.advance(0, 128, 1); + EXPECT_EQ(cur.fields_data(), fb->fields + 0); + EXPECT_EQ(al.bytes_pending(), kBytes * ids.size() - 128); + ASSERT_EQ(cur.remaining_fields(), ids.size()); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, WithFields_FullRectSingleField) + { + AddressList al; + std::vector ids = {1, 2, 3, 4}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + make_1d_entry(al, kBytes, ids.size()); + + AddressListCursor cur; + cur.set_addrlist(&al); + + // full rect for 1 field + cur.advance(0, kBytes, 1); + EXPECT_EQ(cur.fields_data(), fb->fields + 1); + // entry still pending until all 4 fields done + EXPECT_EQ(al.bytes_pending(), kBytes * ids.size() - kBytes); + ASSERT_EQ(cur.remaining_fields(), ids.size() - 1); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, WithFields_FullRectMultipleFieldsAtOnce) + { + AddressList al; + std::vector ids = {1, 2, 3, 4}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + make_1d_entry(al, kBytes, ids.size()); + + AddressListCursor cur; + cur.set_addrlist(&al); + + // consume 2 fields in one go + cur.advance(0, kBytes, 2); + EXPECT_EQ(cur.fields_data(), fb->fields + 2); + EXPECT_EQ(al.bytes_pending(), kBytes * ids.size() - 2 * kBytes); + ASSERT_EQ(cur.remaining_fields(), ids.size() - 2); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, WithFields_ConsumeAllFieldsAtOnce) + { + AddressList al; + std::vector ids = {1, 2, 3, 4}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + make_1d_entry(al, kBytes, ids.size()); + + AddressListCursor cur; + cur.set_addrlist(&al); + + // consume all fields => entry consumed + cur.advance(0, kBytes, ids.size()); + EXPECT_EQ(al.bytes_pending(), 0); + EXPECT_EQ(cur.fields_data(), fb->fields); + ASSERT_EQ(cur.remaining_fields(), ids.size()); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, WithFields_SequentialFullRect) + { + AddressList al; + std::vector ids = {1, 2, 3, 4}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + make_1d_entry(al, kBytes, ids.size()); + + AddressListCursor cur; + cur.set_addrlist(&al); + + // call advance(full,1) four times + for(int i = 0; i < (int)ids.size(); i++) { + EXPECT_EQ(cur.fields_data(), (i < (int)ids.size() ? fb->fields + i : fb->fields)); + cur.advance(0, kBytes, 1); + } + EXPECT_EQ(al.bytes_pending(), 0); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, MultiDim_WithFields) + { + AddressList al; + std::vector ids = {7, 8, 9}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + + size_t *entry = al.begin_entry(3); + entry[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(kBytes, 3); + entry[AddressList::SLOT_BASE] = 0; // base offset + entry[AddressList::DIM_SLOTS * 1] = 8; // dim1 count + entry[AddressList::DIM_SLOTS * 1 + 1] = 1024; // dim1 stride + entry[AddressList::DIM_SLOTS * 2] = 2; // dim2 count + entry[AddressList::DIM_SLOTS * 2 + 1] = 8192; // dim2 stride + const size_t volume = kBytes * 8 * 2; + al.commit_entry(3, volume); + + AddressListCursor cur; + cur.set_addrlist(&al); + + EXPECT_EQ(al.bytes_pending(), volume * ids.size()); + cur.advance(2, 2); + EXPECT_EQ(al.bytes_pending(), volume * (ids.size() - 1)); + + cur.advance(2, 2); + EXPECT_EQ(al.bytes_pending(), volume * (ids.size() - 2)); + + cur.advance(2, 2); + EXPECT_EQ(al.bytes_pending(), volume * (ids.size() - 3)); + + heap.free_obj(fb); + } + + TEST(AdvanceTests, MultiDim_WithFieldsSingleAdvance) + { + AddressList al; + std::vector ids = {7, 8, 9}; + MockHeap heap; + auto *fb = FieldBlock::create(heap, ids.data(), ids.size()); + al.attach_field_block(fb); + + size_t *entry = al.begin_entry(3); + entry[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(kBytes, 3); + entry[AddressList::SLOT_BASE] = 0; // base offset + entry[AddressList::DIM_SLOTS * 1] = 8; // dim1 count + entry[AddressList::DIM_SLOTS * 1 + 1] = 1024; // dim1 stride + entry[AddressList::DIM_SLOTS * 2] = 2; // dim2 count + entry[AddressList::DIM_SLOTS * 2 + 1] = 8192; // dim2 stride + const size_t volume = kBytes * 8 * 2; + al.commit_entry(3, volume); + + AddressListCursor cur; + cur.set_addrlist(&al); + + EXPECT_EQ(al.bytes_pending(), volume * ids.size()); + cur.advance(2, 2, ids.size()); + EXPECT_EQ(al.bytes_pending(), 0); + + heap.free_obj(fb); + } + + TEST(AddressListTests, Basic1DEntryNoPayload) + { + AddressList addrlist; + const int dim = 1; + + size_t *entry = addrlist.begin_entry(dim); + ASSERT_NE(entry, nullptr); + entry[0] = AddressList::pack_entry_header(kBytes, dim); + addrlist.commit_entry(dim, kBytes); + + AddressListCursor cursor; + cursor.set_addrlist(&addrlist); + + EXPECT_EQ(cursor.get_dim(), 1); + EXPECT_EQ(cursor.remaining(0), kBytes); + EXPECT_EQ(cursor.get_offset(), 0); + + cursor.advance(0, kStride); + EXPECT_EQ(cursor.remaining(0), kBytes - kStride); + + cursor.skip_bytes(kStride); + EXPECT_EQ(cursor.remaining(0), kBytes - 2 * kStride); + + cursor.advance(0, kBytes - 2 * kStride); + EXPECT_EQ(addrlist.bytes_pending(), 0); + } + + TEST(AddressListTests, Multiple1DEntries) + { + AddressList addrlist; + const size_t entries = 10; + + for(size_t i = 0; i < entries; ++i) { + size_t *entry = addrlist.begin_entry(1); + ASSERT_NE(entry, nullptr); + entry[0] = AddressList::pack_entry_header(kBytes, 1); + addrlist.commit_entry(1, kBytes); + } + + EXPECT_EQ(addrlist.bytes_pending(), entries * kBytes); + + AddressListCursor cursor; + cursor.set_addrlist(&addrlist); + for(size_t i = 0; i < entries; ++i) { + EXPECT_EQ(cursor.remaining(0), kBytes); + cursor.advance(0, kBytes); + } + EXPECT_EQ(addrlist.bytes_pending(), 0); + } + + TEST(AddressListTests, AppendComplex3DEntry) + { + AddressList addrlist; + + std::unordered_map> count_strides; + count_strides[1] = {8, 1024}; + count_strides[2] = {2, 8192}; + bool commited = addrlist.append_entry(3, kBytes, kBytes * 8 * 2, /*base_offset=*/16, + count_strides); + ASSERT_TRUE(commited); + + AddressListCursor cursor; + cursor.set_addrlist(&addrlist); + + EXPECT_EQ(cursor.remaining(0), kBytes); + EXPECT_EQ(cursor.remaining(1), 8); + EXPECT_EQ(cursor.remaining(2), 2); + EXPECT_EQ(cursor.get_offset(), 16); + + cursor.advance(2, 2); + EXPECT_EQ(addrlist.bytes_pending(), 0); + } + + TEST(AddressListTests, Complex3DEntry) + { + AddressList addrlist; + + size_t *entry = addrlist.begin_entry(3); + entry[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(kBytes, 3); + entry[AddressList::SLOT_BASE] = 0; // base offset + entry[AddressList::DIM_SLOTS * 1] = 8; // dim1 count + entry[AddressList::DIM_SLOTS * 1 + 1] = 1024; // dim1 stride + entry[AddressList::DIM_SLOTS * 2] = 2; // dim2 count + entry[AddressList::DIM_SLOTS * 2 + 1] = 8192; // dim2 stride + const size_t volume = kBytes * 8 * 2; + addrlist.commit_entry(3, volume); + + AddressListCursor cursor; + cursor.set_addrlist(&addrlist); + + EXPECT_EQ(cursor.remaining(0), kBytes); + EXPECT_EQ(cursor.remaining(1), 8); + EXPECT_EQ(cursor.remaining(2), 2); + EXPECT_EQ(cursor.get_offset(), 0); + + cursor.advance(2, 2); + EXPECT_EQ(addrlist.bytes_pending(), 0); + } + + TEST(AddressListTests, WraparoundBufferSafety) + { + const size_t max_entries = 16; + AddressList addrlist(max_entries); + + size_t successful = 0; + while(true) { + size_t *entry = addrlist.begin_entry(1); + if(!entry) + break; + entry[0] = AddressList::pack_entry_header(kBytes, 1); + addrlist.commit_entry(1, kBytes); + successful++; + } + + EXPECT_GT(successful, 0); + EXPECT_LE(successful, max_entries); + + AddressListCursor cursor; + cursor.set_addrlist(&addrlist); + for(size_t i = 0; i < successful; ++i) { + cursor.advance(0, kBytes); + } + EXPECT_EQ(addrlist.bytes_pending(), 0); + } + + TEST(AddressListTests, FullFieldBytes1D) + { + AddressList addrlist; + + // Create a simple 1-D entry with kBytes contiguous bytes + size_t *entry = addrlist.begin_entry(1); + ASSERT_NE(entry, nullptr); + entry[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(kBytes, 1); + addrlist.commit_entry(1, kBytes); + + // full_field_bytes should return the contiguous byte count for 1-D entries + EXPECT_EQ(addrlist.full_field_bytes(), kBytes); + } + + TEST(AddressListTests, FullFieldBytes3D) + { + AddressList addrlist; + + // Build a 3-D entry: dim0 has kBytes contiguous bytes, dim1 has 8 elements, + // and dim2 has 2 elements. The expected volume (per field) is kBytes * 8 * 2. + size_t *entry = addrlist.begin_entry(3); + ASSERT_NE(entry, nullptr); + entry[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(kBytes, 3); + entry[AddressList::SLOT_BASE] = 0; // base offset (unused by full_field_bytes) + entry[AddressList::DIM_SLOTS * 1] = 8; // dim1 count + entry[AddressList::DIM_SLOTS * 1 + 1] = + 1024; // dim1 stride (unused by full_field_bytes) + entry[AddressList::DIM_SLOTS * 2] = 2; // dim2 count + entry[AddressList::DIM_SLOTS * 2 + 1] = + 8192; // dim2 stride (unused by full_field_bytes) + const size_t expected_volume = kBytes * 8 * 2; + addrlist.commit_entry(3, expected_volume); + + // full_field_bytes should compute the volume across all dimensions + EXPECT_EQ(addrlist.full_field_bytes(), expected_volume); + } + + TEST(AddressListTests, WithFields_2D_PartialLine_NoFieldAdvance) + { + constexpr size_t NUM_BYTES = 64; + constexpr size_t NUM_LINES = 3; + AddressList al; + std::vector f = {1, 2, 3}; + MockHeap h; + FieldBlock *fb = FieldBlock::create(h, f.data(), f.size()); + al.attach_field_block(fb); + + size_t *e = al.begin_entry(2); + e[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(NUM_BYTES, 2); + e[AddressList::DIM_SLOTS * 1] = NUM_LINES; + e[AddressList::DIM_SLOTS * 1 + 1] = NUM_BYTES; + al.commit_entry(2, NUM_BYTES * NUM_LINES); + + AddressListCursor c; + c.set_addrlist(&al); + + // Consume half a line – must NOT change field index + c.advance(0, NUM_BYTES / 2, 1); + EXPECT_EQ(c.fields_data(), fb->fields); + EXPECT_EQ(al.bytes_pending(), NUM_BYTES * NUM_LINES * f.size() - NUM_BYTES / 2); + + h.free_obj(fb); + } + + TEST(AddressListTests, WithFields_PartialThenFullRect) + { + constexpr size_t NUM_BYTES = 32; + AddressList al; + std::vector f = {0, 1, 2}; + MockHeap h; + FieldBlock *fb = FieldBlock::create(h, f.data(), f.size()); + al.attach_field_block(fb); + make_1d_entry(al, NUM_BYTES); + + AddressListCursor cur; + cur.set_addrlist(&al); + + // (1) move full rect for two fields at once + cur.advance(0, NUM_BYTES, 2); + EXPECT_EQ(cur.fields_data(), fb->fields + 2); + EXPECT_EQ(al.bytes_pending(), NUM_BYTES * (f.size() - 2)); + + // (2) move half-rect – must stay on field #2 + cur.advance(0, NUM_BYTES / 2, 1); + EXPECT_EQ(cur.fields_data(), fb->fields + 2); + EXPECT_EQ(al.bytes_pending(), NUM_BYTES * (f.size() - 2) - NUM_BYTES / 2); + + // (3) finish the rect – now field pointer wraps to beginning + cur.advance(0, NUM_BYTES / 2, 1); + EXPECT_EQ(cur.fields_data(), fb->fields); + EXPECT_EQ(al.bytes_pending(), 0u); + + h.free_obj(fb); + } + + TEST(AddressListTests, WithFields_3D_PlaneWisePartial) + { + constexpr size_t NUM_BYTES = 16; + constexpr size_t NUM_LINES = 2; + constexpr size_t NUM_PLANES = 3; + AddressList al; + std::vector f = {9, 9}; + MockHeap h; + FieldBlock *fb = FieldBlock::create(h, f.data(), f.size()); + al.attach_field_block(fb); + + size_t *e = al.begin_entry(3); + e[AddressList::SLOT_HEADER] = AddressList::pack_entry_header(NUM_BYTES, 3); + e[AddressList::DIM_SLOTS * 1] = NUM_LINES; + e[AddressList::DIM_SLOTS * 1 + 1] = NUM_BYTES; + e[AddressList::DIM_SLOTS * 2] = NUM_PLANES; + e[AddressList::DIM_SLOTS * 2 + 1] = NUM_BYTES * NUM_LINES; + al.commit_entry(3, NUM_BYTES * NUM_LINES * NUM_PLANES); + + AddressListCursor c; + c.set_addrlist(&al); + + // consume one plane at a time – still field 0 + for(size_t p = 0; p < NUM_PLANES; ++p) { + c.advance(2, 1); + } + + EXPECT_EQ(al.bytes_pending(), NUM_BYTES * NUM_LINES * NUM_PLANES * (f.size() - 1)); + EXPECT_EQ(c.fields_data(), fb->fields + 1); + + // now full rect for one field + c.advance(2, NUM_PLANES, 1); + EXPECT_EQ(al.bytes_pending(), 0u); + EXPECT_EQ(c.fields_data(), fb->fields); + + h.free_obj(fb); + } + +} // namespace diff --git a/tests/unit_tests/transfer_utils_test.cc b/tests/unit_tests/transfer_utils_test.cc index 2bd5bb20565..93e39abad24 100644 --- a/tests/unit_tests/transfer_utils_test.cc +++ b/tests/unit_tests/transfer_utils_test.cc @@ -283,3 +283,113 @@ std::vector allTestCases() INSTANTIATE_TEST_SUITE_P(TestAllDimensions, ComputeTargetSubrectTest, testing::ValuesIn(allTestCases())); + +// Helper to build an AffineLayoutPiece with given strides +template +static AffineLayoutPiece make_affine(const Point &strides) +{ + AffineLayoutPiece piece; + piece.strides = strides; + piece.offset = 0; + // set trivial bounds just to satisfy invariants (not used by function) + piece.bounds = Rect(Point::ZEROES(), Point::ONES()); + return piece; +} + +// Test 1: fully contiguous 2‑D row‑major layout +TEST(FlattenAffineDimensionsTest, Contiguous2D) +{ + constexpr int N = 2; + const size_t field_size = 4; // 4 bytes per element + const int dim_order[N] = {0, 1}; + + /* layout: strides = {4, 16} (row major for 4 columns) + * subrect: 4 x 5 -> total 80 bytes are completely contiguous + */ + AffineLayoutPiece affine = + make_affine(Point<2, size_t>(field_size, field_size * 4)); + + Rect subrect(Point<2, int>(0, 0), Point<2, int>(3, 4)); // 4x5 rectangle + + size_t total_bytes = 0; + size_t contig_bytes = 0; + std::unordered_map> count_strides; + + int ndims = compact_affine_dims(&affine, subrect, dim_order, field_size, total_bytes, + contig_bytes, count_strides); + + EXPECT_EQ(ndims, 1); // flattened to one contiguous dim + EXPECT_EQ(contig_bytes, 4 * 4 * 5); // 80 + EXPECT_EQ(total_bytes, contig_bytes); + EXPECT_TRUE(count_strides.empty()); +} + +// Test 2: 2‑D layout with padding between rows (non‑contiguous second dim) +TEST(FlattenAffineDimensionsTest, Padded2D) +{ + constexpr int N = 2; + const size_t field_size = 4; + const int dim_order[N] = {0, 1}; + + /* layout: stride0 = 4 (element) + * stride1 = 64 (padded row) + * subrect counts: 4 cols x 5 rows + * Expect: contiguous along dim0 only (16 bytes) + * then 5 rows via stride 64 => total 80 bytes + */ + AffineLayoutPiece affine = make_affine(Point<2, size_t>(field_size, 64)); + + Rect subrect(Point<2, int>(0, 0), Point<2, int>(3, 4)); + + size_t total_bytes = 0; + size_t contig_bytes = 0; + std::unordered_map> count_strides; + + int ndims = compact_affine_dims(&affine, subrect, dim_order, field_size, total_bytes, + contig_bytes, count_strides); + + EXPECT_EQ(ndims, 2); + EXPECT_EQ(contig_bytes, field_size * 4); // 16 + EXPECT_EQ(total_bytes, contig_bytes * 5); // 80 + ASSERT_EQ(count_strides.size(), 1u); + auto it = count_strides.find(1); + ASSERT_NE(it, count_strides.end()); + EXPECT_EQ(it->second.first, 5u); // count + EXPECT_EQ(it->second.second, 64u); // stride +} + +#if REALM_MAX_DIM > 2 +// Test 3: 3‑D layout where rows padded but planes contiguous with row padding +TEST(FlattenAffineDimensionsTest, Grouped3D) +{ + constexpr int N = 3; + const size_t field_size = 4; + const int dim_order[N] = {0, 1, 2}; + + /* counts : dim0=4, dim1=5, dim2=6 + * strides : 4 (elem), 64 (row), 64*5=320 (plane) + * Expect : contig dim0 only (16 bytes) + * dims1+2 grouped: count=5*6=30, stride=64 + */ + Point<3, size_t> strides(field_size, 64, 64 * 5); + AffineLayoutPiece affine = make_affine<3>(strides); + + Rect<3, int> subrect(Point<3, int>(0, 0, 0), Point<3, int>(3, 4, 5)); + + size_t total_bytes = 0; + size_t contig_bytes = 0; + std::unordered_map> count_strides; + + int ndims = compact_affine_dims(&affine, subrect, dim_order, field_size, total_bytes, + contig_bytes, count_strides); + + EXPECT_EQ(ndims, 2); + EXPECT_EQ(contig_bytes, field_size * 4); // 16 + EXPECT_EQ(total_bytes, contig_bytes * 30); // 480 + ASSERT_EQ(count_strides.size(), 1u); + auto it = count_strides.find(1); + ASSERT_NE(it, count_strides.end()); + EXPECT_EQ(it->second.first, 30u); // count = 5*6 + EXPECT_EQ(it->second.second, 64u); // stride +} +#endif From b6d0cc3e390991fa60a310ccb7539a089a05646c Mon Sep 17 00:00:00 2001 From: Micha Dery Date: Mon, 3 Nov 2025 16:38:36 +0200 Subject: [PATCH 2/6] initial impl --- src/CMakeLists.txt | 2 +- src/realm/cuda/cuda_internal.cc | 198 +++++------ src/realm/transfer/addrsplit_channel.h | 2 +- src/realm/transfer/channel.cc | 134 ++++---- src/realm/transfer/channel.h | 6 +- src/realm/transfer/lowlevel_dma.cc | 25 +- src/realm/transfer/lowlevel_dma.h | 2 +- src/realm/transfer/memcpy_channel.cc | 48 +-- src/realm/transfer/memory_span.cc | 297 ++++++++++++++++ src/realm/transfer/memory_span.h | 120 +++++++ src/realm/transfer/transfer.cc | 59 ++-- src/realm/transfer/transfer.h | 6 +- tests/CMakeLists.txt | 9 +- tests/unit_tests/addrsplit_channel_test.cc | 2 +- tests/unit_tests/memcpy_channel_test.cc | 4 +- tests/unit_tests/memory_span_test.cc | 374 +++++++++++++++++++++ tests/unit_tests/transfer_iterator_test.cc | 24 +- 17 files changed, 1057 insertions(+), 255 deletions(-) create mode 100644 src/realm/transfer/memory_span.cc create mode 100644 src/realm/transfer/memory_span.h create mode 100644 tests/unit_tests/memory_span_test.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 66817da9bc2..83bb1109bee 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,7 +50,7 @@ set(REALM_SOURCES realm_c.cc fragmented_message.cc transfer/lowlevel_disk.cc - transfer/address_list.cc + transfer/memory_span.cc transfer/channel_common.cc transfer/memcpy_channel.cc transfer/channel.cc diff --git a/src/realm/cuda/cuda_internal.cc b/src/realm/cuda/cuda_internal.cc index 0df4e33dd8a..6d2e3060d46 100644 --- a/src/realm/cuda/cuda_internal.cc +++ b/src/realm/cuda/cuda_internal.cc @@ -219,7 +219,7 @@ namespace Realm { } static void get_nonaffine_strides(size_t &pitch, size_t &height, - AddressInfoCudaArray &ainfo, AddressListCursor &alc, + AddressInfoCudaArray &ainfo, SpanIterator &iter, size_t bytes) { bool shape_ok = false; @@ -227,66 +227,66 @@ namespace Realm { height = 1; // Don't set these to zero, as the driver will patch a pitch that may be // non-sensical if(ainfo.dim <= 2) { - if((alc.get_dim() == 1) || (alc.remaining(0) >= bytes)) { + if((iter.dim() == 1) || (iter.remaining(0) >= bytes)) { // contiguous input range pitch = ainfo.width_in_bytes; height = ainfo.height; - alc.advance(0, bytes); + iter.advance(0, bytes); } else { // width has to match - assert(alc.remaining(0) == ainfo.width_in_bytes); - assert(alc.remaining(1) >= ainfo.height); - pitch = alc.get_stride(1); + assert(iter.remaining(0) == ainfo.width_in_bytes); + assert(iter.remaining(1) >= ainfo.height); + pitch = iter.stride(1); height = ainfo.height; - alc.advance(1, ainfo.height); + iter.advance(1, ainfo.height); } shape_ok = true; } else { - if((alc.get_dim() == 1) || (alc.remaining(0) >= bytes)) { + if((iter.dim() == 1) || (iter.remaining(0) >= bytes)) { // contiguous input range pitch = ainfo.width_in_bytes; height = ainfo.height; - alc.advance(0, bytes); + iter.advance(0, bytes); shape_ok = true; } else { // if it's not contiguous, width must be exactly what // we need for either 1 or 2 leading dimensions - if(alc.remaining(0) == ainfo.width_in_bytes) { - if((alc.get_dim() == 2) || - (alc.remaining(1) >= (ainfo.height * ainfo.depth))) { + if(iter.remaining(0) == ainfo.width_in_bytes) { + if((iter.dim() == 2) || + (iter.remaining(1) >= (ainfo.height * ainfo.depth))) { // input dim 1 covers output 1 and 2 - pitch = alc.get_stride(1); + pitch = iter.stride(1); height = ainfo.height; - alc.advance(1, (ainfo.height * ainfo.depth)); + iter.advance(1, (ainfo.height * ainfo.depth)); shape_ok = true; } else { // for a full 3 dimensions, we need need dim 1 to // match exactly AND the stride for dim 2 has to // be a multiple of dim 1's stride due to // cuMemcpy3D restrictions - if((alc.remaining(1) == ainfo.height) && (alc.get_dim() >= 3) && - (alc.remaining(2) >= ainfo.depth) && - ((alc.get_stride(2) % alc.get_stride(1)) == 0)) { - pitch = alc.get_stride(1); - height = (alc.get_stride(2) / alc.get_stride(1)); - alc.advance(2, ainfo.depth); + if((iter.remaining(1) == ainfo.height) && (iter.dim() >= 3) && + (iter.remaining(2) >= ainfo.depth) && + ((iter.stride(2) % iter.stride(1)) == 0)) { + pitch = iter.stride(1); + height = (iter.stride(2) / iter.stride(1)); + iter.advance(2, ainfo.depth); shape_ok = true; } } } else { - if((alc.remaining(0) == (ainfo.width_in_bytes * ainfo.height)) && - (alc.remaining(1) >= ainfo.depth) && - ((alc.get_stride(1) % ainfo.width_in_bytes) == 0)) { + if((iter.remaining(0) == (ainfo.width_in_bytes * ainfo.height)) && + (iter.remaining(1) >= ainfo.depth) && + ((iter.stride(1) % ainfo.width_in_bytes) == 0)) { pitch = ainfo.width_in_bytes; - height = (alc.get_stride(1) / ainfo.width_in_bytes); - alc.advance(1, ainfo.depth); + height = (iter.stride(1) / ainfo.width_in_bytes); + iter.advance(1, ainfo.depth); shape_ok = true; } } } } if(!shape_ok) { - log_gpudma.fatal() << "array copy shape mismatch: alc=" << alc + log_gpudma.fatal() << "array copy shape mismatch: iter.dim=" << iter.dim() << " ainfo=" << ainfo.width_in_bytes << "x" << ainfo.height << "x" << ainfo.depth; abort(); @@ -314,20 +314,20 @@ namespace Realm { static size_t populate_affine_copy_info(AffineCopyInfo<3> ©_infos, size_t &min_align, MemcpyTransposeInfo &transpose_info, - AddressListCursor &in_alc, uintptr_t in_base, - GPU *in_gpu, AddressListCursor &out_alc, + SpanIterator &in_iter, uintptr_t in_base, + GPU *in_gpu, SpanIterator &out_iter, uintptr_t out_base, GPU *out_gpu, size_t bytes_left) { AffineCopyPair<3> ©_info = copy_infos.subrects[copy_infos.num_rects++]; - uintptr_t in_offset = in_alc.get_offset(); - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t in_offset = in_iter.offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); - size_t icount = in_alc.remaining(0); - size_t ocount = out_alc.remaining(0); + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); + size_t icount = in_iter.remaining(0); + size_t ocount = out_iter.remaining(0); // contig bytes is always the min of the first dimensions size_t contig_bytes = std::min(std::min(icount, ocount), bytes_left); @@ -357,8 +357,8 @@ namespace Realm { copy_info.dst.strides[0] = contig_bytes; copy_info.volume = contig_bytes; - in_alc.advance(0, contig_bytes); - out_alc.advance(0, contig_bytes); + in_iter.advance(0, contig_bytes); + out_iter.advance(0, contig_bytes); return contig_bytes; } @@ -378,8 +378,8 @@ namespace Realm { } else { assert(in_dim > 1); id = 1; - icount = in_alc.remaining(id); - in_lstride = in_alc.get_stride(id); + icount = in_iter.remaining(id); + in_lstride = in_iter.stride(id); iscale = 1; } @@ -398,8 +398,8 @@ namespace Realm { } else { assert(out_dim > 1); od = 1; - ocount = out_alc.remaining(od); - out_lstride = out_alc.get_stride(od); + ocount = out_iter.remaining(od); + out_lstride = out_iter.stride(od); oscale = 1; } @@ -423,8 +423,8 @@ namespace Realm { copy_info.extents[1] = lines; copy_info.volume = lines * contig_bytes; - in_alc.advance(id, lines * iscale); - out_alc.advance(od, lines * oscale); + in_iter.advance(id, lines * iscale); + out_iter.advance(od, lines * oscale); return lines * contig_bytes; } @@ -440,8 +440,8 @@ namespace Realm { } else { id++; assert(in_dim > id); - icount = in_alc.remaining(id); - in_pstride = in_alc.get_stride(id); + icount = in_iter.remaining(id); + in_pstride = in_iter.stride(id); iscale = 1; } @@ -456,8 +456,8 @@ namespace Realm { } else { od++; assert(out_dim > od); - ocount = out_alc.remaining(od); - out_pstride = out_alc.get_stride(od); + ocount = out_iter.remaining(od); + out_pstride = out_iter.stride(od); oscale = 1; } @@ -492,8 +492,8 @@ namespace Realm { copy_info.volume = planes * lines * contig_bytes; } - in_alc.advance(id, planes * iscale); - out_alc.advance(od, planes * oscale); + in_iter.advance(id, planes * iscale); + out_iter.advance(od, planes * oscale); return planes * lines * contig_bytes; } @@ -626,10 +626,10 @@ namespace Realm { if(in_port == 0 || out_port == 0) { if(in_port) { - in_port->addrcursor.skip_bytes(max_bytes); + in_port->span_iter.skip_bytes(max_bytes); rseqcache.add_span(input_control.current_io_port, in_span_start, max_bytes); } else if(out_port) { - out_port->addrcursor.skip_bytes(max_bytes); + out_port->span_iter.skip_bytes(max_bytes); } else { wseqcache.add_span(output_control.current_io_port, out_span_start, max_bytes); } @@ -673,8 +673,8 @@ namespace Realm { // iterating the address list cursor for each and figure out what copy we can do // that best fits the layout of the source and destinations while(bytes_left > 0 && copy_infos.num_rects < AffineCopyInfo<3>::MAX_NUM_RECTS) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; if(!in_nonaffine && !out_nonaffine) { log_gpudma.info() << "Affine -> Affine"; // limit transfer size for host<->device copies @@ -687,7 +687,7 @@ namespace Realm { } const size_t bytes_to_copy = populate_affine_copy_info( - copy_infos, min_align, transpose_copy, in_alc, in_base, in_gpu, out_alc, + copy_infos, min_align, transpose_copy, in_iter, in_base, in_gpu, out_iter, out_base, out_gpu, bytes_left); // Either src or dst can't be accessed with a kernel, so just break out and @@ -718,9 +718,9 @@ namespace Realm { cuda_copy.srcZ = ainfo.pos[2]; cuda_copy.dstMemoryType = CU_MEMORYTYPE_UNIFIED; cuda_copy.dstDevice = - static_cast(out_base + out_alc.get_offset()); + static_cast(out_base + out_iter.offset()); get_nonaffine_strides(cuda_copy.dstPitch, cuda_copy.dstHeight, ainfo, - out_alc, bytes); + out_iter, bytes); } else { assert(!in_nonaffine); log_gpudma.info() << "Affine -> Array"; @@ -734,9 +734,9 @@ namespace Realm { cuda_copy.dstZ = ainfo.pos[2]; cuda_copy.srcMemoryType = CU_MEMORYTYPE_UNIFIED; cuda_copy.srcDevice = - static_cast(in_base + in_alc.get_offset()); + static_cast(in_base + in_iter.offset()); get_nonaffine_strides(cuda_copy.srcPitch, cuda_copy.srcHeight, ainfo, - in_alc, bytes); + in_iter, bytes); } cuda_copy.WidthInBytes = ainfo.width_in_bytes; cuda_copy.Height = ainfo.height; @@ -1016,8 +1016,8 @@ namespace Realm { size_t addr_size = 0; - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; size_t write_ind_bytes = 0; uintptr_t dst_ind_base = 0; @@ -1031,9 +1031,9 @@ namespace Realm { out_port->iter->get_base_offset(), 0)); out_base += addr_info.base_offset; - dst_ind_base += (out_alc.get_offset() / addr_info.bytes_per_chunk) * addr_size; + dst_ind_base += (out_iter.offset() / addr_info.bytes_per_chunk) * addr_size; } else { - out_base += out_alc.get_offset(); + out_base += out_iter.offset(); } size_t read_ind_bytes = 0; @@ -1048,9 +1048,9 @@ namespace Realm { in_port->iter->get_base_offset(), 0)); in_base += addr_info.base_offset; - src_ind_base += (in_alc.get_offset() / addr_info.bytes_per_chunk) * addr_size; + src_ind_base += (in_iter.offset() / addr_info.bytes_per_chunk) * addr_size; } else { - in_base += in_alc.get_offset(); + in_base += in_iter.offset(); } log_gpudma.info() << "cuda gathe/scatter bytes_per_chunk=" @@ -1097,8 +1097,8 @@ namespace Realm { max_bytes, strides, in_base, out_base, src_ind_base, dst_ind_base); - in_alc.advance(0, max_bytes); - out_alc.advance(0, max_bytes); + in_iter.advance(0, max_bytes); + out_iter.advance(0, max_bytes); // TODO(apryakhin@): Add control flow total_bytes += max_bytes; @@ -1862,17 +1862,17 @@ namespace Realm { fill_info.num_rects = 0; while(total_bytes < max_bytes) { - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int out_dim = out_alc.get_dim(); + int out_dim = out_iter.dim(); if((reduced_fill_size < sizeof(fill_info.fill_value)) && ((reduced_fill_size & (reduced_fill_size - 1)) == 0)) { - const size_t bytes = std::min(out_alc.remaining(0), max_bytes); - size_t lines = (out_dim > 1 ? out_alc.remaining(1) : 1); + const size_t bytes = std::min(out_iter.remaining(0), max_bytes); + size_t lines = (out_dim > 1 ? out_iter.remaining(1) : 1); if((lines * bytes) > max_bytes) { lines = std::max(1, max_bytes / bytes); } @@ -1885,7 +1885,7 @@ namespace Realm { bytes / reduced_fill_size; fill_info.subrects[fill_info.num_rects].extents[1] = lines; fill_info.subrects[fill_info.num_rects].strides[0] = - (out_dim > 1 ? out_alc.get_stride(1) : bytes) / reduced_fill_size; + (out_dim > 1 ? out_iter.stride(1) : bytes) / reduced_fill_size; fill_info.num_rects++; total_info_bytes += bytes; @@ -1901,12 +1901,12 @@ namespace Realm { } total_bytes += bytes * lines; - out_alc.advance((out_dim == 1 ? 0 : 1), (out_dim == 1 ? bytes : lines)); + out_iter.advance((out_dim == 1 ? 0 : 1), (out_dim == 1 ? bytes : lines)); } else { // more general approach - use strided 2d copies to fill the first // line, and then we can use logarithmic doublings to deal with // multiple lines and/or planes - size_t bytes = out_alc.remaining(0); + size_t bytes = out_iter.remaining(0); size_t elems = bytes / reduced_fill_size; #ifdef DEBUG_REALM assert((bytes % reduced_fill_size) == 0); @@ -1969,11 +1969,11 @@ namespace Realm { if(out_dim == 1) { // all done - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); total_bytes += bytes; } else { - size_t lines = out_alc.remaining(1); - size_t lstride = out_alc.get_stride(1); + size_t lines = out_iter.remaining(1); + size_t lstride = out_iter.stride(1); CUDA_MEMCPY2D copy2d; copy2d.srcMemoryType = CU_MEMORYTYPE_DEVICE; @@ -2001,11 +2001,11 @@ namespace Realm { } if(out_dim == 2) { - out_alc.advance(1, lines); + out_iter.advance(1, lines); total_bytes += bytes * lines; } else { - size_t planes = out_alc.remaining(2); - size_t pstride = out_alc.get_stride(2); + size_t planes = out_iter.remaining(2); + size_t pstride = out_iter.stride(2); // logarithmic version requires that pstride be a multiple of // lstride @@ -2043,7 +2043,7 @@ namespace Realm { planes_done += todo; } - out_alc.advance(2, planes); + out_iter.advance(2, planes); total_bytes += bytes * lines * planes; } else { // plane-at-a-time fallback - can reuse most of copy2d @@ -2056,7 +2056,7 @@ namespace Realm { CHECK_CU(CUDA_DRIVER_FNPTR(cuMemcpy2DAsync)(©2d, stream->get_stream())); } - out_alc.advance(2, planes); + out_iter.advance(2, planes); total_bytes += bytes * lines * planes; } } @@ -2316,7 +2316,7 @@ namespace Realm { output_control.remaining_count / out_elem_size); if(in_port != 0) { max_elems = - std::min(max_elems, in_port->addrlist.bytes_pending() / in_elem_size); + std::min(max_elems, in_port->span_list.bytes_pending() / in_elem_size); if(in_port->peer_guid != XFERDES_NO_GUID) { size_t read_bytes_avail = in_port->seq_remote.span_exists( in_port->local_bytes_total, (max_elems * in_elem_size)); @@ -2325,7 +2325,7 @@ namespace Realm { } if(out_port != 0) { max_elems = - std::min(max_elems, out_port->addrlist.bytes_pending() / out_elem_size); + std::min(max_elems, out_port->span_list.bytes_pending() / out_elem_size); // no support for reducing into an intermediate buffer assert(out_port->peer_guid == XFERDES_NO_GUID); } @@ -2365,36 +2365,36 @@ namespace Realm { assert(channel->gpu->can_access_peer(in_gpu)); while(total_elems < max_elems) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t in_offset = in_alc.get_offset(); - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t in_offset = in_iter.offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); // the current reduction op interface can reduce multiple elements // with a fixed address stride, which looks to us like either // 1D (stride = elem_size), or 2D with 1 elem/line - size_t icount = in_alc.remaining(0) / in_elem_size; - size_t ocount = out_alc.remaining(0) / out_elem_size; + size_t icount = in_iter.remaining(0) / in_elem_size; + size_t ocount = out_iter.remaining(0) / out_elem_size; size_t istride, ostride; if((in_dim > 1) && (icount == 1)) { in_dim = 2; - icount = in_alc.remaining(1); - istride = in_alc.get_stride(1); + icount = in_iter.remaining(1); + istride = in_iter.stride(1); } else { in_dim = 1; istride = in_elem_size; } if((out_dim > 1) && (ocount == 1)) { out_dim = 2; - ocount = out_alc.remaining(1); - ostride = out_alc.get_stride(1); + ocount = out_iter.remaining(1); + ostride = out_iter.stride(1); } else { out_dim = 1; ostride = out_elem_size; @@ -2453,8 +2453,8 @@ namespace Realm { in_span_start += elems * in_elem_size; out_span_start += elems * out_elem_size; - in_alc.advance(in_dim - 1, elems * ((in_dim == 1) ? in_elem_size : 1)); - out_alc.advance(out_dim - 1, elems * ((out_dim == 1) ? out_elem_size : 1)); + in_iter.advance(in_dim - 1, elems * ((in_dim == 1) ? in_elem_size : 1)); + out_iter.advance(out_dim - 1, elems * ((out_dim == 1) ? out_elem_size : 1)); #ifdef DEBUG_REALM assert(elems <= elems_left); @@ -2470,7 +2470,7 @@ namespace Realm { } else { // input but no output, so skip input bytes total_elems = max_elems; - in_port->addrcursor.skip_bytes(total_elems * in_elem_size); + in_port->span_iter.skip_bytes(total_elems * in_elem_size); rseqcache.add_span(input_control.current_io_port, in_span_start, total_elems * in_elem_size); @@ -2480,7 +2480,7 @@ namespace Realm { if(out_port != 0) { // output but no input, so skip output bytes total_elems = max_elems; - out_port->addrcursor.skip_bytes(total_elems * out_elem_size); + out_port->span_iter.skip_bytes(total_elems * out_elem_size); wseqcache.add_span(output_control.current_io_port, out_span_start, total_elems * out_elem_size); diff --git a/src/realm/transfer/addrsplit_channel.h b/src/realm/transfer/addrsplit_channel.h index c5c943208ce..d31ee89922d 100644 --- a/src/realm/transfer/addrsplit_channel.h +++ b/src/realm/transfer/addrsplit_channel.h @@ -24,7 +24,7 @@ namespace Realm { class XferDes; - class AddressList; + class SpanList; class AddressSplitChannel; diff --git a/src/realm/transfer/channel.cc b/src/realm/transfer/channel.cc index 9f2c609abf1..3c2edaf058f 100644 --- a/src/realm/transfer/channel.cc +++ b/src/realm/transfer/channel.cc @@ -489,7 +489,7 @@ namespace Realm { p.remote_bytes_total.store(size_t(-1)); p.ib_offset = ii.ib_offset; p.ib_size = ii.ib_size; - p.addrcursor.set_addrlist(&p.addrlist); + p.span_iter = SpanIterator(&p.span_list); switch(ii.port_type) { case XferDesPortInfo::GATHER_CONTROL_PORT: gather_control_port = i; @@ -555,7 +555,7 @@ namespace Realm { p.remote_bytes_total.store(size_t(-1)); p.ib_offset = oi.ib_offset; p.ib_size = oi.ib_size; - p.addrcursor.set_addrlist(&p.addrlist); + p.span_iter = SpanIterator(&p.span_list); // if we're writing into an IB, the first 'ib_size' byte // locations can be freely written @@ -803,10 +803,10 @@ namespace Realm { XferPort *in_port = &input_ports[input_control.current_io_port]; // do we need more addresses? - size_t read_bytes_avail = in_port->addrlist.bytes_pending(); + size_t read_bytes_avail = in_port->span_list.bytes_pending(); if(read_bytes_avail < min_xfer_size) { - bool flush = in_port->iter->get_addresses(in_port->addrlist, in_nonaffine); - read_bytes_avail = in_port->addrlist.bytes_pending(); + bool flush = in_port->iter->get_addresses(in_port->span_list, in_nonaffine); + read_bytes_avail = in_port->span_list.bytes_pending(); if(flush) { if(read_bytes_avail > 0) { // ignore a nonaffine piece as we still have some affine bytes @@ -855,10 +855,10 @@ namespace Realm { XferPort *out_port = &output_ports[output_control.current_io_port]; // do we need more addresses? - size_t write_bytes_avail = out_port->addrlist.bytes_pending(); + size_t write_bytes_avail = out_port->span_list.bytes_pending(); if(write_bytes_avail < min_xfer_size) { - bool flush = out_port->iter->get_addresses(out_port->addrlist, out_nonaffine); - write_bytes_avail = out_port->addrlist.bytes_pending(); + bool flush = out_port->iter->get_addresses(out_port->span_list, out_nonaffine); + write_bytes_avail = out_port->span_list.bytes_pending(); // TODO(apryakhin@): We add this to handle scatter when both // indirection and source are coming from IB and this needs @@ -927,7 +927,7 @@ namespace Realm { in_port->local_bytes_cons.fetch_add(total_read_bytes); if(in_port->peer_guid == XFERDES_NO_GUID) - in_done = ((in_port->addrlist.bytes_pending() == 0) && in_port->iter->done()); + in_done = ((in_port->span_list.bytes_pending() == 0) && in_port->iter->done()); else in_done = (in_port->local_bytes_total == in_port->remote_bytes_total.load_acquire()); @@ -941,7 +941,7 @@ namespace Realm { out_port->local_bytes_cons.fetch_add(total_write_bytes); if(out_port->peer_guid == XFERDES_NO_GUID) - out_done = ((out_port->addrlist.bytes_pending() == 0) && out_port->iter->done()); + out_done = ((out_port->span_list.bytes_pending() == 0) && out_port->iter->done()); } input_control.remaining_count -= total_read_bytes; @@ -2062,13 +2062,13 @@ namespace Realm { reinterpret_cast(out_port->mem->get_direct_ptr(0, 0)); while(total_bytes < max_bytes) { - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int out_dim = out_alc.get_dim(); + int out_dim = out_iter.dim(); size_t bytes = 0; size_t bytes_left = max_bytes - total_bytes; @@ -2086,7 +2086,7 @@ namespace Realm { } if(out_dim > 0) { - size_t ocount = out_alc.remaining(0); + size_t ocount = out_iter.remaining(0); // contig bytes is always the first dimension size_t contig_bytes = std::min(ocount, bytes_left); @@ -2096,18 +2096,18 @@ namespace Realm { ((contig_bytes == ocount) && (out_dim == 1))) { bytes = contig_bytes; memset_1d(out_base + out_offset, contig_bytes, fill_data, fill_size); - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); } else { // grow to a 2D fill - ocount = out_alc.remaining(1); - uintptr_t out_lstride = out_alc.get_stride(1); + ocount = out_iter.remaining(1); + uintptr_t out_lstride = out_iter.stride(1); size_t lines = std::min(ocount, bytes_left / contig_bytes); bytes = contig_bytes * lines; memset_2d(out_base + out_offset, out_lstride, contig_bytes, lines, fill_data, fill_size); - out_alc.advance(1, lines); + out_iter.advance(1, lines); } } else { // scatter adddress list @@ -2212,7 +2212,7 @@ namespace Realm { output_control.remaining_count / out_elem_size); if(in_port != 0) { max_elems = - std::min(max_elems, in_port->addrlist.bytes_pending() / in_elem_size); + std::min(max_elems, in_port->span_list.bytes_pending() / in_elem_size); if(in_port->peer_guid != XFERDES_NO_GUID) { size_t read_bytes_avail = in_port->seq_remote.span_exists( in_port->local_bytes_total, (max_elems * in_elem_size)); @@ -2221,7 +2221,7 @@ namespace Realm { } if(out_port != 0) { max_elems = - std::min(max_elems, out_port->addrlist.bytes_pending() / out_elem_size); + std::min(max_elems, out_port->span_list.bytes_pending() / out_elem_size); // no support for reducing into an intermediate buffer assert(out_port->peer_guid == XFERDES_NO_GUID); } @@ -2240,36 +2240,36 @@ namespace Realm { reinterpret_cast(out_port->mem->get_direct_ptr(0, 0)); while(total_elems < max_elems) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t in_offset = in_alc.get_offset(); - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t in_offset = in_iter.offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); // the current reduction op interface can reduce multiple elements // with a fixed address stride, which looks to us like either // 1D (stride = elem_size), or 2D with 1 elem/line - size_t icount = in_alc.remaining(0) / in_elem_size; - size_t ocount = out_alc.remaining(0) / out_elem_size; + size_t icount = in_iter.remaining(0) / in_elem_size; + size_t ocount = out_iter.remaining(0) / out_elem_size; size_t istride, ostride; if((in_dim > 1) && (icount == 1)) { in_dim = 2; - icount = in_alc.remaining(1); - istride = in_alc.get_stride(1); + icount = in_iter.remaining(1); + istride = in_iter.stride(1); } else { in_dim = 1; istride = in_elem_size; } if((out_dim > 1) && (ocount == 1)) { out_dim = 2; - ocount = out_alc.remaining(1); - ostride = out_alc.get_stride(1); + ocount = out_iter.remaining(1); + ostride = out_iter.stride(1); } else { out_dim = 1; ostride = out_elem_size; @@ -2297,8 +2297,8 @@ namespace Realm { redop->userdata); } - in_alc.advance(in_dim - 1, elems * ((in_dim == 1) ? in_elem_size : 1)); - out_alc.advance(out_dim - 1, elems * ((out_dim == 1) ? out_elem_size : 1)); + in_iter.advance(in_dim - 1, elems * ((in_dim == 1) ? in_elem_size : 1)); + out_iter.advance(out_dim - 1, elems * ((out_dim == 1) ? out_elem_size : 1)); #ifdef DEBUG_REALM assert(elems <= elems_left); @@ -2313,13 +2313,13 @@ namespace Realm { } else { // input but no output, so skip input bytes total_elems = max_elems; - in_port->addrcursor.skip_bytes(total_elems * in_elem_size); + in_port->span_iter.skip_bytes(total_elems * in_elem_size); } } else { if(out_port != 0) { // output but no input, so skip output bytes total_elems = max_elems; - out_port->addrcursor.skip_bytes(total_elems * out_elem_size); + out_port->span_iter.skip_bytes(total_elems * out_elem_size); } else { // skipping both input and output is possible for simultaneous // gather+scatter @@ -2543,12 +2543,12 @@ namespace Realm { << " max=" << max_bytes; while(total_bytes < max_bytes) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); - size_t icount = in_alc.remaining(0); - size_t ocount = out_alc.remaining(0); + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); + size_t icount = in_iter.remaining(0); + size_t ocount = out_iter.remaining(0); size_t bytes = 0; size_t bytes_left = max_bytes - total_bytes; @@ -2559,7 +2559,7 @@ namespace Realm { size_t dst_2d_maxbytes = (((out_dim > 1) && (ocount <= (MAX_ASSEMBLY_SIZE / 2))) ? (ocount * - std::min(MAX_ASSEMBLY_SIZE / ocount, out_alc.remaining(1))) + std::min(MAX_ASSEMBLY_SIZE / ocount, out_iter.remaining(1))) : 0); // would have to scan forward through the dst address list to // get the exact number of bytes that we can fit into @@ -2577,12 +2577,12 @@ namespace Realm { // 1D target NodeID dst_node = ID(out_port->mem->me).memory_owner_node(); RemoteAddress dst_buf; - bool ok = out_port->mem->get_remote_addr(out_alc.get_offset(), dst_buf); + bool ok = out_port->mem->get_remote_addr(out_iter.offset(), dst_buf); assert(ok); // now look at the input LocalAddress src_buf; - ok = in_port->mem->get_local_addr(in_alc.get_offset(), src_buf); + ok = in_port->mem->get_local_addr(in_iter.offset(), src_buf); assert(ok); size_t src_1d_maxbytes = 0; if(in_dim > 0) { @@ -2595,9 +2595,9 @@ namespace Realm { // TODO: permit if source memory is cpu-accessible? #ifdef ALLOW_RDMA_SOURCE_2D if(in_dim > 1) { - size_t lines = in_alc.remaining(1); + size_t lines = in_iter.remaining(1); size_t rec_bytes = ActiveMessage::recommended_max_payload( - dst_node, src_buf, icount, lines, in_alc.get_stride(1), dst_buf, + dst_node, src_buf, icount, lines, in_iter.stride(1), dst_buf, true /*w/ congestion*/); // round the recommendation down to a multiple of the line size rec_bytes -= (rec_bytes % icount); @@ -2653,15 +2653,15 @@ namespace Realm { out_span_start += bytes; amsg.commit(); - in_alc.advance(0, bytes); - out_alc.advance(0, bytes); + in_iter.advance(0, bytes); + out_iter.advance(0, bytes); } else if(src_2d_maxbytes >= src_ga_maxbytes) { // 2D source size_t bytes_per_line = icount; size_t lines = src_2d_maxbytes / icount; bytes = bytes_per_line * lines; assert(bytes == src_2d_maxbytes); - size_t src_stride = in_alc.get_stride(1); + size_t src_stride = in_iter.stride(1); // log_xd.info() << "remote write 2d: guid=" << guid // << " src=" << src_buf << " dst=" << dst_buf // << " bytes=" << bytes << " lines=" << lines @@ -2688,8 +2688,8 @@ namespace Realm { out_span_start += bytes; amsg.commit(); - in_alc.advance(1, lines); - out_alc.advance(0, bytes); + in_iter.advance(1, lines); + out_iter.advance(0, bytes); } else { // gather: assemble data bytes = src_ga_maxbytes; @@ -2704,35 +2704,35 @@ namespace Realm { if((icount >= todo / 2) || (in_dim == 1)) { size_t chunk = std::min(todo, icount); uintptr_t src = reinterpret_cast( - in_port->mem->get_direct_ptr(in_alc.get_offset(), chunk)); + in_port->mem->get_direct_ptr(in_iter.offset(), chunk)); uintptr_t dst = reinterpret_cast(amsg.payload_ptr(chunk)); memcpy_1d(dst, src, chunk); - in_alc.advance(0, chunk); + in_iter.advance(0, chunk); todo -= chunk; } else { - size_t lines = std::min(todo / icount, in_alc.remaining(1)); + size_t lines = std::min(todo / icount, in_iter.remaining(1)); if(((icount * lines) >= todo / 2) || (in_dim == 2)) { uintptr_t src = reinterpret_cast( - in_port->mem->get_direct_ptr(in_alc.get_offset(), icount)); + in_port->mem->get_direct_ptr(in_iter.offset(), icount)); uintptr_t dst = reinterpret_cast(amsg.payload_ptr(icount * lines)); - memcpy_2d(dst, icount /*lstride*/, src, in_alc.get_stride(1), + memcpy_2d(dst, icount /*lstride*/, src, in_iter.stride(1), icount, lines); - in_alc.advance(1, lines); + in_iter.advance(1, lines); todo -= icount * lines; } else { size_t planes = - std::min(todo / (icount * lines), in_alc.remaining(2)); + std::min(todo / (icount * lines), in_iter.remaining(2)); uintptr_t src = reinterpret_cast( - in_port->mem->get_direct_ptr(in_alc.get_offset(), icount)); + in_port->mem->get_direct_ptr(in_iter.offset(), icount)); uintptr_t dst = reinterpret_cast( amsg.payload_ptr(icount * lines * planes)); memcpy_3d(dst, icount /*lstride*/, (icount * lines) /*pstride*/, - src, in_alc.get_stride(1), in_alc.get_stride(2), icount, + src, in_iter.stride(1), in_iter.stride(2), icount, lines, planes); - in_alc.advance(2, planes); + in_iter.advance(2, planes); todo -= icount * lines * planes; } } @@ -2744,8 +2744,8 @@ namespace Realm { break; // read next entry - in_dim = in_alc.get_dim(); - icount = in_alc.remaining(0); + in_dim = in_iter.dim(); + icount = in_iter.remaining(0); } // the write isn't complete until it's ack'd by the target @@ -2760,7 +2760,7 @@ namespace Realm { rseqcache.add_span(input_control.current_io_port, in_span_start, bytes); in_span_start += bytes; - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); } } else if(dst_2d_maxbytes >= dst_sc_maxbytes) { // 2D target @@ -2783,7 +2783,7 @@ namespace Realm { } else { // input but no output, so skip input bytes total_bytes = max_bytes; - in_port->addrcursor.skip_bytes(total_bytes); + in_port->span_iter.skip_bytes(total_bytes); rseqcache.add_span(input_control.current_io_port, in_span_start, total_bytes); in_span_start += total_bytes; } @@ -2791,7 +2791,7 @@ namespace Realm { if(out_port != 0) { // output but no input, so skip output bytes total_bytes = max_bytes; - out_port->addrcursor.skip_bytes(total_bytes); + out_port->span_iter.skip_bytes(total_bytes); wseqcache.add_span(output_control.current_io_port, out_span_start, total_bytes); out_span_start += total_bytes; } else { diff --git a/src/realm/transfer/channel.h b/src/realm/transfer/channel.h index e693d0ff01a..fec93637f24 100644 --- a/src/realm/transfer/channel.h +++ b/src/realm/transfer/channel.h @@ -42,7 +42,7 @@ #include "realm/inst_impl.h" #include "realm/bgwork.h" #include "realm/utils.h" -#include "realm/transfer/address_list.h" +#include "realm/transfer/memory_span.h" namespace Realm { @@ -314,8 +314,8 @@ namespace Realm { // to complete) Memory ib_mem; size_t ib_offset, ib_size; - AddressList addrlist; - AddressListCursor addrcursor; + SpanList span_list; + SpanIterator span_iter; }; std::vector input_ports, output_ports; struct ControlPortState { diff --git a/src/realm/transfer/lowlevel_dma.cc b/src/realm/transfer/lowlevel_dma.cc index c2495f4925d..127aaa1a835 100644 --- a/src/realm/transfer/lowlevel_dma.cc +++ b/src/realm/transfer/lowlevel_dma.cc @@ -847,7 +847,7 @@ namespace Realm { tentative_valid = false; } - bool WrappingFIFOIterator::get_addresses(AddressList &addrlist, + bool WrappingFIFOIterator::get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine) { nonaffine = 0; @@ -857,18 +857,19 @@ namespace Realm { // the same location twice) size_t lines = std::max((1 << 30) / size, 1); int dim = (lines > 1) ? 2 : 1; - size_t *data = addrlist.begin_entry(dim); - if(!data) - return true; // can't add more until some is consumed - - // 1-D span from [base,base+size) - data[0] = (size << 4) + 2 /*dim*/; - data[1] = base; - if(dim == 2) { - data[2] = lines; - data[3] = 0; // stride + + // Create span with single dummy field (wrapping FIFO doesn't use real fields) + std::vector fields = {FieldID(0)}; + + if(dim == 1) { + uint32_t extents[] = {static_cast(size)}; + size_t strides[] = {1}; + span_list.append(base, fields, 1, extents, strides); + } else { + uint32_t extents[] = {static_cast(size), static_cast(lines)}; + size_t strides[] = {1, 0}; // stride of 0 for wrapping + span_list.append(base, fields, 2, extents, strides); } - addrlist.commit_entry(dim, size * lines); return false; // we can add more if asked } diff --git a/src/realm/transfer/lowlevel_dma.h b/src/realm/transfer/lowlevel_dma.h index 3209d130fce..b129a0ad4c2 100644 --- a/src/realm/transfer/lowlevel_dma.h +++ b/src/realm/transfer/lowlevel_dma.h @@ -187,7 +187,7 @@ namespace Realm { virtual void confirm_step(void); virtual void cancel_step(void); - virtual bool get_addresses(AddressList &addrlist, + virtual bool get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine); static Serialization::PolymorphicSerdezSubclass(out_port->mem->get_direct_ptr(0, 0)); while(total_bytes < max_bytes) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t in_offset = in_alc.get_offset(); - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t in_offset = in_iter.offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); size_t bytes = 0; size_t bytes_left = max_bytes - total_bytes; @@ -203,8 +203,8 @@ namespace Realm { if(in_dim > 0) { if(out_dim > 0) { - size_t icount = in_alc.remaining(0); - size_t ocount = out_alc.remaining(0); + size_t icount = in_iter.remaining(0); + size_t ocount = out_iter.remaining(0); // contig bytes is always the min of the first dimensions size_t contig_bytes = std::min(std::min(icount, ocount), bytes_left); @@ -215,8 +215,8 @@ namespace Realm { ((contig_bytes == ocount) && (out_dim == 1))) { bytes = contig_bytes; memcpy_1d(out_base + out_offset, in_base + in_offset, bytes); - in_alc.advance(0, bytes); - out_alc.advance(0, bytes); + in_iter.advance(0, bytes); + out_iter.advance(0, bytes); } else { // grow to a 2D copy int id; @@ -235,8 +235,8 @@ namespace Realm { } else { assert(in_dim > 1); id = 1; - icount = in_alc.remaining(id); - in_lstride = in_alc.get_stride(id); + icount = in_iter.remaining(id); + in_lstride = in_iter.stride(id); iscale = 1; } @@ -256,8 +256,8 @@ namespace Realm { } else { assert(out_dim > 1); od = 1; - ocount = out_alc.remaining(od); - out_lstride = out_alc.get_stride(od); + ocount = out_iter.remaining(od); + out_lstride = out_iter.stride(od); oscale = 1; } @@ -271,8 +271,8 @@ namespace Realm { bytes = contig_bytes * lines; memcpy_2d(out_base + out_offset, out_lstride, in_base + in_offset, in_lstride, contig_bytes, lines); - in_alc.advance(id, lines * iscale); - out_alc.advance(od, lines * oscale); + in_iter.advance(id, lines * iscale); + out_iter.advance(od, lines * oscale); } else { uintptr_t in_pstride; if(lines < icount) { @@ -285,8 +285,8 @@ namespace Realm { } else { id++; assert(in_dim > id); - icount = in_alc.remaining(id); - in_pstride = in_alc.get_stride(id); + icount = in_iter.remaining(id); + in_pstride = in_iter.stride(id); iscale = 1; } @@ -301,8 +301,8 @@ namespace Realm { } else { od++; assert(out_dim > od); - ocount = out_alc.remaining(od); - out_pstride = out_alc.get_stride(od); + ocount = out_iter.remaining(od); + out_pstride = out_iter.stride(od); oscale = 1; } @@ -312,8 +312,8 @@ namespace Realm { memcpy_3d(out_base + out_offset, out_lstride, out_pstride, in_base + in_offset, in_lstride, in_pstride, contig_bytes, lines, planes); - in_alc.advance(id, planes * iscale); - out_alc.advance(od, planes * oscale); + in_iter.advance(id, planes * iscale); + out_iter.advance(od, planes * oscale); } } } else { @@ -344,13 +344,13 @@ namespace Realm { } else { // input but no output, so skip input bytes total_bytes = max_bytes; - in_port->addrcursor.skip_bytes(total_bytes); + in_port->span_iter.skip_bytes(total_bytes); } } else { if(out_port != 0) { // output but no input, so skip output bytes total_bytes = max_bytes; - out_port->addrcursor.skip_bytes(total_bytes); + out_port->span_iter.skip_bytes(total_bytes); } else { // skipping both input and output is possible for simultaneous // gather+scatter diff --git a/src/realm/transfer/memory_span.cc b/src/realm/transfer/memory_span.cc new file mode 100644 index 00000000000..02303fe4744 --- /dev/null +++ b/src/realm/transfer/memory_span.cc @@ -0,0 +1,297 @@ +/* + * Copyright 2025 Los Alamos National Laboratory, Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "realm/transfer/memory_span.h" + +#include + +namespace Realm { + + // ================================================================================================= + // Span + // ================================================================================================= + + size_t Span::total_bytes() const { + if (num_dims == 0) { + return 0; + } + + // Calculate bytes for geometry + size_t bytes = extents[0]; + for (int d = 1; d < num_dims; d++) { + bytes *= extents[d]; + } + + // Multiply by number of fields + return bytes * field_ids.size(); + } + + bool Span::is_contiguous() const { + if (num_dims == 0) { + return true; + } + + // Check if strides match expected contiguous layout + size_t expected_stride = 1; + for (int d = 0; d < num_dims; d++) { + if (strides[d] != expected_stride) { + return false; + } + expected_stride *= extents[d]; + } + return true; + } + + // ================================================================================================= + // SpanList + // ================================================================================================= + + void SpanList::append(const Span& span) { + spans_.push_back(span); + } + + void SpanList::append(size_t base, const std::vector& fields, uint8_t dims, + const uint32_t* extents, const size_t* strides) { + Span span; + span.base_offset = base; + span.field_ids = fields; + span.num_dims = dims; + + for (int d = 0; d < dims && d < 3; d++) { + span.extents[d] = extents[d]; + span.strides[d] = strides[d]; + } + // Zero out unused dimensions + for (int d = dims; d < 3; d++) { + span.extents[d] = 0; + span.strides[d] = 0; + } + + spans_.push_back(span); + } + + size_t SpanList::total_bytes() const { + size_t total = 0; + for (const auto& span : spans_) { + total += span.total_bytes(); + } + return total; + } + + // ================================================================================================= + // SpanIterator + // ================================================================================================= + + SpanIterator::SpanIterator(const SpanList* list) + : list_(list) + , bytes_consumed_(0) + { + memset(pos_, 0, sizeof(pos_)); + } + + size_t SpanIterator::offset() const { + if (!list_ || done()) { + return 0; + } + + const Span& s = (*list_)[span_idx_]; + + size_t offset = s.base_offset; + + // Add position within geometry + for (int d = 0; d < s.num_dims; d++) { + offset += pos_[d] * s.strides[d]; + } + + // If multiple fields, add field offset + if (s.field_ids.size() > 1 && field_idx_ > 0) { + size_t bytes_per_field = s.total_bytes() / s.field_ids.size(); + offset += field_idx_ * bytes_per_field; + } + + return offset; + } + + size_t SpanIterator::stride(int dim) const { + if (done()) { + return 0; + } + + const Span& s = (*list_)[span_idx_]; + assert(dim >= 0 && dim < s.num_dims); + return s.strides[dim]; + } + + size_t SpanIterator::remaining(int dim) const { + if (done()) { + return 0; + } + + const Span& s = (*list_)[span_idx_]; + assert(dim >= 0 && dim < s.num_dims); + return s.extents[dim] - pos_[dim]; + } + + int SpanIterator::dim() const { + if (done()) { + return 0; + } + + const Span& s = (*list_)[span_idx_]; + return s.num_dims; + } + + FieldID SpanIterator::current_field() const { + if (done()) { + return FieldID(-1); + } + + const Span& s = (*list_)[span_idx_]; + assert(field_idx_ < s.field_ids.size()); + return s.field_ids[field_idx_]; + } + + size_t SpanIterator::remaining_fields() const { + if (done()) { + return 0; + } + + const Span& s = (*list_)[span_idx_]; + return s.field_ids.size() - field_idx_; + } + + void SpanIterator::advance(int dim, size_t count) { + if (done()) { + return; + } + + const Span& s = (*list_)[span_idx_]; + + size_t bytes_to_consume = count; + for (int d = 0; d < dim; d++) { + bytes_to_consume *= s.extents[d]; + } + + // Update bytes_consumed + bytes_consumed_ += bytes_to_consume; + + pos_[dim] += count; + + // Carry to higher dimensions - handle multi-level overflow + bool geometry_exhausted = false; + for (int d = dim; d < s.num_dims; d++) { + if (pos_[d] >= s.extents[d]) { + if (d + 1 < s.num_dims) { + // Calculate how many times we overflow this dimension + size_t carries = pos_[d] / s.extents[d]; + pos_[d] = pos_[d] % s.extents[d]; // Remainder stays in this dimension + pos_[d + 1] += carries; // Propagate carries to next dimension + // Continue loop to handle cascading carries + } else { + // Reached the end of the highest dimension + geometry_exhausted = true; + break; + } + } else { + break; // No more carries needed + } + } + + // Check if geometry exhausted + if (geometry_exhausted) { + // Move to next field if multi-field span + if (field_idx_ + 1 < s.field_ids.size()) { + field_idx_++; + memset(pos_, 0, sizeof(pos_)); + } else { + // Move to next span + span_idx_++; + field_idx_ = 0; + memset(pos_, 0, sizeof(pos_)); + } + } + } + + void SpanIterator::advance_fields(size_t num_fields) { + if (done()) { + return; + } + + const Span& s = (*list_)[span_idx_]; + + // Calculate bytes per field for current span + size_t elements_per_field = s.extents[0]; + for (int d = 1; d < s.num_dims; d++) { + elements_per_field *= s.extents[d]; + } + + // Increment bytes consumed for fields we're skipping in current span + size_t fields_in_current_span = std::min(num_fields, s.field_ids.size() - field_idx_); + bytes_consumed_ += fields_in_current_span * elements_per_field; + + field_idx_ += num_fields; + + // If we've exhausted fields in current span, move to next span + while (field_idx_ >= s.field_ids.size() && !done()) { + field_idx_ -= s.field_ids.size(); + span_idx_++; + memset(pos_, 0, sizeof(pos_)); + + if (!done() && field_idx_ >= (*list_)[span_idx_].field_ids.size()) { + // Continue to next span if still have fields to skip + // Increment bytes consumed for skipped fields in next span + const Span& next_s = (*list_)[span_idx_]; + size_t next_elements_per_field = next_s.extents[0]; + for (int d = 1; d < next_s.num_dims; d++) { + next_elements_per_field *= next_s.extents[d]; + } + size_t next_fields_skipped = std::min(field_idx_, next_s.field_ids.size()); + bytes_consumed_ += next_fields_skipped * next_elements_per_field; + continue; + } + break; + } + } + + void SpanIterator::skip_bytes(size_t bytes) { + while (bytes > 0 && !done()) { + int d = dim(); + + if (d == 0) { + // No geometry to skip through + return; + } + + size_t chunk = remaining(0); + if (chunk <= bytes) { + // Skip entire remaining chunk in dimension 0 + advance(0, chunk); + bytes -= chunk; + } else { + // Skip partial chunk + advance(0, bytes); + bytes = 0; + } + } + } + + bool SpanIterator::done() const { + return !list_ || span_idx_ >= list_->size(); + } + +} // namespace Realm + diff --git a/src/realm/transfer/memory_span.h b/src/realm/transfer/memory_span.h new file mode 100644 index 00000000000..58dfb8a8ceb --- /dev/null +++ b/src/realm/transfer/memory_span.h @@ -0,0 +1,120 @@ +/* + * Copyright 2025 Los Alamos National Laboratory, Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEMORY_SPAN_H +#define MEMORY_SPAN_H + +#include "realm/realm_config.h" +#include "realm/indexspace.h" + +#include +#include +#include + +namespace Realm { + + // ================================================================================================= + // Span + // ================================================================================================= + + struct Span { + size_t base_offset; // Base memory offset + std::vector field_ids; // Field ID(s) - one or more + uint8_t num_dims; // 1-3 + uint32_t extents[3]; // Elements per dimension + size_t strides[3]; // Byte stride per dimension + + // Computed properties + size_t total_bytes() const; + bool is_contiguous() const; + }; + + // ================================================================================================= + // SpanList + // ================================================================================================= + + class SpanIterator; // Forward declaration + + class SpanList { + std::vector spans_; + + public: + SpanList() = default; + + // Append span (works for single or multiple fields) + void append(const Span& span); + + // Convenience builder + void append(size_t base, const std::vector& fields, uint8_t dims, + const uint32_t* extents, const size_t* strides); + + // Access + size_t size() const { return spans_.size(); } + const Span& operator[](size_t idx) const { return spans_[idx]; } + bool empty() const { return spans_.empty(); } + + // Compute total bytes pending across all spans + size_t total_bytes() const; + size_t bytes_pending() const { return total_bytes(); } // Total bytes (no iterator) + }; + + // ================================================================================================= + // SpanIterator + // ================================================================================================= + + class SpanIterator { + const SpanList* list_ = nullptr; + size_t span_idx_ = 0; + size_t field_idx_ = 0; // Current field within span + uint32_t pos_[3] = {}; + size_t bytes_consumed_ = 0; // Total bytes consumed so far + + public: + SpanIterator() = default; + explicit SpanIterator(const SpanList* list); + + // Geometry queries + size_t offset() const; // Offset for current field at current position + size_t stride(int dim) const; + size_t remaining(int dim) const; + int dim() const; + + // Field queries + FieldID current_field() const; + size_t remaining_fields() const; // Fields left in current span + + // Progress tracking - dynamically accounts for list growth! + size_t bytes_pending() const { + return list_ ? (list_->total_bytes() - bytes_consumed_) : 0; + } + + // Navigation + void advance(int dim, size_t count); + void advance_fields(size_t num_fields); // Move to next field(s) + void skip_bytes(size_t bytes); // Helper for compatibility + bool done() const; + + // Access to current state (for debugging/testing) + size_t current_span_index() const { return span_idx_; } + size_t current_field_index() const { return field_idx_; } + const uint32_t* position() const { return pos_; } + }; + +} // namespace Realm + +#endif // MEMORY_SPAN_H + diff --git a/src/realm/transfer/transfer.cc b/src/realm/transfer/transfer.cc index 54020a174fb..e4a6febc990 100644 --- a/src/realm/transfer/transfer.cc +++ b/src/realm/transfer/transfer.cc @@ -505,7 +505,7 @@ namespace Realm { template bool - TransferIteratorBase::get_addresses(AddressList &addrlist, + TransferIteratorBase::get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine) { #ifdef DEBUG_REALM @@ -521,13 +521,6 @@ namespace Realm { return false; // no more addresses at the moment, but expect more later } - // we may be able to compact dimensions, but ask for space to write a - // an address record of the maximum possible dimension (i.e. N) - size_t *addr_data = addrlist.begin_entry(N); - if(!addr_data) { - return true; // out of space for now - } - // find the layout piece the current point is in const InstanceLayoutPiece *layout_piece; size_t field_rel_offset; @@ -577,13 +570,16 @@ namespace Realm { const AffineLayoutPiece *affine = static_cast *>(layout_piece); - // offset of initial entry is easy to compute - addr_data[1] = (inst_impl->metadata.inst_offset + affine->offset + - affine->strides.dot(target_subrect.lo) + field_rel_offset); + // Prepare span data + Span span; + span.base_offset = (inst_impl->metadata.inst_offset + affine->offset + + affine->strides.dot(target_subrect.lo) + field_rel_offset); + span.field_ids.push_back(cur_field_id); size_t bytes = cur_field_size; - int cur_dim = 1; + int cur_dim = 0; int di = 0; + // compact any dimensions that are contiguous first for(; di < N; di++) { // follow the agreed-upon dimension ordering @@ -603,6 +599,11 @@ namespace Realm { bytes *= (target_subrect.hi[d] - target_subrect.lo[d] + 1); } + // First dimension is the contiguous bytes + span.extents[cur_dim] = bytes; + span.strides[cur_dim] = 1; + cur_dim++; + // if any dimensions are left, they need to become count/stride pairs size_t total_bytes = bytes; while(di < N) { @@ -625,8 +626,8 @@ namespace Realm { total_count *= count; } - addr_data[cur_dim * 2] = total_count; - addr_data[cur_dim * 2 + 1] = stride; + span.extents[cur_dim] = total_count; + span.strides[cur_dim] = stride; log_dma.debug() << "Add addr data dim=" << cur_dim << " total_count=" << total_count << " stride=" << stride; @@ -634,11 +635,10 @@ namespace Realm { cur_dim++; } - // now that we know the compacted dimension, we can finish the address - // record - addr_data[0] = (bytes << 4) + cur_dim; - addrlist.commit_entry(cur_dim, total_bytes); - log_dma.debug() << "Finalize addr data dim=" << cur_dim << " total_bytes" + span.num_dims = cur_dim; + span_list.append(span); + + log_dma.debug() << "Finalize addr data dim=" << cur_dim << " total_bytes=" << total_bytes; } else { assert(0 && "no support for non-affine pieces yet"); @@ -843,7 +843,7 @@ namespace Realm { virtual void set_indirect_input_port(XferDes *xd, int port_idx, TransferIterator *inner_iter); - virtual bool get_addresses(AddressList &addrlist, + virtual bool get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine); virtual size_t get_base_offset(void) const; @@ -1013,7 +1013,7 @@ namespace Realm { template bool WrappingTransferIteratorIndirect::get_addresses( - AddressList &addrlist, const InstanceLayoutPieceBase *&nonaffine) + SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine) { nonaffine = 0; @@ -1022,16 +1022,19 @@ namespace Realm { return false; } - size_t *addr_data = addrlist.begin_entry(1); - if(!addr_data) { - return true; - } - int cur_dim = 1; size_t total_bytes = this->cur_rect.volume() * this->cur_field_size; this->have_rect = false; - addr_data[0] = ((total_bytes) << 4) + cur_dim; - addrlist.commit_entry(cur_dim, total_bytes); + + // Create a 1D span for the indirect address + Span span; + span.base_offset = 0; // Will be filled in by indirect addressing + span.field_ids.push_back(this->cur_field_id); + span.num_dims = 1; + span.extents[0] = total_bytes; + span.strides[0] = 1; + + span_list.append(span); log_dma.debug() << "Finalize gather/scatter addr data dim=" << cur_dim << " total_bytes=" << total_bytes; break; diff --git a/src/realm/transfer/transfer.h b/src/realm/transfer/transfer.h index 8a68a240c1c..79ca2d12aba 100644 --- a/src/realm/transfer/transfer.h +++ b/src/realm/transfer/transfer.h @@ -36,7 +36,7 @@ namespace Realm { // can hold an arbitrary IndexSpace and dispatch based on its type class XferDes; - class AddressList; + class SpanList; class TransferIterator { public: @@ -112,7 +112,7 @@ namespace Realm { virtual void confirm_step(void) = 0; virtual void cancel_step(void) = 0; - virtual bool get_addresses(AddressList &addrlist, + virtual bool get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine) = 0; }; @@ -142,7 +142,7 @@ namespace Realm { virtual size_t get_base_offset(void) const; - virtual bool get_addresses(AddressList &addrlist, + virtual bool get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine); protected: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77bc2d6efba..ccfd404f04e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -101,7 +101,7 @@ list( event_test.cc path_cache_test.cc transfer_utils_test.cc - address_list_test.cc + memory_span_test.cc cmdline_parser_test.cc sequence_assembler_test.cc dynamic_table_test.cc @@ -131,6 +131,13 @@ target_link_libraries( realm_unit_tests PRIVATE realm_obj GTest::gmock_main ${REALM_LIBRARIES} ${REALM_EXTRA_OBJS} ) +# Standalone memory_span_test for debugging +add_executable(memory_span_test ${REALM_TEST_DIR}/unit_tests/memory_span_test.cc) +target_compile_features(memory_span_test PUBLIC cxx_std_20) +target_link_libraries( + memory_span_test PRIVATE realm_obj GTest::gmock_main ${REALM_LIBRARIES} ${REALM_EXTRA_OBJS} +) + # TODO(wei): To be merged into realm_unit_tests once runtime singleton is # removed from all unit tests list( diff --git a/tests/unit_tests/addrsplit_channel_test.cc b/tests/unit_tests/addrsplit_channel_test.cc index 2f2033d4394..b8b9c2b8502 100644 --- a/tests/unit_tests/addrsplit_channel_test.cc +++ b/tests/unit_tests/addrsplit_channel_test.cc @@ -130,7 +130,7 @@ class MockIterator : public TransferIterator { size_t get_base_offset(void) const override { return 0; } - bool get_addresses(AddressList &addrlist, + bool get_addresses(SpanList &span_list, const InstanceLayoutPieceBase *&nonaffine) override { nonaffine = 0; diff --git a/tests/unit_tests/memcpy_channel_test.cc b/tests/unit_tests/memcpy_channel_test.cc index 80f8697ee8b..c78cac2fc0e 100644 --- a/tests/unit_tests/memcpy_channel_test.cc +++ b/tests/unit_tests/memcpy_channel_test.cc @@ -231,7 +231,7 @@ void run_test_case(const MemcpyXferTestCaseData &test_case) input_port.mem = input_mem.get(); input_port.peer_port_idx = 0; input_port.iter = src_it; - input_port.addrcursor.set_addrlist(&input_port.addrlist); + input_port.span_iter = SpanIterator(&input_port.span_list); TransferIteratorIndexSpace *dst_it = new TransferIteratorIndexSpace( test_case.dim_order.data(), test_case.field_ids, test_case.field_offsets, @@ -251,7 +251,7 @@ void run_test_case(const MemcpyXferTestCaseData &test_case) output_port.peer_port_idx = 0; output_port.iter = dst_it; - output_port.addrcursor.set_addrlist(&output_port.addrlist); + output_port.span_iter = SpanIterator(&output_port.span_list); while(xfer_desc->progress_xd(channel.get(), TimeLimit::relative(10000000))) { } diff --git a/tests/unit_tests/memory_span_test.cc b/tests/unit_tests/memory_span_test.cc new file mode 100644 index 00000000000..25f4010f60e --- /dev/null +++ b/tests/unit_tests/memory_span_test.cc @@ -0,0 +1,374 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "realm/transfer/memory_span.h" +#include + +using namespace Realm; + +namespace { + + constexpr size_t kStride = 8; + constexpr size_t kBytes = 1024; + + TEST(MemorySpanTests, SingleFieldBasic) { + // Single field, 1D span + std::vector fields = {100}; + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + spans.append(0x1000, fields, 1, extents, strides); + + EXPECT_EQ(spans.size(), 1); + EXPECT_EQ(spans[0].field_ids.size(), 1); + EXPECT_EQ(spans[0].field_ids[0], 100); + EXPECT_EQ(spans[0].total_bytes(), kBytes); + + SpanIterator it(&spans); + EXPECT_FALSE(it.done()); + EXPECT_EQ(it.current_field(), 100); + EXPECT_EQ(it.offset(), 0x1000); + EXPECT_EQ(it.remaining(0), kBytes); + EXPECT_EQ(it.dim(), 1); + } + + TEST(MemorySpanTests, SingleFieldAdvance) { + std::vector fields = {100}; + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + spans.append(0, fields, 1, extents, strides); + + SpanIterator it(&spans); + + // Advance partway + it.advance(0, 128); + EXPECT_EQ(it.offset(), 128); + EXPECT_EQ(it.remaining(0), kBytes - 128); + EXPECT_FALSE(it.done()); + + // Advance more + it.advance(0, 128); + EXPECT_EQ(it.offset(), 256); + EXPECT_EQ(it.remaining(0), kBytes - 256); + + // Advance to end + it.advance(0, kBytes - 256); + EXPECT_TRUE(it.done()); + } + + TEST(MemorySpanTests, MultiFieldSingleSpan) { + // Multiple fields in one span (contiguous case) + std::vector fields = {10, 11, 12, 13}; + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + spans.append(0x1000, fields, 1, extents, strides); + + EXPECT_EQ(spans.size(), 1); + EXPECT_EQ(spans[0].field_ids.size(), 4); + EXPECT_EQ(spans[0].total_bytes(), kBytes * 4); + + SpanIterator it(&spans); + + // Start at first field + EXPECT_EQ(it.current_field(), 10); + EXPECT_EQ(it.remaining_fields(), 4); + EXPECT_EQ(it.offset(), 0x1000); + + // Advance through geometry for first field + it.advance(0, 128); + EXPECT_EQ(it.current_field(), 10); + EXPECT_EQ(it.remaining(0), kBytes - 128); + + // Advance more + it.advance(0, 128); + EXPECT_EQ(it.remaining(0), kBytes - 256); + + // Complete first field + it.advance(0, kBytes - 256); + // Should auto-advance to next field + EXPECT_FALSE(it.done()); + EXPECT_EQ(it.current_field(), 11); + EXPECT_EQ(it.remaining_fields(), 3); + EXPECT_EQ(it.remaining(0), kBytes); + } + + TEST(MemorySpanTests, MultiFieldAdvanceAll) { + std::vector fields = {7, 8, 9}; + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + spans.append(0, fields, 1, extents, strides); + + SpanIterator it(&spans); + + // Advance through all fields at once + for (size_t f = 0; f < 3; f++) { + EXPECT_EQ(it.current_field(), FieldID(7 + f)); + it.advance(0, kBytes); + } + + EXPECT_TRUE(it.done()); + } + + TEST(MemorySpanTests, TwoDimensionalSingle) { + std::vector fields = {100}; + uint32_t extents[] = {1024, 100}; // 1024 bytes × 100 lines + size_t strides[] = {1, 1024}; + + SpanList spans; + spans.append(0, fields, 2, extents, strides); + + SpanIterator it(&spans); + EXPECT_EQ(it.dim(), 2); + EXPECT_EQ(it.remaining(0), 1024); + EXPECT_EQ(it.remaining(1), 100); + EXPECT_EQ(it.stride(1), 1024); + + // Advance one line + it.advance(0, 1024); + EXPECT_EQ(it.remaining(0), 1024); + EXPECT_EQ(it.remaining(1), 99); + + // Advance rest of lines + it.advance(1, 99); + EXPECT_TRUE(it.done()); + } + + TEST(MemorySpanTests, TwoDimensionalMultiField) { + std::vector fields = {10, 11, 12}; + uint32_t extents[] = {1024, 100}; + size_t strides[] = {1, 1024}; + + SpanList spans; + spans.append(0, fields, 2, extents, strides); + + SpanIterator it(&spans); + + size_t total_bytes = 1024 * 100 * 3; + EXPECT_EQ(spans.total_bytes(), total_bytes); + + // Process first field + EXPECT_EQ(it.current_field(), 10); + it.advance(1, 100); // Advance all lines + + // Should move to second field + EXPECT_FALSE(it.done()); + EXPECT_EQ(it.current_field(), 11); + EXPECT_EQ(it.remaining(1), 100); + } + + TEST(MemorySpanTests, MultipleSpansSeparateFields) { + // Three spans, one field each (non-contiguous case) + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + std::vector fields1 = {10}; + std::vector fields2 = {20}; + std::vector fields3 = {30}; + spans.append(0x1000, fields1, 1, extents, strides); + spans.append(0x5000, fields2, 1, extents, strides); + spans.append(0x9000, fields3, 1, extents, strides); + + EXPECT_EQ(spans.size(), 3); + EXPECT_EQ(spans.total_bytes(), kBytes * 3); + + SpanIterator it(&spans); + + // First span + EXPECT_EQ(it.current_field(), 10); + EXPECT_EQ(it.offset(), 0x1000); + it.advance(0, kBytes); + + // Second span + EXPECT_FALSE(it.done()); + EXPECT_EQ(it.current_field(), 20); + EXPECT_EQ(it.offset(), 0x5000); + it.advance(0, kBytes); + + // Third span + EXPECT_FALSE(it.done()); + EXPECT_EQ(it.current_field(), 30); + EXPECT_EQ(it.offset(), 0x9000); + it.advance(0, kBytes); + + EXPECT_TRUE(it.done()); + } + + TEST(MemorySpanTests, ThreeDimensional) { + std::vector fields = {100}; + uint32_t extents[] = {64, 8, 2}; // 64 bytes × 8 × 2 + size_t strides[] = {1, 64, 512}; + + SpanList spans; + spans.append(0, fields, 3, extents, strides); + + SpanIterator it(&spans); + EXPECT_EQ(it.dim(), 3); + EXPECT_EQ(it.remaining(0), 64); + EXPECT_EQ(it.remaining(1), 8); + EXPECT_EQ(it.remaining(2), 2); + + // Advance through highest dimension + it.advance(2, 2); + EXPECT_TRUE(it.done()); + } + + TEST(MemorySpanTests, SkipBytes) { + std::vector fields = {100}; + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + spans.append(0, fields, 1, extents, strides); + + SpanIterator it(&spans); + + // Skip some bytes + it.skip_bytes(128); + EXPECT_EQ(it.offset(), 128); + EXPECT_EQ(it.remaining(0), kBytes - 128); + + // Skip more + it.skip_bytes(128); + EXPECT_EQ(it.offset(), 256); + } + + TEST(MemorySpanTests, AdvanceFieldsExplicit) { + std::vector fields = {10, 11, 12, 13}; + uint32_t extents[] = {kBytes}; + size_t strides[] = {1}; + + SpanList spans; + spans.append(0, fields, 1, extents, strides); + + SpanIterator it(&spans); + + // At first field + EXPECT_EQ(it.current_field(), 10); + EXPECT_EQ(it.current_field_index(), 0); + + // Advance to skip geometry for 2 fields + it.advance(0, kBytes); // Complete field 10 + EXPECT_EQ(it.current_field(), 11); + + it.advance(0, kBytes); // Complete field 11 + EXPECT_EQ(it.current_field(), 12); + EXPECT_EQ(it.remaining_fields(), 2); + } + + TEST(MemorySpanTests, IsContiguous) { + // Contiguous span + std::vector fields = {100}; + uint32_t extents1[] = {64, 100}; + size_t strides1[] = {1, 64}; + + Span span1; + span1.base_offset = 0; + span1.field_ids = fields; + span1.num_dims = 2; + span1.extents[0] = extents1[0]; + span1.extents[1] = extents1[1]; + span1.strides[0] = strides1[0]; + span1.strides[1] = strides1[1]; + + EXPECT_TRUE(span1.is_contiguous()); + + // Non-contiguous span + Span span2; + span2.base_offset = 0; + span2.field_ids = fields; + span2.num_dims = 2; + span2.extents[0] = 64; + span2.extents[1] = 100; + span2.strides[0] = 1; + span2.strides[1] = 128; // Stride doesn't match (has padding) + + EXPECT_FALSE(span2.is_contiguous()); + } + + TEST(MemorySpanTests, EmptySpanList) { + SpanList spans; + EXPECT_TRUE(spans.empty()); + EXPECT_EQ(spans.size(), 0); + EXPECT_EQ(spans.total_bytes(), 0); + + SpanIterator it(&spans); + EXPECT_TRUE(it.done()); + } + + TEST(MemorySpanTests, PartialAdvanceTwoDim) { + std::vector fields = {100}; + uint32_t extents[] = {64, 10}; // 64 bytes × 10 lines + size_t strides[] = {1, 64}; + + SpanList spans; + spans.append(0, fields, 2, extents, strides); + + SpanIterator it(&spans); + + // Advance half a line + it.advance(0, 32); + EXPECT_EQ(it.remaining(0), 32); + EXPECT_EQ(it.remaining(1), 10); + EXPECT_EQ(it.offset(), 32); + + // Complete the line + it.advance(0, 32); + EXPECT_EQ(it.remaining(0), 64); + EXPECT_EQ(it.remaining(1), 9); + EXPECT_EQ(it.offset(), 64); + } + + TEST(MemorySpanTests, AdvanceMultiLevelOverflow) + { + // Test advancing by an amount that overflows multiple times + // 2D: 10 bytes per line, 5 lines = 50 bytes total + SpanList spans; + uint32_t extents[] = {10, 5}; + size_t strides[] = {1, 10}; + spans.append(0x1000, {100}, 2, extents, strides); + + SpanIterator it(&spans); + EXPECT_EQ(it.dim(), 2); + EXPECT_EQ(it.remaining(0), 10); + EXPECT_EQ(it.remaining(1), 5); + + // Advance by 25 bytes in dimension 0 + // Should overflow twice: 25 / 10 = 2 carries, 25 % 10 = 5 remainder + // Expected: pos[0] = 5, pos[1] = 2 + it.advance(0, 25); + + EXPECT_EQ(it.remaining(0), 5); // 10 - 5 = 5 remaining in dim 0 + EXPECT_EQ(it.remaining(1), 3); // 5 - 2 = 3 remaining in dim 1 + EXPECT_EQ(it.offset(), 0x1000 + 2 * 10 + 5); // base + 2 lines + 5 bytes + EXPECT_FALSE(it.done()); + + // Advance by another 25 bytes to complete the span + // Current: pos[0]=5, pos[1]=2 (consumed 25 bytes, 25 remaining) + it.advance(0, 25); + EXPECT_TRUE(it.done()); + } + +} // namespace + diff --git a/tests/unit_tests/transfer_iterator_test.cc b/tests/unit_tests/transfer_iterator_test.cc index 92c657a5ee6..7969aac8b5c 100644 --- a/tests/unit_tests/transfer_iterator_test.cc +++ b/tests/unit_tests/transfer_iterator_test.cc @@ -75,15 +75,15 @@ void run_test_case(const TrasferItTestCaseData &test_case) test_case.field_sizes), test_case.domain, impl.get()); const InstanceLayoutPieceBase *nonaffine; - AddressList addrlist; - AddressListCursor cursor; + SpanList span_list; + SpanIterator span_iter; - bool ok = it->get_addresses(addrlist, nonaffine); + bool ok = it->get_addresses(span_list, nonaffine); ASSERT_TRUE(ok); ASSERT_TRUE(it->done()); - cursor.set_addrlist(&addrlist); + span_iter = SpanIterator(&span_list); size_t total_volume = 0; for(const auto &rect : test_case.expected) { total_volume += rect.volume(); @@ -94,20 +94,20 @@ void run_test_case(const TrasferItTestCaseData &test_case) bytes_pending += total_volume * size; } - ASSERT_EQ(addrlist.bytes_pending(), bytes_pending); + ASSERT_EQ(span_iter.bytes_pending(), bytes_pending); - if(bytes_pending > 0 && cursor.get_dim() == 1) { - // TODO(apryakhin:@): Find better way to analyze the adddress list - // ASSERT_EQ(cursor.get_dim(), 1); + if(bytes_pending > 0 && span_iter.dim() == 1) { + // TODO(apryakhin:@): Find better way to analyze the address list + // ASSERT_EQ(span_iter.dim(), 1); for(const size_t field_size : test_case.field_sizes) { for(const auto &rect : test_case.expected) { - int dim = cursor.get_dim(); - ASSERT_EQ(cursor.remaining(dim - 1), rect.volume() * field_size); - cursor.advance(dim - 1, cursor.remaining(dim - 1)); + int dim = span_iter.dim(); + ASSERT_EQ(span_iter.remaining(dim - 1), rect.volume() * field_size); + span_iter.advance(dim - 1, span_iter.remaining(dim - 1)); } } - ASSERT_EQ(addrlist.bytes_pending(), 0); + ASSERT_EQ(span_iter.bytes_pending(), 0); } } From a93641b47333ac7eb5aef48a239a55514997093e Mon Sep 17 00:00:00 2001 From: Micha Dery Date: Mon, 3 Nov 2025 18:58:35 +0200 Subject: [PATCH 3/6] bytes_panding: span_list -> span_iter --- src/realm/transfer/channel.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/realm/transfer/channel.cc b/src/realm/transfer/channel.cc index 3c2edaf058f..230c8dc1b9a 100644 --- a/src/realm/transfer/channel.cc +++ b/src/realm/transfer/channel.cc @@ -803,10 +803,10 @@ namespace Realm { XferPort *in_port = &input_ports[input_control.current_io_port]; // do we need more addresses? - size_t read_bytes_avail = in_port->span_list.bytes_pending(); + size_t read_bytes_avail = in_port->span_iter.bytes_pending(); if(read_bytes_avail < min_xfer_size) { bool flush = in_port->iter->get_addresses(in_port->span_list, in_nonaffine); - read_bytes_avail = in_port->span_list.bytes_pending(); + read_bytes_avail = in_port->span_iter.bytes_pending(); if(flush) { if(read_bytes_avail > 0) { // ignore a nonaffine piece as we still have some affine bytes @@ -855,10 +855,10 @@ namespace Realm { XferPort *out_port = &output_ports[output_control.current_io_port]; // do we need more addresses? - size_t write_bytes_avail = out_port->span_list.bytes_pending(); + size_t write_bytes_avail = out_port->span_iter.bytes_pending(); if(write_bytes_avail < min_xfer_size) { bool flush = out_port->iter->get_addresses(out_port->span_list, out_nonaffine); - write_bytes_avail = out_port->span_list.bytes_pending(); + write_bytes_avail = out_port->span_iter.bytes_pending(); // TODO(apryakhin@): We add this to handle scatter when both // indirection and source are coming from IB and this needs @@ -927,7 +927,7 @@ namespace Realm { in_port->local_bytes_cons.fetch_add(total_read_bytes); if(in_port->peer_guid == XFERDES_NO_GUID) - in_done = ((in_port->span_list.bytes_pending() == 0) && in_port->iter->done()); + in_done = ((in_port->span_iter.bytes_pending() == 0) && in_port->iter->done()); else in_done = (in_port->local_bytes_total == in_port->remote_bytes_total.load_acquire()); @@ -941,7 +941,7 @@ namespace Realm { out_port->local_bytes_cons.fetch_add(total_write_bytes); if(out_port->peer_guid == XFERDES_NO_GUID) - out_done = ((out_port->span_list.bytes_pending() == 0) && out_port->iter->done()); + out_done = ((out_port->span_iter.bytes_pending() == 0) && out_port->iter->done()); } input_control.remaining_count -= total_read_bytes; @@ -2212,7 +2212,7 @@ namespace Realm { output_control.remaining_count / out_elem_size); if(in_port != 0) { max_elems = - std::min(max_elems, in_port->span_list.bytes_pending() / in_elem_size); + std::min(max_elems, in_port->span_iter.bytes_pending() / in_elem_size); if(in_port->peer_guid != XFERDES_NO_GUID) { size_t read_bytes_avail = in_port->seq_remote.span_exists( in_port->local_bytes_total, (max_elems * in_elem_size)); @@ -2221,7 +2221,7 @@ namespace Realm { } if(out_port != 0) { max_elems = - std::min(max_elems, out_port->span_list.bytes_pending() / out_elem_size); + std::min(max_elems, out_port->span_iter.bytes_pending() / out_elem_size); // no support for reducing into an intermediate buffer assert(out_port->peer_guid == XFERDES_NO_GUID); } From f21c773797c9d70f39ec8690e35c805020b98f09 Mon Sep 17 00:00:00 2001 From: Micha Dery Date: Tue, 4 Nov 2025 17:01:35 +0200 Subject: [PATCH 4/6] Extending to 4 dimension to cover address list contiguous block tracking --- src/realm/transfer/memory_span.cc | 33 +++++++++++++++++++++++++++++-- src/realm/transfer/memory_span.h | 8 ++++---- src/realm/transfer/transfer.cc | 3 +++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/realm/transfer/memory_span.cc b/src/realm/transfer/memory_span.cc index 02303fe4744..809cc6987c4 100644 --- a/src/realm/transfer/memory_span.cc +++ b/src/realm/transfer/memory_span.cc @@ -71,12 +71,12 @@ namespace Realm { span.field_ids = fields; span.num_dims = dims; - for (int d = 0; d < dims && d < 3; d++) { + for (int d = 0; d < dims && d < 4; d++) { span.extents[d] = extents[d]; span.strides[d] = strides[d]; } // Zero out unused dimensions - for (int d = dims; d < 3; d++) { + for (int d = dims; d < 4; d++) { span.extents[d] = 0; span.strides[d] = 0; } @@ -152,6 +152,35 @@ namespace Realm { } const Span& s = (*list_)[span_idx_]; + + // Return effective dimensionality based on current position + // This matches AddressListCursor behavior: when we're partially through + // a multi-dimensional space, we report dimensions starting from the + // lowest dimension that has been partially consumed. + // + // Find the lowest dimension where we've made partial progress + for (int d = 0; d < s.num_dims; d++) { + if (pos_[d] > 0 && pos_[d] < s.extents[d]) { + // We're partway through dimension d + // Return d+1 (dimensionality includes this dimension) + int result = d + 1; + fprintf(stderr, "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], extents=[%u,%u,%u,%u], partial at d=%d, returning %d\n", + span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], s.extents[2], s.extents[3], d, result); + return result; + } + if (pos_[d] >= s.extents[d]) { + // This dimension is exhausted, should have carried to next + // (this shouldn't happen in well-formed iterator state) + continue; + } + } + + // No partial progress yet, return full dimensionality + fprintf(stderr, "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], extents=[%u,%u,%u,%u], num_dims=%d, no partial, returning %d\n", + span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], s.extents[2], s.extents[3], (int)s.num_dims, (int)s.num_dims); + if (s.num_dims > 4) { + fprintf(stderr, "ERROR: num_dims=%d is > 4, this is a bug!\n", (int)s.num_dims); + } return s.num_dims; } diff --git a/src/realm/transfer/memory_span.h b/src/realm/transfer/memory_span.h index 58dfb8a8ceb..ed56b117e58 100644 --- a/src/realm/transfer/memory_span.h +++ b/src/realm/transfer/memory_span.h @@ -34,9 +34,9 @@ namespace Realm { struct Span { size_t base_offset; // Base memory offset std::vector field_ids; // Field ID(s) - one or more - uint8_t num_dims; // 1-3 - uint32_t extents[3]; // Elements per dimension - size_t strides[3]; // Byte stride per dimension + uint8_t num_dims; // 1-4 (extended from 3 to 4 for complex layouts) + uint32_t extents[4]; // Elements per dimension + size_t strides[4]; // Byte stride per dimension // Computed properties size_t total_bytes() const; @@ -80,7 +80,7 @@ namespace Realm { const SpanList* list_ = nullptr; size_t span_idx_ = 0; size_t field_idx_ = 0; // Current field within span - uint32_t pos_[3] = {}; + uint32_t pos_[4] = {}; size_t bytes_consumed_ = 0; // Total bytes consumed so far public: diff --git a/src/realm/transfer/transfer.cc b/src/realm/transfer/transfer.cc index e4a6febc990..5d377191955 100644 --- a/src/realm/transfer/transfer.cc +++ b/src/realm/transfer/transfer.cc @@ -626,6 +626,9 @@ namespace Realm { total_count *= count; } + // Span now supports up to 4 dimensions + assert(cur_dim < 4 && "Span dimension limit (4) exceeded"); + span.extents[cur_dim] = total_count; span.strides[cur_dim] = stride; From 29174f0c0654938011b96c425d5a11c1557958cb Mon Sep 17 00:00:00 2001 From: Micha Dery Date: Tue, 4 Nov 2025 21:58:15 +0200 Subject: [PATCH 5/6] clang-format --- src/realm/cuda/cuda_internal.cc | 6 +- src/realm/transfer/channel.cc | 4 +- src/realm/transfer/lowlevel_dma.cc | 4 +- src/realm/transfer/memory_span.cc | 256 ++++++++++++++------------- src/realm/transfer/memory_span.h | 76 ++++---- src/realm/transfer/transfer.cc | 16 +- tests/unit_tests/memory_span_test.cc | 78 ++++---- 7 files changed, 232 insertions(+), 208 deletions(-) diff --git a/src/realm/cuda/cuda_internal.cc b/src/realm/cuda/cuda_internal.cc index 6d2e3060d46..03f33da7c96 100644 --- a/src/realm/cuda/cuda_internal.cc +++ b/src/realm/cuda/cuda_internal.cc @@ -252,8 +252,7 @@ namespace Realm { // if it's not contiguous, width must be exactly what // we need for either 1 or 2 leading dimensions if(iter.remaining(0) == ainfo.width_in_bytes) { - if((iter.dim() == 2) || - (iter.remaining(1) >= (ainfo.height * ainfo.depth))) { + if((iter.dim() == 2) || (iter.remaining(1) >= (ainfo.height * ainfo.depth))) { // input dim 1 covers output 1 and 2 pitch = iter.stride(1); height = ainfo.height; @@ -733,8 +732,7 @@ namespace Realm { cuda_copy.dstY = ainfo.pos[1]; cuda_copy.dstZ = ainfo.pos[2]; cuda_copy.srcMemoryType = CU_MEMORYTYPE_UNIFIED; - cuda_copy.srcDevice = - static_cast(in_base + in_iter.offset()); + cuda_copy.srcDevice = static_cast(in_base + in_iter.offset()); get_nonaffine_strides(cuda_copy.srcPitch, cuda_copy.srcHeight, ainfo, in_iter, bytes); } diff --git a/src/realm/transfer/channel.cc b/src/realm/transfer/channel.cc index 230c8dc1b9a..61f07482371 100644 --- a/src/realm/transfer/channel.cc +++ b/src/realm/transfer/channel.cc @@ -2718,8 +2718,8 @@ namespace Realm { in_port->mem->get_direct_ptr(in_iter.offset(), icount)); uintptr_t dst = reinterpret_cast(amsg.payload_ptr(icount * lines)); - memcpy_2d(dst, icount /*lstride*/, src, in_iter.stride(1), - icount, lines); + memcpy_2d(dst, icount /*lstride*/, src, in_iter.stride(1), icount, + lines); in_iter.advance(1, lines); todo -= icount * lines; } else { diff --git a/src/realm/transfer/lowlevel_dma.cc b/src/realm/transfer/lowlevel_dma.cc index 127aaa1a835..deb2ffe9e8a 100644 --- a/src/realm/transfer/lowlevel_dma.cc +++ b/src/realm/transfer/lowlevel_dma.cc @@ -860,14 +860,14 @@ namespace Realm { // Create span with single dummy field (wrapping FIFO doesn't use real fields) std::vector fields = {FieldID(0)}; - + if(dim == 1) { uint32_t extents[] = {static_cast(size)}; size_t strides[] = {1}; span_list.append(base, fields, 1, extents, strides); } else { uint32_t extents[] = {static_cast(size), static_cast(lines)}; - size_t strides[] = {1, 0}; // stride of 0 for wrapping + size_t strides[] = {1, 0}; // stride of 0 for wrapping span_list.append(base, fields, 2, extents, strides); } diff --git a/src/realm/transfer/memory_span.cc b/src/realm/transfer/memory_span.cc index 809cc6987c4..d279518d559 100644 --- a/src/realm/transfer/memory_span.cc +++ b/src/realm/transfer/memory_span.cc @@ -24,31 +24,33 @@ namespace Realm { // ================================================================================================= // Span // ================================================================================================= - - size_t Span::total_bytes() const { - if (num_dims == 0) { + + size_t Span::total_bytes() const + { + if(num_dims == 0) { return 0; } - + // Calculate bytes for geometry size_t bytes = extents[0]; - for (int d = 1; d < num_dims; d++) { + for(int d = 1; d < num_dims; d++) { bytes *= extents[d]; } - + // Multiply by number of fields return bytes * field_ids.size(); } - - bool Span::is_contiguous() const { - if (num_dims == 0) { + + bool Span::is_contiguous() const + { + if(num_dims == 0) { return true; } - + // Check if strides match expected contiguous layout size_t expected_stride = 1; - for (int d = 0; d < num_dims; d++) { - if (strides[d] != expected_stride) { + for(int d = 0; d < num_dims; d++) { + if(strides[d] != expected_stride) { return false; } expected_stride *= extents[d]; @@ -59,176 +61,189 @@ namespace Realm { // ================================================================================================= // SpanList // ================================================================================================= - - void SpanList::append(const Span& span) { - spans_.push_back(span); - } - - void SpanList::append(size_t base, const std::vector& fields, uint8_t dims, - const uint32_t* extents, const size_t* strides) { + + void SpanList::append(const Span &span) { spans_.push_back(span); } + + void SpanList::append(size_t base, const std::vector &fields, uint8_t dims, + const uint32_t *extents, const size_t *strides) + { Span span; span.base_offset = base; span.field_ids = fields; span.num_dims = dims; - - for (int d = 0; d < dims && d < 4; d++) { + + for(int d = 0; d < dims && d < 4; d++) { span.extents[d] = extents[d]; span.strides[d] = strides[d]; } // Zero out unused dimensions - for (int d = dims; d < 4; d++) { + for(int d = dims; d < 4; d++) { span.extents[d] = 0; span.strides[d] = 0; } - + spans_.push_back(span); } - - size_t SpanList::total_bytes() const { + + size_t SpanList::total_bytes() const + { size_t total = 0; - for (const auto& span : spans_) { + for(const auto &span : spans_) { total += span.total_bytes(); } return total; } - + // ================================================================================================= // SpanIterator // ================================================================================================= - - SpanIterator::SpanIterator(const SpanList* list) + + SpanIterator::SpanIterator(const SpanList *list) : list_(list) , bytes_consumed_(0) { memset(pos_, 0, sizeof(pos_)); } - - size_t SpanIterator::offset() const { - if (!list_ || done()) { + + size_t SpanIterator::offset() const + { + if(!list_ || done()) { return 0; } - - const Span& s = (*list_)[span_idx_]; - + + const Span &s = (*list_)[span_idx_]; + size_t offset = s.base_offset; - + // Add position within geometry - for (int d = 0; d < s.num_dims; d++) { + for(int d = 0; d < s.num_dims; d++) { offset += pos_[d] * s.strides[d]; } - + // If multiple fields, add field offset - if (s.field_ids.size() > 1 && field_idx_ > 0) { + if(s.field_ids.size() > 1 && field_idx_ > 0) { size_t bytes_per_field = s.total_bytes() / s.field_ids.size(); offset += field_idx_ * bytes_per_field; } - + return offset; } - - size_t SpanIterator::stride(int dim) const { - if (done()) { + + size_t SpanIterator::stride(int dim) const + { + if(done()) { return 0; } - - const Span& s = (*list_)[span_idx_]; + + const Span &s = (*list_)[span_idx_]; assert(dim >= 0 && dim < s.num_dims); return s.strides[dim]; } - - size_t SpanIterator::remaining(int dim) const { - if (done()) { + + size_t SpanIterator::remaining(int dim) const + { + if(done()) { return 0; } - - const Span& s = (*list_)[span_idx_]; + + const Span &s = (*list_)[span_idx_]; assert(dim >= 0 && dim < s.num_dims); return s.extents[dim] - pos_[dim]; } - - int SpanIterator::dim() const { - if (done()) { + + int SpanIterator::dim() const + { + if(done()) { return 0; } - - const Span& s = (*list_)[span_idx_]; - + + const Span &s = (*list_)[span_idx_]; + // Return effective dimensionality based on current position // This matches AddressListCursor behavior: when we're partially through // a multi-dimensional space, we report dimensions starting from the // lowest dimension that has been partially consumed. // // Find the lowest dimension where we've made partial progress - for (int d = 0; d < s.num_dims; d++) { - if (pos_[d] > 0 && pos_[d] < s.extents[d]) { + for(int d = 0; d < s.num_dims; d++) { + if(pos_[d] > 0 && pos_[d] < s.extents[d]) { // We're partway through dimension d // Return d+1 (dimensionality includes this dimension) int result = d + 1; - fprintf(stderr, "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], extents=[%u,%u,%u,%u], partial at d=%d, returning %d\n", - span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], s.extents[2], s.extents[3], d, result); + fprintf(stderr, + "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], " + "extents=[%u,%u,%u,%u], partial at d=%d, returning %d\n", + span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], + s.extents[2], s.extents[3], d, result); return result; } - if (pos_[d] >= s.extents[d]) { + if(pos_[d] >= s.extents[d]) { // This dimension is exhausted, should have carried to next // (this shouldn't happen in well-formed iterator state) continue; } } - + // No partial progress yet, return full dimensionality - fprintf(stderr, "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], extents=[%u,%u,%u,%u], num_dims=%d, no partial, returning %d\n", - span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], s.extents[2], s.extents[3], (int)s.num_dims, (int)s.num_dims); - if (s.num_dims > 4) { + fprintf(stderr, + "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], " + "extents=[%u,%u,%u,%u], num_dims=%d, no partial, returning %d\n", + span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], + s.extents[2], s.extents[3], (int)s.num_dims, (int)s.num_dims); + if(s.num_dims > 4) { fprintf(stderr, "ERROR: num_dims=%d is > 4, this is a bug!\n", (int)s.num_dims); } return s.num_dims; } - - FieldID SpanIterator::current_field() const { - if (done()) { + + FieldID SpanIterator::current_field() const + { + if(done()) { return FieldID(-1); } - - const Span& s = (*list_)[span_idx_]; + + const Span &s = (*list_)[span_idx_]; assert(field_idx_ < s.field_ids.size()); return s.field_ids[field_idx_]; } - - size_t SpanIterator::remaining_fields() const { - if (done()) { + + size_t SpanIterator::remaining_fields() const + { + if(done()) { return 0; } - - const Span& s = (*list_)[span_idx_]; + + const Span &s = (*list_)[span_idx_]; return s.field_ids.size() - field_idx_; } - - void SpanIterator::advance(int dim, size_t count) { - if (done()) { + + void SpanIterator::advance(int dim, size_t count) + { + if(done()) { return; } - - const Span& s = (*list_)[span_idx_]; - + + const Span &s = (*list_)[span_idx_]; + size_t bytes_to_consume = count; - for (int d = 0; d < dim; d++) { + for(int d = 0; d < dim; d++) { bytes_to_consume *= s.extents[d]; } - + // Update bytes_consumed bytes_consumed_ += bytes_to_consume; - + pos_[dim] += count; - + // Carry to higher dimensions - handle multi-level overflow bool geometry_exhausted = false; - for (int d = dim; d < s.num_dims; d++) { - if (pos_[d] >= s.extents[d]) { - if (d + 1 < s.num_dims) { + for(int d = dim; d < s.num_dims; d++) { + if(pos_[d] >= s.extents[d]) { + if(d + 1 < s.num_dims) { // Calculate how many times we overflow this dimension size_t carries = pos_[d] / s.extents[d]; - pos_[d] = pos_[d] % s.extents[d]; // Remainder stays in this dimension - pos_[d + 1] += carries; // Propagate carries to next dimension + pos_[d] = pos_[d] % s.extents[d]; // Remainder stays in this dimension + pos_[d + 1] += carries; // Propagate carries to next dimension // Continue loop to handle cascading carries } else { // Reached the end of the highest dimension @@ -236,14 +251,14 @@ namespace Realm { break; } } else { - break; // No more carries needed + break; // No more carries needed } } - + // Check if geometry exhausted - if (geometry_exhausted) { + if(geometry_exhausted) { // Move to next field if multi-field span - if (field_idx_ + 1 < s.field_ids.size()) { + if(field_idx_ + 1 < s.field_ids.size()) { field_idx_++; memset(pos_, 0, sizeof(pos_)); } else { @@ -254,38 +269,39 @@ namespace Realm { } } } - - void SpanIterator::advance_fields(size_t num_fields) { - if (done()) { + + void SpanIterator::advance_fields(size_t num_fields) + { + if(done()) { return; } - - const Span& s = (*list_)[span_idx_]; - + + const Span &s = (*list_)[span_idx_]; + // Calculate bytes per field for current span size_t elements_per_field = s.extents[0]; - for (int d = 1; d < s.num_dims; d++) { + for(int d = 1; d < s.num_dims; d++) { elements_per_field *= s.extents[d]; } - + // Increment bytes consumed for fields we're skipping in current span size_t fields_in_current_span = std::min(num_fields, s.field_ids.size() - field_idx_); bytes_consumed_ += fields_in_current_span * elements_per_field; - + field_idx_ += num_fields; - + // If we've exhausted fields in current span, move to next span - while (field_idx_ >= s.field_ids.size() && !done()) { + while(field_idx_ >= s.field_ids.size() && !done()) { field_idx_ -= s.field_ids.size(); span_idx_++; memset(pos_, 0, sizeof(pos_)); - - if (!done() && field_idx_ >= (*list_)[span_idx_].field_ids.size()) { + + if(!done() && field_idx_ >= (*list_)[span_idx_].field_ids.size()) { // Continue to next span if still have fields to skip // Increment bytes consumed for skipped fields in next span - const Span& next_s = (*list_)[span_idx_]; + const Span &next_s = (*list_)[span_idx_]; size_t next_elements_per_field = next_s.extents[0]; - for (int d = 1; d < next_s.num_dims; d++) { + for(int d = 1; d < next_s.num_dims; d++) { next_elements_per_field *= next_s.extents[d]; } size_t next_fields_skipped = std::min(field_idx_, next_s.field_ids.size()); @@ -295,18 +311,19 @@ namespace Realm { break; } } - - void SpanIterator::skip_bytes(size_t bytes) { - while (bytes > 0 && !done()) { + + void SpanIterator::skip_bytes(size_t bytes) + { + while(bytes > 0 && !done()) { int d = dim(); - - if (d == 0) { + + if(d == 0) { // No geometry to skip through return; } - + size_t chunk = remaining(0); - if (chunk <= bytes) { + if(chunk <= bytes) { // Skip entire remaining chunk in dimension 0 advance(0, chunk); bytes -= chunk; @@ -317,10 +334,7 @@ namespace Realm { } } } - - bool SpanIterator::done() const { - return !list_ || span_idx_ >= list_->size(); - } -} // namespace Realm + bool SpanIterator::done() const { return !list_ || span_idx_ >= list_->size(); } +} // namespace Realm diff --git a/src/realm/transfer/memory_span.h b/src/realm/transfer/memory_span.h index ed56b117e58..ebfb88ffc14 100644 --- a/src/realm/transfer/memory_span.h +++ b/src/realm/transfer/memory_span.h @@ -30,14 +30,14 @@ namespace Realm { // ================================================================================================= // Span // ================================================================================================= - + struct Span { - size_t base_offset; // Base memory offset - std::vector field_ids; // Field ID(s) - one or more - uint8_t num_dims; // 1-4 (extended from 3 to 4 for complex layouts) - uint32_t extents[4]; // Elements per dimension - size_t strides[4]; // Byte stride per dimension - + size_t base_offset; + std::vector field_ids; + uint8_t num_dims; + uint32_t extents[4]; + size_t strides[4]; + // Computed properties size_t total_bytes() const; bool is_contiguous() const; @@ -46,75 +46,75 @@ namespace Realm { // ================================================================================================= // SpanList // ================================================================================================= - - class SpanIterator; // Forward declaration - + + class SpanIterator; // Forward declaration + class SpanList { std::vector spans_; - + public: SpanList() = default; - + // Append span (works for single or multiple fields) - void append(const Span& span); - + void append(const Span &span); + // Convenience builder - void append(size_t base, const std::vector& fields, uint8_t dims, - const uint32_t* extents, const size_t* strides); - + void append(size_t base, const std::vector &fields, uint8_t dims, + const uint32_t *extents, const size_t *strides); + // Access size_t size() const { return spans_.size(); } - const Span& operator[](size_t idx) const { return spans_[idx]; } + const Span &operator[](size_t idx) const { return spans_[idx]; } bool empty() const { return spans_.empty(); } - + // Compute total bytes pending across all spans size_t total_bytes() const; - size_t bytes_pending() const { return total_bytes(); } // Total bytes (no iterator) + size_t bytes_pending() const { return total_bytes(); } // Total bytes (no iterator) }; // ================================================================================================= // SpanIterator // ================================================================================================= - + class SpanIterator { - const SpanList* list_ = nullptr; + const SpanList *list_ = nullptr; size_t span_idx_ = 0; - size_t field_idx_ = 0; // Current field within span + size_t field_idx_ = 0; // Current field within span uint32_t pos_[4] = {}; - size_t bytes_consumed_ = 0; // Total bytes consumed so far - + size_t bytes_consumed_ = 0; // Total bytes consumed so far + public: SpanIterator() = default; - explicit SpanIterator(const SpanList* list); - + explicit SpanIterator(const SpanList *list); + // Geometry queries - size_t offset() const; // Offset for current field at current position + size_t offset() const; // Offset for current field at current position size_t stride(int dim) const; size_t remaining(int dim) const; int dim() const; - + // Field queries FieldID current_field() const; - size_t remaining_fields() const; // Fields left in current span - + size_t remaining_fields() const; // Fields left in current span + // Progress tracking - dynamically accounts for list growth! - size_t bytes_pending() const { + size_t bytes_pending() const + { return list_ ? (list_->total_bytes() - bytes_consumed_) : 0; } - + // Navigation void advance(int dim, size_t count); - void advance_fields(size_t num_fields); // Move to next field(s) - void skip_bytes(size_t bytes); // Helper for compatibility + void advance_fields(size_t num_fields); // Move to next field(s) + void skip_bytes(size_t bytes); // Helper for compatibility bool done() const; - + // Access to current state (for debugging/testing) size_t current_span_index() const { return span_idx_; } size_t current_field_index() const { return field_idx_; } - const uint32_t* position() const { return pos_; } + const uint32_t *position() const { return pos_; } }; } // namespace Realm #endif // MEMORY_SPAN_H - diff --git a/src/realm/transfer/transfer.cc b/src/realm/transfer/transfer.cc index 5d377191955..55f8a74c01d 100644 --- a/src/realm/transfer/transfer.cc +++ b/src/realm/transfer/transfer.cc @@ -579,7 +579,7 @@ namespace Realm { size_t bytes = cur_field_size; int cur_dim = 0; int di = 0; - + // compact any dimensions that are contiguous first for(; di < N; di++) { // follow the agreed-upon dimension ordering @@ -628,7 +628,7 @@ namespace Realm { // Span now supports up to 4 dimensions assert(cur_dim < 4 && "Span dimension limit (4) exceeded"); - + span.extents[cur_dim] = total_count; span.strides[cur_dim] = stride; @@ -640,9 +640,9 @@ namespace Realm { span.num_dims = cur_dim; span_list.append(span); - - log_dma.debug() << "Finalize addr data dim=" << cur_dim << " total_bytes=" - << total_bytes; + + log_dma.debug() << "Finalize addr data dim=" << cur_dim + << " total_bytes=" << total_bytes; } else { assert(0 && "no support for non-affine pieces yet"); } @@ -1028,15 +1028,15 @@ namespace Realm { int cur_dim = 1; size_t total_bytes = this->cur_rect.volume() * this->cur_field_size; this->have_rect = false; - + // Create a 1D span for the indirect address Span span; - span.base_offset = 0; // Will be filled in by indirect addressing + span.base_offset = 0; span.field_ids.push_back(this->cur_field_id); span.num_dims = 1; span.extents[0] = total_bytes; span.strides[0] = 1; - + span_list.append(span); log_dma.debug() << "Finalize gather/scatter addr data dim=" << cur_dim << " total_bytes=" << total_bytes; diff --git a/tests/unit_tests/memory_span_test.cc b/tests/unit_tests/memory_span_test.cc index 25f4010f60e..aaf5fcd67af 100644 --- a/tests/unit_tests/memory_span_test.cc +++ b/tests/unit_tests/memory_span_test.cc @@ -25,7 +25,8 @@ namespace { constexpr size_t kStride = 8; constexpr size_t kBytes = 1024; - TEST(MemorySpanTests, SingleFieldBasic) { + TEST(MemorySpanTests, SingleFieldBasic) + { // Single field, 1D span std::vector fields = {100}; uint32_t extents[] = {kBytes}; @@ -47,7 +48,8 @@ namespace { EXPECT_EQ(it.dim(), 1); } - TEST(MemorySpanTests, SingleFieldAdvance) { + TEST(MemorySpanTests, SingleFieldAdvance) + { std::vector fields = {100}; uint32_t extents[] = {kBytes}; size_t strides[] = {1}; @@ -56,7 +58,7 @@ namespace { spans.append(0, fields, 1, extents, strides); SpanIterator it(&spans); - + // Advance partway it.advance(0, 128); EXPECT_EQ(it.offset(), 128); @@ -73,7 +75,8 @@ namespace { EXPECT_TRUE(it.done()); } - TEST(MemorySpanTests, MultiFieldSingleSpan) { + TEST(MemorySpanTests, MultiFieldSingleSpan) + { // Multiple fields in one span (contiguous case) std::vector fields = {10, 11, 12, 13}; uint32_t extents[] = {kBytes}; @@ -87,7 +90,7 @@ namespace { EXPECT_EQ(spans[0].total_bytes(), kBytes * 4); SpanIterator it(&spans); - + // Start at first field EXPECT_EQ(it.current_field(), 10); EXPECT_EQ(it.remaining_fields(), 4); @@ -111,7 +114,8 @@ namespace { EXPECT_EQ(it.remaining(0), kBytes); } - TEST(MemorySpanTests, MultiFieldAdvanceAll) { + TEST(MemorySpanTests, MultiFieldAdvanceAll) + { std::vector fields = {7, 8, 9}; uint32_t extents[] = {kBytes}; size_t strides[] = {1}; @@ -122,7 +126,7 @@ namespace { SpanIterator it(&spans); // Advance through all fields at once - for (size_t f = 0; f < 3; f++) { + for(size_t f = 0; f < 3; f++) { EXPECT_EQ(it.current_field(), FieldID(7 + f)); it.advance(0, kBytes); } @@ -130,9 +134,10 @@ namespace { EXPECT_TRUE(it.done()); } - TEST(MemorySpanTests, TwoDimensionalSingle) { + TEST(MemorySpanTests, TwoDimensionalSingle) + { std::vector fields = {100}; - uint32_t extents[] = {1024, 100}; // 1024 bytes × 100 lines + uint32_t extents[] = {1024, 100}; // 1024 bytes × 100 lines size_t strides[] = {1, 1024}; SpanList spans; @@ -154,7 +159,8 @@ namespace { EXPECT_TRUE(it.done()); } - TEST(MemorySpanTests, TwoDimensionalMultiField) { + TEST(MemorySpanTests, TwoDimensionalMultiField) + { std::vector fields = {10, 11, 12}; uint32_t extents[] = {1024, 100}; size_t strides[] = {1, 1024}; @@ -163,21 +169,22 @@ namespace { spans.append(0, fields, 2, extents, strides); SpanIterator it(&spans); - + size_t total_bytes = 1024 * 100 * 3; EXPECT_EQ(spans.total_bytes(), total_bytes); // Process first field EXPECT_EQ(it.current_field(), 10); - it.advance(1, 100); // Advance all lines - + it.advance(1, 100); // Advance all lines + // Should move to second field EXPECT_FALSE(it.done()); EXPECT_EQ(it.current_field(), 11); EXPECT_EQ(it.remaining(1), 100); } - TEST(MemorySpanTests, MultipleSpansSeparateFields) { + TEST(MemorySpanTests, MultipleSpansSeparateFields) + { // Three spans, one field each (non-contiguous case) uint32_t extents[] = {kBytes}; size_t strides[] = {1}; @@ -215,9 +222,10 @@ namespace { EXPECT_TRUE(it.done()); } - TEST(MemorySpanTests, ThreeDimensional) { + TEST(MemorySpanTests, ThreeDimensional) + { std::vector fields = {100}; - uint32_t extents[] = {64, 8, 2}; // 64 bytes × 8 × 2 + uint32_t extents[] = {64, 8, 2}; // 64 bytes × 8 × 2 size_t strides[] = {1, 64, 512}; SpanList spans; @@ -234,7 +242,8 @@ namespace { EXPECT_TRUE(it.done()); } - TEST(MemorySpanTests, SkipBytes) { + TEST(MemorySpanTests, SkipBytes) + { std::vector fields = {100}; uint32_t extents[] = {kBytes}; size_t strides[] = {1}; @@ -243,7 +252,7 @@ namespace { spans.append(0, fields, 1, extents, strides); SpanIterator it(&spans); - + // Skip some bytes it.skip_bytes(128); EXPECT_EQ(it.offset(), 128); @@ -254,7 +263,8 @@ namespace { EXPECT_EQ(it.offset(), 256); } - TEST(MemorySpanTests, AdvanceFieldsExplicit) { + TEST(MemorySpanTests, AdvanceFieldsExplicit) + { std::vector fields = {10, 11, 12, 13}; uint32_t extents[] = {kBytes}; size_t strides[] = {1}; @@ -263,21 +273,22 @@ namespace { spans.append(0, fields, 1, extents, strides); SpanIterator it(&spans); - + // At first field EXPECT_EQ(it.current_field(), 10); EXPECT_EQ(it.current_field_index(), 0); // Advance to skip geometry for 2 fields - it.advance(0, kBytes); // Complete field 10 + it.advance(0, kBytes); // Complete field 10 EXPECT_EQ(it.current_field(), 11); - - it.advance(0, kBytes); // Complete field 11 + + it.advance(0, kBytes); // Complete field 11 EXPECT_EQ(it.current_field(), 12); EXPECT_EQ(it.remaining_fields(), 2); } - TEST(MemorySpanTests, IsContiguous) { + TEST(MemorySpanTests, IsContiguous) + { // Contiguous span std::vector fields = {100}; uint32_t extents1[] = {64, 100}; @@ -302,12 +313,13 @@ namespace { span2.extents[0] = 64; span2.extents[1] = 100; span2.strides[0] = 1; - span2.strides[1] = 128; // Stride doesn't match (has padding) + span2.strides[1] = 128; // Stride doesn't match (has padding) EXPECT_FALSE(span2.is_contiguous()); } - TEST(MemorySpanTests, EmptySpanList) { + TEST(MemorySpanTests, EmptySpanList) + { SpanList spans; EXPECT_TRUE(spans.empty()); EXPECT_EQ(spans.size(), 0); @@ -317,9 +329,10 @@ namespace { EXPECT_TRUE(it.done()); } - TEST(MemorySpanTests, PartialAdvanceTwoDim) { + TEST(MemorySpanTests, PartialAdvanceTwoDim) + { std::vector fields = {100}; - uint32_t extents[] = {64, 10}; // 64 bytes × 10 lines + uint32_t extents[] = {64, 10}; // 64 bytes × 10 lines size_t strides[] = {1, 64}; SpanList spans; @@ -358,10 +371,10 @@ namespace { // Should overflow twice: 25 / 10 = 2 carries, 25 % 10 = 5 remainder // Expected: pos[0] = 5, pos[1] = 2 it.advance(0, 25); - - EXPECT_EQ(it.remaining(0), 5); // 10 - 5 = 5 remaining in dim 0 - EXPECT_EQ(it.remaining(1), 3); // 5 - 2 = 3 remaining in dim 1 - EXPECT_EQ(it.offset(), 0x1000 + 2 * 10 + 5); // base + 2 lines + 5 bytes + + EXPECT_EQ(it.remaining(0), 5); // 10 - 5 = 5 remaining in dim 0 + EXPECT_EQ(it.remaining(1), 3); // 5 - 2 = 3 remaining in dim 1 + EXPECT_EQ(it.offset(), 0x1000 + 2 * 10 + 5); // base + 2 lines + 5 bytes EXPECT_FALSE(it.done()); // Advance by another 25 bytes to complete the span @@ -371,4 +384,3 @@ namespace { } } // namespace - From 6382c0fee63bb30fdfbe71950155a5ae7a3e1014 Mon Sep 17 00:00:00 2001 From: Micha Dery Date: Tue, 4 Nov 2025 23:47:49 +0200 Subject: [PATCH 6/6] complete address list refactor --- src/realm/cuda/cuda_internal.cc | 4 +- src/realm/hip/hip_internal.cc | 130 ++++++++++----------- src/realm/transfer/memory_span.cc | 38 +++--- src/realm/transfer/memory_span.h | 12 +- tests/unit_tests/indirect_iterator_test.cc | 15 ++- 5 files changed, 97 insertions(+), 102 deletions(-) diff --git a/src/realm/cuda/cuda_internal.cc b/src/realm/cuda/cuda_internal.cc index 03f33da7c96..6b01202c39f 100644 --- a/src/realm/cuda/cuda_internal.cc +++ b/src/realm/cuda/cuda_internal.cc @@ -2314,7 +2314,7 @@ namespace Realm { output_control.remaining_count / out_elem_size); if(in_port != 0) { max_elems = - std::min(max_elems, in_port->span_list.bytes_pending() / in_elem_size); + std::min(max_elems, in_port->span_iter.bytes_pending() / in_elem_size); if(in_port->peer_guid != XFERDES_NO_GUID) { size_t read_bytes_avail = in_port->seq_remote.span_exists( in_port->local_bytes_total, (max_elems * in_elem_size)); @@ -2323,7 +2323,7 @@ namespace Realm { } if(out_port != 0) { max_elems = - std::min(max_elems, out_port->span_list.bytes_pending() / out_elem_size); + std::min(max_elems, out_port->span_iter.bytes_pending() / out_elem_size); // no support for reducing into an intermediate buffer assert(out_port->peer_guid == XFERDES_NO_GUID); } diff --git a/src/realm/hip/hip_internal.cc b/src/realm/hip/hip_internal.cc index c3d3ea07e64..7ce5afdedb5 100644 --- a/src/realm/hip/hip_internal.cc +++ b/src/realm/hip/hip_internal.cc @@ -197,16 +197,16 @@ namespace Realm { size_t bytes_to_fence = 0; while(total_bytes < max_bytes) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t in_offset = in_alc.get_offset(); - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t in_offset = in_iter.offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); size_t bytes = 0; size_t bytes_left = max_bytes - total_bytes; @@ -219,8 +219,8 @@ namespace Realm { assert(in_dim > 0); assert(out_dim > 0); - size_t icount = in_alc.remaining(0); - size_t ocount = out_alc.remaining(0); + size_t icount = in_iter.remaining(0); + size_t ocount = out_iter.remaining(0); // contig bytes is always the min of the first dimensions size_t contig_bytes = std::min(std::min(icount, ocount), bytes_left); @@ -257,8 +257,8 @@ namespace Realm { << " src=" << (in_base + in_offset) << std::dec << " bytes=" << bytes << " stream=" << stream << " kind=" << memcpy_kind; - in_alc.advance(0, bytes); - out_alc.advance(0, bytes); + in_iter.advance(0, bytes); + out_iter.advance(0, bytes); bytes_to_fence += bytes; // TODO: fence on a threshold @@ -279,8 +279,8 @@ namespace Realm { } else { assert(in_dim > 1); id = 1; - icount = in_alc.remaining(id); - in_lstride = in_alc.get_stride(id); + icount = in_iter.remaining(id); + in_lstride = in_iter.stride(id); iscale = 1; } @@ -299,8 +299,8 @@ namespace Realm { } else { assert(out_dim > 1); od = 1; - ocount = out_alc.remaining(od); - out_lstride = out_alc.get_stride(od); + ocount = out_iter.remaining(od); + out_lstride = out_iter.stride(od); oscale = 1; } @@ -344,8 +344,8 @@ namespace Realm { contig_bytes, lines, copy_type, stream->get_stream())); - in_alc.advance(id, lines * iscale); - out_alc.advance(od, lines * oscale); + in_iter.advance(id, lines * iscale); + out_iter.advance(od, lines * oscale); bytes_to_fence += bytes; // TODO: fence on a threshold @@ -361,8 +361,8 @@ namespace Realm { } else { id++; assert(in_dim > id); - icount = in_alc.remaining(id); - in_pstride = in_alc.get_stride(id); + icount = in_iter.remaining(id); + in_pstride = in_iter.stride(id); iscale = 1; } @@ -377,8 +377,8 @@ namespace Realm { } else { od++; assert(out_dim > od); - ocount = out_alc.remaining(od); - out_pstride = out_alc.get_stride(od); + ocount = out_iter.remaining(od); + out_pstride = out_iter.stride(od); oscale = 1; } @@ -434,8 +434,8 @@ namespace Realm { << " stream=" << stream << " kind=" << memcpy_kind; bytes = contig_bytes * lines * act_planes; - in_alc.advance(id, act_planes * iscale); - out_alc.advance(od, act_planes * oscale); + in_iter.advance(id, act_planes * iscale); + out_iter.advance(od, act_planes * oscale); bytes_to_fence += bytes; // TODO: fence on a threshold @@ -468,7 +468,7 @@ namespace Realm { } else { // input but no output, so skip input bytes total_bytes = max_bytes; - in_port->addrcursor.skip_bytes(total_bytes); + in_port->span_iter.skip_bytes(total_bytes); rseqcache.add_span(input_control.current_io_port, in_span_start, total_bytes); in_span_start += total_bytes; @@ -477,7 +477,7 @@ namespace Realm { if(out_port != 0) { // output but no input, so skip output bytes total_bytes = max_bytes; - out_port->addrcursor.skip_bytes(total_bytes); + out_port->span_iter.skip_bytes(total_bytes); } else { // skipping both input and output is possible for simultaneous // gather+scatter @@ -763,13 +763,13 @@ namespace Realm { GPUStream *stream = channel->gpu->get_next_d2d_stream(); while(total_bytes < max_bytes) { - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int out_dim = out_alc.get_dim(); + int out_dim = out_iter.dim(); #ifdef DEBUG_REALM // since HIP does not support 12/32 bit 2D memset, we need to @@ -785,19 +785,19 @@ namespace Realm { uint8_t fill_u8; memcpy(&fill_u8, fill_data, 1); if(out_dim == 1) { - size_t bytes = out_alc.remaining(0); + size_t bytes = out_iter.remaining(0); CHECK_HIP(hipMemsetD8Async((hipDeviceptr_t)(out_base + out_offset), fill_u8, bytes, stream->get_stream())); - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); total_bytes += bytes; } else { - size_t bytes = out_alc.remaining(0); - size_t lines = out_alc.remaining(1); + size_t bytes = out_iter.remaining(0); + size_t lines = out_iter.remaining(1); CHECK_HIP(hipMemset2DAsync((void *)(out_base + out_offset), - out_alc.get_stride(1), + out_iter.stride(1), *reinterpret_cast(fill_data), bytes, lines, stream->get_stream())); - out_alc.advance(1, lines); + out_iter.advance(1, lines); total_bytes += bytes * lines; } break; @@ -809,13 +809,13 @@ namespace Realm { uint16_t fill_u16; memcpy(&fill_u16, fill_data, 2); if(out_dim == 1) { - size_t bytes = out_alc.remaining(0); + size_t bytes = out_iter.remaining(0); #ifdef DEBUG_REALM assert((bytes & 1) == 0); #endif CHECK_HIP(hipMemsetD16Async((hipDeviceptr_t)(out_base + out_offset), fill_u16, bytes >> 1, stream->get_stream())); - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); total_bytes += bytes; } else { #ifdef DEBUG_REALM @@ -832,13 +832,13 @@ namespace Realm { uint32_t fill_u32; memcpy(&fill_u32, fill_data, 4); if(out_dim == 1) { - size_t bytes = out_alc.remaining(0); + size_t bytes = out_iter.remaining(0); #ifdef DEBUG_REALM assert((bytes & 3) == 0); #endif CHECK_HIP(hipMemsetD32Async((hipDeviceptr_t)(out_base + out_offset), fill_u32, bytes >> 2, stream->get_stream())); - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); total_bytes += bytes; } else { #ifdef DEBUG_REALM @@ -862,13 +862,13 @@ namespace Realm { case 2: { assert((bytes & 1) == 0); - assert((out_alc.get_stride(1) & 1) == 0); + assert((out_iter.stride(1) & 1) == 0); break; } case 4: { assert((bytes & 3) == 0); - assert((out_alc.get_stride(1) & 3) == 0); + assert((out_iter.stride(1) & 3) == 0); break; } default: @@ -923,11 +923,11 @@ namespace Realm { if(out_dim == 1) { // all done - out_alc.advance(0, bytes); + out_iter.advance(0, bytes); total_bytes += bytes; } else { - size_t lines = out_alc.remaining(1); - size_t lstride = out_alc.get_stride(1); + size_t lines = out_iter.remaining(1); + size_t lstride = out_iter.stride(1); void *srcDevice = (void *)(out_base + out_offset); @@ -943,11 +943,11 @@ namespace Realm { } if(out_dim == 2) { - out_alc.advance(1, lines); + out_iter.advance(1, lines); total_bytes += bytes * lines; } else { - size_t planes = out_alc.remaining(2); - size_t pstride = out_alc.get_stride(2); + size_t planes = out_iter.remaining(2); + size_t pstride = out_iter.stride(2); // logarithmic version requires that pstride be a multiple of // lstride @@ -976,7 +976,7 @@ namespace Realm { planes_done += todo; } - out_alc.advance(2, planes); + out_iter.advance(2, planes); total_bytes += bytes * lines * planes; } else { // plane-at-a-time fallback - can reuse most of copy2d @@ -988,7 +988,7 @@ namespace Realm { bytes, lines, hipMemcpyDeviceToDevice, stream->get_stream())); } - out_alc.advance(2, planes); + out_iter.advance(2, planes); total_bytes += bytes * lines * planes; } } @@ -1168,7 +1168,7 @@ namespace Realm { output_control.remaining_count / out_elem_size); if(in_port != 0) { max_elems = - std::min(max_elems, in_port->addrlist.bytes_pending() / in_elem_size); + std::min(max_elems, in_port->span_iter.bytes_pending() / in_elem_size); if(in_port->peer_guid != XFERDES_NO_GUID) { size_t read_bytes_avail = in_port->seq_remote.span_exists( in_port->local_bytes_total, (max_elems * in_elem_size)); @@ -1177,7 +1177,7 @@ namespace Realm { } if(out_port != 0) { max_elems = - std::min(max_elems, out_port->addrlist.bytes_pending() / out_elem_size); + std::min(max_elems, out_port->span_iter.bytes_pending() / out_elem_size); // no support for reducing into an intermediate buffer assert(out_port->peer_guid == XFERDES_NO_GUID); } @@ -1196,36 +1196,36 @@ namespace Realm { reinterpret_cast(out_port->mem->get_direct_ptr(0, 0)); while(total_elems < max_elems) { - AddressListCursor &in_alc = in_port->addrcursor; - AddressListCursor &out_alc = out_port->addrcursor; + SpanIterator &in_iter = in_port->span_iter; + SpanIterator &out_iter = out_port->span_iter; - uintptr_t in_offset = in_alc.get_offset(); - uintptr_t out_offset = out_alc.get_offset(); + uintptr_t in_offset = in_iter.offset(); + uintptr_t out_offset = out_iter.offset(); // the reported dim is reduced for partially consumed address // ranges - whatever we get can be assumed to be regular - int in_dim = in_alc.get_dim(); - int out_dim = out_alc.get_dim(); + int in_dim = in_iter.dim(); + int out_dim = out_iter.dim(); // the current reduction op interface can reduce multiple elements // with a fixed address stride, which looks to us like either // 1D (stride = elem_size), or 2D with 1 elem/line - size_t icount = in_alc.remaining(0) / in_elem_size; - size_t ocount = out_alc.remaining(0) / out_elem_size; + size_t icount = in_iter.remaining(0) / in_elem_size; + size_t ocount = out_iter.remaining(0) / out_elem_size; size_t istride, ostride; if((in_dim > 1) && (icount == 1)) { in_dim = 2; - icount = in_alc.remaining(1); - istride = in_alc.get_stride(1); + icount = in_iter.remaining(1); + istride = in_iter.stride(1); } else { in_dim = 1; istride = in_elem_size; } if((out_dim > 1) && (ocount == 1)) { out_dim = 2; - ocount = out_alc.remaining(1); - ostride = out_alc.get_stride(1); + ocount = out_iter.remaining(1); + ostride = out_iter.stride(1); } else { out_dim = 1; ostride = out_elem_size; @@ -1302,8 +1302,8 @@ namespace Realm { in_span_start += elems * in_elem_size; out_span_start += elems * out_elem_size; - in_alc.advance(in_dim - 1, elems * ((in_dim == 1) ? in_elem_size : 1)); - out_alc.advance(out_dim - 1, elems * ((out_dim == 1) ? out_elem_size : 1)); + in_iter.advance(in_dim - 1, elems * ((in_dim == 1) ? in_elem_size : 1)); + out_iter.advance(out_dim - 1, elems * ((out_dim == 1) ? out_elem_size : 1)); #ifdef DEBUG_REALM assert(elems <= elems_left); @@ -1319,7 +1319,7 @@ namespace Realm { } else { // input but no output, so skip input bytes total_elems = max_elems; - in_port->addrcursor.skip_bytes(total_elems * in_elem_size); + in_port->span_iter.skip_bytes(total_elems * in_elem_size); rseqcache.add_span(input_control.current_io_port, in_span_start, total_elems * in_elem_size); @@ -1329,7 +1329,7 @@ namespace Realm { if(out_port != 0) { // output but no input, so skip output bytes total_elems = max_elems; - out_port->addrcursor.skip_bytes(total_elems * out_elem_size); + out_port->span_iter.skip_bytes(total_elems * out_elem_size); wseqcache.add_span(output_control.current_io_port, out_span_start, total_elems * out_elem_size); diff --git a/src/realm/transfer/memory_span.cc b/src/realm/transfer/memory_span.cc index d279518d559..685fea844af 100644 --- a/src/realm/transfer/memory_span.cc +++ b/src/realm/transfer/memory_span.cc @@ -18,6 +18,7 @@ #include "realm/transfer/memory_span.h" #include +#include namespace Realm { @@ -62,22 +63,38 @@ namespace Realm { // SpanList // ================================================================================================= - void SpanList::append(const Span &span) { spans_.push_back(span); } + void SpanList::append(const Span &span) + { + // Validate dimensions upfront + if(span.num_dims > REALM_MAX_DIM + 1) { + fprintf(stderr, "ERROR: num_dims=%d exceeds REALM_MAX_DIM+1=%d\n", + (int)span.num_dims, REALM_MAX_DIM + 1); + assert(span.num_dims <= REALM_MAX_DIM + 1); + } + spans_.push_back(span); + } void SpanList::append(size_t base, const std::vector &fields, uint8_t dims, const uint32_t *extents, const size_t *strides) { + // Validate dimensions upfront + if(dims > REALM_MAX_DIM + 1) { + fprintf(stderr, "ERROR: num_dims=%d exceeds REALM_MAX_DIM+1=%d\n", (int)dims, + REALM_MAX_DIM + 1); + assert(dims <= REALM_MAX_DIM + 1); + } + Span span; span.base_offset = base; span.field_ids = fields; span.num_dims = dims; - for(int d = 0; d < dims && d < 4; d++) { + for(int d = 0; d < dims; d++) { span.extents[d] = extents[d]; span.strides[d] = strides[d]; } // Zero out unused dimensions - for(int d = dims; d < 4; d++) { + for(int d = dims; d < REALM_MAX_DIM + 1; d++) { span.extents[d] = 0; span.strides[d] = 0; } @@ -170,11 +187,6 @@ namespace Realm { // We're partway through dimension d // Return d+1 (dimensionality includes this dimension) int result = d + 1; - fprintf(stderr, - "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], " - "extents=[%u,%u,%u,%u], partial at d=%d, returning %d\n", - span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], - s.extents[2], s.extents[3], d, result); return result; } if(pos_[d] >= s.extents[d]) { @@ -184,15 +196,6 @@ namespace Realm { } } - // No partial progress yet, return full dimensionality - fprintf(stderr, - "DEBUG SpanIterator::dim(): span_idx=%zu, pos=[%u,%u,%u,%u], " - "extents=[%u,%u,%u,%u], num_dims=%d, no partial, returning %d\n", - span_idx_, pos_[0], pos_[1], pos_[2], pos_[3], s.extents[0], s.extents[1], - s.extents[2], s.extents[3], (int)s.num_dims, (int)s.num_dims); - if(s.num_dims > 4) { - fprintf(stderr, "ERROR: num_dims=%d is > 4, this is a bug!\n", (int)s.num_dims); - } return s.num_dims; } @@ -318,7 +321,6 @@ namespace Realm { int d = dim(); if(d == 0) { - // No geometry to skip through return; } diff --git a/src/realm/transfer/memory_span.h b/src/realm/transfer/memory_span.h index ebfb88ffc14..08f2e706b0f 100644 --- a/src/realm/transfer/memory_span.h +++ b/src/realm/transfer/memory_span.h @@ -18,7 +18,6 @@ #ifndef MEMORY_SPAN_H #define MEMORY_SPAN_H -#include "realm/realm_config.h" #include "realm/indexspace.h" #include @@ -35,8 +34,8 @@ namespace Realm { size_t base_offset; std::vector field_ids; uint8_t num_dims; - uint32_t extents[4]; - size_t strides[4]; + uint32_t extents[REALM_MAX_DIM + 1]; + size_t strides[REALM_MAX_DIM + 1]; // Computed properties size_t total_bytes() const; @@ -47,18 +46,14 @@ namespace Realm { // SpanList // ================================================================================================= - class SpanIterator; // Forward declaration - class SpanList { std::vector spans_; public: SpanList() = default; - // Append span (works for single or multiple fields) void append(const Span &span); - // Convenience builder void append(size_t base, const std::vector &fields, uint8_t dims, const uint32_t *extents, const size_t *strides); @@ -69,7 +64,6 @@ namespace Realm { // Compute total bytes pending across all spans size_t total_bytes() const; - size_t bytes_pending() const { return total_bytes(); } // Total bytes (no iterator) }; // ================================================================================================= @@ -80,7 +74,7 @@ namespace Realm { const SpanList *list_ = nullptr; size_t span_idx_ = 0; size_t field_idx_ = 0; // Current field within span - uint32_t pos_[4] = {}; + uint32_t pos_[REALM_MAX_DIM + 1] = {}; size_t bytes_consumed_ = 0; // Total bytes consumed so far public: diff --git a/tests/unit_tests/indirect_iterator_test.cc b/tests/unit_tests/indirect_iterator_test.cc index 2c40a91b316..09d81e55309 100644 --- a/tests/unit_tests/indirect_iterator_test.cc +++ b/tests/unit_tests/indirect_iterator_test.cc @@ -55,9 +55,8 @@ template void run_test_case(const TestCaseData &test_case) { using T = int; - AddressList addrlist; - AddressListCursor cursor; - cursor.set_addrlist(&addrlist); + SpanList span_list; + SpanIterator span_iter(&span_list); constexpr size_t elem_size = sizeof(int); const InstanceLayoutPieceBase *nonaffine; const size_t bytes = sizeof(Point) * test_case.indirection.size(); @@ -93,21 +92,21 @@ void run_test_case(const TestCaseData &test_case) it->set_indirect_input_port(xd.get(), /*indirect_port_idx=*/0, addr_it.get()); bool done_early = it->done(); - bool ok = it->get_addresses(addrlist, nonaffine); + bool ok = it->get_addresses(span_list, nonaffine); bool done_later = it->done(); ASSERT_FALSE(done_early); ASSERT_TRUE(ok); ASSERT_TRUE(done_later); ASSERT_EQ(nonaffine, nullptr); - ASSERT_EQ(addrlist.bytes_pending(), buffer.size() * elem_size); + ASSERT_EQ(span_iter.bytes_pending(), buffer.size() * elem_size); for(size_t offset : test_case.expected) { - EXPECT_EQ(offset, cursor.get_offset()); - cursor.advance(0, cursor.remaining(0)); + EXPECT_EQ(offset, span_iter.offset()); + span_iter.advance(0, span_iter.remaining(0)); } - EXPECT_EQ(addrlist.bytes_pending(), 0); + EXPECT_EQ(span_iter.bytes_pending(), 0); } TEST_P(IndirectGetAddressesTest, Base)