From 5f1cc4ef35ce403be4ec5665c08143b92662b13c Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Wed, 26 Aug 2026 18:57:44 +0200 Subject: [PATCH 01/24] =?UTF-8?q?perf(mpi):=20=F0=9F=A7=AD=20route=20throu?= =?UTF-8?q?gh=20one=20Router,=20GF(2)-linear=20across=20ranks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Routing was two inline copies of `monomial_hash(M) % P` -- one in Scan.h's query emission, one in `find_rank` -- which had to agree exactly or ownership splits silently. Both now go through `routing::Router::dest` and nothing else. The reason for an abstraction at all is that splitmix's full avalanche is what makes the exchange dense: a gate maps M to M^G, and with an avalanching owner function a rank's queries for ONE generator spray across all R ranks, so the message count grows as R*(R-1). If the RANK index is instead a GF(2)-linear function of the support, h(M^G) = h(M) ^ h(G), and -- since I own M -- every query I emit for G lands on exactly one peer, my_rank ^ h(G). So the router is two-level, because the two levels cost differently: across MPI ranks the cost is the message COUNT, within a rank partitions talk through shared memory where fanout is free and only balance matters. part = q % S q = monomial_hash(M), unchanged hi = (q / S) % (R >> d) rank = (a & (2^d - 1)) | (hi << d) a = linear_hash(M) flat = rank * S + part d is a dial: fanout is R >> d, d = 0 collapses to today's `q % (R*S)` bit for bit (the default, and the regression gate), d = log2(R) is fanout 1. A non-power-of-two rank count has no XOR structure and falls back to d = 0. Nothing yet exploits the sparsity -- the transport is still the dense pair. This commit only makes the sparsity exist and proves it costs nothing to have. mpi::geometry() is new because size() alone cannot tell an inter-rank message (a network hop) from an inter-partition one (a memcpy), and the split is exactly what the routing needs. Measured, 20-site Hubbard cutoff 8, 5 steps, against a pristine build of the same HEAD: d = 0 is bit-identical to today at S=1, S=8, world 2 and world 4. With linear bits on (world 4 x {1,2} partitions, d = 1, 2) the term count is exact -- 17,148 in every configuration -- and the expectation value moves by at most 1 ULP, which is the reassociation the baseline already shows across worlds. Co-Authored-By: Claude Opus 5 (1M context) --- .../detail/evolution/layer_build/Engine.h | 4 + .../detail/evolution/layer_build/Scan.h | 10 +- .../MonomialPropagator.inl | 9 +- cpp/monoprop/detail/mpi/CMakeLists.txt | 1 + cpp/monoprop/detail/mpi/HybridComm.h | 2 + cpp/monoprop/detail/mpi/MPICompat.cpp | 14 ++ cpp/monoprop/detail/mpi/MPICompat.h | 9 + cpp/monoprop/detail/mpi/MPIUtils.h | 17 +- cpp/monoprop/detail/mpi/Routing.h | 222 ++++++++++++++++++ cpp/tests/routing_tests.cpp | 211 +++++++++++++++++ 10 files changed, 490 insertions(+), 9 deletions(-) create mode 100644 cpp/monoprop/detail/mpi/Routing.h create mode 100644 cpp/tests/routing_tests.cpp diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 0f03f4af..a70bb3f0 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -600,6 +600,9 @@ auto build_layer(MPOperator &local_op, validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); const size_t my_rank = static_cast(mpi::rank(comm)); const size_t R = static_cast(mpi::size(comm)); + // R is the FLAT world (ranks x partitions); the router is what splits it back into the two levels. + const auto router = router_for(comm); + assert(router.flat_world() == R); // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); @@ -630,6 +633,7 @@ auto build_layer(MPOperator &local_op, only_rotate_len_k, R, my_rank, + router, /*capture_values=*/use_fused, sweep_ptr, cos_build); diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index cce972b0..117dae93 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -237,7 +237,7 @@ struct FusedScanResult { }; // Classify, cut off and emit in one pass over the anticommuting terms. Queries go to the owner of -// M'=M⊕G (hash%R; self at R==1) in ascending source-index order, so resolve and index assignment are +// M'=M⊕G (routing::Router; self at R==1) in ascending source-index order, so resolve and index assignment are // deterministic. `fused_scale_coeffs` (no length cap only; must alias coeffs.data()) scales every anticommuting // coeff in place by `fused_scale_cos`=cos(2·build_angle), so no cosine set is built and a hit's stored // value is post-cos (resolve recovers it via 1/cos). @@ -250,6 +250,7 @@ auto fused_find_and_collect(const MPOperator &op, std::optional only_rotate_len_k, size_t rank_count, size_t my_rank, + const routing::Router &router, bool capture_values = false, double *fused_scale_coeffs = nullptr, double fused_scale_cos = 1.0) -> FusedScanResult { @@ -343,11 +344,12 @@ auto fused_find_and_collect(const MPOperator &op, double v_src, bool is_follower) { // Single rank: every partner is self-owned, skip the O(W) hash; multi-rank routes by owner. - // Must be the SAME function find_rank computes (MPIUtils.h) or a term is placed and queried - // on different ranks, which duplicates a row silently; mpi_utils_tests.cpp asserts it. + // routing::Router is the ONLY owner function -- find_rank (MPIUtils.h) must stay in step + // with it, or a term is placed and queried on different ranks, which duplicates a row + // silently; mpi_utils_tests.cpp asserts the agreement. size_t r_prime = my_rank; if (rank_count != 1) { - r_prime = monomial_hash(dense) % rank_count; + r_prime = router.dest(dense); } if (r_prime == my_rank) { (is_follower ? res.follower_self : res.leader_self).push(pos, k, phase); diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index b2e0aae1..16b5d935 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -143,6 +143,7 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope const size_t num_ranks = static_cast(mpi::size(comm_)); const size_t my_rank = static_cast(mpi::rank(comm_)); + const auto router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term MonomialList local_heisenberg_terms; double core_term = 0.0; @@ -155,7 +156,7 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope core_term = encoded_coeff; continue; } - if (my_rank == find_rank(majorana_bitset, num_ranks)) { + if (my_rank == find_rank(majorana_bitset, router)) { mp_op_.init_op_map[majorana_bitset] = encoded_coeff; local_heisenberg_terms.push_back(majorana_bitset); } @@ -177,7 +178,7 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope // The initial monomials are distinct, so emplace (insert-if-absent) is an assigning insert here. for (size_t r = 0; r < op.size(); ++r) { const auto &mono = materialize_row(op, r); - if (my_rank == find_rank(mono, num_ranks)) { + if (my_rank == find_rank(mono, router)) { mp_op_.append_term(mono); mp_op_.store->emplace(mono, i++); } @@ -366,7 +367,7 @@ auto MonomialPropagator::apply_initial_operator_(const OperatorDict &o for_each_partition_([&](MonomialPropagator &s) { s.update_initial_operator(op_dict); }); return {}; } - const size_t num_ranks = static_cast(mpi::size(comm_)); + const auto router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term const size_t my_rank = static_cast(mpi::rank(comm_)); OperatorDict new_op; @@ -376,7 +377,7 @@ auto MonomialPropagator::apply_initial_operator_(const OperatorDict &o core_term_ = algebra_encode_coeff(basis_, coeff, mono); continue; } - if (my_rank == find_rank(mono, num_ranks)) { + if (my_rank == find_rank(mono, router)) { const auto mono_indices = bitset_to_indices(mono); new_op[mono_indices] = coeff; } diff --git a/cpp/monoprop/detail/mpi/CMakeLists.txt b/cpp/monoprop/detail/mpi/CMakeLists.txt index 5b9969b0..29775bca 100644 --- a/cpp/monoprop/detail/mpi/CMakeLists.txt +++ b/cpp/monoprop/detail/mpi/CMakeLists.txt @@ -12,6 +12,7 @@ target_sources( "MPICompat.h" "MPIUtils.h" "PartitionBarrier.h" + "Routing.h" "ShmComm.h" ) diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 47a79880..a91c85ea 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -97,6 +97,8 @@ class HybridComm { auto operator=(const HybridComm &) -> HybridComm & = delete; auto size() const -> int { return r_ * s_; } + auto ranks() const -> int { return r_; } + auto partitions() const -> int { return s_; } auto global_rank(int local_partition) const -> int { return mpi_rank_ * s_ + local_partition; } auto alltoall_counts(int local_partition, const int *send_counts /*[P]*/, int *recv_counts /*[P]*/) -> void { diff --git a/cpp/monoprop/detail/mpi/MPICompat.cpp b/cpp/monoprop/detail/mpi/MPICompat.cpp index 133a9fb7..625b1725 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.cpp +++ b/cpp/monoprop/detail/mpi/MPICompat.cpp @@ -84,6 +84,20 @@ auto size(const Comm &comm) -> int { #endif } +auto geometry(const Comm &comm) -> Geometry { + if (comm.kind == Comm::Kind::Shm) { + return {.ranks = 1, .partitions = comm.shm->size()}; + } +#ifdef monoprop_ENABLE_MPI + if (comm.kind == Comm::Kind::Hybrid) { + return {.ranks = comm.hyb->ranks(), .partitions = comm.hyb->partitions()}; + } + return {.ranks = size(comm), .partitions = 1}; +#else + return {.ranks = 1, .partitions = 1}; +#endif +} + auto allreduce_sum_inplace(VecD &values, Comm comm) -> void { if (comm.kind == Comm::Kind::Shm) { comm.shm->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index 288ee6c8..c3e74db9 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -90,6 +90,15 @@ inline auto finalize() -> void {} auto rank(const Comm &comm) -> int; auto size(const Comm &comm) -> int; +// How the flat world of size() is actually built: ranks * partitions. Routing needs the split, because +// an inter-rank message costs a network hop while an inter-partition one is a shared-memory copy -- +// size() alone cannot tell them apart. ranks * partitions == size() for every Kind. +struct Geometry { + int ranks = 1; + int partitions = 1; +}; +auto geometry(const Comm &comm) -> Geometry; + template inline auto allreduce_sum(T local_val, Comm comm) -> T { if (comm.kind == Comm::Kind::Shm) { diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index 3a2d20ab..0ae8ca80 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -22,6 +22,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" #include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/Routing.h" namespace monoprop::mpi_detail { @@ -49,12 +50,26 @@ inline auto read_monomial_from_words(const VecZ &buffer, size_t start) -> Monomi namespace monoprop { // Stateless and identical on every rank, so all ranks agree on a term's owner without communication. +// Both overloads go through routing::Router::dest and nothing else: this and Scan.h's query emission +// must return the same slot for the same monomial, and a divergence splits ownership silently. +template +auto find_rank(const Monomial &mono, const routing::Router &router) -> size_t { + return router.dest(mono); +} + +// Flat-world overload, for callers that hold no geometry: the splitmix router (d = 0). template auto find_rank(const Monomial &mono, const size_t n_ranks) -> size_t { if (n_ranks == 0) { return 0; } - return monomial_hash(mono) % n_ranks; + return routing::Router::splitmix(n_ranks).dest(mono); +} + +// The router this communicator's geometry implies, honouring monoprop_ROUTING / _ROUTE_LINEAR_BITS. +inline auto router_for(const mpi::Comm &comm) -> routing::Router { + const auto geom = mpi::geometry(comm); + return routing::make_router(static_cast(geom.ranks), static_cast(geom.partitions)); } } // namespace monoprop diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h new file mode 100644 index 00000000..4048cebd --- /dev/null +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -0,0 +1,222 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/core/Monomial.h" + +// The single home for "which flat slot owns this monomial". Two call sites depend on agreeing exactly +// (Scan.h emits queries by it, MonomialPropagator seeds the operator by it), and a disagreement splits +// ownership silently rather than crashing -- so both go through Router::dest and nothing else. +// +// WHY there is a choice to make here at all +// ---------------------------------------- +// A gate turns a term M into M^G (symmetric difference of Majorana support). With today's splitmix +// destination -- full avalanche -- the owner of M^G is unrelated to the owner of M, so a rank's queries +// for ONE generator spray across all R ranks and the exchange is a dense all-to-all whose message count +// grows as R*(R-1). If instead the RANK index is a GF(2)-LINEAR function of the support, +// +// h(M) = XOR of v_i over i in support(M) => h(M ^ G) = h(M) ^ h(G) +// +// then, since I own M, every query I emit for G goes to exactly one rank: my_rank ^ h(G). XOR is an +// involution, so that peer sends to me in the same round: the exchange becomes a pairwise Sendrecv. +// +// The routing is TWO-LEVEL, because the two levels have different costs: across MPI ranks the cost is +// the message COUNT (make it structured), within a rank partitions talk through shared memory where +// fanout is free and only balance matters (keep full avalanche). +// +// part = q % S q = monomial_hash(M) (splitmix, unchanged) +// hi = (q / S) % (R >> d) the R>>d splitmix-chosen high rank bits +// rank = (a & (2^d - 1)) | (hi << d) a = linear_hash(M); d = linear_bits +// flat = rank * S + part +// +// d is a dial, not a cliff: fanout is R >> d, so d = 0 is EXACTLY today's `q % (R*S)` (see dest()) and +// d = log2(R) is fanout 1. Non-power-of-two R has no XOR structure at all and falls back to d = 0. +// +// Knobs: +// monoprop_ROUTING splitmix (default) | linear -- linear defaults d to log2(R) +// monoprop_ROUTE_LINEAR_BITS explicit d, clamped to [0, log2(R)] +// monoprop_ROUTE_SEED uint64 seed for the linear basis (default kDefaultSeed) + +namespace monoprop::routing { + +inline constexpr uint64_t kDefaultSeed = 0x5DEE'CE66'D0C6'2517ULL; + +inline constexpr auto mix64(uint64_t x) noexcept -> uint64_t { + x += 0x9E37'79B9'7F4A'7C15ULL; + x = (x ^ (x >> 30)) * 0xBF58'476D'1CE4'E5B9ULL; + x = (x ^ (x >> 27)) * 0x94D0'49BB'1331'11EBULL; + return x ^ (x >> 31); +} + +inline auto seed_from_env() -> uint64_t { + static const uint64_t seed = [] { + const char *text = std::getenv("monoprop_ROUTE_SEED"); + return (text == nullptr || *text == '\0') ? kDefaultSeed : std::strtoull(text, nullptr, 10); + }(); + return seed; +} + +// One 64-bit vector per Majorana mode. Deterministic from the seed alone, so every rank builds the +// same table with no communication -- the property find_rank's contract rests on. +template +inline auto linear_basis() -> const std::array & { + static const auto table = [] { + std::array v{}; + const uint64_t seed = seed_from_env(); + for (size_t i = 0; i < NumBits; ++i) { + v[i] = mix64(mix64(seed) + (static_cast(i) * 0x9E37'79B9'7F4A'7C15ULL)); + } + return v; + }(); + return table; +} + +// Terms are sparse under a length cutoff (popcount <= ~2*cutoff), so iterating set bits beats a +// word-wise byte table. A CLMUL form would be popcount-independent; it is also a DIFFERENT linear map, +// so switching to it means re-measuring balance, not just re-benchmarking. +template +[[nodiscard]] inline auto linear_hash(const monoprop::Bitset &bits) noexcept -> uint64_t { + const auto &v = linear_basis(); + uint64_t h = 0; + for (size_t i = bits.find_first(); i < NumBits; i = bits.find_next(i)) { + h ^= v[i]; + } + return h; +} + +// GF(2) rank of a set of 64-bit vectors, by Gaussian elimination over the bit columns. The per-generator +// rank shifts must span at least log2(R) dimensions or the reachable destination ranks form a strict +// subspace and some ranks stay empty -- a load-balance failure, not a correctness one, which is why this +// is a diagnostic (measured: rank 32 for the 60-site Hubbard's 416 distinct shifts, against the 7 bits +// R = 128 needs) rather than a runtime gate. +[[nodiscard]] inline auto gf2_rank(std::vector vectors) noexcept -> size_t { + size_t rank = 0; + for (size_t bit = 0; bit < 64 && rank < vectors.size(); ++bit) { + const uint64_t probe = uint64_t{1} << bit; + size_t pivot = vectors.size(); + for (size_t i = rank; i < vectors.size(); ++i) { + if ((vectors[i] & probe) != 0) { + pivot = i; + break; + } + } + if (pivot == vectors.size()) { + continue; + } + std::swap(vectors[rank], vectors[pivot]); + for (size_t i = 0; i < vectors.size(); ++i) { + if (i != rank && (vectors[i] & probe) != 0) { + vectors[i] ^= vectors[rank]; + } + } + ++rank; + } + return rank; +} + +enum class Mode : uint8_t { Splitmix, Linear }; + +// Trivially copyable and cheap to build; hold one per build_layer call rather than per term. +class Router final { +public: + // ranks x partitions == the flat world the destinations index. linear_bits is clamped here, so a + // caller may pass anything. + constexpr Router(size_t ranks, size_t partitions, size_t linear_bits) noexcept + : ranks_(ranks == 0 ? 1 : ranks), + parts_(partitions == 0 ? 1 : partitions), + flat_(ranks_ * parts_), + bits_(clamp_bits_(ranks_, linear_bits)), + lin_mask_((uint64_t{1} << bits_) - 1), + hi_mask_((ranks_ >> bits_) - 1) {} + + // Today's routing: one flat world, full avalanche, no linear bits. Also what d = 0 collapses to. + static constexpr auto splitmix(size_t flat_world) noexcept -> Router { return Router{flat_world, 1, 0}; } + + [[nodiscard]] constexpr auto ranks() const noexcept -> size_t { return ranks_; } + [[nodiscard]] constexpr auto partitions() const noexcept -> size_t { return parts_; } + [[nodiscard]] constexpr auto flat_world() const noexcept -> size_t { return flat_; } + [[nodiscard]] constexpr auto linear_bits() const noexcept -> size_t { return bits_; } + // Distinct destination RANKS one rank's queries for a single generator reach. 1 == pairwise. + [[nodiscard]] constexpr auto fanout() const noexcept -> size_t { return ranks_ >> bits_; } + [[nodiscard]] constexpr auto mode() const noexcept -> Mode { return bits_ == 0 ? Mode::Splitmix : Mode::Linear; } + + // Flat destination slot in [0, flat_world). Branch is on a member, so it is perfectly predicted. + template + [[nodiscard]] [[gnu::always_inline]] inline auto dest(const Monomial &mono) const noexcept -> size_t { + const uint64_t q = monomial_hash(mono); + if (bits_ == 0) { + return static_cast(q % flat_); // bit-for-bit today's `hash % P` + } + const uint64_t part = q % parts_; + const uint64_t hi = (q / parts_) & hi_mask_; // ranks_>>bits_ is a power of two, so a mask + const uint64_t lin = linear_hash<2 * NumModes>(mono) & lin_mask_; + return static_cast(((lin | (hi << bits_)) * parts_) + part); + } + + // The rank-level shift a generator induces: rank(M^G) low bits == rank(M) low bits ^ shift(G). + // Zero for every G iff bits_ == 0. This is what makes the destination predictable. + template + [[nodiscard]] auto rank_shift(const Monomial &gen) const noexcept -> size_t { + return static_cast(linear_hash<2 * NumModes>(gen) & lin_mask_); + } + +private: + static constexpr auto clamp_bits_(size_t ranks, size_t requested) noexcept -> size_t { + if (!std::has_single_bit(ranks)) { + return 0; // no XOR structure without a power-of-two rank count + } + const size_t max_bits = static_cast(std::countr_zero(ranks)); + return requested < max_bits ? requested : max_bits; + } + + size_t ranks_; + size_t parts_; + size_t flat_; + size_t bits_; + uint64_t lin_mask_; + uint64_t hi_mask_; +}; + +// The requested linear-bit count, before clamping to a particular geometry. Parsed once. +inline auto requested_linear_bits() -> size_t { + static const size_t bits = [] { + const char *explicit_bits = std::getenv("monoprop_ROUTE_LINEAR_BITS"); + if (explicit_bits != nullptr && *explicit_bits != '\0') { + const long value = std::strtol(explicit_bits, nullptr, 10); + return value > 0 ? static_cast(value) : size_t{0}; + } + const char *mode = std::getenv("monoprop_ROUTING"); + if (mode != nullptr && std::string_view{mode} == "linear") { + return ~size_t{0}; // "as many as this geometry allows" -- Router clamps to log2(R) + } + return size_t{0}; + }(); + return bits; +} + +inline auto make_router(size_t ranks, size_t partitions) -> Router { + return Router{ranks, partitions, requested_linear_bits()}; +} + +} // namespace monoprop::routing diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp new file mode 100644 index 00000000..01fc7701 --- /dev/null +++ b/cpp/tests/routing_tests.cpp @@ -0,0 +1,211 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// routing::Router -- the term -> flat-slot map. Two properties carry the whole design: +// * d = 0 is bit-for-bit today's `monomial_hash % P`, so the refactor cannot move a single term; +// * at d = log2(R) the destination RANK of M^G is rank(M) ^ shift(G), which is what turns the dense +// all-to-all into a pairwise exchange. A break here is silent -- wrong owner, not a crash -- so the +// shift identity and the Scan/find_rank agreement are both asserted explicitly. +// +// Flat cases with a shared prefix, no suite nesting (suites break Boost's ctest discovery here). + +#include + +#include +#include +#include +#include + +#include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/mpi/Routing.h" + +using namespace monoprop; +using monoprop::routing::Router; + +namespace { + +constexpr size_t kN = 64; // 2N = 128 bits -> 2 words, so the multi-word hash path is exercised + +auto random_monomials(size_t count, size_t weight, uint64_t seed) -> std::vector> { + std::mt19937_64 rng(seed); + std::uniform_int_distribution slot(0, 2 * kN - 1); + std::vector> out; + out.reserve(count); + for (size_t i = 0; i < count; ++i) { + VecZ inds; + for (size_t k = 0; k < weight; ++k) { + inds.push_back(slot(rng)); + } + out.push_back(indices_to_bitset(inds)); + } + return out; +} + +} // namespace + +// d = 0 must not move a single term relative to `monomial_hash % P`: this is the regression gate that +// licenses everything else. +BOOST_AUTO_TEST_CASE(routing_zero_bits_is_bit_identical_to_splitmix) { + const auto monos = random_monomials(500, 5, 0xC0FFEEULL); + for (const size_t flat : {size_t{1}, size_t{2}, size_t{7}, size_t{112}, size_t{1792}}) { + const auto router = Router::splitmix(flat); + BOOST_TEST(router.linear_bits() == 0U); + for (const auto &m : monos) { + const size_t expected = monomial_hash(m) % flat; + BOOST_TEST(router.dest(m) == expected); + BOOST_TEST(find_rank(m, flat) == expected); + BOOST_TEST(find_rank(m, router) == expected); + } + } +} + +// A two-level router with d = 0 is still today's routing, even though it knows about partitions: +// ((q/S) % R)*S + q%S == q % (R*S). +BOOST_AUTO_TEST_CASE(routing_zero_bits_two_level_collapses_to_flat_modulo) { + const auto monos = random_monomials(300, 6, 0xBEEF01ULL); + for (const auto [r, s] : {std::pair{8, 14}, {4, 28}, {128, 14}, {64, 28}}) { + const Router router{r, s, 0}; + for (const auto &m : monos) { + BOOST_TEST(router.dest(m) == monomial_hash(m) % (r * s)); + } + } +} + +BOOST_AUTO_TEST_CASE(routing_linear_hash_is_gf2_linear) { + const auto a = random_monomials(200, 5, 0x1234ULL); + const auto b = random_monomials(200, 3, 0x5678ULL); + for (size_t i = 0; i < a.size(); ++i) { + const uint64_t ha = routing::linear_hash<2 * kN>(a[i]); + const uint64_t hb = routing::linear_hash<2 * kN>(b[i]); + BOOST_TEST(routing::linear_hash<2 * kN>(a[i] ^ b[i]) == (ha ^ hb)); + } + // The empty monomial is the identity of the group, so its hash must be the identity of the codomain. + BOOST_TEST(routing::linear_hash<2 * kN>(Monomial{}) == 0ULL); +} + +// The load-bearing identity: at full linear bits the destination RANK of M^G is rank(M) ^ shift(G), +// so one rank's queries for one generator all land on one peer. +BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_at_full_bits) { + constexpr size_t kRanks = 16; + constexpr size_t kParts = 14; + const Router router{kRanks, kParts, 64}; // clamped to log2(16) == 4 + BOOST_REQUIRE(router.linear_bits() == 4U); + BOOST_REQUIRE(router.fanout() == 1U); + + const auto terms = random_monomials(400, 6, 0xAAAA01ULL); + const auto gens = random_monomials(40, 4, 0xBBBB02ULL); + for (const auto &g : gens) { + const size_t shift = router.rank_shift(g); + for (const auto &m : terms) { + const size_t rank_m = router.dest(m) / kParts; + const size_t rank_mg = router.dest(m ^ g) / kParts; + BOOST_TEST(rank_mg == (rank_m ^ shift)); + } + } +} + +// The consequence that the transport will rely on: every term a rank owns sends its query for one +// generator to exactly ONE peer rank -- and the partition index within that peer still spreads. +BOOST_AUTO_TEST_CASE(routing_fanout_is_one_at_full_bits) { + constexpr size_t kRanks = 8; + constexpr size_t kParts = 14; + const Router router{kRanks, kParts, 3}; + const auto terms = random_monomials(4000, 6, 0xCCCC03ULL); + const auto gens = random_monomials(12, 4, 0xDDDD04ULL); + + std::vector>> owned(kRanks); + for (const auto &m : terms) { + owned[router.dest(m) / kParts].push_back(m); + } + for (size_t src = 0; src < kRanks; ++src) { + BOOST_REQUIRE(!owned[src].empty()); + for (const auto &g : gens) { + std::set dest_ranks; + std::set dest_parts; + for (const auto &m : owned[src]) { + const size_t flat = router.dest(m ^ g); + dest_ranks.insert(flat / kParts); + dest_parts.insert(flat % kParts); + } + BOOST_TEST(dest_ranks.size() == 1U); + BOOST_TEST(dest_parts.size() > 1U); // partitions keep full avalanche + } + } +} + +// d in between: the low d bits shift deterministically and the high log2(R)-d are splitmix, so the +// realised fanout is R >> d -- the dial the balance/fanout trade is made on. +BOOST_AUTO_TEST_CASE(routing_partial_bits_give_fanout_ranks_over_two_to_the_d) { + constexpr size_t kRanks = 32; + constexpr size_t kParts = 14; + const auto terms = random_monomials(20000, 6, 0xEEEE05ULL); + const auto gen = random_monomials(1, 4, 0xFFFF06ULL).front(); + for (size_t d = 0; d <= 5; ++d) { + const Router router{kRanks, kParts, d}; + BOOST_TEST(router.fanout() == (kRanks >> d)); + std::vector> dest_of(kRanks); + for (const auto &m : terms) { + const size_t src = router.dest(m) / kParts; + dest_of[src].insert(router.dest(m ^ gen) / kParts); + } + for (size_t src = 0; src < kRanks; ++src) { + BOOST_TEST(dest_of[src].size() <= router.fanout()); + } + } +} + +// Without a power-of-two rank count there is no XOR structure to exploit, so the router must fall back +// to today's routing rather than silently produce a lopsided or out-of-range slot. +BOOST_AUTO_TEST_CASE(routing_non_power_of_two_ranks_falls_back_to_zero_bits) { + for (const size_t r : {size_t{3}, size_t{7}, size_t{12}, size_t{112}}) { + const Router router{r, 14, 8}; + BOOST_TEST(router.linear_bits() == 0U); + BOOST_TEST(router.fanout() == r); + } + const Router pow2{64, 14, 8}; + BOOST_TEST(pow2.linear_bits() == 6U); // clamped to log2(64), not 8 +} + +BOOST_AUTO_TEST_CASE(routing_dest_is_in_range_and_deterministic) { + const auto monos = random_monomials(1000, 7, 0x9999ULL); + for (const auto [r, s] : {std::pair{1, 1}, {1, 112}, {8, 14}, {128, 14}, {64, 28}}) { + for (size_t d = 0; d <= 7; ++d) { + const Router router{r, s, d}; + for (const auto &m : monos) { + const size_t slot = router.dest(m); + BOOST_TEST(slot < r * s); + BOOST_TEST(slot == router.dest(m)); // stateless + } + } + } +} + +// gf2_rank is the coverage diagnostic: shifts that span fewer than log2(R) dimensions leave ranks empty. +BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { + BOOST_TEST(routing::gf2_rank({}) == 0U); + BOOST_TEST(routing::gf2_rank({0ULL, 0ULL}) == 0U); + BOOST_TEST(routing::gf2_rank({0b001ULL, 0b010ULL, 0b011ULL}) == 2U); // third is the XOR of the first two + BOOST_TEST(routing::gf2_rank({0b001ULL, 0b010ULL, 0b100ULL}) == 3U); + + // The real generator shifts must span at least log2(R) dimensions or the reachable ranks are a + // strict subspace of the rank space. + constexpr size_t kRanks = 128; + const Router router{kRanks, 14, 7}; + std::vector shifts; + for (const auto &g : random_monomials(200, 4, 0x7777ULL)) { + shifts.push_back(static_cast(router.rank_shift(g))); + } + BOOST_TEST(routing::gf2_rank(shifts) == 7U); // == log2(128): every rank is reachable +} From 72db27d2aad904fc993317aaf82a2a60e8507494 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Wed, 26 Aug 2026 19:03:28 +0200 Subject: [PATCH 02/24] =?UTF-8?q?perf(mpi):=20=E2=8F=AD=EF=B8=8F=20skip=20?= =?UTF-8?q?the=20exchange=20for=20identity=20generators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An empty Majorana generator anticommutes with nothing, so the scan already returned zero queries on every rank -- but run_exchange's collectives fire on payload size zero all the same, and each of the two passes costs three of them. These generators are not an edge case. A gate whose every term falls below its atol expands to the identity monomial (circuit.py's deliberate `or [((), 0.0)]`), and a zero chemical potential alone contributes 60 of the 60-site Hubbard's 476 generators per Trotter layer: 360 of 2,856 collectives per rank per layer, 12.6%, moving nothing. Measured at both sizes: 60/476 at 60 sites, 20/156 at 20. `gen.none()` is unanimous -- the generator list is replicated on every rank -- so skipping needs no agreement and no collective to decide it. This is not gate fusion: no gate is merged, a no-op gate is simply not exchanged for, and the layer is still built (the graph's gate and parameter bookkeeping is untouched). finish() reads neither the query streams nor the response streams, so the skipped path lands in exactly the state the empty-payload exchange left it in. Measured on the 20-site Hubbard (cutoff 8, 5 steps, 20 of 156 generators empty): term count and expectation value bit-identical to the previous commit in all seven configurations -- S=1, S=8, and worlds 2 and 4 crossed with d = 0, 1, 2. Co-Authored-By: Claude Opus 5 (1M context) --- .../detail/evolution/layer_build/Engine.h | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index a70bb3f0..aaf5b039 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -662,16 +662,24 @@ auto build_layer(MPOperator &local_op, matched_scratch, /*combined_size=*/local_op.store->size(), std::move(sink)); - eng.run_exchange(/*is_leader_pass=*/true, - std::move(fused.leader_queries), - std::move(fused.leader_src), - std::move(fused.leader_val), - std::move(fused.leader_self)); - eng.run_exchange(/*is_leader_pass=*/false, - std::move(fused.follower_queries), - std::move(fused.follower_src), - std::move(fused.follower_val), - std::move(fused.follower_self)); + // An empty generator anticommutes with nothing, so the scan already returned zero queries on + // every rank -- but run_exchange's collectives fire regardless of payload, and each pass costs + // three of them. The generator list is replicated, so `gen.none()` is unanimous and skipping + // needs no agreement. (These are the identity monomials a gate whose every term fell below its + // atol expands to; a zero chemical potential alone contributes 60 of the 60-site Hubbard's 476 + // generators per Trotter layer.) No gate is merged: a no-op gate is simply not exchanged for. + if (gen.any()) { + eng.run_exchange(/*is_leader_pass=*/true, + std::move(fused.leader_queries), + std::move(fused.leader_src), + std::move(fused.leader_val), + std::move(fused.leader_self)); + eng.run_exchange(/*is_leader_pass=*/false, + std::move(fused.follower_queries), + std::move(fused.follower_src), + std::move(fused.follower_val), + std::move(fused.follower_self)); + } return eng.finish(std::move(cos_all), out_cos); }; From 1927fe632d70de64ad887a832342c3f6d4c380b1 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Wed, 26 Aug 2026 19:28:30 +0200 Subject: [PATCH 03/24] =?UTF-8?q?perf(mpi):=20=F0=9F=95=B8=EF=B8=8F=20exch?= =?UTF-8?q?ange=20point-to-point=20over=20the=20peers=20routing=20can=20re?= =?UTF-8?q?ach?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linear routing makes the destination RANK of a query predictable, but until now nothing used that: the exchange still called MPI_Alltoall on the counts and MPI_Alltoallv on the payload, so the message count stayed R*(R-1) whatever the data looked like. mpi::PeerPlan{bits, shift} carries the structure from where it is known (build_layer, which holds the generator) to where it is spent (HybridComm, and the plain-MPI path for S == 1). Its two accessors are written so that bits == 0 degenerates exactly: peer(k) == k and count == ranks, so every loop below walks all ranks in the old order and the verbs take their collective path unchanged. What the plan buys, in the order the campaign's evidence ranks them: * the SERIAL sweeps shrink. pack_count_matrix_, size_staging_send_'s two passes, fill_recv_col_from_counts_recv_ and the scatter are all O(R*S^2) and all run on partition 0 while the other S-1 park at a barrier -- ~25k int ops per verb at R=128, S=14 that nothing overlaps. Restricted to the f = R>>bits reachable ranks they become O(f*S^2), so 128x less at fanout 1. * the messages shrink. One MPI_Alltoall + one MPI_Alltoallv become f Isend/Irecv pairs; a self peer (a generator whose rank shift is zero) is a memcpy and costs no message at all. check_routing_agreement() allreduces the resolved configuration once at construction. This is not defensive tidiness: under linear routing each rank posts receives from the peers ITS OWN bits imply, so a rank that never saw monoprop_ROUTING would hang rather than answer differently, and a hang at 128 ranks is much harder to read than an exception. Measured, 20-site Hubbard cutoff 8, 5 steps, across 11 configurations -- world 4 x S in {1,2,3} x d in {0,1,2}, world 2 S=4, world 8 d=3 (fanout 1): term count 17,148 in every one, expectation value within +-1 ULP of the dense arm, no hangs. 252/252 ctest including two new sparse-plan cases that check a non-peer never appears in the delivery and that both the fused-counts and known-counts paths pair correctly; the Python suite passes at world 4 under both routings. Co-Authored-By: Claude Opus 5 (1M context) --- .../detail/evolution/layer_build/Engine.h | 20 +- .../MonomialPropagator.inl | 1 + cpp/monoprop/detail/mpi/Comm.h | 28 ++ cpp/monoprop/detail/mpi/HybridComm.h | 281 +++++++++++++----- cpp/monoprop/detail/mpi/MPICompat.cpp | 29 +- cpp/monoprop/detail/mpi/MPICompat.h | 63 +++- cpp/monoprop/detail/mpi/MPIUtils.h | 32 ++ cpp/tests/hybrid_comm_tests.cpp | 101 +++++++ 8 files changed, 459 insertions(+), 96 deletions(-) diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index aaf5b039..8d640187 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -350,6 +350,9 @@ struct LayerBuildEngine { std::vector> src_val_r; // Fused query+value send scratch (ContractSink, R>1): shared by a gate's two exchange passes. std::vector combined_qv_; + // Which destination ranks this gate's queries can reach. Dense unless the router is GF(2)-linear; + // see mpi::PeerPlan. Derived once per layer in build_layer, never per query. + mpi::PeerPlan plan; Sink sink; LayerBuildEngine(MPOperator &local_op_, @@ -358,7 +361,8 @@ struct LayerBuildEngine { size_t my_rank_, MatchedEpochSet &matched_scratch, size_t combined_size_, - Sink &&sink_) + Sink &&sink_, + mpi::PeerPlan plan_ = {}) // dense by default: the tests build the engine directly : local_op(local_op_), comm(comm_), R(R_), @@ -367,6 +371,7 @@ struct LayerBuildEngine { combined_size(combined_size_), queries_r(R_), src_idx_r(R_), + plan(plan_), sink(std::move(sink_)) { matched.begin_gate(combined_size); } @@ -414,11 +419,12 @@ struct LayerBuildEngine { } std::vector &send = sink.send_buffer(queries_r, src_val_r, combined_qv_); std::vector> inc_q; - mpi::begin_alltoallv(send, comm).wait_into(inc_q); + mpi::begin_alltoallv(send, comm, /*skip_self=*/false, /*known_recv_counts=*/nullptr, plan).wait_into(inc_q); auto resp = resolve_incoming(inc_q, local_op, R, is_leader_pass, matched, combined_size, sink); std::vector resp_recv = response_recv_counts(); std::vector> inc_r; - mpi::begin_alltoallv(resp, comm, /*skip_self=*/false, &resp_recv).wait_into(inc_r); + // The answers retrace the queries, and the pairing is an XOR involution, so the same plan holds. + mpi::begin_alltoallv(resp, comm, /*skip_self=*/false, &resp_recv, plan).wait_into(inc_r); process_responses(inc_r, src_idx_r, queries_r, R, my_rank, sink); } @@ -603,6 +609,11 @@ auto build_layer(MPOperator &local_op, // R is the FLAT world (ranks x partitions); the router is what splits it back into the two levels. const auto router = router_for(comm); assert(router.flat_world() == R); + // Under linear routing every query for THIS generator lands on a rank whose low `linear_bits` are + // this rank's own XOR rank_shift(gen), so the exchange knows its peers before it starts. Dense + // (bits == 0) otherwise, which is today's collective. + const auto plan = mpi::PeerPlan{.bits = static_cast(router.linear_bits()), + .shift = static_cast(router.rank_shift(gen))}; // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); @@ -661,7 +672,8 @@ auto build_layer(MPOperator &local_op, my_rank, matched_scratch, /*combined_size=*/local_op.store->size(), - std::move(sink)); + std::move(sink), + plan); // An empty generator anticommutes with nothing, so the scan already returned zero queries on // every rank -- but run_exchange's collectives fire regardless of payload, and each pass costs // three of them. The generator list is replicated, so `gen.none()` is unanimous and skipping diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index 16b5d935..dc242f39 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -143,6 +143,7 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope const size_t num_ranks = static_cast(mpi::size(comm_)); const size_t my_rank = static_cast(mpi::rank(comm_)); + check_routing_agreement(comm_); // a disagreement here would hang the first exchange, not corrupt it const auto router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term MonomialList local_heisenberg_terms; diff --git a/cpp/monoprop/detail/mpi/Comm.h b/cpp/monoprop/detail/mpi/Comm.h index 9ff3a14c..2354bf0c 100644 --- a/cpp/monoprop/detail/mpi/Comm.h +++ b/cpp/monoprop/detail/mpi/Comm.h @@ -65,6 +65,34 @@ struct Comm { } }; +// Which destination RANKS a round can touch, when the caller knows. Under GF(2)-linear routing +// (routing::Router) the low `bits` of the destination rank are determined by the generator: they are +// this rank's own low bits XOR `shift`, so the peers are +// +// peer(k) = ((me & (2^bits - 1)) ^ shift) | (k << bits), k in [0, ranks >> bits) +// +// -- `ranks >> bits` of them instead of all `ranks`, and the relation is symmetric (XOR is an +// involution), so every rank derives the same pairing with no communication. That is what lets a verb +// replace a dense collective with point-to-point over the peers it can actually reach. +// +// bits == 0 is the dense default: peer(k) == k and count == ranks, so the same loops walk every rank +// and the verbs take their collective path. A caller that gets `shift` wrong does not corrupt data -- +// the counts for a non-peer are zero -- it deadlocks, which is why the plan is derived in one place. +struct PeerPlan { + int bits = 0; + int shift = 0; + + [[nodiscard]] constexpr auto dense() const -> bool { return bits == 0; } + [[nodiscard]] constexpr auto count(int ranks) const -> int { return bits == 0 ? ranks : (ranks >> bits); } + [[nodiscard]] constexpr auto peer(int me, int k) const -> int { + if (bits == 0) { + return k; + } + const int mask = (1 << bits) - 1; + return ((me & mask) ^ shift) | (k << bits); + } +}; + // Argument bundles for the variable all-to-all verbs, deliberately here rather than in HybridComm.h: // ShmComm.h takes the resolve bundle and compiles in non-MPI builds, so neither bundle may name an // MPI-only type. MPI_Datatype therefore stays a separate parameter on the HybridComm verbs that need diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index a91c85ea..9cd3c10a 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -101,27 +101,33 @@ class HybridComm { auto partitions() const -> int { return s_; } auto global_rank(int local_partition) const -> int { return mpi_rank_ * s_ + local_partition; } - auto alltoall_counts(int local_partition, const int *send_counts /*[P]*/, int *recv_counts /*[P]*/) -> void { - guard_partition0_(local_partition, "alltoall_counts", [this, local_partition, send_counts, recv_counts] { - alltoall_counts_impl_(local_partition, send_counts, recv_counts); + auto alltoall_counts(int local_partition, + const int *send_counts /*[P]*/, + int *recv_counts /*[P]*/, + PeerPlan plan = {}) -> void { + guard_partition0_(local_partition, "alltoall_counts", [this, local_partition, send_counts, recv_counts, plan] { + alltoall_counts_impl_(local_partition, send_counts, recv_counts, plan); }); } // See AlltoallvArgs for the send-buffer lifetime and the element-vs-byte convention; `dt` is the MPI // datatype whose extent is args.elem, and it stays a separate argument because the bundle is shared // with the non-MPI-capable transport. - auto alltoallv(int local_partition, const AlltoallvArgs &args, MPI_Datatype dt) -> void { - guard_partition0_(local_partition, "alltoallv", [this, local_partition, &args, dt] { - alltoallv_impl_(local_partition, args, dt); + auto alltoallv(int local_partition, const AlltoallvArgs &args, MPI_Datatype dt, PeerPlan plan = {}) -> void { + guard_partition0_(local_partition, "alltoallv", [this, local_partition, &args, dt, plan] { + alltoallv_impl_(local_partition, args, dt, plan); }); } // See AlltoallvResolveArgs: the recv side is an output, and args.recv is resized here. template - auto alltoallv_resolve(int local_partition, const AlltoallvResolveArgs &args, MPI_Datatype dt) -> void { + auto alltoallv_resolve(int local_partition, + const AlltoallvResolveArgs &args, + MPI_Datatype dt, + PeerPlan plan = {}) -> void { // `args` by reference, not by value: the impl resizes args.recv and then writes through it. - guard_partition0_(local_partition, "alltoallv_resolve", [this, local_partition, &args, dt] { - alltoallv_resolve_impl_(local_partition, args, dt); + guard_partition0_(local_partition, "alltoallv_resolve", [this, local_partition, &args, dt, plan] { + alltoallv_resolve_impl_(local_partition, args, dt, plan); }); } @@ -172,18 +178,29 @@ class HybridComm { } } - // recv_counts[g] = amount global partition g sends to this partition. 2 barriers + one S*S-int MPI_Alltoall. - auto alltoall_counts_impl_(int local_partition, const int *send_counts /*[P]*/, int *recv_counts /*[P]*/) -> void { + // recv_counts[g] = amount global partition g sends to this partition. 2 barriers + one S*S-int + // MPI_Alltoall -- or, under a sparse plan, `f = R>>bits` S*S-int point-to-point pairs. + auto alltoall_counts_impl_(int local_partition, + const int *send_counts /*[P]*/, + int *recv_counts /*[P]*/, + PeerPlan plan) -> void { publish_counts_row_(local_partition, send_counts); sync(); if (local_partition == 0) { - pack_count_matrix_(); - MPI_Alltoall(counts_send_.data(), s_ * s_, MPI_INT, counts_recv_.data(), s_ * s_, MPI_INT, parent_); + pack_count_matrix_(plan); + exchange_count_blocks_(plan); } sync(); // Partition t extracts its row: recv from (rank a, partition su) is contiguous per source rank a. + // Under a plan only the f peer ranks were exchanged, so the rest of the row is zero by definition + // (a non-peer cannot own the partner of any term this rank owns). const int t = local_partition; - for (int a = 0; a < r_; ++a) { + const int f = plan.count(r_); + if (!plan.dense()) { + std::fill(recv_counts, recv_counts + static_cast(r_) * static_cast(s_), 0); + } + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); for (int su = 0; su < s_; ++su) { recv_counts[a * s_ + su] = counts_recv_[counts_idx_(a, t, su)]; } @@ -194,7 +211,7 @@ class HybridComm { } // Flat variable all-to-all over caller-owned buffers; see AlltoallvArgs for the conventions. - auto alltoallv_impl_(int local_partition, const AlltoallvArgs &args, MPI_Datatype dt) -> void { + auto alltoallv_impl_(int local_partition, const AlltoallvArgs &args, MPI_Datatype dt, PeerPlan plan) -> void { const size_t u = static_cast(local_partition); Slot &me = slots_[u]; me.ptr = args.send; @@ -205,27 +222,19 @@ class HybridComm { // B2: partition 0 sizes/reallocates staging; must finish before any partition packs into stage_send_. if (local_partition == 0) { - size_staging_send_(args.elem); - fill_recv_col_from_rows_(); - size_staging_recv_(args.elem); + size_staging_send_(args.elem, plan); + fill_recv_col_from_rows_(plan); + size_staging_recv_(args.elem, plan); } sync(); // B2 // B3: each partition packs its own cross-rank blocks into stage_send_ (disjoint writes). - pack_send_(local_partition, args.elem); + pack_send_(local_partition, args.elem, plan); sync(); // B3 - // B4: partition 0 runs the single MPI_Alltoallv while peers park at the barrier. + // B4: partition 0 moves the payload while peers park at the barrier. if (local_partition == 0) { - MPI_Alltoallv(stage_send_.data(), - mpi_send_counts_.data(), - mpi_send_displs_.data(), - dt, - stage_recv_.data(), - mpi_recv_counts_.data(), - mpi_recv_displs_.data(), - dt, - parent_); + exchange_payload_(dt, plan); } sync(); // B4 @@ -234,7 +243,9 @@ class HybridComm { // block starts from base_recv_ and this partition's own counts. std::byte *dst = args.recv; const int t = local_partition; - for (int a = 0; a < r_; ++a) { + const int f = plan.count(r_); + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); size_t cur = base_recv_[static_cast(a) * static_cast(s_) + static_cast(t)]; for (int su = 0; su < s_; ++su) { const int g = a * s_ + su; @@ -254,7 +265,10 @@ class HybridComm { // B1→B2 window (4 syncs instead of 6). recv_counts / recv_displs and `recv` (resized) are outputs. // Bit-identical to alltoall_counts + alltoallv. template - auto alltoallv_resolve_impl_(int local_partition, const AlltoallvResolveArgs &args, MPI_Datatype dt) -> void { + auto alltoallv_resolve_impl_(int local_partition, + const AlltoallvResolveArgs &args, + MPI_Datatype dt, + PeerPlan plan) -> void { // Typed verb: element bytes are sizeof(T) by construction, so they are derived rather than passed. constexpr size_t elem = sizeof(T); const size_t u = static_cast(local_partition); @@ -268,17 +282,24 @@ class HybridComm { sync(); // B1 if (local_partition == 0) { - pack_count_matrix_(); - MPI_Alltoall(counts_send_.data(), s_ * s_, MPI_INT, counts_recv_.data(), s_ * s_, MPI_INT, parent_); - size_staging_send_(elem); - fill_recv_col_from_counts_recv_(); - size_staging_recv_(elem); + pack_count_matrix_(plan); + exchange_count_blocks_(plan); + size_staging_send_(elem, plan); + fill_recv_col_from_counts_recv_(plan); + size_staging_recv_(elem, plan); } sync(); // B2 const int t = local_partition; + const int f = plan.count(r_); long long total = 0; - for (int a = 0; a < r_; ++a) { + if (!plan.dense()) { + const size_t p = static_cast(r_) * static_cast(s_); + std::fill(args.recv_counts, args.recv_counts + p, 0); + std::fill(args.recv_displs, args.recv_displs + p, 0); + } + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); for (int su = 0; su < s_; ++su) { const int g = a * s_ + su; const int c = counts_recv_[counts_idx_(a, t, su)]; @@ -289,24 +310,17 @@ class HybridComm { } args.recv.resize(static_cast(checked_mpi_count(total, "Total recv count"))); - pack_send_(local_partition, elem); + pack_send_(local_partition, elem, plan); sync(); // B3 if (local_partition == 0) { - MPI_Alltoallv(stage_send_.data(), - mpi_send_counts_.data(), - mpi_send_displs_.data(), - dt, - stage_recv_.data(), - mpi_recv_counts_.data(), - mpi_recv_displs_.data(), - dt, - parent_); + exchange_payload_(dt, plan); } sync(); // B4 std::byte *dst = reinterpret_cast(args.recv.data()); // after the resize: it may reallocate - for (int a = 0; a < r_; ++a) { + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); size_t cur = base_recv_[static_cast(a) * static_cast(s_) + static_cast(t)]; for (int su = 0; su < s_; ++su) { const int g = a * s_ + su; @@ -390,6 +404,11 @@ class HybridComm { // No trailing barrier: red_vec_ is rewritten only inside a future verb's barriered phases. } + // Distinct tags so a count round in flight cannot be matched by a payload receive: partition 0 is + // the only participant per rank, so every message of a verb shares one (source, tag) pair. + static constexpr int kCountTag = 0x6D70; // 'mp' + static constexpr int kPayloadTag = 0x6D71; + static constexpr size_t kLineBytes = 64; static constexpr size_t kIntsPerLine = kLineBytes / sizeof(int); static constexpr size_t kLongsPerLine = kLineBytes / sizeof(long long); @@ -442,10 +461,12 @@ class HybridComm { // Transpose the published count rows into counts_send_, dest-major then source-minor, for the one // S*S-int MPI_Alltoall. Partition 0 only, inside a barriered window. Source partition outer, so the // peer-owned side streams. Every element is written here, so no pre-zeroing. - auto pack_count_matrix_() -> void { + auto pack_count_matrix_(PeerPlan plan) -> void { + const int f = plan.count(r_); for (int su = 0; su < s_; ++su) { const int *row = counts_row_(su); - for (int b = 0; b < r_; ++b) { + for (int k = 0; k < f; ++k) { + const int b = plan.peer(mpi_rank_, k); for (int t = 0; t < s_; ++t) { counts_send_[counts_idx_(b, t, su)] = row[b * s_ + t]; } @@ -453,6 +474,81 @@ class HybridComm { } } + // The count blocks: one S*S-int MPI_Alltoall when dense, else f point-to-point pairs (a self peer + // is a copy, not a message -- with the full linear bits a zero rank shift keeps the whole round + // on-rank). Partition 0 only, inside a barriered window. + auto exchange_count_blocks_(PeerPlan plan) -> void { + const int block = s_ * s_; + if (plan.dense()) { + MPI_Alltoall(counts_send_.data(), block, MPI_INT, counts_recv_.data(), block, MPI_INT, parent_); + return; + } + const int f = plan.count(r_); + grow_(reqs_, static_cast(2 * f)); + int n_req = 0; + for (int k = 0; k < f; ++k) { + const int b = plan.peer(mpi_rank_, k); + const size_t off = static_cast(b) * static_cast(block); + if (b == mpi_rank_) { + std::memcpy(counts_recv_.data() + off, + counts_send_.data() + off, + static_cast(block) * sizeof(int)); + continue; + } + MPI_Irecv(counts_recv_.data() + off, block, MPI_INT, b, kCountTag, parent_, &reqs_[n_req++]); + MPI_Isend(counts_send_.data() + off, block, MPI_INT, b, kCountTag, parent_, &reqs_[n_req++]); + } + MPI_Waitall(n_req, reqs_.data(), MPI_STATUSES_IGNORE); + } + + // The staged payload: one MPI_Alltoallv when dense, else f point-to-point pairs over the same + // per-rank counts and displacements (a non-peer's count is zero, so nothing is dropped). + auto exchange_payload_(MPI_Datatype dt, PeerPlan plan) -> void { + if (plan.dense()) { + MPI_Alltoallv(stage_send_.data(), + mpi_send_counts_.data(), + mpi_send_displs_.data(), + dt, + stage_recv_.data(), + mpi_recv_counts_.data(), + mpi_recv_displs_.data(), + dt, + parent_); + return; + } + // Counts and displacements are in ELEMENTS; the byte offset into staging needs the datatype's + // extent, which the dense MPI_Alltoallv derived for us and point-to-point does not. + MPI_Aint lb = 0; + MPI_Aint extent = 0; + MPI_Type_get_extent(dt, &lb, &extent); + const int f = plan.count(r_); + grow_(reqs_, static_cast(2 * f)); + int n_req = 0; + for (int k = 0; k < f; ++k) { + const int b = plan.peer(mpi_rank_, k); + const auto ub = static_cast(b); + const int sc = mpi_send_counts_[ub]; + const int rc = mpi_recv_counts_[ub]; + std::byte *rbuf = + stage_recv_.data() + static_cast(mpi_recv_displs_[ub]) * static_cast(extent); + const std::byte *sbuf = + stage_send_.data() + static_cast(mpi_send_displs_[ub]) * static_cast(extent); + if (b == mpi_rank_) { + if (rc != 0) { + std::memcpy(rbuf, sbuf, static_cast(rc) * static_cast(extent)); + } + continue; + } + if (rc != 0) { + MPI_Irecv(rbuf, rc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); + } + if (sc != 0) { + MPI_Isend(const_cast(sbuf), sc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); + } + } + MPI_Waitall(n_req, reqs_.data(), MPI_STATUSES_IGNORE); + } + template static auto grow_(V &v, size_t need) -> void { if (v.size() < need) { @@ -462,17 +558,27 @@ class HybridComm { // Partition 0's send-side staging sizing, between B1 and B2, in two sweeps of counts_matrix_. Wire // block order is destination major, source minor: a per-destination base plus a per-source prefix. - auto size_staging_send_(size_t elem) -> void { - const size_t p = static_cast(r_) * static_cast(s_); + // Under a plan the two O(R*S^2) sweeps below shrink to O(f*S^2) -- which is the point as much as the + // message count is: these sweeps run SERIALLY on partition 0 while S-1 partitions park, so at R=128, + // S=14 they are ~25k int ops per verb that nothing else overlaps. + auto size_staging_send_(size_t elem, PeerPlan plan) -> void { + const int f = plan.count(r_); // Pass A: the column sums W over source partitions, u outer so both sides sweep in address order. std::fill(col_sum_.begin(), col_sum_.end(), 0LL); for (int u = 0; u < s_; ++u) { const int *row = counts_row_(u); - for (size_t g = 0; g < p; ++g) { - col_sum_[g] += row[g]; + for (int k = 0; k < f; ++k) { + const size_t base = static_cast(plan.peer(mpi_rank_, k)) * static_cast(s_); + for (int t = 0; t < s_; ++t) { + col_sum_[base + static_cast(t)] += row[base + static_cast(t)]; + } } } - for (int b = 0; b < r_; ++b) { + if (!plan.dense()) { + std::fill(mpi_send_counts_.begin(), mpi_send_counts_.end(), 0); + } + for (int k = 0; k < f; ++k) { + const int b = plan.peer(mpi_rank_, k); const long long *col = col_sum_.data() + static_cast(b) * static_cast(s_); long long send_sum = 0; for (int t = 0; t < s_; ++t) { @@ -486,7 +592,8 @@ class HybridComm { send_running += mpi_send_counts_[static_cast(b)]; } const size_t total_send = static_cast(checked_mpi_count(send_running, "Total send count")); - for (int b = 0; b < r_; ++b) { + for (int k = 0; k < f; ++k) { + const int b = plan.peer(mpi_rank_, k); size_t cur = static_cast(mpi_send_displs_[static_cast(b)]); for (int t = 0; t < s_; ++t) { const size_t g = static_cast(b) * static_cast(s_) + static_cast(t); @@ -499,9 +606,13 @@ class HybridComm { for (int u = 0; u < s_; ++u) { const int *row = counts_row_(u); size_t *off = pack_off_.data() + pack_idx_(u, 0); - for (size_t g = 0; g < p; ++g) { - off[g] = base_send_[g] + static_cast(col_sum_[g]); - col_sum_[g] += row[g]; + for (int k = 0; k < f; ++k) { + const size_t base = static_cast(plan.peer(mpi_rank_, k)) * static_cast(s_); + for (int t = 0; t < s_; ++t) { + const size_t g = base + static_cast(t); + off[g] = base_send_[g] + static_cast(col_sum_[g]); + col_sum_[g] += row[g]; + } } } // Grow-only, no zero-fill: pack_send_'s blocks tile [0, total_send) exactly. @@ -509,16 +620,26 @@ class HybridComm { } // recv_col_[a*S + t] = what partition t receives from rank a: the rows published in Phase P0. - auto fill_recv_col_from_rows_() -> void { - for (int a = 0; a < r_; ++a) { + auto fill_recv_col_from_rows_(PeerPlan plan) -> void { + const int f = plan.count(r_); + if (!plan.dense()) { + std::fill(recv_col_.begin(), recv_col_.end(), 0LL); + } + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); for (int t = 0; t < s_; ++t) { recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)] = row_recv_(t)[a]; } } } - auto fill_recv_col_from_counts_recv_() -> void { - for (int a = 0; a < r_; ++a) { + auto fill_recv_col_from_counts_recv_(PeerPlan plan) -> void { + const int f = plan.count(r_); + if (!plan.dense()) { + std::fill(recv_col_.begin(), recv_col_.end(), 0LL); + } + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); for (int t = 0; t < s_; ++t) { const int *blk = counts_recv_.data() + counts_idx_(a, t, 0); long long sum = 0; @@ -532,8 +653,13 @@ class HybridComm { // Partition 0's recv-side staging sizing, from recv_col_. Only the per-(rank, partition) base; the // post-B4 scatter re-derives the per-source offsets as it walks (a, su). - auto size_staging_recv_(size_t elem) -> void { - for (int a = 0; a < r_; ++a) { + auto size_staging_recv_(size_t elem, PeerPlan plan) -> void { + const int f = plan.count(r_); + if (!plan.dense()) { + std::fill(mpi_recv_counts_.begin(), mpi_recv_counts_.end(), 0); + } + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); long long recv_sum = 0; for (int t = 0; t < s_; ++t) { recv_sum += recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)]; @@ -546,7 +672,8 @@ class HybridComm { recv_running += mpi_recv_counts_[static_cast(a)]; } const size_t total_recv = static_cast(checked_mpi_count(recv_running, "Total recv count")); - for (int a = 0; a < r_; ++a) { + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); size_t cur = static_cast(mpi_recv_displs_[static_cast(a)]); for (int t = 0; t < s_; ++t) { const size_t g = static_cast(a) * static_cast(s_) + static_cast(t); @@ -557,7 +684,7 @@ class HybridComm { grow_(stage_recv_, total_recv * elem); } - auto pack_send_(int local_partition, size_t elem) -> void { + auto pack_send_(int local_partition, size_t elem, PeerPlan plan) -> void { const int u = local_partition; // Own slot only — no peer's published send buffer is read here, which is what lets every // partition pack concurrently in the B2→B3 window. @@ -565,13 +692,17 @@ class HybridComm { const int *my_send_counts = counts_row_(u); const int *my_send_displs = slots_[static_cast(u)].send_displs; const size_t *off = pack_off_.data() + pack_idx_(u, 0); - const int p = r_ * s_; - for (int g = 0; g < p; ++g) { - const int cnt = my_send_counts[g]; - if (cnt != 0) { - std::memcpy(stage_send_.data() + off[g] * elem, - src + static_cast(my_send_displs[g]) * elem, - static_cast(cnt) * elem); + const int f = plan.count(r_); + for (int k = 0; k < f; ++k) { + const int base = plan.peer(mpi_rank_, k) * s_; + for (int t = 0; t < s_; ++t) { + const int g = base + t; + const int cnt = my_send_counts[g]; + if (cnt != 0) { + std::memcpy(stage_send_.data() + off[g] * elem, + src + static_cast(my_send_displs[g]) * elem, + static_cast(cnt) * elem); + } } } } @@ -629,6 +760,8 @@ class HybridComm { double red_f64_ = 0.0; uint64_t red_u64_ = 0; std::vector red_vec_; + // Point-to-point request scratch for the sparse paths; grown on demand, partition 0 only. + std::vector reqs_; PartitionBarrier barrier_; }; diff --git a/cpp/monoprop/detail/mpi/MPICompat.cpp b/cpp/monoprop/detail/mpi/MPICompat.cpp index 625b1725..dabb2e7c 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.cpp +++ b/cpp/monoprop/detail/mpi/MPICompat.cpp @@ -14,6 +14,7 @@ #include "monoprop/detail/mpi/Exchange.h" +#include #include #include #include @@ -114,19 +115,43 @@ auto allreduce_sum_inplace(VecD &values, Comm comm) -> void { #endif } -auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm) -> void { +auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, PeerPlan plan) -> void { if (comm.kind == Comm::Kind::Shm) { comm.shm->alltoall_counts(comm.shm_rank, send_counts, recv_counts); return; } #ifdef monoprop_ENABLE_MPI if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoall_counts(comm.shm_rank, send_counts, recv_counts); + comm.hyb->alltoall_counts(comm.shm_rank, send_counts, recv_counts, plan); + return; + } + if (!plan.dense()) { + // S == 1 world: exchange one int with each reachable peer; the rest of the row is zero by + // definition, so it must be cleared rather than left from a previous round. + int me = 0; + MPI_Comm_rank(comm.mpi, &me); + std::fill(recv_counts, recv_counts + n, 0); + const int f = plan.count(n); + std::vector reqs; + reqs.reserve(static_cast(2 * f)); + for (int k = 0; k < f; ++k) { + const int b = plan.peer(me, k); + if (b == me) { + recv_counts[b] = send_counts[b]; + continue; + } + reqs.emplace_back(); + MPI_Irecv(&recv_counts[b], 1, MPI_INT, b, 0x6D73, comm.mpi, &reqs.back()); + reqs.emplace_back(); + MPI_Isend(&send_counts[b], 1, MPI_INT, b, 0x6D73, comm.mpi, &reqs.back()); + } + MPI_Waitall(static_cast(reqs.size()), reqs.data(), MPI_STATUSES_IGNORE); return; } (void)n; MPI_Alltoall(send_counts, 1, MPI_INT, recv_counts, 1, MPI_INT, comm.mpi); #else + (void)plan; // single participant: nothing to narrow for (int i = 0; i < n; ++i) { recv_counts[i] = send_counts[i]; } diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index c3e74db9..0970f2c5 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -118,8 +118,9 @@ inline auto allreduce_sum(T local_val, Comm comm) -> T { auto allreduce_sum_inplace(VecD &values, Comm comm) -> void; -// `n` is the comm size. -auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm) -> void; +// `n` is the comm size. `plan` narrows the exchange to the destination ranks it can reach (see PeerPlan); +// the default is dense, i.e. today's collective. +auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, PeerPlan plan = {}) -> void; // In-flight variable-size all-to-all owning its buffers + layout, so several can be in flight. // recv_counts is valid on return from begin_alltoallv; wait_into completes the payload transfer (a @@ -161,7 +162,8 @@ template inline auto begin_alltoallv(const std::vector> &send_data, Comm comm, bool skip_self = false, - const std::vector *known_recv_counts = nullptr) -> PendingAlltoallv { + const std::vector *known_recv_counts = nullptr, + PeerPlan plan = {}) -> PendingAlltoallv { const int num_ranks = size(comm); if (static_cast(send_data.size()) != num_ranks) { throw CollectiveArgumentError( @@ -218,7 +220,7 @@ inline auto begin_alltoallv(const std::vector> &send_data, } #ifdef monoprop_ENABLE_MPI if (known_recv_counts == nullptr && comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoallv_resolve(comm.shm_rank, resolve_args, datatype::get()); + comm.hyb->alltoallv_resolve(comm.shm_rank, resolve_args, datatype::get(), plan); return h; } #endif @@ -233,7 +235,7 @@ inline auto begin_alltoallv(const std::vector> &send_data, } } else { - alltoall_counts(h.send_counts.data(), h.recv_counts.data(), num_ranks, comm); + alltoall_counts(h.send_counts.data(), h.recv_counts.data(), num_ranks, comm, plan); } // Wide accumulator + checked narrowing: see checked_mpi_count. @@ -264,21 +266,50 @@ inline auto begin_alltoallv(const std::vector> &send_data, } #ifdef monoprop_ENABLE_MPI else if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoallv(comm.shm_rank, flat, datatype::get()); + comm.hyb->alltoallv(comm.shm_rank, flat, datatype::get(), plan); } #endif else { #ifdef monoprop_ENABLE_MPI - MPI_Ialltoallv(h.send_buffer.data(), - h.send_counts.data(), - h.send_displs.data(), - datatype::get(), - h.recv_buffer.data(), - h.recv_counts.data(), - h.recv_displs.data(), - datatype::get(), - comm.mpi, - &h.request); + if (plan.dense()) { + MPI_Ialltoallv(h.send_buffer.data(), + h.send_counts.data(), + h.send_displs.data(), + datatype::get(), + h.recv_buffer.data(), + h.recv_counts.data(), + h.recv_displs.data(), + datatype::get(), + comm.mpi, + &h.request); + } + else { + // S == 1 world: the same pairing as the Hybrid path, one message per reachable peer. Blocking + // here rather than through the Ticket, because the request set is per-peer, not one handle. + const int me = rank(comm); + const int f = plan.count(num_ranks); + std::vector reqs; + reqs.reserve(static_cast(2 * f)); + for (int k = 0; k < f; ++k) { + const int b = plan.peer(me, k); + const auto ub = static_cast(b); + T *rbuf = h.recv_buffer.data() + h.recv_displs[ub]; + const T *sbuf = h.send_buffer.data() + h.send_displs[ub]; + if (b == me) { + std::copy(sbuf, sbuf + h.recv_counts[ub], rbuf); + continue; + } + if (h.recv_counts[ub] != 0) { + reqs.emplace_back(); + MPI_Irecv(rbuf, h.recv_counts[ub], datatype::get(), b, 0x6D72, comm.mpi, &reqs.back()); + } + if (h.send_counts[ub] != 0) { + reqs.emplace_back(); + MPI_Isend(sbuf, h.send_counts[ub], datatype::get(), b, 0x6D72, comm.mpi, &reqs.back()); + } + } + MPI_Waitall(static_cast(reqs.size()), reqs.data(), MPI_STATUSES_IGNORE); + } #else h.recv_buffer = h.send_buffer; // single participant: self round-trip (layouts identical) #endif diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index 0ae8ca80..f6c69439 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include "monoprop/MPGraph.h" @@ -72,4 +74,34 @@ inline auto router_for(const mpi::Comm &comm) -> routing::Router { return routing::make_router(static_cast(geom.ranks), static_cast(geom.partitions)); } +class RoutingDisagreement : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// Every participant must resolve the SAME router, and the failure mode if they do not is a hang, not a +// wrong answer: linear routing makes each rank post receives from the peers its own bits imply, so a +// rank whose monoprop_ROUTING or _ROUTE_SEED did not reach it waits forever on a peer that never sends. +// One allreduce at construction turns that into an exception. Called once, never per gate. +inline auto check_routing_agreement(const mpi::Comm &comm) -> void { + const auto router = router_for(comm); + const size_t world = static_cast(mpi::size(comm)); + if (world <= 1) { + return; + } + const uint64_t mine = + routing::mix64((static_cast(router.linear_bits()) << 40) ^ routing::seed_from_env()); + const uint64_t total = mpi::allreduce_sum(mine, comm); + if (total != mine * static_cast(world)) { + throw RoutingDisagreement( + std::format("routing configuration differs across the {} participants (this one: linear_bits={}, " + "seed={}). monoprop_ROUTING / monoprop_ROUTE_LINEAR_BITS / monoprop_ROUTE_SEED must " + "reach every rank identically -- under linear routing a disagreement deadlocks the " + "exchange rather than corrupting it.", + world, + router.linear_bits(), + routing::seed_from_env())); + } +} + } // namespace monoprop diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index bb6ca426..8430881f 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -23,6 +23,7 @@ #ifdef monoprop_ENABLE_MPI #include +#include #include #include #include @@ -484,4 +485,104 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_poison_releases_waiters) { } } +// A sparse PeerPlan replaces the collectives with point-to-point over the peers the plan names, so the +// two failure modes it can have are DROPPED data and a HANG -- neither of which a dense-path test can +// see. Every rank derives the same pairing from the same (bits, shift), and a block whose destination is +// not a peer must be empty: send only to the plan's peer and check the delivery is exactly that. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_delivers_only_to_its_peers) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; // the XOR pairing needs a power-of-two rank count + } + const int bits = std::countr_zero(static_cast(R)); + for (const int S : {1, 2, 3}) { + const int P = R * S; + for (int shift = 0; shift < R; ++shift) { + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const int peer = plan.peer(world_rank(), 0); + BOOST_REQUIRE_EQUAL(plan.count(R), 1); // full bits => pairwise + std::vector>> recv(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + Comm c = Comm::make_hybrid(&hyb, u); + const int g = monoprop::mpi::rank(c); + std::vector> send(static_cast(P)); + for (int t = 0; t < S; ++t) { + auto &blk = send[static_cast(peer * S + t)]; + for (int j = 0; j <= t; ++j) { + blk.push_back(g * 1000 + t * 10 + j); + } + } + auto h = monoprop::mpi::begin_alltoallv(send, + c, + /*skip_self=*/false, + /*known_recv_counts=*/nullptr, + plan); + std::vector> out; + h.wait_into(out); + recv[static_cast(u)] = out; + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + // XOR is an involution, so whoever I send to sends to me: my sources are exactly `peer`'s + // partitions, and the block from (peer, su) to my partition t has t+1 entries. + for (int t = 0; t < S; ++t) { + const auto &out = recv[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int src = 0; src < P; ++src) { + const auto &blk = out[static_cast(src)]; + if (src / S != peer) { + BOOST_CHECK(blk.empty()); // a non-peer must not appear at all + continue; + } + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), t + 1); + for (int j = 0; j <= t; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], src * 1000 + t * 10 + j); + } + } + } + } + } +} + +// Same contract on the S == 1 world, which takes the pure-MPI branch (MPI_Ialltoallv vs Isend/Irecv) +// rather than HybridComm's staged one, and on the KNOWN-recv-counts path the response round uses. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_on_the_plain_mpi_path) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + Comm c{MPI_COMM_WORLD}; + for (int shift = 0; shift < R; ++shift) { + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const int peer = plan.peer(world_rank(), 0); + std::vector> send(static_cast(R)); + for (int j = 0; j < 4; ++j) { + send[static_cast(peer)].push_back(world_rank() * 1000 + j); + } + // Unknown recv layout: the counts round is point-to-point too. + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); + for (int src = 0; src < R; ++src) { + if (src != peer) { + BOOST_CHECK(out[static_cast(src)].empty()); + continue; + } + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), 4); + for (int j = 0; j < 4; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], src * 1000 + j); + } + } + // Known recv layout (the response round): counts are the transpose, so peer-only again. + std::vector known(static_cast(R), 0); + known[static_cast(peer)] = 4; + std::vector> out2; + monoprop::mpi::begin_alltoallv(send, c, false, &known, plan).wait_into(out2); + BOOST_REQUIRE_EQUAL(static_cast(out2.size()), R); + BOOST_CHECK(out2[static_cast(peer)] == out[static_cast(peer)]); + } +} + #endif // monoprop_ENABLE_MPI From 40041c1027ce202bb1a3538847970dc4918464f7 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Wed, 26 Aug 2026 23:19:37 +0200 Subject: [PATCH 04/24] =?UTF-8?q?perf(mpi):=20=F0=9F=A7=AD=20route=20GF(2)?= =?UTF-8?q?-linear=20by=20default=20where=20the=20geometry=20allows=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `monoprop_ROUTING` now defaults to linear; `splitmix` is the explicit opt-out, and an unrecognised value falls back to the default, matching EnvConfig.h's parse_positive_int convention. The Router still clamps to log2(R) and to d=0 when R is not a power of two, so a geometry without XOR structure keeps the dense path untouched. Measured at the production point (60 sites, cutoff 10, atol 2.6e-06, 1,569,152,761 terms): fanout 1 costs nothing on balance -- rank occupancy max/mean 1.001 at R=128 with all 128 ranks used, and the flat (memory) imbalance is slightly better than splitmix's -- while messages per rank per layer fall from 362,712 to 1,397, i.e. from proportional-to-R to flat. Co-Authored-By: Claude Opus 5 (1M context) --- cpp/monoprop/detail/mpi/Routing.h | 11 ++++++++--- cpp/tests/routing_tests.cpp | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index 4048cebd..f01e4a06 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -207,10 +207,15 @@ inline auto requested_linear_bits() -> size_t { return value > 0 ? static_cast(value) : size_t{0}; } const char *mode = std::getenv("monoprop_ROUTING"); - if (mode != nullptr && std::string_view{mode} == "linear") { - return ~size_t{0}; // "as many as this geometry allows" -- Router clamps to log2(R) + if (mode != nullptr && std::string_view{mode} == "splitmix") { + return size_t{0}; // full avalanche across the flat world: every rank talks to every rank } - return size_t{0}; + // Default. "As many bits as this geometry allows" -- Router clamps to log2(R), and to 0 + // when R is not a power of two, so a geometry without XOR structure keeps the dense path. + // Measured at the production point: fanout 1 costs nothing on balance (rank occupancy + // max/mean 1.001 at R=128, all ranks used) and takes messages per rank per layer from + // 362,712 to 1,397, i.e. from proportional-to-R to flat. + return ~size_t{0}; }(); return bits; } diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp index 01fc7701..2444a349 100644 --- a/cpp/tests/routing_tests.cpp +++ b/cpp/tests/routing_tests.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -209,3 +210,25 @@ BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { } BOOST_TEST(routing::gf2_rank(shifts) == 7U); // == log2(128): every rank is reachable } + +// The SHIPPED default. Flipping this is the whole point of the change, so it is pinned by a test +// rather than left to a comment: with no environment override, a power-of-two rank count routes at +// fanout 1, and a geometry with no XOR structure keeps the dense path instead of silently losing +// ranks. Skipped when the environment does override it, because then the default is not what is +// under test. +BOOST_AUTO_TEST_CASE(routing_default_is_linear_where_the_geometry_allows_it) { + const char *mode = std::getenv("monoprop_ROUTING"); + const char *bits = std::getenv("monoprop_ROUTE_LINEAR_BITS"); + if ((mode != nullptr && *mode != '\0') || (bits != nullptr && *bits != '\0')) { + BOOST_TEST_MESSAGE("routing overridden in the environment; default not under test"); + return; + } + BOOST_TEST(routing::make_router(8, 14).fanout() == 1U); + BOOST_TEST(routing::make_router(128, 14).fanout() == 1U); + BOOST_TEST(routing::make_router(1, 112).fanout() == 1U); // single rank: nothing to route between + + // 6 and 12 are not powers of two: no XOR structure, so Router clamps to d = 0 and every rank + // stays reachable through splitmix rather than a subspace of them. + BOOST_TEST(routing::make_router(6, 14).fanout() == 6U); + BOOST_TEST(routing::make_router(12, 28).fanout() == 12U); +} From 0204d9c490379056e1bd7c6560517f14b777cfbf Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Wed, 26 Aug 2026 23:22:20 +0200 Subject: [PATCH 05/24] =?UTF-8?q?test(mpi):=20=F0=9F=A7=AD=20check=20scan/?= =?UTF-8?q?find=5Frank=20agreement=20under=20both=20routers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ownership oracle in mpi_utils_scan_routing_agrees_with_find_rank hashed with the flat-world splitmix overload of find_rank, which is only the owner function when the router resolves to d=0. The scan calls Router::dest, so on a linear geometry the case asserted an invariant the code no longer has. Thread the Router through both helpers and run the case twice per rank count -- splitmix and fanout-1 linear -- so the agreement is checked as a property of the pair rather than of either hash. The floors are untouched; they were never tight. Co-Authored-By: Claude Opus 5 (1M context) --- cpp/tests/mpi_utils_tests.cpp | 63 +++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 45fdaf9e..08fe37b7 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -28,6 +28,7 @@ #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Scan.h" #include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/mpi/Routing.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/OperatorIndex.h" @@ -111,7 +112,8 @@ auto build_op(const std::vector> &terms) -> detail::MPOperator<32> return op; } -auto check_bucket_ownership(const std::vector &buckets, size_t ranks, size_t &checked) -> void { +auto check_bucket_ownership(const std::vector &buckets, const routing::Router &router, size_t &checked) + -> void { // Every offset comes from the codec's walk: the record is VARIABLE WIDTH, so a hardcoded stride // would compare a monomial decoded at the wrong offset against the wrong rank. using QC = detail::QueryCodec<32>; @@ -122,7 +124,7 @@ auto check_bucket_ownership(const std::vector &buckets, size_t ranks, size Monomial<32> mono; int phase = 0; QC::read_mono(buckets[r], off, mono, phase); - BOOST_REQUIRE_EQUAL(find_rank<32>(mono, ranks), r); + BOOST_REQUIRE_EQUAL(find_rank<32>(mono, router), r); off = QC::next_off(buckets[r], layout, off); ++checked; } @@ -132,14 +134,16 @@ auto check_bucket_ownership(const std::vector &buckets, size_t ranks, size // The self-owned bucket is staged as positions, not encoded, so it is invisible to the walk above -- // without this the r == my_rank arm of the routing decision goes unchecked. -auto check_self_ownership(const detail::SelfQueryStage<32> &stage, size_t ranks, size_t my_rank, size_t &checked) - -> void { +auto check_self_ownership(const detail::SelfQueryStage<32> &stage, + const routing::Router &router, + size_t my_rank, + size_t &checked) -> void { for (size_t q = 0; q < stage.size(); ++q) { Monomial<32> mono; for (size_t j = 0; j < stage.k_of[q]; ++j) { mono.set(static_cast(stage.pos_flat[stage.pos_off[q] + j])); } - BOOST_REQUIRE_EQUAL(find_rank<32>(mono, ranks), my_rank); + BOOST_REQUIRE_EQUAL(find_rank<32>(mono, router), my_rank); ++checked; } } @@ -171,31 +175,40 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { size_t checked = 0; size_t self_checked = 0; for (const size_t ranks : {2U, 4U, 8U}) { - const auto res = detail::fused_find_and_collect>(op, - gen, - eval, - cut, - coeffs, - std::nullopt, - ranks, - 0, - false, - nullptr, - 1.0); - BOOST_REQUIRE_EQUAL(res.leader_queries.size(), ranks); - // The scan routes a self-owned partner to the stage, so bucket 0 must be empty here. - BOOST_REQUIRE(res.leader_queries[0].empty()); - BOOST_REQUIRE(res.follower_queries[0].empty()); - check_bucket_ownership(res.leader_queries, ranks, checked); - check_bucket_ownership(res.follower_queries, ranks, checked); - check_self_ownership(res.leader_self, ranks, /*my_rank=*/0, self_checked); - check_self_ownership(res.follower_self, ranks, /*my_rank=*/0, self_checked); + // BOTH routers, because the agreement is a property of the pair and not of either hash: the + // scan calls Router::dest and find_rank calls the same Router, so a divergence introduced by + // one of them shows up here whichever routing the geometry resolves to. bits=~0 asks for as + // many linear bits as log2(ranks) allows, i.e. fanout 1. + for (const size_t bits : {size_t{0}, ~size_t{0}}) { + const routing::Router router{ranks, /*partitions=*/1, bits}; + const auto res = detail::fused_find_and_collect>(op, + gen, + eval, + cut, + coeffs, + std::nullopt, + ranks, + 0, + router, + false, + nullptr, + 1.0); + BOOST_REQUIRE_EQUAL(res.leader_queries.size(), ranks); + // The scan routes a self-owned partner to the stage, so bucket 0 must be empty here. + BOOST_REQUIRE(res.leader_queries[0].empty()); + BOOST_REQUIRE(res.follower_queries[0].empty()); + check_bucket_ownership(res.leader_queries, router, checked); + check_bucket_ownership(res.follower_queries, router, checked); + check_self_ownership(res.leader_self, router, /*my_rank=*/0, self_checked); + check_self_ownership(res.follower_self, router, /*my_rank=*/0, self_checked); + } } // Without this the loop above passes trivially if the scan emitted nothing. The floor is on the SUM // because that is what is invariant across the split: the encoded counter alone fell to 797 of 1161 // when the self-owned partners moved into the stage, with nothing going unchecked. Each arm still // carries its own floor -- a routing bug sending everything one way leaves the sum intact -- and the - // message prints the measured 797/364 so those can be re-grounded rather than guessed. + // message prints the measured counts so those can be re-grounded rather than guessed. The floors + // are unchanged although the loop now runs twice (one router each): they were never tight. BOOST_TEST_MESSAGE("encoded=" << checked << " staged=" << self_checked); BOOST_TEST(checked + self_checked > 1000U); BOOST_TEST(checked > 500U); From 27da5fadc341c1bc11fbc8b45de69f1c03ab1385 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 09:47:50 +0100 Subject: [PATCH 06/24] =?UTF-8?q?style(tests):=20=F0=9F=8E=A8=20clang-form?= =?UTF-8?q?at=20mpi=5Futils=5Ftests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 --- cpp/tests/mpi_utils_tests.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 08fe37b7..6d97043e 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -112,8 +112,7 @@ auto build_op(const std::vector> &terms) -> detail::MPOperator<32> return op; } -auto check_bucket_ownership(const std::vector &buckets, const routing::Router &router, size_t &checked) - -> void { +auto check_bucket_ownership(const std::vector &buckets, const routing::Router &router, size_t &checked) -> void { // Every offset comes from the codec's walk: the record is VARIABLE WIDTH, so a hardcoded stride // would compare a monomial decoded at the wrong offset against the wrong rank. using QC = detail::QueryCodec<32>; From ac5dcdb0f7887773ba89f0389a1d155163e5c035 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 10:29:44 +0100 Subject: [PATCH 07/24] =?UTF-8?q?docs(parallelism):=20=F0=9F=93=9D=20docum?= =?UTF-8?q?ent=20GF(2)-linear=20rank=20routing=20and=20its=20knobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The routing change flips a shipped default and adds three environment variables, none of which appeared anywhere in docs/. The rank hash is a homomorphism of the group the gate acts by, so state the identity, what follows from it (fanout, the involution, cosets of the kernel), and the one condition it rests on -- that the generator shifts span the rank space, which is a load-balance property and not a correctness one. Distinguishes the scheme from the block-sum-mod-N map of [@Broers2025-or]: that sum is additive modulo the rank count while the gate acts by XOR, so carries leave the destination dependent on the operand's bits and only bound the fanout. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/tests/README.md | 6 ++- docs/bibliography.bib | 13 +++++ docs/content/docs/features/parallelism.mdx | 57 ++++++++++++++++++++++ docs/content/docs/references.mdx | 7 +++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/cpp/tests/README.md b/cpp/tests/README.md index 6cd35461..dc4c1e2b 100644 --- a/cpp/tests/README.md +++ b/cpp/tests/README.md @@ -87,8 +87,10 @@ name and cannot address suite-nested cases, tests use flat `pauli_algebra_tests.cpp`, `majorana_cutoff_tests.cpp` (length/support cutoff, CutoffEvaluator, interleave phase, coeff encode/decode), `validation_tests.cpp` (parameter validators), `mpi_utils_tests.cpp` (find_rank, word serialization, - scan routing agreement), `evolution_detail_tests.cpp` (MatchedEpochSet + - CutoffContext), + scan routing agreement), `routing_tests.cpp` (Router term -> flat-slot map: + the `d = 0` splitmix equivalence, the linear shift identity, the gf2_rank + coverage diagnostic, and the shipped default), + `evolution_detail_tests.cpp` (MatchedEpochSet + CutoffContext), `row_accessor_tests.cpp` (dense vs OperatorIndex row accessors), `sparse_monomial_tests.cpp` (the `(k, d)` cutoff predicates vs their bitset forms). diff --git a/docs/bibliography.bib b/docs/bibliography.bib index 290f83b3..79964d64 100644 --- a/docs/bibliography.bib +++ b/docs/bibliography.bib @@ -89,3 +89,16 @@ @ARTICLE{chakraborty2026scalablequantumcircuitgeneration primaryClass={quant-ph}, url={https://arxiv.org/abs/2603.23444}, } + +@ARTICLE{Broers2025-or, + title = "{Scalable Simulation of Quantum Many-Body Dynamics with Or-Represented Quantum Algebra}", + author = "Broers, Lukas and Sun, Rong-Yang and Yunoki, Seiji", + journal = "Phys. Rev. Applied", + volume = 26, + pages = "024046", + year = 2026, + url = "https://arxiv.org/abs/2506.13241", + archivePrefix = "arXiv", + primaryClass = "quant-ph", + eprint = "2506.13241" +} diff --git a/docs/content/docs/features/parallelism.mdx b/docs/content/docs/features/parallelism.mdx index e9e56ed5..ac55871f 100644 --- a/docs/content/docs/features/parallelism.mdx +++ b/docs/content/docs/features/parallelism.mdx @@ -30,6 +30,9 @@ whose partner lives in another partition are resolved through a per-gate exchang | `monoprop_NUM_THREADS` | one partition per physical core | Caps the number of partitions. Set it to run fewer partitions than cores. | | `monoprop_PARTITIONS` | `auto` | `auto` = one partition per core (capped by `monoprop_NUM_THREADS`); an integer `N` = exactly `N` partitions; `off` = one partition holding the whole operator. | +Three further variables control which MPI rank owns a term; they belong with the +distribution axis and are documented under [Rank routing](#rank-routing) below. + ```bash # Run 8 partitions instead of one-per-core: export monoprop_NUM_THREADS=8 @@ -61,6 +64,60 @@ partitioning into one flat world of `R × S` partitions (`R` ranks, `S` partitio MPI communication is serialised through each rank's first partition, bracketed by the intra-rank barriers. +### Rank routing + +For $N$ modes, monomials under a gate form the group $(\mathbb{F}_2^{2N}, \oplus)$: a generator $G$ sends +$M \mapsto M \oplus G$. The rank index is chosen to be a homomorphism of that group — +$h(M) = \bigoplus_{i \in \mathrm{supp}(M)} v_i$, over one fixed vector $v_i$ per Majorana +slot, with $h_d$ its low $d$ bits — so that + +$$ +h(M \oplus G) = h(M) \oplus h(G). +$$ + +Three consequences. A rank owning the fibre $h_d^{-1}(r)$ sends every query for $G$ to +$r \oplus h_d(G)$: one peer, independent of $R$ and of $|\mathrm{supp}\,G|$, where a +full-avalanche hash sprays the same queries across all $R$ ranks. XOR is an involution, so +the peer relation is symmetric and both sides derive the pairing without communicating. +And the fibres are cosets of $\ker h_d$, all of size $2^{2N-d}$, so a uniformly drawn +monomial is balanced by construction; imbalance can only come from the operator's support +being non-uniform, which is why balance is measured rather than proved — rank occupancy +max/mean 1.001 at $R = 128$, with every rank used. + +The one condition is that the per-generator shifts $\{h_d(G)\}$ span $\mathbb{F}_2^{d}$. +If they span only $\rho < d$ dimensions, the ranks a query can reach form a coset of a +$\rho$-dimensional subspace: only $2^{\rho}$ of the $2^{d}$ are ever used and the rest stay +empty. That is a load-balance failure and not a wrong answer, so it is left as a diagnostic +rather than enforced at runtime — measured $\rho = 32$ over the 60-site Hubbard's 416 +distinct shifts, against the $d = 7$ that $R = 128$ needs. + +Within a rank, partitions keep the full-avalanche `splitmix` hash: fanout across shared +memory is free, so only balance matters there. Routing is therefore two-level. Writing $q$ +for the `splitmix` hash of $M$, the partition is $q \bmod S$, the low $d$ bits of the rank +are $h_d(M)$, and the remaining $\log_2 R - d$ rank bits come from $q$: + +$$ +\mathrm{flat}(M) = \bigl[\,h_d(M) + 2^{d}\bigl(\lfloor q/S \rfloor \bmod (R/2^{d})\bigr)\,\bigr]\,S + (q \bmod S). +$$ + +$d$ is a dial and not a switch: the fanout is $R/2^{d}$, so $d = 0$ reproduces the dense +`hash % (R × S)` bit for bit, and $d = \log_2 R$ — the default — gives fanout 1 and +$\mathrm{flat}(M) = h_d(M)\,S + (q \bmod S)$. A rank count that is not a power of two has +no XOR structure and falls back to $d = 0$. + +A related distributed scheme maps an index by summing its $k$-bit blocks modulo the rank +count [@Broers2025-or]. That sum is additive modulo that count while the gate acts by XOR, +so the carries make $f(I \oplus J)$ differ from $f(I)$ by $\pm 2^{J_j \bmod k}$ terms whose +signs depend on the bits of $I$, bounding the destinations at $2^{2|J|+1}$ rather than +collapsing them to one. Being linear over the same group the gate acts by is what turns +that bound into an identity. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `monoprop_ROUTING` | `linear` | `splitmix` (the literal string) forces $d = 0$, the dense all-to-all. Any other value, including unset, selects linear routing with $d$ as large as the geometry allows. | +| `monoprop_ROUTE_LINEAR_BITS` | unset | An explicit $d$, clamped to $[0, \log_2 R]$. Takes precedence over `monoprop_ROUTING`; a value that is zero, negative or unparseable means $d = 0$. | +| `monoprop_ROUTE_SEED` | `6768574230969066775` | Decimal `uint64` from which every rank derives the same basis $\{v_i\}$ with no communication. The same value must reach every rank: a mismatch in any of these three variables is caught by one allreduce at propagator construction and raised, because under linear routing it deadlocks the exchange instead of corrupting it. | + ### Single-node (`MPI.COMM_SELF`) ```python notest diff --git a/docs/content/docs/references.mdx b/docs/content/docs/references.mdx index 887489ee..410f3596 100644 --- a/docs/content/docs/references.mdx +++ b/docs/content/docs/references.mdx @@ -50,4 +50,11 @@ described in [@Miller2023-bonsai]. Other representative large-scale applications arXiv [quant-ph], 2026. arXiv:2605.04025 +
  • + L. Broers, R.-Y. Sun, and S. Yunoki, + "Scalable simulation of quantum many-body dynamics with or-represented + quantum algebra," + Phys. Rev. Applied, vol. 26, p. 024046, 2026. + arXiv:2506.13241 +
    • From c2554554f0b3e25acaa72b4663bc7ef1199d2882 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 11:55:53 +0100 Subject: [PATCH 08/24] =?UTF-8?q?fix(mpi):=20=F0=9F=90=9B=20mask=20known?= =?UTF-8?q?=20recv=20counts=20through=20the=20peer=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under a sparse plan begin_alltoallv copied the caller's known_recv_counts verbatim, zeroing only the self slot, while alltoall_counts already masked the counts it exchanged. A non-zero count for a rank outside the peer set therefore sized recv_buffer for bytes no Irecv ever writes, and wait_into handed the caller uninitialised memory with no error. PeerPlan::contains answers membership as a low-bits equality, so the mask costs no allocation. publish_recv_rows_ takes the plan for the same reason: it summed over every rank while its one reader masked to peers. The self slot is a copy rather than a message and its two counts are each other's transpose, so both point-to-point loops now assert that instead of reading the send buffer through the recv count -- reachable whenever a generator's shift is zero, which happens in essentially every layer. Ranks that all agree on a WRONG shift stay symmetric and never hang; they drop the blocks outside the peer set silently. pack_count_matrix_ asserts the non-peer remainder is empty, and Comm.h no longer claims a deadlock is the only failure mode. Routing knobs move into EnvConfig.h, which already owned this job: raw strtol left monoprop_ROUTE_LINEAR_BITS=abc parsing as 0, turning linear routing off with no diagnostic. Unparseable and out-of-range values now throw. monoprop_ROUTING's documented default was also backwards -- linear has been the default since 40041c10. check_routing_agreement reduces two independent digests rather than one: allreduce_sum is the only collective here and a sum is not an equality test. Partitions join the digest, since S enters Router::dest and two ranks differing only in S agreed before and still routed apart. Drops the flat-world find_rank overload. It had no production caller left and answered splitmix during a linear run, which is precisely the silent ownership split its own comment warned about. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/EnvConfig.h | 77 ++++++++++++++++++++++ cpp/monoprop/detail/mpi/Comm.h | 34 ++++++++-- cpp/monoprop/detail/mpi/HybridComm.h | 34 ++++++++-- cpp/monoprop/detail/mpi/MPICompat.h | 15 +++++ cpp/monoprop/detail/mpi/MPIUtils.h | 53 ++++++++------- cpp/monoprop/detail/mpi/Routing.h | 46 +++++-------- cpp/tests/env_config_tests.cpp | 42 ++++++++++++ cpp/tests/hybrid_comm_tests.cpp | 76 +++++++++++++++++++++ cpp/tests/mpi_utils_tests.cpp | 15 +++-- cpp/tests/routing_tests.cpp | 3 +- docs/content/docs/features/parallelism.mdx | 6 +- 11 files changed, 329 insertions(+), 72 deletions(-) diff --git a/cpp/monoprop/detail/EnvConfig.h b/cpp/monoprop/detail/EnvConfig.h index 30120417..6c4e755d 100644 --- a/cpp/monoprop/detail/EnvConfig.h +++ b/cpp/monoprop/detail/EnvConfig.h @@ -14,18 +14,36 @@ #pragma once +#include #include +#include #include #include +#include +#include +#include // Single home for runtime environment configuration. Kept dependency-free by design, because it is // pulled into hot-path headers. // // monoprop_NUM_THREADS positive int (1..1e6), else ignored → num_threads // monoprop_PARTITIONS int N | "auto" | "off"; parsed where it is used (resolve_partition_count_) +// monoprop_ROUTING "splitmix" | "linear" → routing_mode +// monoprop_ROUTE_LINEAR_BITS int in [0, 64], 0 meaning dense → route_linear_bits +// monoprop_ROUTE_SEED decimal uint64 basis seed → route_seed +// +// The three routing knobs THROW on a malformed value instead of falling back: each silently changes the +// transport, so a typo that defaulted would stay invisible until a performance postmortem. namespace monoprop::config { +class EnvConfigError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +enum class RoutingMode : std::uint8_t { Splitmix, Linear }; + namespace detail { inline auto parse_positive_int(const char *text) -> std::optional { @@ -43,10 +61,65 @@ inline auto parse_positive_int(const char *text) -> std::optional { return static_cast(value); } +[[noreturn]] inline auto reject_env(std::string_view name, const char *text, std::string_view expected) -> void { + throw EnvConfigError(std::string{name} + "=\"" + text + "\" is not " + std::string{expected} + + "; correct it or leave the variable unset."); +} + +// Unset and empty are both nullopt; a value that is present but unparseable throws. +inline auto parse_uint64(std::string_view name, const char *text) -> std::optional { + if (text == nullptr || *text == '\0') { + return std::nullopt; + } + // strtoull WRAPS a negative literal to a huge unsigned rather than failing, so '-' is rejected here. + if (std::string_view{text}.find('-') != std::string_view::npos) { + reject_env(name, text, "a decimal uint64"); + } + errno = 0; + char *end = nullptr; + const unsigned long long value = std::strtoull(text, &end, 10); + if (end == text || *end != '\0' || errno == ERANGE) { + reject_env(name, text, "a decimal uint64"); + } + return static_cast(value); +} + +// 0 is legal here (it means dense routing), so "unset" must stay distinguishable from "0" -- hence the +// optional rather than a sentinel. 64 is the width of the linear hash: no further bits exist to ask for. +inline auto parse_bit_count(std::string_view name, const char *text) -> std::optional { + if (text == nullptr || *text == '\0') { + return std::nullopt; + } + errno = 0; + char *end = nullptr; + const long value = std::strtol(text, &end, 10); + if (end == text || *end != '\0' || errno == ERANGE || value < 0 || value > 64) { + reject_env(name, text, "an integer in [0, 64]"); + } + return static_cast(value); +} + +inline auto parse_routing_mode(std::string_view name, const char *text) -> std::optional { + if (text == nullptr || *text == '\0') { + return std::nullopt; + } + const std::string_view value{text}; + if (value == "splitmix") { + return RoutingMode::Splitmix; + } + if (value == "linear") { + return RoutingMode::Linear; + } + reject_env(name, text, "one of \"splitmix\" or \"linear\""); +} + } // namespace detail struct Settings { std::optional num_threads; + std::optional routing_mode; + std::optional route_linear_bits; // takes precedence over routing_mode when both are set + std::optional route_seed; }; // Parse the environment once; the Settings are cached and shared across TUs. @@ -54,6 +127,10 @@ inline auto get() -> const Settings & { static const Settings settings = [] { Settings s; s.num_threads = detail::parse_positive_int(std::getenv("monoprop_NUM_THREADS")); + s.routing_mode = detail::parse_routing_mode("monoprop_ROUTING", std::getenv("monoprop_ROUTING")); + s.route_linear_bits = + detail::parse_bit_count("monoprop_ROUTE_LINEAR_BITS", std::getenv("monoprop_ROUTE_LINEAR_BITS")); + s.route_seed = detail::parse_uint64("monoprop_ROUTE_SEED", std::getenv("monoprop_ROUTE_SEED")); return s; }(); return settings; diff --git a/cpp/monoprop/detail/mpi/Comm.h b/cpp/monoprop/detail/mpi/Comm.h index 2354bf0c..582aceb7 100644 --- a/cpp/monoprop/detail/mpi/Comm.h +++ b/cpp/monoprop/detail/mpi/Comm.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -76,20 +77,43 @@ struct Comm { // replace a dense collective with point-to-point over the peers it can actually reach. // // bits == 0 is the dense default: peer(k) == k and count == ranks, so the same loops walk every rank -// and the verbs take their collective path. A caller that gets `shift` wrong does not corrupt data -- -// the counts for a non-peer are zero -- it deadlocks, which is why the plan is derived in one place. +// and the verbs take their collective path. +// +// Two distinct failure modes if `shift` is wrong, which is why the plan is derived in one place. Ranks +// that DISAGREE deadlock: the pairing stops being symmetric and someone waits on a send never posted. +// Ranks that all agree on the same wrong shift stay symmetric and never hang -- they silently DROP the +// blocks outside the peer set, because pack_count_matrix_ / size_staging_send_ / pack_send_ only ever +// touch peers. pack_count_matrix_ asserts the non-peer remainder is empty to catch that one. struct PeerPlan { int bits = 0; int shift = 0; [[nodiscard]] constexpr auto dense() const -> bool { return bits == 0; } - [[nodiscard]] constexpr auto count(int ranks) const -> int { return bits == 0 ? ranks : (ranks >> bits); } + // A plan too narrow for the world would yield 0 peers and turn the exchange into a silent no-op. + [[nodiscard]] constexpr auto count(int ranks) const -> int { + if (bits == 0) { + return ranks; + } + assert(bits > 0 && bits < 31 && (ranks >> static_cast(bits)) > 0); + return ranks >> static_cast(bits); + } + // Unsigned shifts: `bits` is a public field, and 1 << 31 on a signed int is UB. [[nodiscard]] constexpr auto peer(int me, int k) const -> int { if (bits == 0) { return k; } - const int mask = (1 << bits) - 1; - return ((me & mask) ^ shift) | (k << bits); + assert(bits > 0 && bits < 31); + const auto ubits = static_cast(bits); + const auto mask = static_cast((1U << ubits) - 1U); + return ((me & mask) ^ shift) | static_cast(static_cast(k) << ubits); + } + // Membership without a search: by the XOR structure every peer shares the same low `bits`. + [[nodiscard]] constexpr auto contains(int me, int b) const -> bool { + if (bits == 0) { + return true; + } + const auto mask = static_cast((1U << static_cast(bits)) - 1U); + return (b & mask) == ((me & mask) ^ shift); } }; diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 9cd3c10a..04eb5cf2 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -217,7 +217,7 @@ class HybridComm { me.ptr = args.send; me.send_displs = args.send_displs; publish_counts_row_(local_partition, args.send_counts); - publish_recv_rows_(local_partition, args.recv_counts); + publish_recv_rows_(local_partition, args.recv_counts, plan); sync(); // B1 // B2: partition 0 sizes/reallocates staging; must finish before any partition packs into stage_send_. @@ -446,13 +446,17 @@ class HybridComm { static_cast(r_) * static_cast(s_) * sizeof(int)); } - // long long: it sums S int counts. - auto publish_recv_rows_(int local_partition, const int *recv_counts) -> void { + // long long: it sums S int counts. Masked through the plan, symmetric with the one reader + // (fill_recv_col_from_rows_): a non-peer's row is zero by definition, and a caller who left a count + // there would otherwise size staging for a block no receive is posted for. + auto publish_recv_rows_(int local_partition, const int *recv_counts, PeerPlan plan) -> void { long long *rr = row_recv_(local_partition); for (int a = 0; a < r_; ++a) { long long sum = 0; - for (int su = 0; su < s_; ++su) { - sum += recv_counts[a * s_ + su]; + if (plan.contains(mpi_rank_, a)) { + for (int su = 0; su < s_; ++su) { + sum += recv_counts[a * s_ + su]; + } } rr[a] = sum; } @@ -463,6 +467,7 @@ class HybridComm { // peer-owned side streams. Every element is written here, so no pre-zeroing. auto pack_count_matrix_(PeerPlan plan) -> void { const int f = plan.count(r_); + assert(narrowing_is_lossless_(plan)); // a wrong shift every rank agrees on drops blocks silently for (int su = 0; su < s_; ++su) { const int *row = counts_row_(su); for (int k = 0; k < f; ++k) { @@ -474,6 +479,19 @@ class HybridComm { } } + // Do the published rows put anything outside the plan's peers? If so the narrowing silently drops it. + auto narrowing_is_lossless_(PeerPlan plan) const -> bool { + for (int su = 0; su < s_; ++su) { + const int *row = counts_matrix_ + static_cast(su) * counts_stride_; + for (int g = 0; g < r_ * s_; ++g) { + if (row[g] != 0 && !plan.contains(mpi_rank_, g / s_)) { + return false; + } + } + } + return true; + } + // The count blocks: one S*S-int MPI_Alltoall when dense, else f point-to-point pairs (a self peer // is a copy, not a message -- with the full linear bits a zero rank shift keeps the whole round // on-rank). Partition 0 only, inside a barriered window. @@ -531,9 +549,11 @@ class HybridComm { const int rc = mpi_recv_counts_[ub]; std::byte *rbuf = stage_recv_.data() + static_cast(mpi_recv_displs_[ub]) * static_cast(extent); - const std::byte *sbuf = + std::byte *sbuf = stage_send_.data() + static_cast(mpi_send_displs_[ub]) * static_cast(extent); if (b == mpi_rank_) { + // The self slot is a copy, not a message: its two counts are each other's transpose. + assert(sc == rc); if (rc != 0) { std::memcpy(rbuf, sbuf, static_cast(rc) * static_cast(extent)); } @@ -543,7 +563,7 @@ class HybridComm { MPI_Irecv(rbuf, rc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); } if (sc != 0) { - MPI_Isend(const_cast(sbuf), sc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); + MPI_Isend(sbuf, sc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); } } MPI_Waitall(n_req, reqs_.data(), MPI_STATUSES_IGNORE); diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index 0970f2c5..86ee34e7 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include @@ -233,6 +234,18 @@ inline auto begin_alltoallv(const std::vector> &send_data, if (self >= 0) { h.recv_counts[static_cast(self)] = 0; } + // Mask the caller's array through the plan, as alltoall_counts already does for the counts it + // exchanges: no receive is ever posted for a non-peer, so a non-zero count there sizes + // recv_buffer for bytes nothing writes and wait_into hands the caller uninitialised memory. + if (!plan.dense()) { + const auto geom = geometry(comm); + const int me = rank(comm) / geom.partitions; + for (int g = 0; g < num_ranks; ++g) { + if (!plan.contains(me, g / geom.partitions)) { + h.recv_counts[static_cast(g)] = 0; + } + } + } } else { alltoall_counts(h.send_counts.data(), h.recv_counts.data(), num_ranks, comm, plan); @@ -296,6 +309,8 @@ inline auto begin_alltoallv(const std::vector> &send_data, T *rbuf = h.recv_buffer.data() + h.recv_displs[ub]; const T *sbuf = h.send_buffer.data() + h.send_displs[ub]; if (b == me) { + // The self slot is a copy, not a message: its two counts are each other's transpose. + assert(h.send_counts[ub] == h.recv_counts[ub]); std::copy(sbuf, sbuf + h.recv_counts[ub], rbuf); continue; } diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index f6c69439..feabc322 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -52,22 +52,14 @@ inline auto read_monomial_from_words(const VecZ &buffer, size_t start) -> Monomi namespace monoprop { // Stateless and identical on every rank, so all ranks agree on a term's owner without communication. -// Both overloads go through routing::Router::dest and nothing else: this and Scan.h's query emission -// must return the same slot for the same monomial, and a divergence splits ownership silently. +// Goes through routing::Router::dest and nothing else: this and Scan.h's query emission must return the +// same slot for the same monomial, and a divergence splits ownership silently. There is deliberately no +// rank-count overload -- it would answer splitmix during a linear run, which is exactly that split. template auto find_rank(const Monomial &mono, const routing::Router &router) -> size_t { return router.dest(mono); } -// Flat-world overload, for callers that hold no geometry: the splitmix router (d = 0). -template -auto find_rank(const Monomial &mono, const size_t n_ranks) -> size_t { - if (n_ranks == 0) { - return 0; - } - return routing::Router::splitmix(n_ranks).dest(mono); -} - // The router this communicator's geometry implies, honouring monoprop_ROUTING / _ROUTE_LINEAR_BITS. inline auto router_for(const mpi::Comm &comm) -> routing::Router { const auto geom = mpi::geometry(comm); @@ -82,25 +74,42 @@ class RoutingDisagreement : public std::runtime_error { // Every participant must resolve the SAME router, and the failure mode if they do not is a hang, not a // wrong answer: linear routing makes each rank post receives from the peers its own bits imply, so a // rank whose monoprop_ROUTING or _ROUTE_SEED did not reach it waits forever on a peer that never sends. -// One allreduce at construction turns that into an exception. Called once, never per gate. +// Turning that into an exception at construction costs two allreduces, called once and never per gate. +// +// TWO independent digests, not one: allreduce_sum is the only collective in the tree, and a sum is not +// an equality test -- differing values can add up to mine*world. Both must agree, so a disagreement +// survives at ~2^-128 rather than ~2^-64. Partitions are in the digest because S enters Router::dest: +// two ranks differing only in S agree on linear_bits and the seed and still route apart. inline auto check_routing_agreement(const mpi::Comm &comm) -> void { - const auto router = router_for(comm); const size_t world = static_cast(mpi::size(comm)); if (world <= 1) { return; } - const uint64_t mine = - routing::mix64((static_cast(router.linear_bits()) << 40) ^ routing::seed_from_env()); - const uint64_t total = mpi::allreduce_sum(mine, comm); - if (total != mine * static_cast(world)) { + const auto router = router_for(comm); + const auto parts = static_cast(mpi::geometry(comm).partitions); + const auto bits = static_cast(router.linear_bits()); + const uint64_t seed = routing::seed_from_env(); + const auto digest = [&](uint64_t salt) { + return routing::mix64(routing::mix64(routing::mix64(salt ^ bits) ^ parts) ^ seed); + }; + const uint64_t first = digest(0x9E37'79B9'7F4A'7C15ULL); + const uint64_t second = digest(0xC2B2'AE3D'27D4'EB4FULL); + const auto agrees = [&](uint64_t mine) { + return mpi::allreduce_sum(mine, comm) == mine * static_cast(world); + }; + // Both allreduces run on every participant: short-circuiting the second would itself deadlock. + const bool ok_first = agrees(first); + const bool ok_second = agrees(second); + if (!ok_first || !ok_second) { throw RoutingDisagreement( std::format("routing configuration differs across the {} participants (this one: linear_bits={}, " - "seed={}). monoprop_ROUTING / monoprop_ROUTE_LINEAR_BITS / monoprop_ROUTE_SEED must " - "reach every rank identically -- under linear routing a disagreement deadlocks the " - "exchange rather than corrupting it.", + "partitions={}, seed={}). monoprop_ROUTING / monoprop_ROUTE_LINEAR_BITS / " + "monoprop_ROUTE_SEED must reach every rank identically -- under linear routing a " + "disagreement deadlocks the exchange rather than corrupting it.", world, - router.linear_bits(), - routing::seed_from_env())); + bits, + parts, + seed)); } } diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index f01e4a06..af091962 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -18,12 +18,11 @@ #include #include #include -#include -#include #include #include #include "monoprop/core/Monomial.h" +#include "monoprop/detail/EnvConfig.h" // The single home for "which flat slot owns this monomial". Two call sites depend on agreeing exactly // (Scan.h emits queries by it, MonomialPropagator seeds the operator by it), and a disagreement splits @@ -53,9 +52,9 @@ // d is a dial, not a cliff: fanout is R >> d, so d = 0 is EXACTLY today's `q % (R*S)` (see dest()) and // d = log2(R) is fanout 1. Non-power-of-two R has no XOR structure at all and falls back to d = 0. // -// Knobs: -// monoprop_ROUTING splitmix (default) | linear -- linear defaults d to log2(R) -// monoprop_ROUTE_LINEAR_BITS explicit d, clamped to [0, log2(R)] +// Knobs, parsed and validated in EnvConfig.h: +// monoprop_ROUTING linear (default) | splitmix -- linear defaults d to log2(R) +// monoprop_ROUTE_LINEAR_BITS explicit d, clamped to [0, log2(R)]; overrides monoprop_ROUTING // monoprop_ROUTE_SEED uint64 seed for the linear basis (default kDefaultSeed) namespace monoprop::routing { @@ -70,11 +69,7 @@ inline constexpr auto mix64(uint64_t x) noexcept -> uint64_t { } inline auto seed_from_env() -> uint64_t { - static const uint64_t seed = [] { - const char *text = std::getenv("monoprop_ROUTE_SEED"); - return (text == nullptr || *text == '\0') ? kDefaultSeed : std::strtoull(text, nullptr, 10); - }(); - return seed; + return config::get().route_seed.value_or(kDefaultSeed); } // One 64-bit vector per Majorana mode. Deterministic from the seed alone, so every rank builds the @@ -200,24 +195,19 @@ class Router final { // The requested linear-bit count, before clamping to a particular geometry. Parsed once. inline auto requested_linear_bits() -> size_t { - static const size_t bits = [] { - const char *explicit_bits = std::getenv("monoprop_ROUTE_LINEAR_BITS"); - if (explicit_bits != nullptr && *explicit_bits != '\0') { - const long value = std::strtol(explicit_bits, nullptr, 10); - return value > 0 ? static_cast(value) : size_t{0}; - } - const char *mode = std::getenv("monoprop_ROUTING"); - if (mode != nullptr && std::string_view{mode} == "splitmix") { - return size_t{0}; // full avalanche across the flat world: every rank talks to every rank - } - // Default. "As many bits as this geometry allows" -- Router clamps to log2(R), and to 0 - // when R is not a power of two, so a geometry without XOR structure keeps the dense path. - // Measured at the production point: fanout 1 costs nothing on balance (rank occupancy - // max/mean 1.001 at R=128, all ranks used) and takes messages per rank per layer from - // 362,712 to 1,397, i.e. from proportional-to-R to flat. - return ~size_t{0}; - }(); - return bits; + const auto &env = config::get(); + if (env.route_linear_bits.has_value()) { + return static_cast(*env.route_linear_bits); + } + if (env.routing_mode == config::RoutingMode::Splitmix) { + return 0; // full avalanche across the flat world: every rank talks to every rank + } + // Default. "As many bits as this geometry allows" -- Router clamps to log2(R), and to 0 when R is + // not a power of two, so a geometry without XOR structure keeps the dense path. Measured at the + // production point: fanout 1 costs nothing on balance (rank occupancy max/mean 1.001 at R=128, all + // ranks used) and takes messages per rank per layer from 362,712 to 1,397, i.e. from + // proportional-to-R to flat. + return ~size_t{0}; } inline auto make_router(size_t ranks, size_t partitions) -> Router { diff --git a/cpp/tests/env_config_tests.cpp b/cpp/tests/env_config_tests.cpp index fe46afe7..e8d29dc3 100644 --- a/cpp/tests/env_config_tests.cpp +++ b/cpp/tests/env_config_tests.cpp @@ -14,11 +14,17 @@ #include +#include #include #include "monoprop/detail/EnvConfig.h" +using monoprop::config::EnvConfigError; +using monoprop::config::RoutingMode; +using monoprop::config::detail::parse_bit_count; using monoprop::config::detail::parse_positive_int; +using monoprop::config::detail::parse_routing_mode; +using monoprop::config::detail::parse_uint64; BOOST_AUTO_TEST_CASE(env_config_parse_positive_int_null_and_malformed) { BOOST_CHECK(parse_positive_int(nullptr) == std::nullopt); @@ -44,3 +50,39 @@ BOOST_AUTO_TEST_CASE(env_config_settings_cached_singleton) { // Touch a field so the Settings aggregate is actually read. BOOST_CHECK(a.num_threads == std::nullopt || *a.num_threads >= 1); } + +// The three routing parsers throw where parse_positive_int returns nullopt: a routing knob that +// defaulted silently would change the transport with no diagnostic. +BOOST_AUTO_TEST_CASE(env_config_parse_uint64_unset_valid_and_rejected) { + BOOST_CHECK(parse_uint64("k", nullptr) == std::nullopt); + BOOST_CHECK(parse_uint64("k", "") == std::nullopt); + BOOST_CHECK(parse_uint64("k", "0") == std::optional(0)); + BOOST_CHECK(parse_uint64("k", "18446744073709551615") == std::optional(~std::uint64_t{0})); + BOOST_CHECK_THROW(parse_uint64("k", "abc"), EnvConfigError); + BOOST_CHECK_THROW(parse_uint64("k", "12x"), EnvConfigError); + BOOST_CHECK_THROW(parse_uint64("k", "-1"), EnvConfigError); // strtoull would WRAP it + BOOST_CHECK_THROW(parse_uint64("k", "18446744073709551616"), EnvConfigError); // ERANGE +} + +BOOST_AUTO_TEST_CASE(env_config_parse_bit_count_keeps_zero_distinct_from_unset) { + BOOST_CHECK(parse_bit_count("k", nullptr) == std::nullopt); + BOOST_CHECK(parse_bit_count("k", "") == std::nullopt); + BOOST_CHECK(parse_bit_count("k", "0") == std::optional(0)); // legal: 0 bits is dense routing + BOOST_CHECK(parse_bit_count("k", "7") == std::optional(7)); + BOOST_CHECK(parse_bit_count("k", "64") == std::optional(64)); + BOOST_CHECK_THROW(parse_bit_count("k", "abc"), EnvConfigError); + BOOST_CHECK_THROW(parse_bit_count("k", "7x"), EnvConfigError); + BOOST_CHECK_THROW(parse_bit_count("k", "-1"), EnvConfigError); + BOOST_CHECK_THROW(parse_bit_count("k", "65"), EnvConfigError); + BOOST_CHECK_THROW(parse_bit_count("k", "99999999999999999999"), EnvConfigError); // ERANGE +} + +BOOST_AUTO_TEST_CASE(env_config_parse_routing_mode_rejects_a_typo) { + BOOST_CHECK(parse_routing_mode("k", nullptr) == std::nullopt); + BOOST_CHECK(parse_routing_mode("k", "") == std::nullopt); + BOOST_CHECK(parse_routing_mode("k", "splitmix") == std::optional(RoutingMode::Splitmix)); + BOOST_CHECK(parse_routing_mode("k", "linear") == std::optional(RoutingMode::Linear)); + BOOST_CHECK_THROW(parse_routing_mode("k", "dense"), EnvConfigError); // all three used to mean linear + BOOST_CHECK_THROW(parse_routing_mode("k", "off"), EnvConfigError); + BOOST_CHECK_THROW(parse_routing_mode("k", "Linear"), EnvConfigError); +} diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index 8430881f..26b9622c 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -585,4 +585,80 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_on_the_plain_mpi_path) { } } +// known_recv_counts is CALLER-supplied, so it can carry a count for a rank the plan does not name -- +// the response round's transpose is only as masked as whatever produced it. No receive is ever posted +// for a non-peer, so an unmasked count sizes recv_buffer for bytes nothing writes and wait_into would +// hand that slot to the caller as data. Both the plain-MPI and the staged HybridComm path must drop it. +BOOST_AUTO_TEST_CASE(hybrid_comm_known_recv_counts_are_masked_through_the_plan) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + constexpr int kReal = 4; + constexpr int kBogus = 7; // what a stale or unmasked transpose would claim a non-peer is sending + for (int shift = 0; shift < R; ++shift) { + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const int peer = plan.peer(world_rank(), 0); + const int bad = (peer + 1) % R; // at full bits the peer set is exactly {peer} + BOOST_REQUIRE(bad != peer); + + // S == 1: the plain-MPI Isend/Irecv branch of begin_alltoallv. + { + Comm c{MPI_COMM_WORLD}; + std::vector> send(static_cast(R)); + for (int j = 0; j < kReal; ++j) { + send[static_cast(peer)].push_back(world_rank() * 1000 + j); + } + std::vector known(static_cast(R), 0); + known[static_cast(peer)] = kReal; + known[static_cast(bad)] = kBogus; + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, &known, plan).wait_into(out); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); + BOOST_CHECK(out[static_cast(bad)].empty()); // unmasked, this holds kBogus elements + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(peer)].size()), kReal); + for (int j = 0; j < kReal; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(peer)][static_cast(j)], peer * 1000 + j); + } + } + + // S == 2: the same array through HybridComm's staged alltoallv. + constexpr int S = 2; + const int P = R * S; + std::vector>> recv(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + Comm c = Comm::make_hybrid(&hyb, u); + const int g = monoprop::mpi::rank(c); + std::vector> send(static_cast(P)); + std::vector known(static_cast(P), 0); + for (int t = 0; t < S; ++t) { + for (int j = 0; j < kReal; ++j) { + send[static_cast((peer * S) + t)].push_back((g * 1000) + j); + } + known[static_cast((peer * S) + t)] = kReal; + known[static_cast((bad * S) + t)] = kBogus; + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, &known, plan).wait_into(out); + recv[static_cast(u)] = out; + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const auto &out = recv[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int su = 0; su < S; ++su) { + BOOST_CHECK(out[static_cast((bad * S) + su)].empty()); + const auto &blk = out[static_cast((peer * S) + su)]; + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), kReal); + for (int j = 0; j < kReal; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], (((peer * S) + su) * 1000) + j); + } + } + } + } +} + #endif // monoprop_ENABLE_MPI diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 6d97043e..3bb9ba04 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -34,8 +34,9 @@ using namespace monoprop; -// find_rank is splitmix over the dense words modulo the rank count, and nothing else, so the oracle -// is asserted unconditionally rather than as one of several permitted hashes. +// Under the splitmix router find_rank is the dense words modulo the rank count and nothing else, so the +// oracle is asserted unconditionally rather than as one of several permitted hashes. The router is +// constructed explicitly: there is no rank-count overload to reach it by accident. BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_range_and_hash_mod) { constexpr size_t N = 32; std::mt19937_64 rng(0x9E3779B9ULL); @@ -47,19 +48,21 @@ BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_range_and_hash_mod) { } const auto mono = indices_to_bitset(inds); for (size_t n_ranks : {size_t{1}, size_t{2}, size_t{3}, size_t{7}}) { - const size_t r = find_rank(mono, n_ranks); + const auto router = routing::Router::splitmix(n_ranks); + const size_t r = find_rank(mono, router); BOOST_TEST(r == monomial_hash(mono) % n_ranks); BOOST_TEST(r < n_ranks); - BOOST_TEST(r == find_rank(mono, n_ranks)); // deterministic + BOOST_TEST(r == find_rank(mono, router)); // deterministic } } } -// n_ranks == 0 is degenerate: owner is rank 0, not a modulo by zero. +// A zero-rank world is degenerate: Router clamps it to one slot, so the owner is rank 0 rather than a +// modulo by zero. BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_zero_ranks) { constexpr size_t N = 32; const auto mono = indices_to_bitset(VecZ{0, 3, 5}); - BOOST_TEST(find_rank(mono, 0) == 0U); + BOOST_TEST(find_rank(mono, routing::Router::splitmix(0)) == 0U); } BOOST_AUTO_TEST_CASE(mpi_utils_monomial_words_roundtrip) { diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp index 2444a349..2c5bb68e 100644 --- a/cpp/tests/routing_tests.cpp +++ b/cpp/tests/routing_tests.cpp @@ -66,7 +66,6 @@ BOOST_AUTO_TEST_CASE(routing_zero_bits_is_bit_identical_to_splitmix) { for (const auto &m : monos) { const size_t expected = monomial_hash(m) % flat; BOOST_TEST(router.dest(m) == expected); - BOOST_TEST(find_rank(m, flat) == expected); BOOST_TEST(find_rank(m, router) == expected); } } @@ -208,6 +207,8 @@ BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { for (const auto &g : random_monomials(200, 4, 0x7777ULL)) { shifts.push_back(static_cast(router.rank_shift(g))); } + // Seed-independent, so this is NOT skipped when monoprop_ROUTE_SEED is overridden: 200 vectors fail + // to span F_2^7 with probability ~2^-194, whatever basis the seed picks. BOOST_TEST(routing::gf2_rank(shifts) == 7U); // == log2(128): every rank is reachable } diff --git a/docs/content/docs/features/parallelism.mdx b/docs/content/docs/features/parallelism.mdx index ac55871f..99f92d2d 100644 --- a/docs/content/docs/features/parallelism.mdx +++ b/docs/content/docs/features/parallelism.mdx @@ -114,9 +114,9 @@ that bound into an identity. | Variable | Default | Meaning | | --- | --- | --- | -| `monoprop_ROUTING` | `linear` | `splitmix` (the literal string) forces $d = 0$, the dense all-to-all. Any other value, including unset, selects linear routing with $d$ as large as the geometry allows. | -| `monoprop_ROUTE_LINEAR_BITS` | unset | An explicit $d$, clamped to $[0, \log_2 R]$. Takes precedence over `monoprop_ROUTING`; a value that is zero, negative or unparseable means $d = 0$. | -| `monoprop_ROUTE_SEED` | `6768574230969066775` | Decimal `uint64` from which every rank derives the same basis $\{v_i\}$ with no communication. The same value must reach every rank: a mismatch in any of these three variables is caught by one allreduce at propagator construction and raised, because under linear routing it deadlocks the exchange instead of corrupting it. | +| `monoprop_ROUTING` | `linear` | `splitmix` forces $d = 0$, the dense all-to-all; `linear`, or unset, takes $d$ as large as the geometry allows. Any other value is rejected at startup rather than silently defaulting. | +| `monoprop_ROUTE_LINEAR_BITS` | unset | An explicit $d$, clamped to $[0, \log_2 R]$. Takes precedence over `monoprop_ROUTING`. `0` selects the dense path; a negative, out-of-range or unparseable value is rejected at startup. | +| `monoprop_ROUTE_SEED` | `6768574230969066775` | Decimal `uint64` from which every rank derives the same basis $\{v_i\}$ with no communication. The same value must reach every rank: a mismatch in any of these three variables, or in the partition count, is caught by two allreduces at propagator construction and raised, because under linear routing it deadlocks the exchange instead of corrupting it. | ### Single-node (`MPI.COMM_SELF`) From 948eae02fd0cf1fd25c10c70256c3ad8df40068e Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 12:59:58 +0100 Subject: [PATCH 09/24] =?UTF-8?q?refactor(mpi):=20=E2=99=BB=EF=B8=8F=20dri?= =?UTF-8?q?ve=20every=20sparse=20exchange=20from=20one=20pairwise=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sparse plan arrived as a second copy of each dense path: four point-to-point loops differing only in member-vs-local request storage, byte-vs-typed pointers and the tag, two byte-identical scatters, two recv-column fills around one loop nest, and eleven plan.dense() branches. Five branches remain and all five earn it -- four are the collectives themselves, where R-1 Isends would be a regression against an Alltoall, and one is the known-recv-count mask, which the dense path must not pay. Pairwise.h also gives the four MPI tags one home. They were bare magic numbers in three files, and the reason Engine.h's two exchange rounds may share one on a single communicator -- non-overtaking within (src, dst, tag, comm), the query round's Waitall preceding any round-2 post, and both ends skipping a zero-count leg on the same value by transpose -- was written nowhere. It is load-bearing, so it is written down. Peers materialise once per verb rather than being recomputed S*f times inside the staging loops, and exchange_payload_ takes the extent its callers already hold instead of asking MPI for it per call. Net line count rises: the extracted helper is a new file, and four copies collapsing into one is the point rather than the arithmetic. Routing.h's derivation moves to the parallelism docs page, keeping only the invariant that Scan.h and find_rank must agree. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/mpi/CMakeLists.txt | 1 + cpp/monoprop/detail/mpi/HybridComm.h | 318 ++++++++++--------------- cpp/monoprop/detail/mpi/MPICompat.cpp | 31 +-- cpp/monoprop/detail/mpi/MPICompat.h | 37 +-- cpp/monoprop/detail/mpi/MPIUtils.h | 2 +- cpp/monoprop/detail/mpi/Pairwise.h | 98 ++++++++ cpp/monoprop/detail/mpi/Routing.h | 33 +-- 7 files changed, 271 insertions(+), 249 deletions(-) create mode 100644 cpp/monoprop/detail/mpi/Pairwise.h diff --git a/cpp/monoprop/detail/mpi/CMakeLists.txt b/cpp/monoprop/detail/mpi/CMakeLists.txt index 29775bca..9128c8ef 100644 --- a/cpp/monoprop/detail/mpi/CMakeLists.txt +++ b/cpp/monoprop/detail/mpi/CMakeLists.txt @@ -11,6 +11,7 @@ target_sources( "HybridComm.h" "MPICompat.h" "MPIUtils.h" + "Pairwise.h" "PartitionBarrier.h" "Routing.h" "ShmComm.h" diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 04eb5cf2..7b7d5076 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -32,6 +32,7 @@ #include "monoprop/detail/mpi/CheckedCount.h" #include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/Pairwise.h" #include "monoprop/detail/mpi/PartitionBarrier.h" // Composes R MPI ranks x S in-process partitions into one flat P=R*S SPMD world. Global id is rank-major @@ -178,8 +179,7 @@ class HybridComm { } } - // recv_counts[g] = amount global partition g sends to this partition. 2 barriers + one S*S-int - // MPI_Alltoall -- or, under a sparse plan, `f = R>>bits` S*S-int point-to-point pairs. + // recv_counts[g] = amount global partition g sends to this partition. 2 barriers + one count round. auto alltoall_counts_impl_(int local_partition, const int *send_counts /*[P]*/, int *recv_counts /*[P]*/, @@ -187,6 +187,7 @@ class HybridComm { publish_counts_row_(local_partition, send_counts); sync(); if (local_partition == 0) { + fill_peers_(plan); pack_count_matrix_(plan); exchange_count_blocks_(plan); } @@ -195,12 +196,8 @@ class HybridComm { // Under a plan only the f peer ranks were exchanged, so the rest of the row is zero by definition // (a non-peer cannot own the partner of any term this rank owns). const int t = local_partition; - const int f = plan.count(r_); - if (!plan.dense()) { - std::fill(recv_counts, recv_counts + static_cast(r_) * static_cast(s_), 0); - } - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); + std::fill(recv_counts, recv_counts + static_cast(r_) * static_cast(s_), 0); + for (const int a : peers_) { for (int su = 0; su < s_; ++su) { recv_counts[a * s_ + su] = counts_recv_[counts_idx_(a, t, su)]; } @@ -222,42 +219,24 @@ class HybridComm { // B2: partition 0 sizes/reallocates staging; must finish before any partition packs into stage_send_. if (local_partition == 0) { - size_staging_send_(args.elem, plan); - fill_recv_col_from_rows_(plan); - size_staging_recv_(args.elem, plan); + fill_peers_(plan); + size_staging_send_(args.elem); + fill_recv_col_([this](int a, int t) { return row_recv_(t)[a]; }); + size_staging_recv_(args.elem); } sync(); // B2 // B3: each partition packs its own cross-rank blocks into stage_send_ (disjoint writes). - pack_send_(local_partition, args.elem, plan); + pack_send_(local_partition, args.elem); sync(); // B3 // B4: partition 0 moves the payload while peers park at the barrier. if (local_partition == 0) { - exchange_payload_(dt, plan); + exchange_payload_(dt, args.elem, plan); } sync(); // B4 - // Scatter each global source's contiguous run from stage_recv_ to recv_displs[g] (all legs, incl. - // self-rank, go through staging). Walks (a, su) in accumulation order, so `cur` re-derives the - // block starts from base_recv_ and this partition's own counts. - std::byte *dst = args.recv; - const int t = local_partition; - const int f = plan.count(r_); - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); - size_t cur = base_recv_[static_cast(a) * static_cast(s_) + static_cast(t)]; - for (int su = 0; su < s_; ++su) { - const int g = a * s_ + su; - const int cnt = args.recv_counts[g]; - if (cnt != 0) { - std::memcpy(dst + static_cast(args.recv_displs[g]) * args.elem, - stage_recv_.data() + cur * args.elem, - static_cast(cnt) * args.elem); - } - cur += static_cast(cnt); - } - } + scatter_recv_(local_partition, args.recv, args.recv_counts, args.recv_displs, args.elem); // No trailing barrier: base_recv_ is rewritten only in a later verb's B1→B2 window. } @@ -282,24 +261,21 @@ class HybridComm { sync(); // B1 if (local_partition == 0) { + fill_peers_(plan); pack_count_matrix_(plan); exchange_count_blocks_(plan); - size_staging_send_(elem, plan); - fill_recv_col_from_counts_recv_(plan); - size_staging_recv_(elem, plan); + size_staging_send_(elem); + fill_recv_col_([this](int a, int t) { return block_sum_(a, t); }); + size_staging_recv_(elem); } sync(); // B2 const int t = local_partition; - const int f = plan.count(r_); long long total = 0; - if (!plan.dense()) { - const size_t p = static_cast(r_) * static_cast(s_); - std::fill(args.recv_counts, args.recv_counts + p, 0); - std::fill(args.recv_displs, args.recv_displs + p, 0); - } - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); + const size_t p = static_cast(r_) * static_cast(s_); + std::fill(args.recv_counts, args.recv_counts + p, 0); + std::fill(args.recv_displs, args.recv_displs + p, 0); + for (const int a : peers_) { for (int su = 0; su < s_; ++su) { const int g = a * s_ + su; const int c = counts_recv_[counts_idx_(a, t, su)]; @@ -310,29 +286,19 @@ class HybridComm { } args.recv.resize(static_cast(checked_mpi_count(total, "Total recv count"))); - pack_send_(local_partition, elem, plan); + pack_send_(local_partition, elem); sync(); // B3 if (local_partition == 0) { - exchange_payload_(dt, plan); + exchange_payload_(dt, elem, plan); } sync(); // B4 - std::byte *dst = reinterpret_cast(args.recv.data()); // after the resize: it may reallocate - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); - size_t cur = base_recv_[static_cast(a) * static_cast(s_) + static_cast(t)]; - for (int su = 0; su < s_; ++su) { - const int g = a * s_ + su; - const int cnt = args.recv_counts[g]; - if (cnt != 0) { - std::memcpy(dst + static_cast(args.recv_displs[g]) * elem, - stage_recv_.data() + cur * elem, - static_cast(cnt) * elem); - } - cur += static_cast(cnt); - } - } + scatter_recv_(local_partition, + reinterpret_cast(args.recv.data()), // after the resize: it may reallocate + args.recv_counts, + args.recv_displs, + elem); // No trailing barrier: same discipline as alltoallv_impl_. } @@ -404,11 +370,6 @@ class HybridComm { // No trailing barrier: red_vec_ is rewritten only inside a future verb's barriered phases. } - // Distinct tags so a count round in flight cannot be matched by a payload receive: partition 0 is - // the only participant per rank, so every message of a verb shares one (source, tag) pair. - static constexpr int kCountTag = 0x6D70; // 'mp' - static constexpr int kPayloadTag = 0x6D71; - static constexpr size_t kLineBytes = 64; static constexpr size_t kIntsPerLine = kLineBytes / sizeof(int); static constexpr size_t kLongsPerLine = kLineBytes / sizeof(long long); @@ -429,6 +390,26 @@ class HybridComm { + static_cast(su); } + // The plan's peer ranks, materialised once per verb: the sizing sweeps index them S times each. + // Written by partition 0 in the B1→B2 window, like base_recv_, so every reader past B2 sees it. + auto fill_peers_(PeerPlan plan) -> void { + const int f = plan.count(r_); + peers_.resize(static_cast(f)); + for (int k = 0; k < f; ++k) { + peers_[static_cast(k)] = plan.peer(mpi_rank_, k); + } + } + + // What rank a's block of the count message holds for partition t, summed over source partitions. + auto block_sum_(int a, int t) const -> long long { + const int *blk = counts_recv_.data() + counts_idx_(a, t, 0); + long long sum = 0; + for (int su = 0; su < s_; ++su) { + sum += blk[su]; + } + return sum; + } + // [S x P] payload offset table: row u is source partition u's staging starts, one per destination g. auto pack_idx_(int u, int g) const -> size_t { return static_cast(u) * static_cast(r_) * static_cast(s_) + static_cast(g); @@ -446,9 +427,9 @@ class HybridComm { static_cast(r_) * static_cast(s_) * sizeof(int)); } - // long long: it sums S int counts. Masked through the plan, symmetric with the one reader - // (fill_recv_col_from_rows_): a non-peer's row is zero by definition, and a caller who left a count - // there would otherwise size staging for a block no receive is posted for. + // long long: it sums S int counts. Masked through the plan, symmetric with its one reader + // (fill_recv_col_): a non-peer's row is zero by definition, and a count left there would size + // staging for a block no receive is posted for. auto publish_recv_rows_(int local_partition, const int *recv_counts, PeerPlan plan) -> void { long long *rr = row_recv_(local_partition); for (int a = 0; a < r_; ++a) { @@ -465,13 +446,11 @@ class HybridComm { // Transpose the published count rows into counts_send_, dest-major then source-minor, for the one // S*S-int MPI_Alltoall. Partition 0 only, inside a barriered window. Source partition outer, so the // peer-owned side streams. Every element is written here, so no pre-zeroing. - auto pack_count_matrix_(PeerPlan plan) -> void { - const int f = plan.count(r_); + auto pack_count_matrix_([[maybe_unused]] PeerPlan plan) -> void { assert(narrowing_is_lossless_(plan)); // a wrong shift every rank agrees on drops blocks silently for (int su = 0; su < s_; ++su) { const int *row = counts_row_(su); - for (int k = 0; k < f; ++k) { - const int b = plan.peer(mpi_rank_, k); + for (const int b : peers_) { for (int t = 0; t < s_; ++t) { counts_send_[counts_idx_(b, t, su)] = row[b * s_ + t]; } @@ -492,36 +471,33 @@ class HybridComm { return true; } - // The count blocks: one S*S-int MPI_Alltoall when dense, else f point-to-point pairs (a self peer - // is a copy, not a message -- with the full linear bits a zero rank shift keeps the whole round - // on-rank). Partition 0 only, inside a barriered window. + // The count blocks: one S*S-int MPI_Alltoall when dense, else a pair per peer (with the full linear + // bits a zero rank shift keeps the whole round on-rank). Partition 0 only, in a barriered window. auto exchange_count_blocks_(PeerPlan plan) -> void { const int block = s_ * s_; if (plan.dense()) { MPI_Alltoall(counts_send_.data(), block, MPI_INT, counts_recv_.data(), block, MPI_INT, parent_); return; } - const int f = plan.count(r_); - grow_(reqs_, static_cast(2 * f)); - int n_req = 0; - for (int k = 0; k < f; ++k) { - const int b = plan.peer(mpi_rank_, k); - const size_t off = static_cast(b) * static_cast(block); - if (b == mpi_rank_) { - std::memcpy(counts_recv_.data() + off, - counts_send_.data() + off, - static_cast(block) * sizeof(int)); - continue; - } - MPI_Irecv(counts_recv_.data() + off, block, MPI_INT, b, kCountTag, parent_, &reqs_[n_req++]); - MPI_Isend(counts_send_.data() + off, block, MPI_INT, b, kCountTag, parent_, &reqs_[n_req++]); - } - MPI_Waitall(n_req, reqs_.data(), MPI_STATUSES_IGNORE); + const PeerLayout blocks{.block = block}; + sparse_pairwise(plan, + mpi_rank_, + r_, + parent_, + kHybridCountTag, + MPI_INT, + sizeof(int), + reinterpret_cast(counts_send_.data()), + blocks, + reinterpret_cast(counts_recv_.data()), + blocks, + reqs_); } - // The staged payload: one MPI_Alltoallv when dense, else f point-to-point pairs over the same - // per-rank counts and displacements (a non-peer's count is zero, so nothing is dropped). - auto exchange_payload_(MPI_Datatype dt, PeerPlan plan) -> void { + // The staged payload: one MPI_Alltoallv when dense, else a pair per peer over the same per-rank + // counts and displacements (a non-peer's count is zero, so nothing is dropped). `elem` is dt's + // extent: needed to reach a block, and known exactly to both callers. + auto exchange_payload_(MPI_Datatype dt, size_t elem, PeerPlan plan) -> void { if (plan.dense()) { MPI_Alltoallv(stage_send_.data(), mpi_send_counts_.data(), @@ -534,39 +510,18 @@ class HybridComm { parent_); return; } - // Counts and displacements are in ELEMENTS; the byte offset into staging needs the datatype's - // extent, which the dense MPI_Alltoallv derived for us and point-to-point does not. - MPI_Aint lb = 0; - MPI_Aint extent = 0; - MPI_Type_get_extent(dt, &lb, &extent); - const int f = plan.count(r_); - grow_(reqs_, static_cast(2 * f)); - int n_req = 0; - for (int k = 0; k < f; ++k) { - const int b = plan.peer(mpi_rank_, k); - const auto ub = static_cast(b); - const int sc = mpi_send_counts_[ub]; - const int rc = mpi_recv_counts_[ub]; - std::byte *rbuf = - stage_recv_.data() + static_cast(mpi_recv_displs_[ub]) * static_cast(extent); - std::byte *sbuf = - stage_send_.data() + static_cast(mpi_send_displs_[ub]) * static_cast(extent); - if (b == mpi_rank_) { - // The self slot is a copy, not a message: its two counts are each other's transpose. - assert(sc == rc); - if (rc != 0) { - std::memcpy(rbuf, sbuf, static_cast(rc) * static_cast(extent)); - } - continue; - } - if (rc != 0) { - MPI_Irecv(rbuf, rc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); - } - if (sc != 0) { - MPI_Isend(sbuf, sc, dt, b, kPayloadTag, parent_, &reqs_[n_req++]); - } - } - MPI_Waitall(n_req, reqs_.data(), MPI_STATUSES_IGNORE); + sparse_pairwise(plan, + mpi_rank_, + r_, + parent_, + kHybridPayloadTag, + dt, + elem, + stage_send_.data(), + PeerLayout{.counts = mpi_send_counts_.data(), .displs = mpi_send_displs_.data()}, + stage_recv_.data(), + PeerLayout{.counts = mpi_recv_counts_.data(), .displs = mpi_recv_displs_.data()}, + reqs_); } template @@ -578,27 +533,22 @@ class HybridComm { // Partition 0's send-side staging sizing, between B1 and B2, in two sweeps of counts_matrix_. Wire // block order is destination major, source minor: a per-destination base plus a per-source prefix. - // Under a plan the two O(R*S^2) sweeps below shrink to O(f*S^2) -- which is the point as much as the - // message count is: these sweeps run SERIALLY on partition 0 while S-1 partitions park, so at R=128, - // S=14 they are ~25k int ops per verb that nothing else overlaps. - auto size_staging_send_(size_t elem, PeerPlan plan) -> void { - const int f = plan.count(r_); + // Both sweeps run SERIALLY here while S-1 partitions park, so narrowing them to the peers matters as + // much as the message count does. + auto size_staging_send_(size_t elem) -> void { // Pass A: the column sums W over source partitions, u outer so both sides sweep in address order. - std::fill(col_sum_.begin(), col_sum_.end(), 0LL); + std::ranges::fill(col_sum_, 0LL); for (int u = 0; u < s_; ++u) { const int *row = counts_row_(u); - for (int k = 0; k < f; ++k) { - const size_t base = static_cast(plan.peer(mpi_rank_, k)) * static_cast(s_); + for (const int b : peers_) { + const size_t base = static_cast(b) * static_cast(s_); for (int t = 0; t < s_; ++t) { col_sum_[base + static_cast(t)] += row[base + static_cast(t)]; } } } - if (!plan.dense()) { - std::fill(mpi_send_counts_.begin(), mpi_send_counts_.end(), 0); - } - for (int k = 0; k < f; ++k) { - const int b = plan.peer(mpi_rank_, k); + std::ranges::fill(mpi_send_counts_, 0); + for (const int b : peers_) { const long long *col = col_sum_.data() + static_cast(b) * static_cast(s_); long long send_sum = 0; for (int t = 0; t < s_; ++t) { @@ -612,8 +562,7 @@ class HybridComm { send_running += mpi_send_counts_[static_cast(b)]; } const size_t total_send = static_cast(checked_mpi_count(send_running, "Total send count")); - for (int k = 0; k < f; ++k) { - const int b = plan.peer(mpi_rank_, k); + for (const int b : peers_) { size_t cur = static_cast(mpi_send_displs_[static_cast(b)]); for (int t = 0; t < s_; ++t) { const size_t g = static_cast(b) * static_cast(s_) + static_cast(t); @@ -622,12 +571,12 @@ class HybridComm { } } // Pass B: the exclusive prefix over source partitions; col_sum_ is free to be reused for it here. - std::fill(col_sum_.begin(), col_sum_.end(), 0LL); + std::ranges::fill(col_sum_, 0LL); for (int u = 0; u < s_; ++u) { const int *row = counts_row_(u); size_t *off = pack_off_.data() + pack_idx_(u, 0); - for (int k = 0; k < f; ++k) { - const size_t base = static_cast(plan.peer(mpi_rank_, k)) * static_cast(s_); + for (const int b : peers_) { + const size_t base = static_cast(b) * static_cast(s_); for (int t = 0; t < s_; ++t) { const size_t g = base + static_cast(t); off[g] = base_send_[g] + static_cast(col_sum_[g]); @@ -639,47 +588,23 @@ class HybridComm { grow_(stage_send_, total_send * elem); } - // recv_col_[a*S + t] = what partition t receives from rank a: the rows published in Phase P0. - auto fill_recv_col_from_rows_(PeerPlan plan) -> void { - const int f = plan.count(r_); - if (!plan.dense()) { - std::fill(recv_col_.begin(), recv_col_.end(), 0LL); - } - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); + // recv_col_[a*S + t] = what partition t receives from rank a. `value(a, t)` reads it from the rows + // published in Phase P0 (alltoallv) or from the count blocks just exchanged (the fused resolve). + template + auto fill_recv_col_(Value &&value) -> void { + std::ranges::fill(recv_col_, 0LL); + for (const int a : peers_) { for (int t = 0; t < s_; ++t) { - recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)] = row_recv_(t)[a]; - } - } - } - - auto fill_recv_col_from_counts_recv_(PeerPlan plan) -> void { - const int f = plan.count(r_); - if (!plan.dense()) { - std::fill(recv_col_.begin(), recv_col_.end(), 0LL); - } - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); - for (int t = 0; t < s_; ++t) { - const int *blk = counts_recv_.data() + counts_idx_(a, t, 0); - long long sum = 0; - for (int su = 0; su < s_; ++su) { - sum += blk[su]; - } - recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)] = sum; + recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)] = value(a, t); } } } // Partition 0's recv-side staging sizing, from recv_col_. Only the per-(rank, partition) base; the // post-B4 scatter re-derives the per-source offsets as it walks (a, su). - auto size_staging_recv_(size_t elem, PeerPlan plan) -> void { - const int f = plan.count(r_); - if (!plan.dense()) { - std::fill(mpi_recv_counts_.begin(), mpi_recv_counts_.end(), 0); - } - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); + auto size_staging_recv_(size_t elem) -> void { + std::ranges::fill(mpi_recv_counts_, 0); + for (const int a : peers_) { long long recv_sum = 0; for (int t = 0; t < s_; ++t) { recv_sum += recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)]; @@ -692,8 +617,7 @@ class HybridComm { recv_running += mpi_recv_counts_[static_cast(a)]; } const size_t total_recv = static_cast(checked_mpi_count(recv_running, "Total recv count")); - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); + for (const int a : peers_) { size_t cur = static_cast(mpi_recv_displs_[static_cast(a)]); for (int t = 0; t < s_; ++t) { const size_t g = static_cast(a) * static_cast(s_) + static_cast(t); @@ -704,7 +628,7 @@ class HybridComm { grow_(stage_recv_, total_recv * elem); } - auto pack_send_(int local_partition, size_t elem, PeerPlan plan) -> void { + auto pack_send_(int local_partition, size_t elem) -> void { const int u = local_partition; // Own slot only — no peer's published send buffer is read here, which is what lets every // partition pack concurrently in the B2→B3 window. @@ -712,9 +636,8 @@ class HybridComm { const int *my_send_counts = counts_row_(u); const int *my_send_displs = slots_[static_cast(u)].send_displs; const size_t *off = pack_off_.data() + pack_idx_(u, 0); - const int f = plan.count(r_); - for (int k = 0; k < f; ++k) { - const int base = plan.peer(mpi_rank_, k) * s_; + for (const int b : peers_) { + const int base = b * s_; for (int t = 0; t < s_; ++t) { const int g = base + t; const int cnt = my_send_counts[g]; @@ -727,6 +650,27 @@ class HybridComm { } } + // Scatter each global source's contiguous run out of stage_recv_ to recv_displs[g] (all legs, incl. + // self-rank, go through staging). Walks (a, su) in accumulation order, so `cur` re-derives the block + // starts from base_recv_ and this partition's own counts. + auto scatter_recv_(int local_partition, std::byte *dst, const int *recv_counts, const int *recv_displs, size_t elem) + -> void { + const int t = local_partition; + for (const int a : peers_) { + size_t cur = base_recv_[static_cast(a) * static_cast(s_) + static_cast(t)]; + for (int su = 0; su < s_; ++su) { + const int g = a * s_ + su; + const int cnt = recv_counts[g]; + if (cnt != 0) { + std::memcpy(dst + static_cast(recv_displs[g]) * elem, + stage_recv_.data() + cur * elem, + static_cast(cnt) * elem); + } + cur += static_cast(cnt); + } + } + } + [[noreturn]] auto abort_rank_(const char *verb, const char *what) -> void { std::print(stderr, "monoprop: rank {} cannot complete the collective '{}' ({}). Its peer ranks are " @@ -782,6 +726,8 @@ class HybridComm { std::vector red_vec_; // Point-to-point request scratch for the sparse paths; grown on demand, partition 0 only. std::vector reqs_; + // This verb's peer ranks; see fill_peers_. + std::vector peers_; PartitionBarrier barrier_; }; diff --git a/cpp/monoprop/detail/mpi/MPICompat.cpp b/cpp/monoprop/detail/mpi/MPICompat.cpp index dabb2e7c..0da7b82e 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.cpp +++ b/cpp/monoprop/detail/mpi/MPICompat.cpp @@ -19,6 +19,10 @@ #include #include +#ifdef monoprop_ENABLE_MPI +#include "monoprop/detail/mpi/Pairwise.h" +#endif + namespace monoprop::mpi { #ifdef monoprop_ENABLE_MPI @@ -131,21 +135,20 @@ auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, int me = 0; MPI_Comm_rank(comm.mpi, &me); std::fill(recv_counts, recv_counts + n, 0); - const int f = plan.count(n); + const PeerLayout one{.block = 1}; std::vector reqs; - reqs.reserve(static_cast(2 * f)); - for (int k = 0; k < f; ++k) { - const int b = plan.peer(me, k); - if (b == me) { - recv_counts[b] = send_counts[b]; - continue; - } - reqs.emplace_back(); - MPI_Irecv(&recv_counts[b], 1, MPI_INT, b, 0x6D73, comm.mpi, &reqs.back()); - reqs.emplace_back(); - MPI_Isend(&send_counts[b], 1, MPI_INT, b, 0x6D73, comm.mpi, &reqs.back()); - } - MPI_Waitall(static_cast(reqs.size()), reqs.data(), MPI_STATUSES_IGNORE); + sparse_pairwise(plan, + me, + n, + comm.mpi, + kFlatCountTag, + MPI_INT, + sizeof(int), + reinterpret_cast(send_counts), + one, + reinterpret_cast(recv_counts), + one, + reqs); return; } (void)n; diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index 86ee34e7..a3575616 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -33,6 +33,7 @@ #include "monoprop/detail/mpi/ShmComm.h" #ifdef monoprop_ENABLE_MPI #include "monoprop/detail/mpi/HybridComm.h" +#include "monoprop/detail/mpi/Pairwise.h" #endif // These includes are here on purpose and should not be moved to the top @@ -299,31 +300,19 @@ inline auto begin_alltoallv(const std::vector> &send_data, else { // S == 1 world: the same pairing as the Hybrid path, one message per reachable peer. Blocking // here rather than through the Ticket, because the request set is per-peer, not one handle. - const int me = rank(comm); - const int f = plan.count(num_ranks); std::vector reqs; - reqs.reserve(static_cast(2 * f)); - for (int k = 0; k < f; ++k) { - const int b = plan.peer(me, k); - const auto ub = static_cast(b); - T *rbuf = h.recv_buffer.data() + h.recv_displs[ub]; - const T *sbuf = h.send_buffer.data() + h.send_displs[ub]; - if (b == me) { - // The self slot is a copy, not a message: its two counts are each other's transpose. - assert(h.send_counts[ub] == h.recv_counts[ub]); - std::copy(sbuf, sbuf + h.recv_counts[ub], rbuf); - continue; - } - if (h.recv_counts[ub] != 0) { - reqs.emplace_back(); - MPI_Irecv(rbuf, h.recv_counts[ub], datatype::get(), b, 0x6D72, comm.mpi, &reqs.back()); - } - if (h.send_counts[ub] != 0) { - reqs.emplace_back(); - MPI_Isend(sbuf, h.send_counts[ub], datatype::get(), b, 0x6D72, comm.mpi, &reqs.back()); - } - } - MPI_Waitall(static_cast(reqs.size()), reqs.data(), MPI_STATUSES_IGNORE); + sparse_pairwise(plan, + rank(comm), + num_ranks, + comm.mpi, + kFlatPayloadTag, + datatype::get(), + sizeof(T), + reinterpret_cast(h.send_buffer.data()), + PeerLayout{.counts = h.send_counts.data(), .displs = h.send_displs.data()}, + reinterpret_cast(h.recv_buffer.data()), + PeerLayout{.counts = h.recv_counts.data(), .displs = h.recv_displs.data()}, + reqs); } #else h.recv_buffer = h.send_buffer; // single participant: self round-trip (layouts identical) diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index feabc322..2f3007d2 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -81,7 +81,7 @@ class RoutingDisagreement : public std::runtime_error { // survives at ~2^-128 rather than ~2^-64. Partitions are in the digest because S enters Router::dest: // two ranks differing only in S agree on linear_bits and the seed and still route apart. inline auto check_routing_agreement(const mpi::Comm &comm) -> void { - const size_t world = static_cast(mpi::size(comm)); + const auto world = static_cast(mpi::size(comm)); if (world <= 1) { return; } diff --git a/cpp/monoprop/detail/mpi/Pairwise.h b/cpp/monoprop/detail/mpi/Pairwise.h new file mode 100644 index 00000000..70ca5ac4 --- /dev/null +++ b/cpp/monoprop/detail/mpi/Pairwise.h @@ -0,0 +1,98 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include + +#include "monoprop/detail/mpi/Comm.h" + +namespace monoprop::mpi { + +// One tag per (transport, verb), all four here so no two can collide unseen: one thread per rank calls +// MPI, so the tag is all that keeps a count round in flight from being matched by a payload receive. +// +// Why Engine.h's run_exchange may post BOTH its begin_alltoallv rounds under kFlatPayloadTag on one +// communicator: MPI does not overtake within a (src, dst, tag, comm); the query round's MPI_Waitall +// completes before round 2 posts; and both ends skip a zero-count leg on the same value -- what one +// sends a peer IS that peer's recv count, by transpose -- so the two posted sequences match element +// for element and a round-2 receive cannot match a round-1 send. +inline constexpr int kHybridCountTag = 0x6D70; // 'mp' +inline constexpr int kHybridPayloadTag = 0x6D71; +inline constexpr int kFlatPayloadTag = 0x6D72; +inline constexpr int kFlatCountTag = 0x6D73; + +// Per-peer element counts and offsets. Null `counts` is the fixed-block case: `block` each, at b*block. +struct PeerLayout { + const int *counts = nullptr; + const int *displs = nullptr; + int block = 0; + + [[nodiscard]] auto count(int b) const -> int { return counts != nullptr ? counts[b] : block; } + [[nodiscard]] auto displ(int b) const -> size_t { + return static_cast(displs != nullptr ? displs[b] : b * block); + } +}; + +// A variable all-to-all as point-to-point over `plan`'s peers: one Irecv/Isend pair each, the self peer +// copied in place. Counts and displacements are in ELEMENTS of `dt`, whose extent must be `elem`. +// `reqs` is caller scratch, grown then INDEXED: MPI holds these pointers until Waitall, so a +// reallocating push_back would dangle them. +inline auto sparse_pairwise(PeerPlan plan, + int me, + int n_ranks, + MPI_Comm comm, + int tag, + MPI_Datatype dt, + size_t elem, + const std::byte *send, + PeerLayout send_lay, + std::byte *recv, + PeerLayout recv_lay, + std::vector &reqs) -> void { + const int f = plan.count(n_ranks); + if (reqs.size() < static_cast(2 * f)) { + reqs.resize(static_cast(2 * f)); + } + int n_req = 0; + for (int k = 0; k < f; ++k) { + const int b = plan.peer(me, k); + const int sc = send_lay.count(b); + const int rc = recv_lay.count(b); + std::byte *rbuf = recv + recv_lay.displ(b) * elem; + const std::byte *sbuf = send + send_lay.displ(b) * elem; + if (b == me) { + // The self slot is a copy, not a message: its two counts are each other's transpose. + assert(sc == rc); + if (rc != 0) { + std::memcpy(rbuf, sbuf, static_cast(rc) * elem); + } + continue; + } + if (rc != 0) { + MPI_Irecv(rbuf, rc, dt, b, tag, comm, &reqs[static_cast(n_req++)]); + } + if (sc != 0) { + MPI_Isend(sbuf, sc, dt, b, tag, comm, &reqs[static_cast(n_req++)]); + } + } + MPI_Waitall(n_req, reqs.data(), MPI_STATUSES_IGNORE); +} + +} // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index af091962..888509de 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -28,29 +28,17 @@ // (Scan.h emits queries by it, MonomialPropagator seeds the operator by it), and a disagreement splits // ownership silently rather than crashing -- so both go through Router::dest and nothing else. // -// WHY there is a choice to make here at all -// ---------------------------------------- -// A gate turns a term M into M^G (symmetric difference of Majorana support). With today's splitmix -// destination -- full avalanche -- the owner of M^G is unrelated to the owner of M, so a rank's queries -// for ONE generator spray across all R ranks and the exchange is a dense all-to-all whose message count -// grows as R*(R-1). If instead the RANK index is a GF(2)-LINEAR function of the support, -// -// h(M) = XOR of v_i over i in support(M) => h(M ^ G) = h(M) ^ h(G) -// -// then, since I own M, every query I emit for G goes to exactly one rank: my_rank ^ h(G). XOR is an -// involution, so that peer sends to me in the same round: the exchange becomes a pairwise Sendrecv. -// -// The routing is TWO-LEVEL, because the two levels have different costs: across MPI ranks the cost is -// the message COUNT (make it structured), within a rank partitions talk through shared memory where -// fanout is free and only balance matters (keep full avalanche). +// Two-level, because the levels cost differently: across MPI ranks the message COUNT is what hurts, so +// the rank index is GF(2)-linear in the support and a generator maps every query to one peer; within a +// rank partitions talk through shared memory, where fanout is free and only balance matters. // // part = q % S q = monomial_hash(M) (splitmix, unchanged) // hi = (q / S) % (R >> d) the R>>d splitmix-chosen high rank bits // rank = (a & (2^d - 1)) | (hi << d) a = linear_hash(M); d = linear_bits // flat = rank * S + part // -// d is a dial, not a cliff: fanout is R >> d, so d = 0 is EXACTLY today's `q % (R*S)` (see dest()) and -// d = log2(R) is fanout 1. Non-power-of-two R has no XOR structure at all and falls back to d = 0. +// The derivation, what d buys and what it costs: docs/content/docs/features/parallelism.mdx, under +// "Rank routing". // // Knobs, parsed and validated in EnvConfig.h: // monoprop_ROUTING linear (default) | splitmix -- linear defaults d to log2(R) @@ -102,9 +90,9 @@ template // GF(2) rank of a set of 64-bit vectors, by Gaussian elimination over the bit columns. The per-generator // rank shifts must span at least log2(R) dimensions or the reachable destination ranks form a strict -// subspace and some ranks stay empty -- a load-balance failure, not a correctness one, which is why this -// is a diagnostic (measured: rank 32 for the 60-site Hubbard's 416 distinct shifts, against the 7 bits -// R = 128 needs) rather than a runtime gate. +// subspace and 2^d - 2^rank ranks stay empty. A balance failure, not a correctness one, so its caller +// (MonomialPropagator::report_routing_coverage_) warns on stderr rather than gating. +// Measured: rank 32 for the 60-site Hubbard's 416 distinct shifts, against the 7 bits R = 128 needs. [[nodiscard]] inline auto gf2_rank(std::vector vectors) noexcept -> size_t { size_t rank = 0; for (size_t bit = 0; bit < 64 && rank < vectors.size(); ++bit) { @@ -130,8 +118,6 @@ template return rank; } -enum class Mode : uint8_t { Splitmix, Linear }; - // Trivially copyable and cheap to build; hold one per build_layer call rather than per term. class Router final { public: @@ -154,7 +140,6 @@ class Router final { [[nodiscard]] constexpr auto linear_bits() const noexcept -> size_t { return bits_; } // Distinct destination RANKS one rank's queries for a single generator reach. 1 == pairwise. [[nodiscard]] constexpr auto fanout() const noexcept -> size_t { return ranks_ >> bits_; } - [[nodiscard]] constexpr auto mode() const noexcept -> Mode { return bits_ == 0 ? Mode::Splitmix : Mode::Linear; } // Flat destination slot in [0, flat_world). Branch is on a member, so it is perfectly predicted. template @@ -181,7 +166,7 @@ class Router final { if (!std::has_single_bit(ranks)) { return 0; // no XOR structure without a power-of-two rank count } - const size_t max_bits = static_cast(std::countr_zero(ranks)); + const auto max_bits = static_cast(std::countr_zero(ranks)); return requested < max_bits ? requested : max_bits; } From f104b59ed976cfa46eff5ed909c79061118921e5 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 13:00:22 +0100 Subject: [PATCH 10/24] =?UTF-8?q?feat(mpi):=20=F0=9F=93=88=20report=20a=20?= =?UTF-8?q?shift=20set=20that=20cannot=20reach=20every=20rank?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit routing::gf2_rank called itself the coverage diagnostic and had no caller outside its own test, so the check it describes never ran. Linear routing reaches only the subspace the per-generator shifts span: at rank rho below linear_bits, 2^d - 2^rho ranks receive nothing all run and the imbalance looks like slow peers rather than a routing property. Not beside check_routing_agreement, where it belongs conceptually -- at construction the gate list does not exist yet. It runs once at the top of the gate loop, where the generators first arrive, and only under linear routing, so splitmix and non-power-of-two geometries pay nothing. A warning rather than a throw: every term still lands on exactly one owner, so the answer is right and only the balance is not. One COMMROUTE line per rank, in COMMPLACE's shape. An out-of-range gate index leaves the report unwritten so it cannot pre-empt build_evolve_result_'s per-gate throw. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/include/monoprop/MonomialPropagator.h | 6 +++ .../MonomialPropagator.inl | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/cpp/include/monoprop/MonomialPropagator.h b/cpp/include/monoprop/MonomialPropagator.h index b684ff58..6edaf6d0 100644 --- a/cpp/include/monoprop/MonomialPropagator.h +++ b/cpp/include/monoprop/MonomialPropagator.h @@ -365,6 +365,8 @@ class MonomialPropagator { // it captures this and rejects a later call once it moves, as it does for a rebuilt graph. size_t initial_operator_epoch_{0}; + bool routing_coverage_reported_{false}; // report_routing_coverage_ speaks once per propagator + size_t logical_num_modes_{NumModes}; CutoffType cutoff_type_; @@ -449,6 +451,10 @@ class MonomialPropagator { const VecD ¶meters, std::optional only_rotate_len_k) -> void; + // Do this call's generator shifts span linear_bits? If not, ranks receive nothing (routing::gf2_rank). + // Not beside check_routing_agreement: at construction the gate list does not exist yet. + auto report_routing_coverage_(const std::vector &majoranas) -> void; + template auto run_gate_loop_(const std::vector &majoranas, std::optional only_rotate_len_k, diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index dc242f39..d3012f99 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -207,6 +207,7 @@ MonomialPropagator::MonomialPropagator(const MonomialPropagator &other upper_atol_(other.upper_atol_), core_term_(other.core_term_), initial_operator_epoch_(other.initial_operator_epoch_), + routing_coverage_reported_(other.routing_coverage_reported_), logical_num_modes_(other.logical_num_modes_), cutoff_type_(other.cutoff_type_), basis_change_(other.basis_change_), @@ -720,11 +721,51 @@ auto MonomialPropagator::propagate(const std::vector &majoranas, evolve_mode_contract_immediately_(majoranas, parameter_mapping, gen_coeffs, parameters, only_rotate_len_k); } +template +auto MonomialPropagator::report_routing_coverage_(const std::vector &majoranas) -> void { + // One report per rank, so only its partition 0 speaks, and only once whatever the outcome. + if (routing_coverage_reported_ || comm_.shm_rank != 0) { + return; + } + routing_coverage_reported_ = true; + const auto router = router_for(comm_); + if (router.linear_bits() == 0) { + return; // splitmix: no subspace to fall short of + } + std::vector shifts; + shifts.reserve(majoranas.size()); + for (const auto &gate : majoranas) { + // An out-of-range index is build_evolve_result_'s to reject, gate by gate: converting the whole + // list up front would pre-empt that throw. + if (std::ranges::any_of(gate, [this](size_t i) { return i >= 2 * logical_num_modes_; })) { + return; + } + shifts.push_back(static_cast(router.rank_shift(indices_to_bitset(gate)))); + } + std::ranges::sort(shifts); + shifts.erase(std::ranges::unique(shifts).begin(), shifts.end()); + const size_t span = routing::gf2_rank(shifts); + if (span >= router.linear_bits()) { + return; + } + // A warning, not a throw: every term still lands on one owner, they just do not cover the ranks. + // COMMPLACE's shape -- greppable prefix, rank-identified, one line. + const auto line = std::format("COMMROUTE rank={} linear_bits={} shift_rank={} shifts={} idle_ranks={}\n", + static_cast(mpi::rank(comm_)) / router.partitions(), + router.linear_bits(), + span, + shifts.size(), + router.ranks() - (router.fanout() << span)); + std::fputs(line.c_str(), stderr); + std::fflush(stderr); +} + template template auto MonomialPropagator::run_gate_loop_(const std::vector &majoranas, std::optional only_rotate_len_k, EvolutionFunc evolution_func) -> void { + report_routing_coverage_(majoranas); // Serial per partition; parallelism comes from partitioning the operator across cores. for (size_t i = 0; i < majoranas.size(); ++i) { const auto idx = !schrodinger_ ? majoranas.size() - 1 - i : i; From 5c9f15f9dd8e2c89630e4e7c5df06ab5cf0f04f1 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 13:00:24 +0100 Subject: [PATCH 11/24] =?UTF-8?q?perf(evolution):=20=E2=8F=AD=EF=B8=8F=20s?= =?UTF-8?q?kip=20the=20scan=20too=20for=20an=20identity=20generator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skip tested gen.any() after the scan had already run. An identity generator anticommutes with nothing, so the scan returns on its empty fold-column set having produced no query, no cosine block and no swept coefficient -- work that was being done to discover it was not needed. Hoisting the test above it also skips the cos-block concatenation. LayerBuildEngine construction stays: its ctor sizes the caller-owned matched scratch, which is reported as matched_scratch_bytes, and skipping it would move that telemetry when a propagator's first gate is identity. The ctor is O(R); everything expensive is now behind the test. Assisted-by: ClaudeCode:claude-opus-5 --- .../detail/evolution/layer_build/Engine.h | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 8d640187..a87ee8f8 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -633,9 +633,18 @@ auto build_layer(MPOperator &local_op, } assert(fused_scale_coeffs == nullptr || (local_coeffs && &local_coeffs->get() == fused_scale_coeffs)); - FusedScanResult fused = [&] { + // An identity generator anticommutes with nothing: the scan returns on its empty fold-column set + // with no query, no cosine block and no coefficient swept, and run_exchange's three collectives per + // pass would carry no payload. The generator list is replicated, so skipping needs no agreement. + // (A zero chemical potential alone contributes 60 of the 60-site Hubbard's 476 generators per + // Trotter layer.) No gate is merged: a no-op gate is simply not exchanged for. + const bool identity_gen = !gen.any(); + + FusedScanResult fused; + CosMask cos_all; + if (!identity_gen) { double *const sweep_ptr = fused_scale ? fused_scale_coeffs->data() : nullptr; - return with_algebra(basis, [&]() { + fused = with_algebra(basis, [&]() { return fused_find_and_collect(local_op, gen, cut_eval, @@ -649,21 +658,19 @@ auto build_layer(MPOperator &local_op, sweep_ptr, cos_build); }); - }(); - - CosMask cos_all; - if (fused.cos_blocks.size() == 1) { - // The serial scan produces a single cosine block set — take it wholesale. - cos_all = std::move(fused.cos_blocks[0]); - } - else { - // Cosine block sets are disjoint and ascending; concatenate in order. - for (const auto &block : fused.cos_blocks) { - cos_all.total_count += block.total_count; - cos_all.blocks.insert(cos_all.blocks.end(), block.blocks.begin(), block.blocks.end()); + if (fused.cos_blocks.size() == 1) { + // The serial scan produces a single cosine block set — take it wholesale. + cos_all = std::move(fused.cos_blocks[0]); + } + else { + // Cosine block sets are disjoint and ascending; concatenate in order. + for (const auto &block : fused.cos_blocks) { + cos_all.total_count += block.total_count; + cos_all.blocks.insert(cos_all.blocks.end(), block.blocks.begin(), block.blocks.end()); + } } + fused.cos_blocks = std::vector{}; } - fused.cos_blocks = std::vector{}; auto run = [&](Sink sink) -> std::shared_ptr { LayerBuildEngine eng(local_op, @@ -674,13 +681,7 @@ auto build_layer(MPOperator &local_op, /*combined_size=*/local_op.store->size(), std::move(sink), plan); - // An empty generator anticommutes with nothing, so the scan already returned zero queries on - // every rank -- but run_exchange's collectives fire regardless of payload, and each pass costs - // three of them. The generator list is replicated, so `gen.none()` is unanimous and skipping - // needs no agreement. (These are the identity monomials a gate whose every term fell below its - // atol expands to; a zero chemical potential alone contributes 60 of the 60-site Hubbard's 476 - // generators per Trotter layer.) No gate is merged: a no-op gate is simply not exchanged for. - if (gen.any()) { + if (!identity_gen) { eng.run_exchange(/*is_leader_pass=*/true, std::move(fused.leader_queries), std::move(fused.leader_src), From effc76ed281b00dc4d0cf94303f65c71e0d3861d Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 14:53:35 +0100 Subject: [PATCH 12/24] =?UTF-8?q?perf(routing):=20=E2=9A=A1=20take=20the?= =?UTF-8?q?=20rank=20bits=20from=20a=20transposed=20basis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dest() walked the monomial's set bits, loading a basis vector and XOR-ing per bit -- a data-dependent chain ~20-28 long under the production cutoff, per term, in the hottest loop here. monomial_hash beside it is a single mix of one word, so the new hash dominated the destination, not the old one. Only the low d bits survive the mask, so transpose: plane j holds bit j of every basis vector, and bit j of the image is parity(popcount(M & plane_j)). Folding the words with XOR before the popcount is the same parity, so it is d popcounts rather than d per word -- 14 branch-free ops at R=128 over 120 slots. The planes key on the seed and the width alone, never the geometry, so one table serves every Router and dest() binds a pointer at construction instead of meeting the static-init guard per term. Bit-identical, which is the acceptance criterion and not an aspiration: routing_transposed_basis_is_bit_identical_to_the_bit_walk pins dest() and rank_shift() against an independent reference of the old walk over 100k monomials across several (R, S, d), and asserts the comparison count so it cannot pass vacuously. The constructor is private now: a router with linear bits has to come through for_modes, because reading planes bound at another width would be silent. splitmix stays width-free -- with d = 0 no plane is ever read. Assisted-by: ClaudeCode:claude-opus-5 --- .../detail/evolution/layer_build/Engine.h | 2 +- .../MonomialPropagator.inl | 8 +- cpp/monoprop/detail/mpi/MPIUtils.h | 10 +- cpp/monoprop/detail/mpi/Routing.h | 115 ++++++++++++++---- cpp/tests/routing_tests.cpp | 110 +++++++++++++++-- 5 files changed, 201 insertions(+), 44 deletions(-) diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index a87ee8f8..0875fb4c 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -607,7 +607,7 @@ auto build_layer(MPOperator &local_op, const size_t my_rank = static_cast(mpi::rank(comm)); const size_t R = static_cast(mpi::size(comm)); // R is the FLAT world (ranks x partitions); the router is what splits it back into the two levels. - const auto router = router_for(comm); + const routing::Router router = router_for(comm); assert(router.flat_world() == R); // Under linear routing every query for THIS generator lands on a rank whose low `linear_bits` are // this rank's own XOR rank_shift(gen), so the exchange knows its peers before it starts. Dense diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index d3012f99..c441103c 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -143,8 +143,8 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope const size_t num_ranks = static_cast(mpi::size(comm_)); const size_t my_rank = static_cast(mpi::rank(comm_)); - check_routing_agreement(comm_); // a disagreement here would hang the first exchange, not corrupt it - const auto router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term + check_routing_agreement(comm_); // a disagreement here would hang the first exchange, not corrupt it + const routing::Router router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term MonomialList local_heisenberg_terms; double core_term = 0.0; @@ -369,7 +369,7 @@ auto MonomialPropagator::apply_initial_operator_(const OperatorDict &o for_each_partition_([&](MonomialPropagator &s) { s.update_initial_operator(op_dict); }); return {}; } - const auto router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term + const routing::Router router = router_for(comm_); // hoisted: geometry() can hit MPI, so never per term const size_t my_rank = static_cast(mpi::rank(comm_)); OperatorDict new_op; @@ -728,7 +728,7 @@ auto MonomialPropagator::report_routing_coverage_(const std::vector(comm_); if (router.linear_bits() == 0) { return; // splitmix: no subspace to fall short of } diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index 2f3007d2..bfaf20f2 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -61,9 +61,11 @@ auto find_rank(const Monomial &mono, const routing::Router &router) -> } // The router this communicator's geometry implies, honouring monoprop_ROUTING / _ROUTE_LINEAR_BITS. +// Templated because the router binds the transposed basis for this monomial width. +template inline auto router_for(const mpi::Comm &comm) -> routing::Router { const auto geom = mpi::geometry(comm); - return routing::make_router(static_cast(geom.ranks), static_cast(geom.partitions)); + return routing::make_router(static_cast(geom.ranks), static_cast(geom.partitions)); } class RoutingDisagreement : public std::runtime_error { @@ -85,9 +87,9 @@ inline auto check_routing_agreement(const mpi::Comm &comm) -> void { if (world <= 1) { return; } - const auto router = router_for(comm); - const auto parts = static_cast(mpi::geometry(comm).partitions); - const auto bits = static_cast(router.linear_bits()); + const auto geom = mpi::geometry(comm); + const auto parts = static_cast(geom.partitions); + const auto bits = static_cast(routing::linear_bits_for(static_cast(geom.ranks))); const uint64_t seed = routing::seed_from_env(); const auto digest = [&](uint64_t salt) { return routing::mix64(routing::mix64(routing::mix64(salt ^ bits) ^ parts) ^ seed); diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index 888509de..0410368f 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -75,9 +76,9 @@ inline auto linear_basis() -> const std::array & { return table; } -// Terms are sparse under a length cutoff (popcount <= ~2*cutoff), so iterating set bits beats a -// word-wise byte table. A CLMUL form would be popcount-independent; it is also a DIFFERENT linear map, -// so switching to it means re-measuring balance, not just re-benchmarking. +// The full 64-bit image, by walking the set bits. Router::dest does NOT use this -- it needs only the +// low d bits and takes the transposed form below -- but the map is defined here, and the GF(2)-linearity +// test and the plane build both read it as the definition. template [[nodiscard]] inline auto linear_hash(const monoprop::Bitset &bits) noexcept -> uint64_t { const auto &v = linear_basis(); @@ -88,6 +89,36 @@ template return h; } +inline constexpr size_t kLinearPlanes = 64; // one per output bit; d <= 63 of them are ever read + +template +inline constexpr size_t kPlaneWords = monoprop::Bitset::num_words(); + +// The basis transposed: plane j, bit i, is bit j of basis vector i. Then bit j of linear_hash(M) is +// popcount(M & plane_j) & 1 -- d * words branch-free AND/XOR/popcount ops instead of a gather whose +// length is the term's popcount (~20-28 under the production cutoff). Same map, bit for bit. +// +// Keyed on the seed and the width alone, never on the geometry, so one table serves every Router; the +// Router binds a pointer to it at construction and dest() never reaches the static-init guard. +template +inline auto linear_planes() -> const std::array> & { + static const auto table = [] { + std::array> planes{}; + const auto &v = linear_basis(); + for (size_t i = 0; i < NumBits; ++i) { + const size_t word = i / 64; + const uint64_t bit = uint64_t{1} << (i % 64); + for (size_t j = 0; j < kLinearPlanes; ++j) { + if (((v[i] >> j) & 1U) != 0) { + planes[(j * kPlaneWords)+word] |= bit; + } + } + } + return planes; + }(); + return table; +} + // GF(2) rank of a set of 64-bit vectors, by Gaussian elimination over the bit columns. The per-generator // rank shifts must span at least log2(R) dimensions or the reachable destination ranks form a strict // subspace and 2^d - 2^rank ranks stay empty. A balance failure, not a correctness one, so its caller @@ -121,19 +152,30 @@ template // Trivially copyable and cheap to build; hold one per build_layer call rather than per term. class Router final { public: - // ranks x partitions == the flat world the destinations index. linear_bits is clamped here, so a - // caller may pass anything. - constexpr Router(size_t ranks, size_t partitions, size_t linear_bits) noexcept - : ranks_(ranks == 0 ? 1 : ranks), - parts_(partitions == 0 ? 1 : partitions), - flat_(ranks_ * parts_), - bits_(clamp_bits_(ranks_, linear_bits)), - lin_mask_((uint64_t{1} << bits_) - 1), - hi_mask_((ranks_ >> bits_) - 1) {} + // Bound to a monomial width: dest() reads the transposed basis for THAT width, and binding the + // pointer here is what keeps the static-init guard out of the per-term path. The only way to a + // router with linear bits, so an unbound one cannot reach dest(). + template + [[nodiscard]] static auto for_modes(size_t ranks, size_t partitions, size_t linear_bits) -> Router { + Router r{ranks, partitions, linear_bits}; + r.planes_ = linear_planes<2 * NumModes>().data(); + r.plane_words_ = kPlaneWords<2 * NumModes>; + return r; + } // Today's routing: one flat world, full avalanche, no linear bits. Also what d = 0 collapses to. + // Width-free: with bits_ == 0 no plane is ever read. static constexpr auto splitmix(size_t flat_world) noexcept -> Router { return Router{flat_world, 1, 0}; } + // The clamp on its own, for callers that need the resolved d without a monomial width. + static constexpr auto clamp_bits(size_t ranks, size_t requested) noexcept -> size_t { + if (!std::has_single_bit(ranks)) { + return 0; // no XOR structure without a power-of-two rank count + } + const auto max_bits = static_cast(std::countr_zero(ranks)); + return requested < max_bits ? requested : max_bits; + } + [[nodiscard]] constexpr auto ranks() const noexcept -> size_t { return ranks_; } [[nodiscard]] constexpr auto partitions() const noexcept -> size_t { return parts_; } [[nodiscard]] constexpr auto flat_world() const noexcept -> size_t { return flat_; } @@ -150,32 +192,54 @@ class Router final { } const uint64_t part = q % parts_; const uint64_t hi = (q / parts_) & hi_mask_; // ranks_>>bits_ is a power of two, so a mask - const uint64_t lin = linear_hash<2 * NumModes>(mono) & lin_mask_; - return static_cast(((lin | (hi << bits_)) * parts_) + part); + return static_cast(((linear_low_(mono) | (hi << bits_)) * parts_) + part); } // The rank-level shift a generator induces: rank(M^G) low bits == rank(M) low bits ^ shift(G). // Zero for every G iff bits_ == 0. This is what makes the destination predictable. template [[nodiscard]] auto rank_shift(const Monomial &gen) const noexcept -> size_t { - return static_cast(linear_hash<2 * NumModes>(gen) & lin_mask_); + return static_cast(linear_low_(gen)); } private: - static constexpr auto clamp_bits_(size_t ranks, size_t requested) noexcept -> size_t { - if (!std::has_single_bit(ranks)) { - return 0; // no XOR structure without a power-of-two rank count + // ranks x partitions == the flat world the destinations index. linear_bits is clamped here, so a + // caller may pass anything. Private: a router with bits_ > 0 must go through for_modes. + constexpr Router(size_t ranks, size_t partitions, size_t linear_bits) noexcept + : ranks_(ranks == 0 ? 1 : ranks), + parts_(partitions == 0 ? 1 : partitions), + flat_(ranks_ * parts_), + bits_(clamp_bits(ranks_, linear_bits)), + hi_mask_((ranks_ >> bits_) - 1) {} + + // linear_hash(M) & lin_mask, one output bit per plane: parity(popcount(M & plane_j)). Folding the + // words with XOR before the popcount is the same parity (popcount(x)+popcount(y) == popcount(x^y) + // mod 2) for one popcount per bit instead of one per word. bits_ == 0 reads no plane, so a + // splitmix router needs none. + template + [[nodiscard]] [[gnu::always_inline]] inline auto linear_low_(const Monomial &m) const noexcept + -> uint64_t { + constexpr size_t kW = kPlaneWords<2 * NumModes>; + assert(bits_ == 0 || (planes_ != nullptr && plane_words_ == kW)); // bound at a different width + uint64_t acc = 0; + for (size_t j = 0; j < bits_; ++j) { + const uint64_t *plane = planes_ + (j * kW); + uint64_t fold = 0; + for (size_t w = 0; w < kW; ++w) { + fold ^= m.word(w) & plane[w]; + } + acc |= static_cast(std::popcount(fold) & 1) << j; } - const auto max_bits = static_cast(std::countr_zero(ranks)); - return requested < max_bits ? requested : max_bits; + return acc; } size_t ranks_; size_t parts_; size_t flat_; size_t bits_; - uint64_t lin_mask_; uint64_t hi_mask_; + const uint64_t *planes_ = nullptr; // [kLinearPlanes x plane_words_], owned by linear_planes() + size_t plane_words_ = 0; }; // The requested linear-bit count, before clamping to a particular geometry. Parsed once. @@ -195,8 +259,15 @@ inline auto requested_linear_bits() -> size_t { return ~size_t{0}; } +template inline auto make_router(size_t ranks, size_t partitions) -> Router { - return Router{ranks, partitions, requested_linear_bits()}; + return Router::for_modes(ranks, partitions, requested_linear_bits()); +} + +// The resolved d for a geometry, without binding a width: the routing-agreement digest needs the +// number and nothing else. +inline auto linear_bits_for(size_t ranks) -> size_t { + return Router::clamp_bits(ranks == 0 ? 1 : ranks, requested_linear_bits()); } } // namespace monoprop::routing diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp index 2c5bb68e..0203065d 100644 --- a/cpp/tests/routing_tests.cpp +++ b/cpp/tests/routing_tests.cpp @@ -22,6 +22,7 @@ #include +#include #include #include #include @@ -39,6 +40,35 @@ namespace { constexpr size_t kN = 64; // 2N = 128 bits -> 2 words, so the multi-word hash path is exercised +// Exactly `weight` distinct bits, so a case can pin the popcount the old bit-walk paid for. +auto monomials_of_weight(size_t count, size_t weight, uint64_t seed) -> std::vector> { + std::mt19937_64 rng(seed); + std::uniform_int_distribution slot(0, 2 * kN - 1); + std::vector> out; + out.reserve(count); + for (size_t i = 0; i < count; ++i) { + Monomial m; + while (m.count() < weight) { + m.set(slot(rng)); + } + out.push_back(m); + } + return out; +} + +// The map Router::dest computed before the basis was transposed: load one 64-bit vector per SET bit and +// XOR. Rebuilt from mix64 and the seed rather than read from linear_basis(), so the plane build and this +// share nothing but the specification. +template +auto linear_hash_reference(const monoprop::Bitset &bits) -> uint64_t { + const uint64_t seed = routing::seed_from_env(); + uint64_t h = 0; + for (size_t i = bits.find_first(); i < NumBits; i = bits.find_next(i)) { + h ^= routing::mix64(routing::mix64(seed) + (static_cast(i) * 0x9E37'79B9'7F4A'7C15ULL)); + } + return h; +} + auto random_monomials(size_t count, size_t weight, uint64_t seed) -> std::vector> { std::mt19937_64 rng(seed); std::uniform_int_distribution slot(0, 2 * kN - 1); @@ -76,7 +106,7 @@ BOOST_AUTO_TEST_CASE(routing_zero_bits_is_bit_identical_to_splitmix) { BOOST_AUTO_TEST_CASE(routing_zero_bits_two_level_collapses_to_flat_modulo) { const auto monos = random_monomials(300, 6, 0xBEEF01ULL); for (const auto [r, s] : {std::pair{8, 14}, {4, 28}, {128, 14}, {64, 28}}) { - const Router router{r, s, 0}; + const auto router = Router::for_modes(r, s, 0); for (const auto &m : monos) { BOOST_TEST(router.dest(m) == monomial_hash(m) % (r * s)); } @@ -100,7 +130,7 @@ BOOST_AUTO_TEST_CASE(routing_linear_hash_is_gf2_linear) { BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_at_full_bits) { constexpr size_t kRanks = 16; constexpr size_t kParts = 14; - const Router router{kRanks, kParts, 64}; // clamped to log2(16) == 4 + const auto router = Router::for_modes(kRanks, kParts, 64); // clamped to log2(16) == 4 BOOST_REQUIRE(router.linear_bits() == 4U); BOOST_REQUIRE(router.fanout() == 1U); @@ -121,7 +151,7 @@ BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_at_full_bits) { BOOST_AUTO_TEST_CASE(routing_fanout_is_one_at_full_bits) { constexpr size_t kRanks = 8; constexpr size_t kParts = 14; - const Router router{kRanks, kParts, 3}; + const auto router = Router::for_modes(kRanks, kParts, 3); const auto terms = random_monomials(4000, 6, 0xCCCC03ULL); const auto gens = random_monomials(12, 4, 0xDDDD04ULL); @@ -153,7 +183,7 @@ BOOST_AUTO_TEST_CASE(routing_partial_bits_give_fanout_ranks_over_two_to_the_d) { const auto terms = random_monomials(20000, 6, 0xEEEE05ULL); const auto gen = random_monomials(1, 4, 0xFFFF06ULL).front(); for (size_t d = 0; d <= 5; ++d) { - const Router router{kRanks, kParts, d}; + const auto router = Router::for_modes(kRanks, kParts, d); BOOST_TEST(router.fanout() == (kRanks >> d)); std::vector> dest_of(kRanks); for (const auto &m : terms) { @@ -170,11 +200,11 @@ BOOST_AUTO_TEST_CASE(routing_partial_bits_give_fanout_ranks_over_two_to_the_d) { // to today's routing rather than silently produce a lopsided or out-of-range slot. BOOST_AUTO_TEST_CASE(routing_non_power_of_two_ranks_falls_back_to_zero_bits) { for (const size_t r : {size_t{3}, size_t{7}, size_t{12}, size_t{112}}) { - const Router router{r, 14, 8}; + const auto router = Router::for_modes(r, 14, 8); BOOST_TEST(router.linear_bits() == 0U); BOOST_TEST(router.fanout() == r); } - const Router pow2{64, 14, 8}; + const auto pow2 = Router::for_modes(64, 14, 8); BOOST_TEST(pow2.linear_bits() == 6U); // clamped to log2(64), not 8 } @@ -182,7 +212,7 @@ BOOST_AUTO_TEST_CASE(routing_dest_is_in_range_and_deterministic) { const auto monos = random_monomials(1000, 7, 0x9999ULL); for (const auto [r, s] : {std::pair{1, 1}, {1, 112}, {8, 14}, {128, 14}, {64, 28}}) { for (size_t d = 0; d <= 7; ++d) { - const Router router{r, s, d}; + const auto router = Router::for_modes(r, s, d); for (const auto &m : monos) { const size_t slot = router.dest(m); BOOST_TEST(slot < r * s); @@ -192,6 +222,60 @@ BOOST_AUTO_TEST_CASE(routing_dest_is_in_range_and_deterministic) { } } +// The transposed basis must be the SAME map, not merely a faster one: a divergence is a silently wrong +// owner. Pin dest() and rank_shift() against the old bit-walk over the geometries the dial spans -- +// d = 0, d < log2(R), d == log2(R) -- and over popcounts from empty to full support. +BOOST_AUTO_TEST_CASE(routing_transposed_basis_is_bit_identical_to_the_bit_walk) { + std::vector> monos; + for (const size_t w : {size_t{0}, + size_t{1}, + size_t{2}, + size_t{3}, + size_t{5}, + size_t{8}, + size_t{13}, + size_t{21}, + size_t{34}, + size_t{2 * kN}}) { + const auto batch = monomials_of_weight(500, w, 0xD15EA5E0ULL + w); + monos.insert(monos.end(), batch.begin(), batch.end()); + } + BOOST_REQUIRE_EQUAL(monos.size(), 5000U); + + // (R, S, d). log2(R) is 7, 4, 6, 3, 10, 1, 12, 5 respectively, so both d < log2(R) and d == log2(R) + // appear, as does d = 0. + const std::vector> geometries{ + {128, 14, 0}, {128, 14, 1}, {128, 14, 3}, {128, 14, 6}, {128, 14, 7}, {16, 1, 0}, {16, 1, 2}, {16, 1, 4}, + {64, 28, 1}, {64, 28, 5}, {64, 28, 6}, {8, 14, 0}, {8, 14, 1}, {8, 14, 2}, {8, 14, 3}, {1024, 1, 5}, + {1024, 1, 10}, {2, 112, 0}, {2, 112, 1}, {4096, 16, 12}, {32, 3, 4}, {32, 3, 5}}; + + size_t checked = 0; + for (const auto &[r, s, d] : geometries) { + const auto router = Router::for_modes(r, s, d); + BOOST_REQUIRE_EQUAL(router.linear_bits(), d); // no clamping in this table + const uint64_t lin_mask = d == 0 ? 0ULL : (uint64_t{1} << d) - 1; + for (const auto &m : monos) { + const uint64_t q = monomial_hash(m); + size_t expected = 0; + if (d == 0) { + expected = static_cast(q % (r * s)); + } + else { + const uint64_t part = q % s; + const uint64_t hi = (q / s) % (r >> d); + const uint64_t lin = linear_hash_reference<2 * kN>(m) & lin_mask; + expected = static_cast(((lin | (hi << d)) * s) + part); + } + BOOST_REQUIRE_EQUAL(router.dest(m), expected); + BOOST_REQUIRE_EQUAL(router.rank_shift(m), + static_cast(linear_hash_reference<2 * kN>(m) & lin_mask)); + ++checked; + } + } + BOOST_TEST_MESSAGE("bit-identity checks: " << checked); + BOOST_TEST(checked >= 100000U); +} + // gf2_rank is the coverage diagnostic: shifts that span fewer than log2(R) dimensions leave ranks empty. BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { BOOST_TEST(routing::gf2_rank({}) == 0U); @@ -202,7 +286,7 @@ BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { // The real generator shifts must span at least log2(R) dimensions or the reachable ranks are a // strict subspace of the rank space. constexpr size_t kRanks = 128; - const Router router{kRanks, 14, 7}; + const auto router = Router::for_modes(kRanks, 14, 7); std::vector shifts; for (const auto &g : random_monomials(200, 4, 0x7777ULL)) { shifts.push_back(static_cast(router.rank_shift(g))); @@ -224,12 +308,12 @@ BOOST_AUTO_TEST_CASE(routing_default_is_linear_where_the_geometry_allows_it) { BOOST_TEST_MESSAGE("routing overridden in the environment; default not under test"); return; } - BOOST_TEST(routing::make_router(8, 14).fanout() == 1U); - BOOST_TEST(routing::make_router(128, 14).fanout() == 1U); - BOOST_TEST(routing::make_router(1, 112).fanout() == 1U); // single rank: nothing to route between + BOOST_TEST(routing::make_router(8, 14).fanout() == 1U); + BOOST_TEST(routing::make_router(128, 14).fanout() == 1U); + BOOST_TEST(routing::make_router(1, 112).fanout() == 1U); // single rank: nothing to route between // 6 and 12 are not powers of two: no XOR structure, so Router clamps to d = 0 and every rank // stays reachable through splitmix rather than a subspace of them. - BOOST_TEST(routing::make_router(6, 14).fanout() == 6U); - BOOST_TEST(routing::make_router(12, 28).fanout() == 12U); + BOOST_TEST(routing::make_router(6, 14).fanout() == 6U); + BOOST_TEST(routing::make_router(12, 28).fanout() == 12U); } From f5428a48ce77fce24950f62951458b9c502b3189 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 14:53:37 +0100 Subject: [PATCH 13/24] =?UTF-8?q?perf(mpi):=20=E2=9A=A1=20complete=20the?= =?UTF-8?q?=20sparse=20exchange=20through=20the=20handle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dense branch posted MPI_Ialltoallv and completed it in wait_into while the sparse branch waited inline, so the two transports differed in a way nothing in the signature showed. The handle now carries the request set and waits with the rest; MPI reads send_buffer and recv_buffer until those complete, and both move with the handle, so the pointers stay good. No win is claimed: both consumers call wait_into immediately, so there is nothing to overlap yet. This is the seam that a later overlap needs, and one path fewer to reason about. Sizing no longer sweeps [0, R) three times per exchange: counts and their prefix fold into one pass, and the known-recv mask copies the f peer blocks into a freshly zeroed array rather than copying all R and zeroing the remainder. At R=128 this is noise; at 4096, against ~1,400 exchanges per rank per layer, it is not. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/mpi/HybridComm.h | 51 +++++++++-------- cpp/monoprop/detail/mpi/MPICompat.cpp | 27 +++++---- cpp/monoprop/detail/mpi/MPICompat.h | 82 +++++++++++++++------------ cpp/monoprop/detail/mpi/Pairwise.h | 32 ++++++----- 4 files changed, 105 insertions(+), 87 deletions(-) diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 7b7d5076..ee507fe1 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -480,18 +480,19 @@ class HybridComm { return; } const PeerLayout blocks{.block = block}; - sparse_pairwise(plan, - mpi_rank_, - r_, - parent_, - kHybridCountTag, - MPI_INT, - sizeof(int), - reinterpret_cast(counts_send_.data()), - blocks, - reinterpret_cast(counts_recv_.data()), - blocks, - reqs_); + const int posted = sparse_pairwise(plan, + mpi_rank_, + r_, + parent_, + kHybridCountTag, + MPI_INT, + sizeof(int), + reinterpret_cast(counts_send_.data()), + blocks, + reinterpret_cast(counts_recv_.data()), + blocks, + reqs_); + MPI_Waitall(posted, reqs_.data(), MPI_STATUSES_IGNORE); } // The staged payload: one MPI_Alltoallv when dense, else a pair per peer over the same per-rank @@ -510,18 +511,20 @@ class HybridComm { parent_); return; } - sparse_pairwise(plan, - mpi_rank_, - r_, - parent_, - kHybridPayloadTag, - dt, - elem, - stage_send_.data(), - PeerLayout{.counts = mpi_send_counts_.data(), .displs = mpi_send_displs_.data()}, - stage_recv_.data(), - PeerLayout{.counts = mpi_recv_counts_.data(), .displs = mpi_recv_displs_.data()}, - reqs_); + const int posted = + sparse_pairwise(plan, + mpi_rank_, + r_, + parent_, + kHybridPayloadTag, + dt, + elem, + stage_send_.data(), + PeerLayout{.counts = mpi_send_counts_.data(), .displs = mpi_send_displs_.data()}, + stage_recv_.data(), + PeerLayout{.counts = mpi_recv_counts_.data(), .displs = mpi_recv_displs_.data()}, + reqs_); + MPI_Waitall(posted, reqs_.data(), MPI_STATUSES_IGNORE); } template diff --git a/cpp/monoprop/detail/mpi/MPICompat.cpp b/cpp/monoprop/detail/mpi/MPICompat.cpp index 0da7b82e..c9285c40 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.cpp +++ b/cpp/monoprop/detail/mpi/MPICompat.cpp @@ -136,19 +136,22 @@ auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, MPI_Comm_rank(comm.mpi, &me); std::fill(recv_counts, recv_counts + n, 0); const PeerLayout one{.block = 1}; + // Eager by contract: recv_counts is caller memory the caller reads on return, so unlike the + // payload round this one cannot be handed on in a handle. std::vector reqs; - sparse_pairwise(plan, - me, - n, - comm.mpi, - kFlatCountTag, - MPI_INT, - sizeof(int), - reinterpret_cast(send_counts), - one, - reinterpret_cast(recv_counts), - one, - reqs); + const int posted = sparse_pairwise(plan, + me, + n, + comm.mpi, + kFlatCountTag, + MPI_INT, + sizeof(int), + reinterpret_cast(send_counts), + one, + reinterpret_cast(recv_counts), + one, + reqs); + MPI_Waitall(posted, reqs.data(), MPI_STATUSES_IGNORE); return; } (void)n; diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index a3575616..82ca7b15 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -137,7 +137,10 @@ struct PendingAlltoallv { std::vector send_buffer; std::vector recv_buffer; #ifdef monoprop_ENABLE_MPI - MPI_Request request = MPI_REQUEST_NULL; // set only on the Kind::Mpi async path + MPI_Request request = MPI_REQUEST_NULL; // set only on the Kind::Mpi dense async path + std::vector requests; // the Kind::Mpi sparse path's pairs; `posted` of them live + int posted = 0; // MPI reads send_buffer/recv_buffer until these complete, + // and both move with the handle, so the pointers hold #endif auto wait_into(std::vector> &recv_data) -> void { @@ -146,6 +149,10 @@ struct PendingAlltoallv { MPI_Wait(&request, MPI_STATUS_IGNORE); request = MPI_REQUEST_NULL; } + if (posted != 0) { + MPI_Waitall(posted, requests.data(), MPI_STATUSES_IGNORE); + posted = 0; + } #endif recv_data.resize(static_cast(num_ranks)); for (int i = 0; i < num_ranks; ++i) { @@ -180,21 +187,17 @@ inline auto begin_alltoallv(const std::vector> &send_data, h.recv_displs.resize(static_cast(num_ranks)); const int self = skip_self ? rank(comm) : -1; - // Wide accumulator + checked narrowing: a wrapped count would size send_buffer short and then feed - // MPI a negative count/displacement. - long long total_send = 0; + // Counts and their prefix in ONE sweep. Wide accumulator + checked narrowing: a wrapped count would + // size send_buffer short and then feed MPI a negative count/displacement. + long long running_send = 0; for (int i = 0; i < num_ranks; ++i) { const size_t n = (i == self) ? 0 : send_data[static_cast(i)].size(); const int c = checked_mpi_count(n, "Send count"); h.send_counts[static_cast(i)] = c; - total_send += c; - } - long long running_send = 0; - for (int i = 0; i < num_ranks; ++i) { h.send_displs[static_cast(i)] = checked_mpi_count(running_send, "Send displacement"); - running_send += h.send_counts[static_cast(i)]; + running_send += c; } - h.send_buffer.resize(static_cast(checked_mpi_count(total_send, "Total send count"))); + h.send_buffer.resize(static_cast(checked_mpi_count(running_send, "Total send count"))); for (int i = 0; i < num_ranks; ++i) { const int c = h.send_counts[static_cast(i)]; if (c == 0) { @@ -228,25 +231,29 @@ inline auto begin_alltoallv(const std::vector> &send_data, #endif if (known_recv_counts != nullptr) { - std::copy( - known_recv_counts->begin(), - known_recv_counts->begin() + std::min(known_recv_counts->size(), static_cast(num_ranks)), - h.recv_counts.begin()); - if (self >= 0) { - h.recv_counts[static_cast(self)] = 0; - } + const auto avail = + static_cast(std::min(known_recv_counts->size(), static_cast(num_ranks))); // Mask the caller's array through the plan, as alltoall_counts already does for the counts it // exchanges: no receive is ever posted for a non-peer, so a non-zero count there sizes - // recv_buffer for bytes nothing writes and wait_into hands the caller uninitialised memory. - if (!plan.dense()) { + // recv_buffer for bytes nothing writes and wait_into hands the caller uninitialised memory. Done + // by copying only the f peer blocks -- recv_counts is freshly zeroed, so the rest is the mask. + if (plan.dense()) { + std::copy_n(known_recv_counts->begin(), avail, h.recv_counts.begin()); + } + else { const auto geom = geometry(comm); const int me = rank(comm) / geom.partitions; - for (int g = 0; g < num_ranks; ++g) { - if (!plan.contains(me, g / geom.partitions)) { - h.recv_counts[static_cast(g)] = 0; + const int f = plan.count(geom.ranks); + for (int k = 0; k < f; ++k) { + const int base = plan.peer(me, k) * geom.partitions; + for (int t = 0; t < geom.partitions && base + t < avail; ++t) { + h.recv_counts[static_cast(base + t)] = (*known_recv_counts)[static_cast(base + t)]; } } } + if (self >= 0) { + h.recv_counts[static_cast(self)] = 0; + } } else { alltoall_counts(h.send_counts.data(), h.recv_counts.data(), num_ranks, comm, plan); @@ -298,21 +305,22 @@ inline auto begin_alltoallv(const std::vector> &send_data, &h.request); } else { - // S == 1 world: the same pairing as the Hybrid path, one message per reachable peer. Blocking - // here rather than through the Ticket, because the request set is per-peer, not one handle. - std::vector reqs; - sparse_pairwise(plan, - rank(comm), - num_ranks, - comm.mpi, - kFlatPayloadTag, - datatype::get(), - sizeof(T), - reinterpret_cast(h.send_buffer.data()), - PeerLayout{.counts = h.send_counts.data(), .displs = h.send_displs.data()}, - reinterpret_cast(h.recv_buffer.data()), - PeerLayout{.counts = h.recv_counts.data(), .displs = h.recv_displs.data()}, - reqs); + // S == 1 world: the same pairing as the Hybrid path, one message per reachable peer, left + // in flight in the handle exactly as MPI_Ialltoallv is. The buffers MPI holds live in `h` + // and travel with it: a vector move keeps its heap block, so returning `h` moves nothing + // MPI is reading. + h.posted = sparse_pairwise(plan, + rank(comm), + num_ranks, + comm.mpi, + kFlatPayloadTag, + datatype::get(), + sizeof(T), + reinterpret_cast(h.send_buffer.data()), + PeerLayout{.counts = h.send_counts.data(), .displs = h.send_displs.data()}, + reinterpret_cast(h.recv_buffer.data()), + PeerLayout{.counts = h.recv_counts.data(), .displs = h.recv_displs.data()}, + h.requests); } #else h.recv_buffer = h.send_buffer; // single participant: self round-trip (layouts identical) diff --git a/cpp/monoprop/detail/mpi/Pairwise.h b/cpp/monoprop/detail/mpi/Pairwise.h index 70ca5ac4..970d4fb9 100644 --- a/cpp/monoprop/detail/mpi/Pairwise.h +++ b/cpp/monoprop/detail/mpi/Pairwise.h @@ -52,20 +52,24 @@ struct PeerLayout { // A variable all-to-all as point-to-point over `plan`'s peers: one Irecv/Isend pair each, the self peer // copied in place. Counts and displacements are in ELEMENTS of `dt`, whose extent must be `elem`. -// `reqs` is caller scratch, grown then INDEXED: MPI holds these pointers until Waitall, so a +// `reqs` is caller storage, grown then INDEXED: MPI holds these pointers until the wait, so a // reallocating push_back would dangle them. -inline auto sparse_pairwise(PeerPlan plan, - int me, - int n_ranks, - MPI_Comm comm, - int tag, - MPI_Datatype dt, - size_t elem, - const std::byte *send, - PeerLayout send_lay, - std::byte *recv, - PeerLayout recv_lay, - std::vector &reqs) -> void { +// +// POSTS ONLY, and returns how many of `reqs` are live. The caller waits, so `send`, `recv` and `reqs` +// must all outlive that wait -- which is what lets a caller hold the round open (PendingAlltoallv) the +// same way the dense branch holds an MPI_Ialltoallv. +[[nodiscard]] inline auto sparse_pairwise(PeerPlan plan, + int me, + int n_ranks, + MPI_Comm comm, + int tag, + MPI_Datatype dt, + size_t elem, + const std::byte *send, + PeerLayout send_lay, + std::byte *recv, + PeerLayout recv_lay, + std::vector &reqs) -> int { const int f = plan.count(n_ranks); if (reqs.size() < static_cast(2 * f)) { reqs.resize(static_cast(2 * f)); @@ -92,7 +96,7 @@ inline auto sparse_pairwise(PeerPlan plan, MPI_Isend(sbuf, sc, dt, b, tag, comm, &reqs[static_cast(n_req++)]); } } - MPI_Waitall(n_req, reqs.data(), MPI_STATUSES_IGNORE); + return n_req; } } // namespace monoprop::mpi From 22ad70491c34f190f1a6903d9347ab487079b2ba Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 14:53:54 +0100 Subject: [PATCH 14/24] =?UTF-8?q?test(mpi):=20=E2=9C=85=20reach=20the=20sp?= =?UTF-8?q?arse=20paths=20a=20single=20peer=20cannot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both sparse cases pinned plan.count(R) == 1, so every `for k in [0, f)` in the transport had only ever run one iteration -- and the rank list defaulted to 2, where a plan with any linear bits can resolve only one peer, so no amount of local testing would have reached f > 1 either. Adding 4 to the list is what makes the multi-peer cases runnable at all. Four gaps closed: several peers, where peer-ordered blocks interleave with the [0, R) prefix sums in the staging sizers; an empty leg, so the zero-count skip is taken on one side only -- the asymmetry that deadlocks; skip_self under shift 0, which is the self-peer slot; and two rounds back to back on one communicator, which is the pattern Pairwise.h's non-overtaking argument claims is safe, now asserted rather than argued. The scan/find_rank floors count per router. They were written for one loop and kept when a second router was added, so each arm's floor was really the pair's. Measured, all six routers agree on a total of 387 while the split runs 196/191 at R=2 to 335/52 at R=8 -- the partner count belongs to the operator and the gate, and routing only moves a partner between the cross-rank and self-owned side. That invariance is the assertion now; the floors only catch a scan that emitted nothing. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/tests/boost-test.cmake | 4 +- cpp/tests/hybrid_comm_tests.cpp | 281 ++++++++++++++++++++++++++++++++ cpp/tests/mpi_utils_tests.cpp | 41 +++-- 3 files changed, 312 insertions(+), 14 deletions(-) diff --git a/cpp/tests/boost-test.cmake b/cpp/tests/boost-test.cmake index 7d0d52c9..19bf4828 100644 --- a/cpp/tests/boost-test.cmake +++ b/cpp/tests/boost-test.cmake @@ -1,6 +1,8 @@ +# 4 as well as 2: at R = 2 the peer plan can only ever resolve one peer, so every `for k in [0, f)` in +# the sparse transport stays a single iteration and the f > 1 cases never run. set( monoprop_MPI_TEST_PROCS - "2" + "2;4" CACHE STRING "Semicolon-separated list of ranks for MPI test variants" ) diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index 26b9622c..c6d26e74 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -661,4 +661,285 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_known_recv_counts_are_masked_through_the_plan) } } +namespace { + +// Varies along BOTH ends and hits 0, so a block landing on the wrong peer or the wrong partition +// changes a length, not just a value. +auto sparse_count(int src, int dst) -> int { + return ((src * 3) + (dst * 5)) % 4; +} +auto sparse_tag(int src, int dst, int j) -> int { + return (((src * 128) + dst) * 1000) + j; +} + +} // namespace + +// f > 1. Both sparse cases above pin bits == log2(R), so `plan.count(R)` is 1 and every `for k in +// [0, f)` in the sparse path has only ever run once -- the interleaving of peer-ordered blocks with the +// [0, R)-ordered prefix sums in size_staging_send_ / size_staging_recv_ is what that leaves unchecked. +// bits < log2(R) is the only way to reach it, and a mis-indexed prefix shows up as a block delivered at +// the wrong offset, i.e. a wrong tag, not a hang. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_with_several_peers) { + const int R = world_size(); + if (R < 4 || (R & (R - 1)) != 0) { + return; + } + const int full = std::countr_zero(static_cast(R)); + const int me = world_rank(); + int cases = 0; + for (const int f : {2, 4}) { + const int bits = full - std::countr_zero(static_cast(f)); + if (bits < 1) { + continue; // bits == 0 is the dense path, which these cases are not about + } + for (int shift = 0; shift < (1 << bits); ++shift) { + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + BOOST_REQUIRE_EQUAL(plan.count(R), f); + ++cases; + + // The S == 1 world first: the plain-MPI Isend/Irecv branch, no staging in the way. + { + Comm c{MPI_COMM_WORLD}; + std::vector> send(static_cast(R)); + for (int k = 0; k < f; ++k) { + const int b = plan.peer(me, k); + for (int j = 0; j < sparse_count(me, b); ++j) { + send[static_cast(b)].push_back(sparse_tag(me, b, j)); + } + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); + for (int src = 0; src < R; ++src) { + const int want = plan.contains(me, src) ? sparse_count(src, me) : 0; + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), want); + for (int j = 0; j < want; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], + sparse_tag(src, me, j)); + } + } + } + + // Then the staged HybridComm path, where the peer-ordered sweeps live. + for (const int S : {1, 2, 3}) { + const int P = R * S; + std::vector>> recv(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + Comm c = Comm::make_hybrid(&hyb, u); + const int g = monoprop::mpi::rank(c); + std::vector> send(static_cast(P)); + for (int k = 0; k < f; ++k) { + const int b = plan.peer(me, k); + for (int t = 0; t < S; ++t) { + const int d = (b * S) + t; + for (int j = 0; j < sparse_count(g, d); ++j) { + send[static_cast(d)].push_back(sparse_tag(g, d, j)); + } + } + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); + recv[static_cast(u)] = out; + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const int g = (me * S) + t; + const auto &out = recv[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int src = 0; src < P; ++src) { + const int want = plan.contains(me, src / S) ? sparse_count(src, g) : 0; + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), want); + for (int j = 0; j < want; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], + sparse_tag(src, g, j)); + } + } + } + } + } + } + BOOST_TEST(cases > 0); // at R < 4 the case is a no-op and must not read as coverage +} + +// A zero-count leg is where a send/recv posting asymmetry deadlocks rather than mis-delivers: both ends +// must skip on the SAME value. Nothing above ever sends an empty block over a real message, so force +// one -- the lower-numbered end of every pair sends nothing while its peer sends four. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_with_an_empty_leg) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + const int me = world_rank(); + constexpr int kLen = 4; + for (int shift = 1; shift < R; ++shift) { // shift 0 is the self peer, covered separately + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const int peer = plan.peer(me, 0); + BOOST_REQUIRE(peer != me); + const int my_len = me < peer ? 0 : kLen; // exactly one end of the pair is silent + const int peer_len = peer < me ? 0 : kLen; + + { + Comm c{MPI_COMM_WORLD}; + std::vector> send(static_cast(R)); + for (int j = 0; j < my_len; ++j) { + send[static_cast(peer)].push_back(sparse_tag(me, peer, j)); + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(peer)].size()), peer_len); + for (int j = 0; j < peer_len; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(peer)][static_cast(j)], sparse_tag(peer, me, j)); + } + } + + for (const int S : {1, 2}) { + const int P = R * S; + std::vector>> recv(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + Comm c = Comm::make_hybrid(&hyb, u); + const int g = monoprop::mpi::rank(c); + std::vector> send(static_cast(P)); + for (int t = 0; t < S; ++t) { + const int d = (peer * S) + t; + for (int j = 0; j < my_len; ++j) { + send[static_cast(d)].push_back(sparse_tag(g, d, j)); + } + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); + recv[static_cast(u)] = out; + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const int g = (me * S) + t; + const auto &out = recv[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int su = 0; su < S; ++su) { + const auto &blk = out[static_cast((peer * S) + su)]; + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), peer_len); + for (int j = 0; j < peer_len; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag((peer * S) + su, g, j)); + } + } + } + } + } +} + +// shift == 0 makes every rank its own and only peer, so the sparse path's self slot is the whole +// exchange -- and skip_self then removes the one leg that would have moved anything for a partition. +// The remaining S-1 in-rank legs must still arrive. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_skip_self_at_shift_zero) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + const int me = world_rank(); + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = 0}; + BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); + + { + Comm c{MPI_COMM_WORLD}; // the self peer is the ONLY peer, and skip_self drops it + std::vector> send(static_cast(R)); + for (int j = 0; j < 5; ++j) { + send[static_cast(me)].push_back(sparse_tag(me, me, j)); + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, /*skip_self=*/true, nullptr, plan).wait_into(out); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); + for (const auto &blk : out) { + BOOST_CHECK(blk.empty()); + } + } + + for (const int S : {2, 3}) { + const int P = R * S; + std::vector>> recv(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + Comm c = Comm::make_hybrid(&hyb, u); + const int g = monoprop::mpi::rank(c); + std::vector> send(static_cast(P)); + for (int t = 0; t < S; ++t) { + const int d = (me * S) + t; + for (int j = 0; j <= t; ++j) { + send[static_cast(d)].push_back(sparse_tag(g, d, j)); + } + } + std::vector> out; + monoprop::mpi::begin_alltoallv(send, c, /*skip_self=*/true, nullptr, plan).wait_into(out); + recv[static_cast(u)] = out; + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const int g = (me * S) + t; + const auto &out = recv[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int src = 0; src < P; ++src) { + const bool in_rank = (src / S) == me; + const int want = (in_rank && src != g) ? t + 1 : 0; // own slot dropped by skip_self + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), want); + for (int j = 0; j < want; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], sparse_tag(src, g, j)); + } + } + } + } +} + +// Engine.h's run_exchange posts both its rounds on ONE communicator under kFlatPayloadTag, and the +// non-overtaking argument in Pairwise.h is what says a round-2 receive cannot match a round-1 send. +// Two back-to-back sparse rounds, the second on the transpose of the first's counts, assert it. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_back_to_back_rounds) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + const int me = world_rank(); + Comm c{MPI_COMM_WORLD}; + for (int shift = 0; shift < R; ++shift) { + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const int peer = plan.peer(me, 0); + const int len = 3 + (me % 2); // asymmetric, so a swapped round is a length mismatch + + std::vector> q(static_cast(R)); + for (int j = 0; j < len; ++j) { + q[static_cast(peer)].push_back(sparse_tag(me, peer, j)); + } + std::vector> q_out; + monoprop::mpi::begin_alltoallv(q, c, false, nullptr, plan).wait_into(q_out); + + // Round 2 immediately, same comm and tag, sized from round 1's transpose -- the response shape. + std::vector known(static_cast(R), 0); + known[static_cast(peer)] = len; + std::vector> r(static_cast(R)); + const int back = static_cast(q_out[static_cast(peer)].size()); + for (int j = 0; j < back; ++j) { + r[static_cast(peer)].push_back(q_out[static_cast(peer)][static_cast(j)] + 7); + } + std::vector> r_out; + monoprop::mpi::begin_alltoallv(r, c, false, &known, plan).wait_into(r_out); + + BOOST_REQUIRE_EQUAL(back, 3 + (peer % 2)); + BOOST_REQUIRE_EQUAL(static_cast(r_out[static_cast(peer)].size()), len); + for (int j = 0; j < len; ++j) { + BOOST_CHECK_EQUAL(r_out[static_cast(peer)][static_cast(j)], sparse_tag(me, peer, j) + 7); + } + for (int src = 0; src < R; ++src) { + if (src != peer) { + BOOST_CHECK(r_out[static_cast(src)].empty()); + } + } + } +} + #endif // monoprop_ENABLE_MPI diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 3bb9ba04..604540fd 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -174,15 +174,23 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { std::nullopt, std::optional{0.3}); - size_t checked = 0; - size_t self_checked = 0; + // The partner count is a property of the operator and the gate, not of where the partners live, so + // it is the same for every router; routing only moves a partner between the encoded (cross-rank) + // and staged (self-owned) side. Pinning that invariance is stronger than a floor: a routing bug + // that drops partners moves the total, and one that misroutes them moves the split. + size_t routers = 0; + std::optional first_total; for (const size_t ranks : {2U, 4U, 8U}) { // BOTH routers, because the agreement is a property of the pair and not of either hash: the // scan calls Router::dest and find_rank calls the same Router, so a divergence introduced by // one of them shows up here whichever routing the geometry resolves to. bits=~0 asks for as // many linear bits as log2(ranks) allows, i.e. fanout 1. for (const size_t bits : {size_t{0}, ~size_t{0}}) { - const routing::Router router{ranks, /*partitions=*/1, bits}; + // Per router, not summed over them: the floors are what stops the loop passing on an empty + // scan, and a sum lets one router carry the other. + size_t checked = 0; + size_t self_checked = 0; + const auto router = routing::Router::for_modes(ranks, /*partitions=*/1, bits); const auto res = detail::fused_find_and_collect>(op, gen, eval, @@ -203,16 +211,23 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { check_bucket_ownership(res.follower_queries, router, checked); check_self_ownership(res.leader_self, router, /*my_rank=*/0, self_checked); check_self_ownership(res.follower_self, router, /*my_rank=*/0, self_checked); + // Measured, all six routers: total 387 every time, with the split running from 196/191 at + // R=2 to 335/52 at R=8 as more partners fall cross-rank. The floors sit below the observed + // minimum of each arm and only catch a scan that emitted nothing. + BOOST_TEST_MESSAGE("ranks=" << ranks << " bits=" << router.linear_bits() << " encoded=" << checked + << " staged=" << self_checked); + const size_t total = checked + self_checked; + if (first_total.has_value()) { + BOOST_TEST(total == *first_total); // routing moves partners, it does not create or lose them + } + else { + first_total = total; + } + BOOST_TEST(total > 300U); + BOOST_TEST(checked > 150U); + BOOST_TEST(self_checked > 40U); + ++routers; } } - // Without this the loop above passes trivially if the scan emitted nothing. The floor is on the SUM - // because that is what is invariant across the split: the encoded counter alone fell to 797 of 1161 - // when the self-owned partners moved into the stage, with nothing going unchecked. Each arm still - // carries its own floor -- a routing bug sending everything one way leaves the sum intact -- and the - // message prints the measured counts so those can be re-grounded rather than guessed. The floors - // are unchanged although the loop now runs twice (one router each): they were never tight. - BOOST_TEST_MESSAGE("encoded=" << checked << " staged=" << self_checked); - BOOST_TEST(checked + self_checked > 1000U); - BOOST_TEST(checked > 500U); - BOOST_TEST(self_checked > 200U); + BOOST_TEST(routers == 6U); // the floors above are per router, so the router count is part of them } From a610f04843a9b2ec7903be826447b7d7a71af739 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 16:23:04 +0100 Subject: [PATCH 15/24] =?UTF-8?q?refactor(routing)!:=20=E2=99=BB=EF=B8=8F?= =?UTF-8?q?=20collapse=20the=20linear-bit=20dial=20to=20a=20boolean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GF(2)-linear rank routing supported any d in [0, log2 R], but only d = 0 (monoprop_ROUTING=splitmix) and d = log2 R (the shipped default) were ever run; the intermediate values existed only to be tested. Router now carries a mode flag: linear takes every rank bit from the hash, splitmix takes none. A rank count that is not a power of two no longer falls back to d = 0 silently under linear routing -- it raises routing::UnroutableGeometry at Router construction. R = 1 is a power of two, takes no rank bit, and stays dense, so every single-rank run keeps the collective transport. PeerPlan follows: {bool sparse; int shift} with one peer, me ^ shift, which removes the 1 << bits signed-shift hazard entirely rather than narrowing it. monoprop_ROUTE_LINEAR_BITS and its parser are gone; monoprop_ROUTING and monoprop_ROUTE_SEED are unchanged, and d = 0 stays bit-identical to monomial_hash(M) % P. BREAKING CHANGE: monoprop_ROUTE_LINEAR_BITS is removed, and a non-power-of-two MPI rank count now raises under the default linear routing instead of falling back to the dense all-to-all. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/include/monoprop/MonomialPropagator.h | 2 +- cpp/monoprop/detail/EnvConfig.h | 21 +- .../detail/evolution/layer_build/Engine.h | 10 +- .../MonomialPropagator.inl | 4 +- cpp/monoprop/detail/mpi/Comm.h | 57 ++---- cpp/monoprop/detail/mpi/MPIUtils.h | 18 +- cpp/monoprop/detail/mpi/Routing.h | 139 +++++++------- cpp/tests/env_config_tests.cpp | 16 +- cpp/tests/hybrid_comm_tests.cpp | 150 ++++----------- cpp/tests/mpi_utils_tests.cpp | 9 +- cpp/tests/routing_tests.cpp | 179 +++++++++--------- docs/content/docs/features/parallelism.mdx | 20 +- 12 files changed, 243 insertions(+), 382 deletions(-) diff --git a/cpp/include/monoprop/MonomialPropagator.h b/cpp/include/monoprop/MonomialPropagator.h index 6edaf6d0..72ead85f 100644 --- a/cpp/include/monoprop/MonomialPropagator.h +++ b/cpp/include/monoprop/MonomialPropagator.h @@ -451,7 +451,7 @@ class MonomialPropagator { const VecD ¶meters, std::optional only_rotate_len_k) -> void; - // Do this call's generator shifts span linear_bits? If not, ranks receive nothing (routing::gf2_rank). + // Do this call's generator shifts span log2(R)? If not, ranks receive nothing (routing::gf2_rank). // Not beside check_routing_agreement: at construction the gate list does not exist yet. auto report_routing_coverage_(const std::vector &majoranas) -> void; diff --git a/cpp/monoprop/detail/EnvConfig.h b/cpp/monoprop/detail/EnvConfig.h index 6c4e755d..81b0786a 100644 --- a/cpp/monoprop/detail/EnvConfig.h +++ b/cpp/monoprop/detail/EnvConfig.h @@ -29,10 +29,9 @@ // monoprop_NUM_THREADS positive int (1..1e6), else ignored → num_threads // monoprop_PARTITIONS int N | "auto" | "off"; parsed where it is used (resolve_partition_count_) // monoprop_ROUTING "splitmix" | "linear" → routing_mode -// monoprop_ROUTE_LINEAR_BITS int in [0, 64], 0 meaning dense → route_linear_bits // monoprop_ROUTE_SEED decimal uint64 basis seed → route_seed // -// The three routing knobs THROW on a malformed value instead of falling back: each silently changes the +// Both routing knobs THROW on a malformed value instead of falling back: each silently changes the // transport, so a typo that defaulted would stay invisible until a performance postmortem. namespace monoprop::config { @@ -84,21 +83,6 @@ inline auto parse_uint64(std::string_view name, const char *text) -> std::option return static_cast(value); } -// 0 is legal here (it means dense routing), so "unset" must stay distinguishable from "0" -- hence the -// optional rather than a sentinel. 64 is the width of the linear hash: no further bits exist to ask for. -inline auto parse_bit_count(std::string_view name, const char *text) -> std::optional { - if (text == nullptr || *text == '\0') { - return std::nullopt; - } - errno = 0; - char *end = nullptr; - const long value = std::strtol(text, &end, 10); - if (end == text || *end != '\0' || errno == ERANGE || value < 0 || value > 64) { - reject_env(name, text, "an integer in [0, 64]"); - } - return static_cast(value); -} - inline auto parse_routing_mode(std::string_view name, const char *text) -> std::optional { if (text == nullptr || *text == '\0') { return std::nullopt; @@ -118,7 +102,6 @@ inline auto parse_routing_mode(std::string_view name, const char *text) -> std:: struct Settings { std::optional num_threads; std::optional routing_mode; - std::optional route_linear_bits; // takes precedence over routing_mode when both are set std::optional route_seed; }; @@ -128,8 +111,6 @@ inline auto get() -> const Settings & { Settings s; s.num_threads = detail::parse_positive_int(std::getenv("monoprop_NUM_THREADS")); s.routing_mode = detail::parse_routing_mode("monoprop_ROUTING", std::getenv("monoprop_ROUTING")); - s.route_linear_bits = - detail::parse_bit_count("monoprop_ROUTE_LINEAR_BITS", std::getenv("monoprop_ROUTE_LINEAR_BITS")); s.route_seed = detail::parse_uint64("monoprop_ROUTE_SEED", std::getenv("monoprop_ROUTE_SEED")); return s; }(); diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 0875fb4c..341fc72d 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -609,11 +609,11 @@ auto build_layer(MPOperator &local_op, // R is the FLAT world (ranks x partitions); the router is what splits it back into the two levels. const routing::Router router = router_for(comm); assert(router.flat_world() == R); - // Under linear routing every query for THIS generator lands on a rank whose low `linear_bits` are - // this rank's own XOR rank_shift(gen), so the exchange knows its peers before it starts. Dense - // (bits == 0) otherwise, which is today's collective. - const auto plan = mpi::PeerPlan{.bits = static_cast(router.linear_bits()), - .shift = static_cast(router.rank_shift(gen))}; + // Under linear routing every query for THIS generator lands on the rank this rank's own index XOR + // rank_shift(gen), so the exchange knows its peer before it starts. Dense otherwise, which is + // today's collective. + const auto plan = + mpi::PeerPlan{.sparse = router.is_linear(), .shift = static_cast(router.rank_shift(gen))}; // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index c441103c..e415d6ff 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -729,7 +729,7 @@ auto MonomialPropagator::report_routing_coverage_(const std::vector(comm_); - if (router.linear_bits() == 0) { + if (!router.is_linear()) { return; // splitmix: no subspace to fall short of } std::vector shifts; @@ -755,7 +755,7 @@ auto MonomialPropagator::report_routing_coverage_(const std::vector #include #include #include @@ -66,18 +65,22 @@ struct Comm { } }; -// Which destination RANKS a round can touch, when the caller knows. Under GF(2)-linear routing -// (routing::Router) the low `bits` of the destination rank are determined by the generator: they are -// this rank's own low bits XOR `shift`, so the peers are +// Which destination RANKS a round can touch, when the caller knows. Two states, matching +// routing::Router: dense, or sparse over the single peer GF(2)-linear routing implies. // -// peer(k) = ((me & (2^bits - 1)) ^ shift) | (k << bits), k in [0, ranks >> bits) +// Sparse means the destination rank of every block is determined by the generator: it is this rank's +// own index XOR `shift`, so // -// -- `ranks >> bits` of them instead of all `ranks`, and the relation is symmetric (XOR is an -// involution), so every rank derives the same pairing with no communication. That is what lets a verb -// replace a dense collective with point-to-point over the peers it can actually reach. +// peer = me ^ shift, count == 1 // -// bits == 0 is the dense default: peer(k) == k and count == ranks, so the same loops walk every rank -// and the verbs take their collective path. +// -- one peer instead of all `ranks`, and the relation is symmetric (XOR is an involution), so every +// rank derives the same pairing with no communication. That is what lets a verb replace a dense +// collective with point-to-point. Linear routing takes ALL log2(ranks) rank bits, so there is no +// intermediate fanout to express here. +// +// Dense is the default: peer(k) == k and count == ranks, so the same loops walk every rank and the +// verbs take their collective path. Every single-rank run is dense (Router::is_linear is false at +// R == 1), so the collectives are not a fallback but the common case. // // Two distinct failure modes if `shift` is wrong, which is why the plan is derived in one place. Ranks // that DISAGREE deadlock: the pairing stops being symmetric and someone waits on a send never posted. @@ -85,36 +88,14 @@ struct Comm { // blocks outside the peer set, because pack_count_matrix_ / size_staging_send_ / pack_send_ only ever // touch peers. pack_count_matrix_ asserts the non-peer remainder is empty to catch that one. struct PeerPlan { - int bits = 0; + bool sparse = false; int shift = 0; - [[nodiscard]] constexpr auto dense() const -> bool { return bits == 0; } - // A plan too narrow for the world would yield 0 peers and turn the exchange into a silent no-op. - [[nodiscard]] constexpr auto count(int ranks) const -> int { - if (bits == 0) { - return ranks; - } - assert(bits > 0 && bits < 31 && (ranks >> static_cast(bits)) > 0); - return ranks >> static_cast(bits); - } - // Unsigned shifts: `bits` is a public field, and 1 << 31 on a signed int is UB. - [[nodiscard]] constexpr auto peer(int me, int k) const -> int { - if (bits == 0) { - return k; - } - assert(bits > 0 && bits < 31); - const auto ubits = static_cast(bits); - const auto mask = static_cast((1U << ubits) - 1U); - return ((me & mask) ^ shift) | static_cast(static_cast(k) << ubits); - } - // Membership without a search: by the XOR structure every peer shares the same low `bits`. - [[nodiscard]] constexpr auto contains(int me, int b) const -> bool { - if (bits == 0) { - return true; - } - const auto mask = static_cast((1U << static_cast(bits)) - 1U); - return (b & mask) == ((me & mask) ^ shift); - } + [[nodiscard]] constexpr auto dense() const -> bool { return !sparse; } + [[nodiscard]] constexpr auto count(int ranks) const -> int { return sparse ? 1 : ranks; } + // `k` indexes the peer set, which is a singleton when sparse. + [[nodiscard]] constexpr auto peer(int me, int k) const -> int { return sparse ? (me ^ shift) : k; } + [[nodiscard]] constexpr auto contains(int me, int b) const -> bool { return !sparse || b == (me ^ shift); } }; // Argument bundles for the variable all-to-all verbs, deliberately here rather than in HybridComm.h: diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index bfaf20f2..6022d2a6 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -60,7 +60,7 @@ auto find_rank(const Monomial &mono, const routing::Router &router) -> return router.dest(mono); } -// The router this communicator's geometry implies, honouring monoprop_ROUTING / _ROUTE_LINEAR_BITS. +// The router this communicator's geometry implies, honouring monoprop_ROUTING. // Templated because the router binds the transposed basis for this monomial width. template inline auto router_for(const mpi::Comm &comm) -> routing::Router { @@ -81,7 +81,7 @@ class RoutingDisagreement : public std::runtime_error { // TWO independent digests, not one: allreduce_sum is the only collective in the tree, and a sum is not // an equality test -- differing values can add up to mine*world. Both must agree, so a disagreement // survives at ~2^-128 rather than ~2^-64. Partitions are in the digest because S enters Router::dest: -// two ranks differing only in S agree on linear_bits and the seed and still route apart. +// two ranks differing only in S agree on the mode and the seed and still route apart. inline auto check_routing_agreement(const mpi::Comm &comm) -> void { const auto world = static_cast(mpi::size(comm)); if (world <= 1) { @@ -89,10 +89,10 @@ inline auto check_routing_agreement(const mpi::Comm &comm) -> void { } const auto geom = mpi::geometry(comm); const auto parts = static_cast(geom.partitions); - const auto bits = static_cast(routing::linear_bits_for(static_cast(geom.ranks))); + const auto linear = static_cast(routing::linear_requested()); const uint64_t seed = routing::seed_from_env(); const auto digest = [&](uint64_t salt) { - return routing::mix64(routing::mix64(routing::mix64(salt ^ bits) ^ parts) ^ seed); + return routing::mix64(routing::mix64(routing::mix64(salt ^ linear) ^ parts) ^ seed); }; const uint64_t first = digest(0x9E37'79B9'7F4A'7C15ULL); const uint64_t second = digest(0xC2B2'AE3D'27D4'EB4FULL); @@ -104,12 +104,12 @@ inline auto check_routing_agreement(const mpi::Comm &comm) -> void { const bool ok_second = agrees(second); if (!ok_first || !ok_second) { throw RoutingDisagreement( - std::format("routing configuration differs across the {} participants (this one: linear_bits={}, " - "partitions={}, seed={}). monoprop_ROUTING / monoprop_ROUTE_LINEAR_BITS / " - "monoprop_ROUTE_SEED must reach every rank identically -- under linear routing a " - "disagreement deadlocks the exchange rather than corrupting it.", + std::format("routing configuration differs across the {} participants (this one: linear={}, " + "partitions={}, seed={}). monoprop_ROUTING / monoprop_ROUTE_SEED must reach every rank " + "identically -- under linear routing a disagreement deadlocks the exchange rather than " + "corrupting it.", world, - bits, + linear, parts, seed)); } diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index 0410368f..44088fea 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include @@ -33,23 +35,33 @@ // the rank index is GF(2)-linear in the support and a generator maps every query to one peer; within a // rank partitions talk through shared memory, where fanout is free and only balance matters. // -// part = q % S q = monomial_hash(M) (splitmix, unchanged) -// hi = (q / S) % (R >> d) the R>>d splitmix-chosen high rank bits -// rank = (a & (2^d - 1)) | (hi << d) a = linear_hash(M); d = linear_bits +// part = q % S q = monomial_hash(M) (splitmix, unchanged) +// rank = a & (R - 1) a = linear_hash(M); all log2(R) rank bits, so fanout is 1 // flat = rank * S + part // -// The derivation, what d buys and what it costs: docs/content/docs/features/parallelism.mdx, under -// "Rank routing". +// Linear or not is a switch and not a dial: the rank takes every bit from the linear hash or none of +// them. None of them is the splitmix router, which is `q % (R*S)` bit for bit, and R == 1 is that case +// by construction. R > 1 must then be a power of two, or there is no XOR structure to route by and the +// geometry is rejected (UnroutableGeometry) rather than silently falling back. +// +// The derivation, what linear routing buys and what it costs: docs/content/docs/features/parallelism.mdx, +// under "Rank routing". // // Knobs, parsed and validated in EnvConfig.h: -// monoprop_ROUTING linear (default) | splitmix -- linear defaults d to log2(R) -// monoprop_ROUTE_LINEAR_BITS explicit d, clamped to [0, log2(R)]; overrides monoprop_ROUTING -// monoprop_ROUTE_SEED uint64 seed for the linear basis (default kDefaultSeed) +// monoprop_ROUTING linear (default) | splitmix +// monoprop_ROUTE_SEED uint64 seed for the linear basis (default kDefaultSeed) namespace monoprop::routing { inline constexpr uint64_t kDefaultSeed = 0x5DEE'CE66'D0C6'2517ULL; +// A rank count linear routing cannot serve. Thrown at Router construction, not at the first term: the +// alternative is a silent fallback to splitmix on some ranks and a deadlocked exchange. +class UnroutableGeometry : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + inline constexpr auto mix64(uint64_t x) noexcept -> uint64_t { x += 0x9E37'79B9'7F4A'7C15ULL; x = (x ^ (x >> 30)) * 0xBF58'476D'1CE4'E5B9ULL; @@ -77,8 +89,8 @@ inline auto linear_basis() -> const std::array & { } // The full 64-bit image, by walking the set bits. Router::dest does NOT use this -- it needs only the -// low d bits and takes the transposed form below -- but the map is defined here, and the GF(2)-linearity -// test and the plane build both read it as the definition. +// low log2(R) bits and takes the transposed form below -- but the map is defined here, and the +// GF(2)-linearity test and the plane build both read it as the definition. template [[nodiscard]] inline auto linear_hash(const monoprop::Bitset &bits) noexcept -> uint64_t { const auto &v = linear_basis(); @@ -89,14 +101,16 @@ template return h; } -inline constexpr size_t kLinearPlanes = 64; // one per output bit; d <= 63 of them are ever read +// One per output bit; a Router reads log2(R) of them. Not trimmed to that: the count is geometry, and +// keying the table on it would build one table per Router instead of one per (seed, width). +inline constexpr size_t kLinearPlanes = 64; template inline constexpr size_t kPlaneWords = monoprop::Bitset::num_words(); // The basis transposed: plane j, bit i, is bit j of basis vector i. Then bit j of linear_hash(M) is -// popcount(M & plane_j) & 1 -- d * words branch-free AND/XOR/popcount ops instead of a gather whose -// length is the term's popcount (~20-28 under the production cutoff). Same map, bit for bit. +// popcount(M & plane_j) & 1 -- log2(R) * words branch-free AND/XOR/popcount ops instead of a gather +// whose length is the term's popcount (~20-28 under the production cutoff). Same map, bit for bit. // // Keyed on the seed and the width alone, never on the geometry, so one table serves every Router; the // Router binds a pointer to it at construction and dest() never reaches the static-init guard. @@ -121,7 +135,7 @@ inline auto linear_planes() -> const std::array vectors) noexcept -> size_t { @@ -154,75 +168,73 @@ class Router final { public: // Bound to a monomial width: dest() reads the transposed basis for THAT width, and binding the // pointer here is what keeps the static-init guard out of the per-term path. The only way to a - // router with linear bits, so an unbound one cannot reach dest(). + // linear router, so an unbound one cannot reach dest(). Throws if `linear` and ranks is not 2^k. template - [[nodiscard]] static auto for_modes(size_t ranks, size_t partitions, size_t linear_bits) -> Router { - Router r{ranks, partitions, linear_bits}; + [[nodiscard]] static auto for_modes(size_t ranks, size_t partitions, bool linear) -> Router { + Router r{ranks, partitions, linear}; r.planes_ = linear_planes<2 * NumModes>().data(); r.plane_words_ = kPlaneWords<2 * NumModes>; return r; } - // Today's routing: one flat world, full avalanche, no linear bits. Also what d = 0 collapses to. - // Width-free: with bits_ == 0 no plane is ever read. - static constexpr auto splitmix(size_t flat_world) noexcept -> Router { return Router{flat_world, 1, 0}; } - - // The clamp on its own, for callers that need the resolved d without a monomial width. - static constexpr auto clamp_bits(size_t ranks, size_t requested) noexcept -> size_t { - if (!std::has_single_bit(ranks)) { - return 0; // no XOR structure without a power-of-two rank count - } - const auto max_bits = static_cast(std::countr_zero(ranks)); - return requested < max_bits ? requested : max_bits; - } + // Today's routing: one flat world, full avalanche, no linear bits. Width-free: no plane is read. + static constexpr auto splitmix(size_t flat_world) noexcept -> Router { return Router{flat_world, 1, false}; } [[nodiscard]] constexpr auto ranks() const noexcept -> size_t { return ranks_; } [[nodiscard]] constexpr auto partitions() const noexcept -> size_t { return parts_; } [[nodiscard]] constexpr auto flat_world() const noexcept -> size_t { return flat_; } - [[nodiscard]] constexpr auto linear_bits() const noexcept -> size_t { return bits_; } - // Distinct destination RANKS one rank's queries for a single generator reach. 1 == pairwise. - [[nodiscard]] constexpr auto fanout() const noexcept -> size_t { return ranks_ >> bits_; } + // False for a splitmix router AND for R == 1, which has no rank bit to take: both route densely. + [[nodiscard]] constexpr auto is_linear() const noexcept -> bool { return linear_; } + // Rank bits read off the linear hash: log2(R), or 0 when not linear. The span a generator set must + // cover for every rank to be reachable. + [[nodiscard]] constexpr auto linear_bits() const noexcept -> size_t { + return linear_ ? static_cast(std::countr_zero(ranks_)) : 0; + } // Flat destination slot in [0, flat_world). Branch is on a member, so it is perfectly predicted. template [[nodiscard]] [[gnu::always_inline]] inline auto dest(const Monomial &mono) const noexcept -> size_t { const uint64_t q = monomial_hash(mono); - if (bits_ == 0) { + if (!linear_) { return static_cast(q % flat_); // bit-for-bit today's `hash % P` } - const uint64_t part = q % parts_; - const uint64_t hi = (q / parts_) & hi_mask_; // ranks_>>bits_ is a power of two, so a mask - return static_cast(((linear_low_(mono) | (hi << bits_)) * parts_) + part); + return static_cast((linear_low_(mono) * parts_) + (q % parts_)); } - // The rank-level shift a generator induces: rank(M^G) low bits == rank(M) low bits ^ shift(G). - // Zero for every G iff bits_ == 0. This is what makes the destination predictable. + // The rank-level shift a generator induces: rank(M^G) == rank(M) ^ shift(G). Zero for every G when + // the router is not linear. This is what makes the destination predictable. template [[nodiscard]] auto rank_shift(const Monomial &gen) const noexcept -> size_t { return static_cast(linear_low_(gen)); } private: - // ranks x partitions == the flat world the destinations index. linear_bits is clamped here, so a - // caller may pass anything. Private: a router with bits_ > 0 must go through for_modes. - constexpr Router(size_t ranks, size_t partitions, size_t linear_bits) noexcept + // ranks x partitions == the flat world the destinations index. Private: a linear router must go + // through for_modes, which binds the basis linear_low_ reads. + constexpr Router(size_t ranks, size_t partitions, bool linear) : ranks_(ranks == 0 ? 1 : ranks), parts_(partitions == 0 ? 1 : partitions), flat_(ranks_ * parts_), - bits_(clamp_bits(ranks_, linear_bits)), - hi_mask_((ranks_ >> bits_) - 1) {} + linear_(linear && ranks_ > 1) { // R == 1 takes no rank bit, so it IS the dense case + if (linear && !std::has_single_bit(ranks_)) { + throw UnroutableGeometry( + std::format("linear routing needs a power-of-two rank count, got {}. Launch 2^k ranks, or set " + "monoprop_ROUTING=splitmix to keep the dense all-to-all.", + ranks_)); + } + } - // linear_hash(M) & lin_mask, one output bit per plane: parity(popcount(M & plane_j)). Folding the + // linear_hash(M) & (R - 1), one output bit per plane: parity(popcount(M & plane_j)). Folding the // words with XOR before the popcount is the same parity (popcount(x)+popcount(y) == popcount(x^y) - // mod 2) for one popcount per bit instead of one per word. bits_ == 0 reads no plane, so a - // splitmix router needs none. + // mod 2) for one popcount per bit instead of one per word. A non-linear router reads no plane. template [[nodiscard]] [[gnu::always_inline]] inline auto linear_low_(const Monomial &m) const noexcept -> uint64_t { constexpr size_t kW = kPlaneWords<2 * NumModes>; - assert(bits_ == 0 || (planes_ != nullptr && plane_words_ == kW)); // bound at a different width + const size_t bits = linear_bits(); + assert(bits == 0 || (planes_ != nullptr && plane_words_ == kW)); // bound at a different width uint64_t acc = 0; - for (size_t j = 0; j < bits_; ++j) { + for (size_t j = 0; j < bits; ++j) { const uint64_t *plane = planes_ + (j * kW); uint64_t fold = 0; for (size_t w = 0; w < kW; ++w) { @@ -236,38 +248,21 @@ class Router final { size_t ranks_; size_t parts_; size_t flat_; - size_t bits_; - uint64_t hi_mask_; + bool linear_; const uint64_t *planes_ = nullptr; // [kLinearPlanes x plane_words_], owned by linear_planes() size_t plane_words_ = 0; }; -// The requested linear-bit count, before clamping to a particular geometry. Parsed once. -inline auto requested_linear_bits() -> size_t { - const auto &env = config::get(); - if (env.route_linear_bits.has_value()) { - return static_cast(*env.route_linear_bits); - } - if (env.routing_mode == config::RoutingMode::Splitmix) { - return 0; // full avalanche across the flat world: every rank talks to every rank - } - // Default. "As many bits as this geometry allows" -- Router clamps to log2(R), and to 0 when R is - // not a power of two, so a geometry without XOR structure keeps the dense path. Measured at the - // production point: fanout 1 costs nothing on balance (rank occupancy max/mean 1.001 at R=128, all - // ranks used) and takes messages per rank per layer from 362,712 to 1,397, i.e. from - // proportional-to-R to flat. - return ~size_t{0}; +// The mode, before any geometry. Linear unless asked otherwise: measured at the production point it +// costs nothing on balance (rank occupancy max/mean 1.001 at R=128, all ranks used) and takes messages +// per rank per layer from 362,712 to 1,397, i.e. from proportional-to-R to flat. +inline auto linear_requested() -> bool { + return config::get().routing_mode.value_or(config::RoutingMode::Linear) == config::RoutingMode::Linear; } template inline auto make_router(size_t ranks, size_t partitions) -> Router { - return Router::for_modes(ranks, partitions, requested_linear_bits()); -} - -// The resolved d for a geometry, without binding a width: the routing-agreement digest needs the -// number and nothing else. -inline auto linear_bits_for(size_t ranks) -> size_t { - return Router::clamp_bits(ranks == 0 ? 1 : ranks, requested_linear_bits()); + return Router::for_modes(ranks, partitions, linear_requested()); } } // namespace monoprop::routing diff --git a/cpp/tests/env_config_tests.cpp b/cpp/tests/env_config_tests.cpp index e8d29dc3..9d7b80e7 100644 --- a/cpp/tests/env_config_tests.cpp +++ b/cpp/tests/env_config_tests.cpp @@ -21,7 +21,6 @@ using monoprop::config::EnvConfigError; using monoprop::config::RoutingMode; -using monoprop::config::detail::parse_bit_count; using monoprop::config::detail::parse_positive_int; using monoprop::config::detail::parse_routing_mode; using monoprop::config::detail::parse_uint64; @@ -51,7 +50,7 @@ BOOST_AUTO_TEST_CASE(env_config_settings_cached_singleton) { BOOST_CHECK(a.num_threads == std::nullopt || *a.num_threads >= 1); } -// The three routing parsers throw where parse_positive_int returns nullopt: a routing knob that +// Both routing parsers throw where parse_positive_int returns nullopt: a routing knob that // defaulted silently would change the transport with no diagnostic. BOOST_AUTO_TEST_CASE(env_config_parse_uint64_unset_valid_and_rejected) { BOOST_CHECK(parse_uint64("k", nullptr) == std::nullopt); @@ -64,19 +63,6 @@ BOOST_AUTO_TEST_CASE(env_config_parse_uint64_unset_valid_and_rejected) { BOOST_CHECK_THROW(parse_uint64("k", "18446744073709551616"), EnvConfigError); // ERANGE } -BOOST_AUTO_TEST_CASE(env_config_parse_bit_count_keeps_zero_distinct_from_unset) { - BOOST_CHECK(parse_bit_count("k", nullptr) == std::nullopt); - BOOST_CHECK(parse_bit_count("k", "") == std::nullopt); - BOOST_CHECK(parse_bit_count("k", "0") == std::optional(0)); // legal: 0 bits is dense routing - BOOST_CHECK(parse_bit_count("k", "7") == std::optional(7)); - BOOST_CHECK(parse_bit_count("k", "64") == std::optional(64)); - BOOST_CHECK_THROW(parse_bit_count("k", "abc"), EnvConfigError); - BOOST_CHECK_THROW(parse_bit_count("k", "7x"), EnvConfigError); - BOOST_CHECK_THROW(parse_bit_count("k", "-1"), EnvConfigError); - BOOST_CHECK_THROW(parse_bit_count("k", "65"), EnvConfigError); - BOOST_CHECK_THROW(parse_bit_count("k", "99999999999999999999"), EnvConfigError); // ERANGE -} - BOOST_AUTO_TEST_CASE(env_config_parse_routing_mode_rejects_a_typo) { BOOST_CHECK(parse_routing_mode("k", nullptr) == std::nullopt); BOOST_CHECK(parse_routing_mode("k", "") == std::nullopt); diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index c6d26e74..d62547a8 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -23,7 +23,6 @@ #ifdef monoprop_ENABLE_MPI #include -#include #include #include #include @@ -485,22 +484,46 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_poison_releases_waiters) { } } -// A sparse PeerPlan replaces the collectives with point-to-point over the peers the plan names, so the -// two failure modes it can have are DROPPED data and a HANG -- neither of which a dense-path test can -// see. Every rank derives the same pairing from the same (bits, shift), and a block whose destination is -// not a peer must be empty: send only to the plan's peer and check the delivery is exactly that. +// The pairing is derived independently on both ends, so it must be an involution: whoever I send to +// sends back to me. A plan on which it fails deadlocks rather than mis-delivers, and no transport case +// below can distinguish the two. +BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_peer_is_an_involution) { + for (const int shift : {0, 1, 2, 3, 5, 8, 13, 255}) { + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; + BOOST_REQUIRE(!plan.dense()); + BOOST_REQUIRE_EQUAL(plan.count(256), 1); + for (int me = 0; me < 256; ++me) { + const int peer = plan.peer(me, 0); + BOOST_REQUIRE_EQUAL(plan.peer(peer, 0), me); + BOOST_REQUIRE(plan.contains(me, peer)); + BOOST_REQUIRE(plan.contains(peer, me)); + BOOST_REQUIRE(!plan.contains(me, peer ^ 1)); // the peer set is a singleton + } + } + const monoprop::mpi::PeerPlan dense_plan{}; + BOOST_REQUIRE(dense_plan.dense()); + BOOST_REQUIRE_EQUAL(dense_plan.count(7), 7); + for (int k = 0; k < 7; ++k) { + BOOST_REQUIRE_EQUAL(dense_plan.peer(3, k), k); // dense ignores `me`: every rank is a peer + BOOST_REQUIRE(dense_plan.contains(3, k)); + } +} + +// A sparse PeerPlan replaces the collectives with point-to-point over the one peer the plan names, so +// the two failure modes it can have are DROPPED data and a HANG -- neither of which a dense-path test +// can see. Every rank derives the same pairing from the same shift, and a block whose destination is +// not the peer must be empty: send only to the plan's peer and check the delivery is exactly that. BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_delivers_only_to_its_peers) { const int R = world_size(); if (R < 2 || (R & (R - 1)) != 0) { return; // the XOR pairing needs a power-of-two rank count } - const int bits = std::countr_zero(static_cast(R)); for (const int S : {1, 2, 3}) { const int P = R * S; for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; const int peer = plan.peer(world_rank(), 0); - BOOST_REQUIRE_EQUAL(plan.count(R), 1); // full bits => pairwise + BOOST_REQUIRE_EQUAL(plan.count(R), 1); // sparse is always pairwise std::vector>> recv(static_cast(S)); auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { Comm c = Comm::make_hybrid(&hyb, u); @@ -552,10 +575,9 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_on_the_plain_mpi_path) { if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); Comm c{MPI_COMM_WORLD}; for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; const int peer = plan.peer(world_rank(), 0); std::vector> send(static_cast(R)); for (int j = 0; j < 4; ++j) { @@ -594,13 +616,12 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_known_recv_counts_are_masked_through_the_plan) if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); constexpr int kReal = 4; constexpr int kBogus = 7; // what a stale or unmasked transpose would claim a non-peer is sending for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; const int peer = plan.peer(world_rank(), 0); - const int bad = (peer + 1) % R; // at full bits the peer set is exactly {peer} + const int bad = (peer + 1) % R; // the peer set is exactly {peer} BOOST_REQUIRE(bad != peer); // S == 1: the plain-MPI Isend/Irecv branch of begin_alltoallv. @@ -663,106 +684,12 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_known_recv_counts_are_masked_through_the_plan) namespace { -// Varies along BOTH ends and hits 0, so a block landing on the wrong peer or the wrong partition -// changes a length, not just a value. -auto sparse_count(int src, int dst) -> int { - return ((src * 3) + (dst * 5)) % 4; -} auto sparse_tag(int src, int dst, int j) -> int { return (((src * 128) + dst) * 1000) + j; } } // namespace -// f > 1. Both sparse cases above pin bits == log2(R), so `plan.count(R)` is 1 and every `for k in -// [0, f)` in the sparse path has only ever run once -- the interleaving of peer-ordered blocks with the -// [0, R)-ordered prefix sums in size_staging_send_ / size_staging_recv_ is what that leaves unchecked. -// bits < log2(R) is the only way to reach it, and a mis-indexed prefix shows up as a block delivered at -// the wrong offset, i.e. a wrong tag, not a hang. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_with_several_peers) { - const int R = world_size(); - if (R < 4 || (R & (R - 1)) != 0) { - return; - } - const int full = std::countr_zero(static_cast(R)); - const int me = world_rank(); - int cases = 0; - for (const int f : {2, 4}) { - const int bits = full - std::countr_zero(static_cast(f)); - if (bits < 1) { - continue; // bits == 0 is the dense path, which these cases are not about - } - for (int shift = 0; shift < (1 << bits); ++shift) { - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; - BOOST_REQUIRE_EQUAL(plan.count(R), f); - ++cases; - - // The S == 1 world first: the plain-MPI Isend/Irecv branch, no staging in the way. - { - Comm c{MPI_COMM_WORLD}; - std::vector> send(static_cast(R)); - for (int k = 0; k < f; ++k) { - const int b = plan.peer(me, k); - for (int j = 0; j < sparse_count(me, b); ++j) { - send[static_cast(b)].push_back(sparse_tag(me, b, j)); - } - } - std::vector> out; - monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); - BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); - for (int src = 0; src < R; ++src) { - const int want = plan.contains(me, src) ? sparse_count(src, me) : 0; - BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), want); - for (int j = 0; j < want; ++j) { - BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], - sparse_tag(src, me, j)); - } - } - } - - // Then the staged HybridComm path, where the peer-ordered sweeps live. - for (const int S : {1, 2, 3}) { - const int P = R * S; - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector> send(static_cast(P)); - for (int k = 0; k < f; ++k) { - const int b = plan.peer(me, k); - for (int t = 0; t < S; ++t) { - const int d = (b * S) + t; - for (int j = 0; j < sparse_count(g, d); ++j) { - send[static_cast(d)].push_back(sparse_tag(g, d, j)); - } - } - } - std::vector> out; - monoprop::mpi::begin_alltoallv(send, c, false, nullptr, plan).wait_into(out); - recv[static_cast(u)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const int g = (me * S) + t; - const auto &out = recv[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int src = 0; src < P; ++src) { - const int want = plan.contains(me, src / S) ? sparse_count(src, g) : 0; - BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), want); - for (int j = 0; j < want; ++j) { - BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], - sparse_tag(src, g, j)); - } - } - } - } - } - } - BOOST_TEST(cases > 0); // at R < 4 the case is a no-op and must not read as coverage -} - // A zero-count leg is where a send/recv posting asymmetry deadlocks rather than mis-delivers: both ends // must skip on the SAME value. Nothing above ever sends an empty block over a real message, so force // one -- the lower-numbered end of every pair sends nothing while its peer sends four. @@ -771,11 +698,10 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_with_an_empty_leg) { if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); const int me = world_rank(); constexpr int kLen = 4; for (int shift = 1; shift < R; ++shift) { // shift 0 is the self peer, covered separately - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; const int peer = plan.peer(me, 0); BOOST_REQUIRE(peer != me); const int my_len = me < peer ? 0 : kLen; // exactly one end of the pair is silent @@ -840,9 +766,8 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_skip_self_at_shift_zero) { if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); const int me = world_rank(); - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = 0}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = 0}; BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); { @@ -903,11 +828,10 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_back_to_back_rounds) { if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); const int me = world_rank(); Comm c{MPI_COMM_WORLD}; for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; const int peer = plan.peer(me, 0); const int len = 3 + (me % 2); // asymmetric, so a swapped round is a length mismatch diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 604540fd..82604cb9 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -183,14 +183,13 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { for (const size_t ranks : {2U, 4U, 8U}) { // BOTH routers, because the agreement is a property of the pair and not of either hash: the // scan calls Router::dest and find_rank calls the same Router, so a divergence introduced by - // one of them shows up here whichever routing the geometry resolves to. bits=~0 asks for as - // many linear bits as log2(ranks) allows, i.e. fanout 1. - for (const size_t bits : {size_t{0}, ~size_t{0}}) { + // one of them shows up here whichever routing the geometry resolves to. + for (const bool linear : {false, true}) { // Per router, not summed over them: the floors are what stops the loop passing on an empty // scan, and a sum lets one router carry the other. size_t checked = 0; size_t self_checked = 0; - const auto router = routing::Router::for_modes(ranks, /*partitions=*/1, bits); + const auto router = routing::Router::for_modes(ranks, /*partitions=*/1, linear); const auto res = detail::fused_find_and_collect>(op, gen, eval, @@ -214,7 +213,7 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { // Measured, all six routers: total 387 every time, with the split running from 196/191 at // R=2 to 335/52 at R=8 as more partners fall cross-rank. The floors sit below the observed // minimum of each arm and only catch a scan that emitted nothing. - BOOST_TEST_MESSAGE("ranks=" << ranks << " bits=" << router.linear_bits() << " encoded=" << checked + BOOST_TEST_MESSAGE("ranks=" << ranks << " linear=" << router.is_linear() << " encoded=" << checked << " staged=" << self_checked); const size_t total = checked + self_checked; if (first_total.has_value()) { diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp index 0203065d..1617a455 100644 --- a/cpp/tests/routing_tests.cpp +++ b/cpp/tests/routing_tests.cpp @@ -12,21 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -// routing::Router -- the term -> flat-slot map. Two properties carry the whole design: -// * d = 0 is bit-for-bit today's `monomial_hash % P`, so the refactor cannot move a single term; -// * at d = log2(R) the destination RANK of M^G is rank(M) ^ shift(G), which is what turns the dense -// all-to-all into a pairwise exchange. A break here is silent -- wrong owner, not a crash -- so the -// shift identity and the Scan/find_rank agreement are both asserted explicitly. +// routing::Router -- the term -> flat-slot map. Two states, two properties that carry the whole design: +// * splitmix is bit-for-bit today's `monomial_hash % P`, so the refactor cannot move a single term; +// * under linear routing the destination RANK of M^G is rank(M) ^ shift(G), which is what turns the +// dense all-to-all into a pairwise exchange. A break here is silent -- wrong owner, not a crash -- +// so the shift identity and the Scan/find_rank agreement are both asserted explicitly. // // Flat cases with a shared prefix, no suite nesting (suites break Boost's ctest discovery here). #include -#include +#include #include #include #include #include +#include #include #include "monoprop/algebra/MajoranaAlgebra.h" @@ -86,13 +87,13 @@ auto random_monomials(size_t count, size_t weight, uint64_t seed) -> std::vector } // namespace -// d = 0 must not move a single term relative to `monomial_hash % P`: this is the regression gate that -// licenses everything else. -BOOST_AUTO_TEST_CASE(routing_zero_bits_is_bit_identical_to_splitmix) { +// The splitmix router must not move a single term relative to `monomial_hash % P`: this is the +// regression gate that licenses everything else. +BOOST_AUTO_TEST_CASE(routing_splitmix_is_bit_identical_to_hash_mod_p) { const auto monos = random_monomials(500, 5, 0xC0FFEEULL); for (const size_t flat : {size_t{1}, size_t{2}, size_t{7}, size_t{112}, size_t{1792}}) { const auto router = Router::splitmix(flat); - BOOST_TEST(router.linear_bits() == 0U); + BOOST_TEST(!router.is_linear()); for (const auto &m : monos) { const size_t expected = monomial_hash(m) % flat; BOOST_TEST(router.dest(m) == expected); @@ -101,12 +102,12 @@ BOOST_AUTO_TEST_CASE(routing_zero_bits_is_bit_identical_to_splitmix) { } } -// A two-level router with d = 0 is still today's routing, even though it knows about partitions: -// ((q/S) % R)*S + q%S == q % (R*S). -BOOST_AUTO_TEST_CASE(routing_zero_bits_two_level_collapses_to_flat_modulo) { +// A two-level splitmix router is still today's routing, even though it knows about partitions: +// ((q/S) % R)*S + q%S == q % (R*S). 12 ranks is here too: splitmix carries no power-of-two condition. +BOOST_AUTO_TEST_CASE(routing_splitmix_two_level_collapses_to_flat_modulo) { const auto monos = random_monomials(300, 6, 0xBEEF01ULL); - for (const auto [r, s] : {std::pair{8, 14}, {4, 28}, {128, 14}, {64, 28}}) { - const auto router = Router::for_modes(r, s, 0); + for (const auto [r, s] : {std::pair{8, 14}, {4, 28}, {128, 14}, {64, 28}, {12, 5}}) { + const auto router = Router::for_modes(r, s, false); for (const auto &m : monos) { BOOST_TEST(router.dest(m) == monomial_hash(m) % (r * s)); } @@ -125,14 +126,14 @@ BOOST_AUTO_TEST_CASE(routing_linear_hash_is_gf2_linear) { BOOST_TEST(routing::linear_hash<2 * kN>(Monomial{}) == 0ULL); } -// The load-bearing identity: at full linear bits the destination RANK of M^G is rank(M) ^ shift(G), +// The load-bearing identity: under linear routing the destination RANK of M^G is rank(M) ^ shift(G), // so one rank's queries for one generator all land on one peer. -BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_at_full_bits) { +BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_under_linear_routing) { constexpr size_t kRanks = 16; constexpr size_t kParts = 14; - const auto router = Router::for_modes(kRanks, kParts, 64); // clamped to log2(16) == 4 - BOOST_REQUIRE(router.linear_bits() == 4U); - BOOST_REQUIRE(router.fanout() == 1U); + const auto router = Router::for_modes(kRanks, kParts, true); + BOOST_REQUIRE(router.is_linear()); + BOOST_REQUIRE(router.linear_bits() == 4U); // every rank bit, log2(16) const auto terms = random_monomials(400, 6, 0xAAAA01ULL); const auto gens = random_monomials(40, 4, 0xBBBB02ULL); @@ -148,10 +149,10 @@ BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_at_full_bits) { // The consequence that the transport will rely on: every term a rank owns sends its query for one // generator to exactly ONE peer rank -- and the partition index within that peer still spreads. -BOOST_AUTO_TEST_CASE(routing_fanout_is_one_at_full_bits) { +BOOST_AUTO_TEST_CASE(routing_fanout_is_one_under_linear_routing) { constexpr size_t kRanks = 8; constexpr size_t kParts = 14; - const auto router = Router::for_modes(kRanks, kParts, 3); + const auto router = Router::for_modes(kRanks, kParts, true); const auto terms = random_monomials(4000, 6, 0xCCCC03ULL); const auto gens = random_monomials(12, 4, 0xDDDD04ULL); @@ -175,44 +176,36 @@ BOOST_AUTO_TEST_CASE(routing_fanout_is_one_at_full_bits) { } } -// d in between: the low d bits shift deterministically and the high log2(R)-d are splitmix, so the -// realised fanout is R >> d -- the dial the balance/fanout trade is made on. -BOOST_AUTO_TEST_CASE(routing_partial_bits_give_fanout_ranks_over_two_to_the_d) { - constexpr size_t kRanks = 32; - constexpr size_t kParts = 14; - const auto terms = random_monomials(20000, 6, 0xEEEE05ULL); - const auto gen = random_monomials(1, 4, 0xFFFF06ULL).front(); - for (size_t d = 0; d <= 5; ++d) { - const auto router = Router::for_modes(kRanks, kParts, d); - BOOST_TEST(router.fanout() == (kRanks >> d)); - std::vector> dest_of(kRanks); - for (const auto &m : terms) { - const size_t src = router.dest(m) / kParts; - dest_of[src].insert(router.dest(m ^ gen) / kParts); - } - for (size_t src = 0; src < kRanks; ++src) { - BOOST_TEST(dest_of[src].size() <= router.fanout()); - } +// Without a power-of-two rank count there is no XOR structure to exploit, and there is no partial dial +// to fall back to, so the geometry is rejected at construction rather than routed on a subspace. +BOOST_AUTO_TEST_CASE(routing_non_power_of_two_ranks_throw_under_linear_routing) { + for (const size_t r : {size_t{3}, size_t{7}, size_t{12}, size_t{112}}) { + BOOST_CHECK_THROW(static_cast(Router::for_modes(r, 14, true)), routing::UnroutableGeometry); + BOOST_CHECK_NO_THROW(static_cast(Router::for_modes(r, 14, false))); // splitmix has no condition } + BOOST_CHECK_NO_THROW(static_cast(Router::for_modes(64, 14, true))); } -// Without a power-of-two rank count there is no XOR structure to exploit, so the router must fall back -// to today's routing rather than silently produce a lopsided or out-of-range slot. -BOOST_AUTO_TEST_CASE(routing_non_power_of_two_ranks_falls_back_to_zero_bits) { - for (const size_t r : {size_t{3}, size_t{7}, size_t{12}, size_t{112}}) { - const auto router = Router::for_modes(r, 14, 8); +// R = 1 is a power of two, so it must NOT throw: it has no rank bit to take, which makes it the dense +// router and keeps the collective transport that serves every single-rank run. +BOOST_AUTO_TEST_CASE(routing_single_rank_is_dense_and_not_an_error) { + const auto monos = random_monomials(200, 5, 0x51A61EULL); + for (const size_t s : {size_t{1}, size_t{14}, size_t{112}}) { + const auto router = Router::for_modes(1, s, true); + BOOST_TEST(!router.is_linear()); BOOST_TEST(router.linear_bits() == 0U); - BOOST_TEST(router.fanout() == r); + for (const auto &m : monos) { + BOOST_TEST(router.dest(m) == monomial_hash(m) % s); + BOOST_TEST(router.rank_shift(m) == 0U); // no bit to shift, so every generator is on-rank + } } - const auto pow2 = Router::for_modes(64, 14, 8); - BOOST_TEST(pow2.linear_bits() == 6U); // clamped to log2(64), not 8 } BOOST_AUTO_TEST_CASE(routing_dest_is_in_range_and_deterministic) { const auto monos = random_monomials(1000, 7, 0x9999ULL); for (const auto [r, s] : {std::pair{1, 1}, {1, 112}, {8, 14}, {128, 14}, {64, 28}}) { - for (size_t d = 0; d <= 7; ++d) { - const auto router = Router::for_modes(r, s, d); + for (const bool linear : {false, true}) { + const auto router = Router::for_modes(r, s, linear); for (const auto &m : monos) { const size_t slot = router.dest(m); BOOST_TEST(slot < r * s); @@ -223,8 +216,8 @@ BOOST_AUTO_TEST_CASE(routing_dest_is_in_range_and_deterministic) { } // The transposed basis must be the SAME map, not merely a faster one: a divergence is a silently wrong -// owner. Pin dest() and rank_shift() against the old bit-walk over the geometries the dial spans -- -// d = 0, d < log2(R), d == log2(R) -- and over popcounts from empty to full support. +// owner. Pin dest() and rank_shift() against the old bit-walk over both routers, every geometry, and +// popcounts from empty to full support. BOOST_AUTO_TEST_CASE(routing_transposed_basis_is_bit_identical_to_the_bit_walk) { std::vector> monos; for (const size_t w : {size_t{0}, @@ -242,34 +235,38 @@ BOOST_AUTO_TEST_CASE(routing_transposed_basis_is_bit_identical_to_the_bit_walk) } BOOST_REQUIRE_EQUAL(monos.size(), 5000U); - // (R, S, d). log2(R) is 7, 4, 6, 3, 10, 1, 12, 5 respectively, so both d < log2(R) and d == log2(R) - // appear, as does d = 0. - const std::vector> geometries{ - {128, 14, 0}, {128, 14, 1}, {128, 14, 3}, {128, 14, 6}, {128, 14, 7}, {16, 1, 0}, {16, 1, 2}, {16, 1, 4}, - {64, 28, 1}, {64, 28, 5}, {64, 28, 6}, {8, 14, 0}, {8, 14, 1}, {8, 14, 2}, {8, 14, 3}, {1024, 1, 5}, - {1024, 1, 10}, {2, 112, 0}, {2, 112, 1}, {4096, 16, 12}, {32, 3, 4}, {32, 3, 5}}; + // (R, S), each run under both routers: log2(R) runs 0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, so the plane + // count varies from none to twelve and R = 1 pins the geometry that is dense under either mode. + const std::vector> geometries{{128, 14}, + {16, 1}, + {64, 28}, + {8, 14}, + {1024, 1}, + {2, 112}, + {4096, 16}, + {32, 3}, + {1, 112}, + {256, 2}, + {2048, 1}}; size_t checked = 0; - for (const auto &[r, s, d] : geometries) { - const auto router = Router::for_modes(r, s, d); - BOOST_REQUIRE_EQUAL(router.linear_bits(), d); // no clamping in this table - const uint64_t lin_mask = d == 0 ? 0ULL : (uint64_t{1} << d) - 1; - for (const auto &m : monos) { - const uint64_t q = monomial_hash(m); - size_t expected = 0; - if (d == 0) { - expected = static_cast(q % (r * s)); - } - else { - const uint64_t part = q % s; - const uint64_t hi = (q / s) % (r >> d); - const uint64_t lin = linear_hash_reference<2 * kN>(m) & lin_mask; - expected = static_cast(((lin | (hi << d)) * s) + part); + for (const auto &[r, s] : geometries) { + for (const bool linear : {false, true}) { + const auto router = Router::for_modes(r, s, linear); + const size_t d = router.linear_bits(); + BOOST_REQUIRE_EQUAL(d, linear ? static_cast(std::countr_zero(r)) : 0U); + const uint64_t lin_mask = d == 0 ? 0ULL : (uint64_t{1} << d) - 1; + for (const auto &m : monos) { + const uint64_t q = monomial_hash(m); + // d == 0 is the splitmix arm, and R = 1 under linear routing lands there too. + const size_t expected = + d == 0 ? static_cast(q % (r * s)) + : static_cast(((linear_hash_reference<2 * kN>(m) & lin_mask) * s) + (q % s)); + BOOST_REQUIRE_EQUAL(router.dest(m), expected); + BOOST_REQUIRE_EQUAL(router.rank_shift(m), + static_cast(linear_hash_reference<2 * kN>(m) & lin_mask)); + ++checked; } - BOOST_REQUIRE_EQUAL(router.dest(m), expected); - BOOST_REQUIRE_EQUAL(router.rank_shift(m), - static_cast(linear_hash_reference<2 * kN>(m) & lin_mask)); - ++checked; } } BOOST_TEST_MESSAGE("bit-identity checks: " << checked); @@ -286,7 +283,7 @@ BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { // The real generator shifts must span at least log2(R) dimensions or the reachable ranks are a // strict subspace of the rank space. constexpr size_t kRanks = 128; - const auto router = Router::for_modes(kRanks, 14, 7); + const auto router = Router::for_modes(kRanks, 14, true); std::vector shifts; for (const auto &g : random_monomials(200, 4, 0x7777ULL)) { shifts.push_back(static_cast(router.rank_shift(g))); @@ -296,24 +293,22 @@ BOOST_AUTO_TEST_CASE(routing_gf2_rank_detects_a_degenerate_shift_set) { BOOST_TEST(routing::gf2_rank(shifts) == 7U); // == log2(128): every rank is reachable } -// The SHIPPED default. Flipping this is the whole point of the change, so it is pinned by a test -// rather than left to a comment: with no environment override, a power-of-two rank count routes at -// fanout 1, and a geometry with no XOR structure keeps the dense path instead of silently losing -// ranks. Skipped when the environment does override it, because then the default is not what is -// under test. +// The SHIPPED default, pinned by a test rather than left to a comment: with no environment override a +// power-of-two rank count routes linearly at fanout 1, a single rank routes densely, and a geometry +// with no XOR structure is refused. Skipped when the environment does override it, because then the +// default is not what is under test. BOOST_AUTO_TEST_CASE(routing_default_is_linear_where_the_geometry_allows_it) { const char *mode = std::getenv("monoprop_ROUTING"); - const char *bits = std::getenv("monoprop_ROUTE_LINEAR_BITS"); - if ((mode != nullptr && *mode != '\0') || (bits != nullptr && *bits != '\0')) { + if (mode != nullptr && *mode != '\0') { BOOST_TEST_MESSAGE("routing overridden in the environment; default not under test"); return; } - BOOST_TEST(routing::make_router(8, 14).fanout() == 1U); - BOOST_TEST(routing::make_router(128, 14).fanout() == 1U); - BOOST_TEST(routing::make_router(1, 112).fanout() == 1U); // single rank: nothing to route between + BOOST_TEST(routing::make_router(8, 14).is_linear()); + BOOST_TEST(routing::make_router(128, 14).linear_bits() == 7U); + BOOST_TEST(!routing::make_router(1, 112).is_linear()); // single rank: nothing to route between - // 6 and 12 are not powers of two: no XOR structure, so Router clamps to d = 0 and every rank - // stays reachable through splitmix rather than a subspace of them. - BOOST_TEST(routing::make_router(6, 14).fanout() == 6U); - BOOST_TEST(routing::make_router(12, 28).fanout() == 12U); + // 6 and 12 are not powers of two: no XOR structure and no partial dial to retreat to, so the + // geometry is rejected instead of silently routing onto a subspace of the ranks. + BOOST_CHECK_THROW(routing::make_router(6, 14), routing::UnroutableGeometry); + BOOST_CHECK_THROW(routing::make_router(12, 28), routing::UnroutableGeometry); } diff --git a/docs/content/docs/features/parallelism.mdx b/docs/content/docs/features/parallelism.mdx index 99f92d2d..068c9a06 100644 --- a/docs/content/docs/features/parallelism.mdx +++ b/docs/content/docs/features/parallelism.mdx @@ -93,17 +93,18 @@ distinct shifts, against the $d = 7$ that $R = 128$ needs. Within a rank, partitions keep the full-avalanche `splitmix` hash: fanout across shared memory is free, so only balance matters there. Routing is therefore two-level. Writing $q$ -for the `splitmix` hash of $M$, the partition is $q \bmod S$, the low $d$ bits of the rank -are $h_d(M)$, and the remaining $\log_2 R - d$ rank bits come from $q$: +for the `splitmix` hash of $M$, the partition is $q \bmod S$ and the rank is $h_d(M)$ with +$d = \log_2 R$ — every rank bit, so the fanout is 1: $$ -\mathrm{flat}(M) = \bigl[\,h_d(M) + 2^{d}\bigl(\lfloor q/S \rfloor \bmod (R/2^{d})\bigr)\,\bigr]\,S + (q \bmod S). +\mathrm{flat}(M) = h_d(M)\,S + (q \bmod S). $$ -$d$ is a dial and not a switch: the fanout is $R/2^{d}$, so $d = 0$ reproduces the dense -`hash % (R × S)` bit for bit, and $d = \log_2 R$ — the default — gives fanout 1 and -$\mathrm{flat}(M) = h_d(M)\,S + (q \bmod S)$. A rank count that is not a power of two has -no XOR structure and falls back to $d = 0$. +Linear routing is a switch and not a dial: the rank takes all $\log_2 R$ bits from $h$ or +none of them, and none of them is `splitmix`, which reproduces the dense `hash % (R × S)` +bit for bit. It therefore requires $R$ to be a power of two; any other rank count has no +XOR structure to route by and is raised at propagator construction rather than silently +routed. $R = 1$ is a power of two, takes no rank bit, and so is the dense case already. A related distributed scheme maps an index by summing its $k$-bit blocks modulo the rank count [@Broers2025-or]. That sum is additive modulo that count while the gate acts by XOR, @@ -114,9 +115,8 @@ that bound into an identity. | Variable | Default | Meaning | | --- | --- | --- | -| `monoprop_ROUTING` | `linear` | `splitmix` forces $d = 0$, the dense all-to-all; `linear`, or unset, takes $d$ as large as the geometry allows. Any other value is rejected at startup rather than silently defaulting. | -| `monoprop_ROUTE_LINEAR_BITS` | unset | An explicit $d$, clamped to $[0, \log_2 R]$. Takes precedence over `monoprop_ROUTING`. `0` selects the dense path; a negative, out-of-range or unparseable value is rejected at startup. | -| `monoprop_ROUTE_SEED` | `6768574230969066775` | Decimal `uint64` from which every rank derives the same basis $\{v_i\}$ with no communication. The same value must reach every rank: a mismatch in any of these three variables, or in the partition count, is caught by two allreduces at propagator construction and raised, because under linear routing it deadlocks the exchange instead of corrupting it. | +| `monoprop_ROUTING` | `linear` | `splitmix` selects the dense all-to-all; `linear`, or unset, takes every rank bit from $h$ and requires a power-of-two $R$. Any other value is rejected at startup rather than silently defaulting. | +| `monoprop_ROUTE_SEED` | `6768574230969066775` | Decimal `uint64` from which every rank derives the same basis $\{v_i\}$ with no communication. The same value must reach every rank: a mismatch in either of these variables, or in the partition count, is caught by two allreduces at propagator construction and raised, because under linear routing it deadlocks the exchange instead of corrupting it. | ### Single-node (`MPI.COMM_SELF`) From b1ee72c920b94503516c898f79d0e534f76b6708 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 16:42:04 +0100 Subject: [PATCH 16/24] =?UTF-8?q?perf(routing):=20=E2=9A=A1=20route=20the?= =?UTF-8?q?=20emit=20path=20from=20the=20generator's=20shift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rank(M^G) == rank(M) ^ rank_shift(G) is exact under linear routing, and build_layer already derives rank_shift(G) once per gate for the PeerPlan -- which is to say the linear planes Scan.h evaluated for every emitted query were recomputing a per-generator constant. Router::dest_from_shift takes the rank bits from this rank's own slot XOR that shift, leaving only the partition index per term; splitmix has no such identity and falls back to dest() bit for bit. Two strength reductions in dest() ride along, both blocked only by parts_ being a runtime member. `q % S` becomes `q & (S - 1)` when S is a power of two, which every production layout is (S in 1,2,4,8,16); and at S == 1 the partition index is 0 for every term, so monomial_hash is not evaluated at all. Both are identities, not approximations. Per term at the production geometry the emit path was tzcnt; log2(R) x (kW loads + kW ands + kW xors + popcnt + shift/or); imul; splitmix mix64; 64-bit divq and is now shrx; xor; imul; splitmix mix64; and. The plane loop, its popcounts and the division are gone; the mix64 stays because the partition index still needs it (and goes too at S == 1). Ownership must not move by a term, so the identity is asserted rather than argued: a debug assert at the fast path against dest(), and a sweep in routing_tests over nine geometries and both modes, including S == 1, S = 3 and S = 14 so the mask path and the division path are both covered. dest_from_shift requires that the local operator hold only terms this rank owns -- the precondition mpi::PeerPlan already carries, since it sends every query for a gate to me ^ shift. The scan/find_rank agreement test was feeding one rank the whole operator, which no rank ever holds; it now distributes by find_rank and runs every rank. The six routers still agree on a total of 387 partners, and the split becomes all-encoded under linear routing (shift 1, 3, 7) against 212/175 to 343/44 under splitmix. Assisted-by: ClaudeCode:claude-opus-5 --- .../detail/evolution/layer_build/Engine.h | 5 +- .../detail/evolution/layer_build/Scan.h | 8 +- cpp/monoprop/detail/mpi/Routing.h | 58 +++++++++- cpp/tests/mpi_utils_tests.cpp | 104 +++++++++++------- cpp/tests/routing_tests.cpp | 39 +++++++ 5 files changed, 167 insertions(+), 47 deletions(-) diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 341fc72d..d13e7759 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -612,8 +612,8 @@ auto build_layer(MPOperator &local_op, // Under linear routing every query for THIS generator lands on the rank this rank's own index XOR // rank_shift(gen), so the exchange knows its peer before it starts. Dense otherwise, which is // today's collective. - const auto plan = - mpi::PeerPlan{.sparse = router.is_linear(), .shift = static_cast(router.rank_shift(gen))}; + const size_t gen_shift = router.rank_shift(gen); + const auto plan = mpi::PeerPlan{.sparse = router.is_linear(), .shift = static_cast(gen_shift)}; // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); @@ -654,6 +654,7 @@ auto build_layer(MPOperator &local_op, R, my_rank, router, + gen_shift, /*capture_values=*/use_fused, sweep_ptr, cos_build); diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index 117dae93..36d135e9 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -241,6 +241,10 @@ struct FusedScanResult { // deterministic. `fused_scale_coeffs` (no length cap only; must alias coeffs.data()) scales every anticommuting // coeff in place by `fused_scale_cos`=cos(2·build_angle), so no cosine set is built and a hit's stored // value is post-cos (resolve recovers it via 1/cos). +// +// `gen_shift` is router.rank_shift(gen), and `op` must hold only terms `my_rank` owns -- then the owner of +// M⊕G is rank(M) ^ gen_shift and the linear planes never run per term. Both are what mpi::PeerPlan +// already assumes; a violation moves ownership silently, so the fast path asserts against dest(). template auto fused_find_and_collect(const MPOperator &op, const Monomial &gen, @@ -251,6 +255,7 @@ auto fused_find_and_collect(const MPOperator &op, size_t rank_count, size_t my_rank, const routing::Router &router, + size_t gen_shift, bool capture_values = false, double *fused_scale_coeffs = nullptr, double fused_scale_cos = 1.0) -> FusedScanResult { @@ -349,7 +354,8 @@ auto fused_find_and_collect(const MPOperator &op, // silently; mpi_utils_tests.cpp asserts the agreement. size_t r_prime = my_rank; if (rank_count != 1) { - r_prime = router.dest(dense); + r_prime = router.dest_from_shift(dense, my_rank, gen_shift); + assert(r_prime == router.dest(dense)); // an identity, not an approximation } if (r_prime == my_rank) { (is_follower ? res.follower_self : res.leader_self).push(pos, k, phase); diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index 44088fea..2cb565d8 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -29,7 +29,8 @@ // The single home for "which flat slot owns this monomial". Two call sites depend on agreeing exactly // (Scan.h emits queries by it, MonomialPropagator seeds the operator by it), and a disagreement splits -// ownership silently rather than crashing -- so both go through Router::dest and nothing else. +// ownership silently rather than crashing -- so both go through this Router and nothing else. Scan.h +// enters at dest_from_shift, which is dest with the per-generator rank bits taken from the shift. // // Two-level, because the levels cost differently: across MPI ranks the message COUNT is what hurts, so // the rank index is GF(2)-linear in the support and a generator maps every query to one peer; within a @@ -194,11 +195,33 @@ class Router final { // Flat destination slot in [0, flat_world). Branch is on a member, so it is perfectly predicted. template [[nodiscard]] [[gnu::always_inline]] inline auto dest(const Monomial &mono) const noexcept -> size_t { - const uint64_t q = monomial_hash(mono); if (!linear_) { - return static_cast(q % flat_); // bit-for-bit today's `hash % P` + return static_cast(monomial_hash(mono) % flat_); // bit-for-bit today's `hash % P` } - return static_cast((linear_low_(mono) * parts_) + (q % parts_)); + const size_t rank = static_cast(linear_low_(mono)); + if (parts_ == 1) { + return rank; // q % 1 == 0 for every q, so S == 1 owes the hash nothing + } + return (rank * parts_) + part_of_(monomial_hash(mono)); + } + + // The emit path's dest, for a query M^G raised on a term M THIS slot owns. rank(M^G) == rank(M) ^ + // shift(G) is exact under linear routing, so the planes are a per-generator constant already in hand + // and only the partition index is per term. `my_flat` must be this slot's own index and `shift` this + // generator's rank_shift, or ownership moves silently -- the same precondition mpi::PeerPlan carries. + // Splitmix has no such identity and falls back to dest(), bit for bit. + template + [[nodiscard]] [[gnu::always_inline]] inline auto dest_from_shift(const Monomial &mono, + size_t my_flat, + size_t shift) const noexcept -> size_t { + if (!linear_) { + return dest(mono); + } + const size_t rank = rank_of_slot_(my_flat) ^ shift; + if (parts_ == 1) { + return rank; + } + return (rank * parts_) + part_of_(monomial_hash(mono)); } // The rank-level shift a generator induces: rank(M^G) == rank(M) ^ shift(G). Zero for every G when @@ -215,7 +238,10 @@ class Router final { : ranks_(ranks == 0 ? 1 : ranks), parts_(partitions == 0 ? 1 : partitions), flat_(ranks_ * parts_), - linear_(linear && ranks_ > 1) { // R == 1 takes no rank bit, so it IS the dense case + linear_(linear && ranks_ > 1), // R == 1 takes no rank bit, so it IS the dense case + parts_pow2_(std::has_single_bit(parts_)), + parts_mask_(parts_ - 1), + parts_log2_(static_cast(std::countr_zero(parts_))) { if (linear && !std::has_single_bit(ranks_)) { throw UnroutableGeometry( std::format("linear routing needs a power-of-two rank count, got {}. Launch 2^k ranks, or set " @@ -224,6 +250,25 @@ class Router final { } } + // q % S. Every production layout has S in {1,2,4,8,16}, where the modulo is a mask -- and parts_ is a + // runtime member, so the compiler cannot strength-reduce it on our behalf. Identical bit for bit. + [[nodiscard]] [[gnu::always_inline]] inline auto part_of_(uint64_t q) const noexcept -> size_t { + if (parts_pow2_) { + assert(static_cast(q & parts_mask_) == static_cast(q % parts_)); + return static_cast(q & parts_mask_); + } + return static_cast(q % parts_); + } + + // Flat slot -> rank index. Flat slots are rank * S + partition (mpi::rank under the hybrid comm). + [[nodiscard]] [[gnu::always_inline]] inline auto rank_of_slot_(size_t flat_slot) const noexcept -> size_t { + if (parts_pow2_) { + assert((flat_slot >> parts_log2_) == flat_slot / parts_); + return flat_slot >> parts_log2_; + } + return flat_slot / parts_; + } + // linear_hash(M) & (R - 1), one output bit per plane: parity(popcount(M & plane_j)). Folding the // words with XOR before the popcount is the same parity (popcount(x)+popcount(y) == popcount(x^y) // mod 2) for one popcount per bit instead of one per word. A non-linear router reads no plane. @@ -249,6 +294,9 @@ class Router final { size_t parts_; size_t flat_; bool linear_; + bool parts_pow2_; // S is 2^k, so `% S` is a mask and `/ S` a shift + size_t parts_mask_; // S - 1, and parts_log2_ == log2(S); both meaningless unless parts_pow2_ + size_t parts_log2_; const uint64_t *planes_ = nullptr; // [kLinearPlanes x plane_words_], owned by linear_planes() size_t plane_words_ = 0; }; diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 82604cb9..a8684850 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -163,58 +163,70 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { for (size_t i = 0; i < 2000; ++i) { terms.push_back(draw_well_formed(rng, kLogical, 1 + (rng() % 6))); } - auto op = build_op(terms); const Monomial gen = draw_well_formed(rng, kLogical, 4); - VecD coeffs(op.store->size(), 1.0); const CutoffFn fn = detail::LengthCutoff{10, kLogical}; const detail::CutoffEvaluator eval(fn); - const auto cut = detail::build_majorana_evolution_cutoff_state(std::nullopt, - std::cref(coeffs), - std::nullopt, - std::optional{0.3}); + // Each rank is handed exactly the terms it owns, as MonomialPropagator seeds it: the emit path routes + // by rank(M) ^ rank_shift(G), which is only the owner of M^G when M really is local. + // // The partner count is a property of the operator and the gate, not of where the partners live, so - // it is the same for every router; routing only moves a partner between the encoded (cross-rank) - // and staged (self-owned) side. Pinning that invariance is stronger than a floor: a routing bug - // that drops partners moves the total, and one that misroutes them moves the split. + // the sum over the ranks is the same for every router; routing only moves a partner between ranks and + // between the encoded (cross-rank) and staged (self-owned) sides. Pinning that invariance is stronger + // than a floor: a routing bug that drops partners moves the total, and one that misroutes them moves + // the split. size_t routers = 0; std::optional first_total; for (const size_t ranks : {2U, 4U, 8U}) { - // BOTH routers, because the agreement is a property of the pair and not of either hash: the - // scan calls Router::dest and find_rank calls the same Router, so a divergence introduced by + // BOTH routers, because the agreement is a property of the pair and not of either hash: the scan + // calls Router::dest_from_shift and find_rank calls Router::dest, so a divergence introduced by // one of them shows up here whichever routing the geometry resolves to. for (const bool linear : {false, true}) { + const auto router = routing::Router::for_modes(ranks, /*partitions=*/1, linear); + const size_t shift = router.rank_shift(gen); // Per router, not summed over them: the floors are what stops the loop passing on an empty // scan, and a sum lets one router carry the other. size_t checked = 0; size_t self_checked = 0; - const auto router = routing::Router::for_modes(ranks, /*partitions=*/1, linear); - const auto res = detail::fused_find_and_collect>(op, - gen, - eval, - cut, - coeffs, - std::nullopt, - ranks, - 0, - router, - false, - nullptr, - 1.0); - BOOST_REQUIRE_EQUAL(res.leader_queries.size(), ranks); - // The scan routes a self-owned partner to the stage, so bucket 0 must be empty here. - BOOST_REQUIRE(res.leader_queries[0].empty()); - BOOST_REQUIRE(res.follower_queries[0].empty()); - check_bucket_ownership(res.leader_queries, router, checked); - check_bucket_ownership(res.follower_queries, router, checked); - check_self_ownership(res.leader_self, router, /*my_rank=*/0, self_checked); - check_self_ownership(res.follower_self, router, /*my_rank=*/0, self_checked); - // Measured, all six routers: total 387 every time, with the split running from 196/191 at - // R=2 to 335/52 at R=8 as more partners fall cross-rank. The floors sit below the observed - // minimum of each arm and only catch a scan that emitted nothing. - BOOST_TEST_MESSAGE("ranks=" << ranks << " linear=" << router.is_linear() << " encoded=" << checked - << " staged=" << self_checked); + for (size_t my_rank = 0; my_rank < ranks; ++my_rank) { + std::vector> owned; + for (const auto &t : terms) { + if (find_rank(t, router) == my_rank) { + owned.push_back(t); + } + } + BOOST_REQUIRE(!owned.empty()); + auto op = build_op(owned); + VecD coeffs(op.store->size(), 1.0); + const auto cut = detail::build_majorana_evolution_cutoff_state(std::nullopt, + std::cref(coeffs), + std::nullopt, + std::optional{0.3}); + const auto res = detail::fused_find_and_collect>(op, + gen, + eval, + cut, + coeffs, + std::nullopt, + ranks, + my_rank, + router, + shift, + false, + nullptr, + 1.0); + BOOST_REQUIRE_EQUAL(res.leader_queries.size(), ranks); + // The scan routes a self-owned partner to the stage, so my own bucket must be empty here. + BOOST_REQUIRE(res.leader_queries[my_rank].empty()); + BOOST_REQUIRE(res.follower_queries[my_rank].empty()); + check_bucket_ownership(res.leader_queries, router, checked); + check_bucket_ownership(res.follower_queries, router, checked); + check_self_ownership(res.leader_self, router, my_rank, self_checked); + check_self_ownership(res.follower_self, router, my_rank, self_checked); + } + BOOST_TEST_MESSAGE("ranks=" << ranks << " linear=" << router.is_linear() << " shift=" << shift + << " encoded=" << checked << " staged=" << self_checked); const size_t total = checked + self_checked; if (first_total.has_value()) { BOOST_TEST(total == *first_total); // routing moves partners, it does not create or lose them @@ -222,9 +234,23 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { else { first_total = total; } + // Measured, all six routers: total 387 every time, with the splitmix split running from + // 212/175 at R=2 to 343/44 at R=8 as more partners fall cross-rank, and the linear arm all + // encoded (shift 1, 3, 7). The floors sit below the observed minimum of each arm and only + // catch a scan that emitted nothing. BOOST_TEST(total > 300U); - BOOST_TEST(checked > 150U); - BOOST_TEST(self_checked > 40U); + if (!router.is_linear()) { + // Splitmix re-hashes the partner, so both sides are populated at every rank count. + BOOST_TEST(checked > 150U); + BOOST_TEST(self_checked > 40U); + } + else if (shift != 0) { + // Fanout 1: every partner leaves for my_rank ^ shift, so nothing stays self-owned. + BOOST_TEST(self_checked == 0U); + } + else { + BOOST_TEST(checked == 0U); // a shift of zero keeps every partner on its own rank + } ++routers; } } diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp index 1617a455..452b9a88 100644 --- a/cpp/tests/routing_tests.cpp +++ b/cpp/tests/routing_tests.cpp @@ -147,6 +147,45 @@ BOOST_AUTO_TEST_CASE(routing_shift_identity_holds_under_linear_routing) { } } +// What the emit path actually calls. rank(M^G) == rank(M) ^ shift(G) is an identity, so the check is +// exact equality against dest() over every geometry and both modes -- including splitmix, where +// dest_from_shift is a pass-through and must not move a term either. S == 1 (no partition index at all), +// S a power of two (the masked modulo) and S = 3, 14 (the division) are all here. +BOOST_AUTO_TEST_CASE(routing_dest_from_shift_agrees_with_dest) { + const auto terms = random_monomials(400, 6, 0x5E1F7A11ULL); + const auto gens = random_monomials(25, 4, 0x9110F7E5ULL); + const std::vector> + geometries{{1, 1}, {8, 1}, {16, 1}, {2, 2}, {32, 16}, {64, 8}, {4, 3}, {8, 14}, {1, 14}}; + size_t checked = 0; + for (const auto &[r, s] : geometries) { + for (const bool linear : {false, true}) { + const auto router = Router::for_modes(r, s, linear); + for (const auto &g : gens) { + const size_t shift = router.rank_shift(g); + for (const auto &m : terms) { + // dest(m) stands in for the flat slot the owning rank would pass in. + const auto partner = m ^ g; + BOOST_REQUIRE_EQUAL(router.dest_from_shift(partner, router.dest(m), shift), + router.dest(partner)); + ++checked; + } + } + } + } + BOOST_TEST_MESSAGE("dest_from_shift checks: " << checked); + BOOST_TEST(checked >= 100000U); +} + +// The S == 1 skip: the partition index is 0 for every term, so under linear routing the destination is +// the rank index alone and monomial_hash is not on the path at all. +BOOST_AUTO_TEST_CASE(routing_single_partition_destination_is_the_rank_index) { + constexpr size_t kRanks = 64; + const auto router = Router::for_modes(kRanks, 1, true); + for (const auto &m : random_monomials(500, 5, 0x0FAE7101ULL)) { + BOOST_REQUIRE_EQUAL(router.dest(m), routing::linear_hash<2 * kN>(m) & (kRanks - 1)); + } +} + // The consequence that the transport will rely on: every term a rank owns sends its query for one // generator to exactly ONE peer rank -- and the partition index within that peer still spreads. BOOST_AUTO_TEST_CASE(routing_fanout_is_one_under_linear_routing) { From 9a14631e618d935ed8527804a74c510d95377fd0 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 16:35:27 +0100 Subject: [PATCH 17/24] =?UTF-8?q?perf(mpi):=20=E2=9A=A1=20pack=20the=20sen?= =?UTF-8?q?d=20side=20under=20the=20count=20round?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exchange_count_blocks_ posted its Isend/Irecv pairs and MPI_Waitall'd on them in the same breath, inside the partition-0 serial section between B1 and B2, while pack_send_ -- the only real per-partition work in the verb -- did not start until after B2. The count block is S*S ints, 1 KB at S=16: an eager message that needs no cooperation from the peer, so nothing about it justified blocking the packing behind it. Split into post_count_blocks_ / wait_count_blocks_, with the wait moved to where the counts are first genuinely read. That reader is fill_recv_col_ via block_sum_, not the send-side sizing: size_staging_send_ works off the rows each partition published before B1, and pack_send_ off that sizing, so both run with the round in flight. The recv-side sizing and the payload exchange follow the wait in the B3->B4 window, and the per-partition extraction of recv_counts / recv_displs moves past B4, which is the first point at which counts_recv_ exists. Still four syncs; no byte moves differently and no delivery order changes. The dense arm is untouched by construction -- MPI_Alltoall is blocking and cannot be split, so plan.dense() completes inside the post and leaves count_posted_ at zero, making the wait a no-op. Only the sparse arm splits. The count requests live in their own count_reqs_, never the payload's reqs_. The current ordering drains the count round before exchange_payload_ posts, so one vector would in fact be safe today; separate storage is what keeps Pairwise.h's resize-once-then-index rule from turning into a use-after- realloc if the wait is ever moved again. Three cases: the same peer-masked layout through the dense and the sparse arm, compared element for element on both against each other and against the tags, so a dropped or torn count block changes a length; the self-peer plan at shift 0, where the count round posts nothing at all and the wait is the no-op path; and a zero-count peer, whose payload legs are skipped while its count block still travels, followed by an all-silent round over the first's staging high-water bytes. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/mpi/HybridComm.h | 91 +++++++---- cpp/tests/hybrid_comm_tests.cpp | 226 ++++++++++++++++++++++++++- 2 files changed, 287 insertions(+), 30 deletions(-) diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index ee507fe1..fa6b1f34 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -241,8 +241,13 @@ class HybridComm { } // Fused count-resolve + payload alltoallv: folds the standalone count exchange into this verb's - // B1→B2 window (4 syncs instead of 6). recv_counts / recv_displs and `recv` (resized) are outputs. - // Bit-identical to alltoall_counts + alltoallv. + // barriered windows (4 syncs instead of 6). recv_counts / recv_displs and `recv` (resized) are + // outputs. Bit-identical to alltoall_counts + alltoallv. + // + // The count round is POSTED in B1→B2 and only waited on in B3→B4, so the whole B2→B3 packing runs + // underneath it. Split, not fused, because nothing before fill_recv_col_ reads counts_recv_: the + // send side sizes from the locally published rows, and pack_send_ from that sizing. The dense arm + // is MPI_Alltoall, blocking, and completes inside post_count_blocks_ regardless. template auto alltoallv_resolve_impl_(int local_partition, const AlltoallvResolveArgs &args, @@ -256,20 +261,34 @@ class HybridComm { // never reconstructs T, so the slot stays type-erased for the untyped alltoallv_impl_ above. me.ptr = reinterpret_cast(args.send); me.send_displs = args.send_displs; - // Count row only: the recv counts do not exist until the count Alltoall in B1→B2. + // Count row only: the recv counts do not exist until the count round is drained in B3→B4. publish_counts_row_(local_partition, args.send_counts); sync(); // B1 if (local_partition == 0) { fill_peers_(plan); pack_count_matrix_(plan); - exchange_count_blocks_(plan); + post_count_blocks_(plan); size_staging_send_(elem); + } + sync(); // B2 + + // B3: each partition packs its own cross-rank blocks into stage_send_, the count round in flight. + pack_send_(local_partition, elem); + sync(); // B3 + + // B4: the counts land here -- fill_recv_col_ is their first reader -- then the payload moves. + if (local_partition == 0) { + wait_count_blocks_(); fill_recv_col_([this](int a, int t) { return block_sum_(a, t); }); size_staging_recv_(elem); + exchange_payload_(dt, elem, plan); } - sync(); // B2 + sync(); // B4 + // Past B4 now, not before B3: counts_recv_ does not exist until the wait above. Partition 0 + // cannot rewrite it before a later verb's B1→B2 window, unreachable until every reader here + // has arrived at that verb's B1. const int t = local_partition; long long total = 0; const size_t p = static_cast(r_) * static_cast(s_); @@ -286,14 +305,6 @@ class HybridComm { } args.recv.resize(static_cast(checked_mpi_count(total, "Total recv count"))); - pack_send_(local_partition, elem); - sync(); // B3 - - if (local_partition == 0) { - exchange_payload_(dt, elem, plan); - } - sync(); // B4 - scatter_recv_(local_partition, reinterpret_cast(args.recv.data()), // after the resize: it may reallocate args.recv_counts, @@ -473,26 +484,45 @@ class HybridComm { // The count blocks: one S*S-int MPI_Alltoall when dense, else a pair per peer (with the full linear // bits a zero rank shift keeps the whole round on-rank). Partition 0 only, in a barriered window. - auto exchange_count_blocks_(PeerPlan plan) -> void { + // + // Sparse arm POSTS ONLY, so counts_recv_ and counts_send_ must stay put until wait_count_blocks_; + // the dense MPI_Alltoall is blocking and has completed on return, which is why plan.dense() leaves + // nothing live. The requests go in count_reqs_, never reqs_: see the member declaration. + auto post_count_blocks_(PeerPlan plan) -> void { + assert(count_posted_ == 0); // an un-drained round would be waited on twice const int block = s_ * s_; if (plan.dense()) { MPI_Alltoall(counts_send_.data(), block, MPI_INT, counts_recv_.data(), block, MPI_INT, parent_); return; } const PeerLayout blocks{.block = block}; - const int posted = sparse_pairwise(plan, - mpi_rank_, - r_, - parent_, - kHybridCountTag, - MPI_INT, - sizeof(int), - reinterpret_cast(counts_send_.data()), - blocks, - reinterpret_cast(counts_recv_.data()), - blocks, - reqs_); - MPI_Waitall(posted, reqs_.data(), MPI_STATUSES_IGNORE); + count_posted_ = sparse_pairwise(plan, + mpi_rank_, + r_, + parent_, + kHybridCountTag, + MPI_INT, + sizeof(int), + reinterpret_cast(counts_send_.data()), + blocks, + reinterpret_cast(counts_recv_.data()), + blocks, + count_reqs_); + } + + // Drains post_count_blocks_. A no-op on the dense arm, and on a plan whose only peer is this rank + // itself -- a count block is a fixed S*S ints, so no other peer can be skipped for a zero count. + auto wait_count_blocks_() -> void { + if (count_posted_ != 0) { + MPI_Waitall(count_posted_, count_reqs_.data(), MPI_STATUSES_IGNORE); + count_posted_ = 0; + } + } + + // Post and drain in one step, for callers with no work to overlap. + auto exchange_count_blocks_(PeerPlan plan) -> void { + post_count_blocks_(plan); + wait_count_blocks_(); } // The staged payload: one MPI_Alltoallv when dense, else a pair per peer over the same per-rank @@ -727,8 +757,13 @@ class HybridComm { double red_f64_ = 0.0; uint64_t red_u64_ = 0; std::vector red_vec_; - // Point-to-point request scratch for the sparse paths; grown on demand, partition 0 only. + // Point-to-point request scratch for the sparse payload round; grown on demand, partition 0 only. std::vector reqs_; + // The count round's own scratch, separate from reqs_ by construction and not merely by the current + // ordering: it stays live across B2 and B3, and Pairwise.h's resize would move the buffer MPI holds + // pointers into the moment a payload post ever preceded the count wait. + std::vector count_reqs_; + int count_posted_ = 0; // live requests in count_reqs_; always 0 on the dense (blocking) arm // This verb's peer ranks; see fill_peers_. std::vector peers_; diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index d62547a8..b2f3ce70 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -23,6 +23,8 @@ #ifdef monoprop_ENABLE_MPI #include +#include +#include #include #include #include @@ -204,8 +206,8 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_repeated_alltoallv_varying_sizes) { BOOST_CHECK_EQUAL(failures.load(), 0); } -// alltoallv_resolve driven directly: it folds the count MPI_Alltoall into the payload verb's B1→B2 -// window and sizes recv itself. +// alltoallv_resolve driven directly: it folds the count round into the payload verb's barriered +// windows and sizes recv itself. BOOST_AUTO_TEST_CASE(hybrid_comm_alltoallv_resolve_fused) { if (world_size() < 2) { return; @@ -866,4 +868,224 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_back_to_back_rounds) { } } +namespace { + +// One fused-resolve round under `plan`: flattens send_blocks, runs the verb, and re-splits the payload +// by global source using the resolved counts, so a case compares delivered BLOCKS rather than offsets. +auto resolve_round(HybridComm &hyb, + int u, + int P, + const std::vector> &send_blocks, + monoprop::mpi::PeerPlan plan) -> std::vector> { + std::vector send; + std::vector sc(static_cast(P)), sd(static_cast(P)); + for (int d = 0; d < P; ++d) { + const auto &blk = send_blocks[static_cast(d)]; + sc[static_cast(d)] = static_cast(blk.size()); + sd[static_cast(d)] = static_cast(send.size()); + send.insert(send.end(), blk.begin(), blk.end()); + } + std::vector recv; + std::vector rc(static_cast(P)), rd(static_cast(P)); + hyb.alltoallv_resolve(u, + {.send = send.data(), + .send_counts = sc.data(), + .send_displs = sd.data(), + .recv = recv, + .recv_counts = rc.data(), + .recv_displs = rd.data()}, + monoprop::mpi::datatype::get(), + plan); + std::vector> out(static_cast(P)); + for (int src = 0; src < P; ++src) { + const auto off = static_cast(rd[static_cast(src)]); + const auto n = static_cast(rc[static_cast(src)]); + out[static_cast(src)].assign(recv.begin() + static_cast(off), + recv.begin() + static_cast(off + n)); + } + return out; +} + +// Every leg INTO an odd rank is empty, so a multi-peer plan always has a peer whose payload legs are +// skipped entirely while its S*S-int count block still travels. Both ends read the same function of +// (src, dst), which is what keeps the skip symmetric. +auto muted_count(int src, int dst, int S) -> int { + return (dst / S) % 2 == 1 ? 0 : sparse_count(src, dst); +} + +} // namespace + +// The fused resolve POSTS its count round in B1→B2 and drains it in B3→B4, with every partition's +// pack_send_ running underneath. Only the sparse arm splits -- plan.dense() is a blocking MPI_Alltoall +// -- so drive the SAME peer-masked layout through both arms and require the delivered blocks to agree +// element for element. A dropped or torn count block changes a length here, not just a value. +BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_matches_the_dense_arm) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; // the XOR pairing needs a power-of-two rank count + } + const int full = std::countr_zero(static_cast(R)); + const int me = world_rank(); + int cases = 0; + for (const int f : {1, 2}) { + const int bits = full - std::countr_zero(static_cast(f)); + if (bits < 1) { + continue; // bits == 0 IS the dense arm, which is the reference here + } + for (int shift = 0; shift < (1 << bits); ++shift) { + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + BOOST_REQUIRE_EQUAL(plan.count(R), f); + ++cases; + for (const int S : {1, 2, 3}) { + const int P = R * S; + // Peer-masked, so the dense plan carries the identical bytes: its non-peer blocks are + // empty rather than absent, and a dense count of zero and a sparse absence must agree. + const auto fill = [&](int g) { + std::vector> send(static_cast(P)); + for (int k = 0; k < f; ++k) { + const int b = plan.peer(me, k); + for (int t = 0; t < S; ++t) { + const int d = (b * S) + t; + for (int j = 0; j < muted_count(g, d, S); ++j) { + send[static_cast(d)].push_back(sparse_tag(g, d, j)); + } + } + } + return send; + }; + std::vector>> dense(static_cast(S)); + std::vector>> sparse(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + const int g = (me * S) + u; + dense[static_cast(u)] = resolve_round(hyb, u, P, fill(g), {}); + sparse[static_cast(u)] = resolve_round(hyb, u, P, fill(g), plan); + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const int g = (me * S) + t; + const auto &got = sparse[static_cast(t)]; + const auto &want_dense = dense[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(got.size()), P); + BOOST_REQUIRE_EQUAL(static_cast(want_dense.size()), P); + for (int src = 0; src < P; ++src) { + const int want = plan.contains(me, src / S) ? muted_count(src, g, S) : 0; + const auto &blk = got[static_cast(src)]; + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), want); + BOOST_REQUIRE_EQUAL(static_cast(want_dense[static_cast(src)].size()), want); + for (int j = 0; j < want; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag(src, g, j)); + BOOST_CHECK_EQUAL(blk[static_cast(j)], + want_dense[static_cast(src)][static_cast(j)]); + } + } + } + } + } + } + BOOST_TEST(cases > 0); // at R < 2 the case is a no-op and must not read as coverage +} + +// shift == 0 at full bits makes this rank its own and only peer, so the count round posts NOTHING: +// sparse_pairwise memcpys the S*S-int block in place and the B3→B4 wait must be a no-op rather than a +// wait on stale requests. The S^2 in-rank payload legs still have to arrive. +BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_self_peer_only) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + const int me = world_rank(); + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = 0}; + BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); + for (const int S : {1, 2, 3}) { + const int P = R * S; + std::vector>> recv(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + const int g = (me * S) + u; + std::vector> send(static_cast(P)); + for (int t = 0; t < S; ++t) { + const int d = (me * S) + t; + for (int j = 0; j <= t; ++j) { + send[static_cast(d)].push_back(sparse_tag(g, d, j)); + } + } + recv[static_cast(u)] = resolve_round(hyb, u, P, send, plan); + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const int g = (me * S) + t; + const auto &out = recv[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int src = 0; src < P; ++src) { + const int want = (src / S) == me ? t + 1 : 0; // block (su -> t) has t+1 entries + const auto &blk = out[static_cast(src)]; + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), want); + for (int j = 0; j < want; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag(src, g, j)); + } + } + } + } +} + +// A peer with nothing to send is where the split can hang: pack_send_ copies zero bytes in B2→B3 while +// the count round is still live, and the payload round posts no leg for it at all. Back-to-back rounds, +// the second silent, so the second also runs over the first's staging high-water bytes. +BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_zero_count_peer) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; + } + const int bits = std::countr_zero(static_cast(R)); + const int me = world_rank(); + for (int shift = 1; shift < R; ++shift) { // shift 0 is the self peer, covered above + const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const int peer = plan.peer(me, 0); + BOOST_REQUIRE(peer != me); + const int my_len = me < peer ? 0 : 4; // exactly one end of the pair is silent + const int peer_len = peer < me ? 0 : 4; + for (const int S : {1, 2}) { + const int P = R * S; + std::vector>> loud(static_cast(S)); + std::vector>> quiet(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + const int g = (me * S) + u; + std::vector> send(static_cast(P)); + for (int t = 0; t < S; ++t) { + const int d = (peer * S) + t; + for (int j = 0; j < my_len; ++j) { + send[static_cast(d)].push_back(sparse_tag(g, d, j)); + } + } + loud[static_cast(u)] = resolve_round(hyb, u, P, send, plan); + quiet[static_cast(u)] = + resolve_round(hyb, u, P, std::vector>(static_cast(P)), plan); + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int t = 0; t < S; ++t) { + const int g = (me * S) + t; + const auto &out = loud[static_cast(t)]; + BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); + for (int su = 0; su < S; ++su) { + const auto &blk = out[static_cast((peer * S) + su)]; + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), peer_len); + for (int j = 0; j < peer_len; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag((peer * S) + su, g, j)); + } + } + // The all-silent round: every resolved count is zero, so a stale staged byte cannot hide. + for (const auto &blk : quiet[static_cast(t)]) { + BOOST_CHECK(blk.empty()); + } + } + } + } +} + #endif // monoprop_ENABLE_MPI From e3f30bb8ebe729c14e38f51848647580738704ef Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 16:22:58 +0100 Subject: [PATCH 18/24] =?UTF-8?q?perf(mpi):=20=E2=9A=A1=20replay=20the=20l?= =?UTF-8?q?ayer=20exchange=20point-to-point?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #296 made rank routing GF(2)-linear and converted the graph BUILD path, but replay still posted MPI_Ialltoallv over all R ranks -- so propagate(), which Hubbard calls 29 times per build_graph, paid a full collective to move one peer's worth of doubles. post_flat_alltoallv now counts the legs carrying a payload and, at or below num_ranks/4 of them, posts Irecv/Isend pairs over those legs instead. No plan and no count round are needed: derive_exchange_layout already hands both sides the same array, so what a rank sends a peer IS that peer's recv count and both ends drop the same legs on the same value. sparse_pairwise drives it unchanged, with a dense PeerPlan walking [0, R) and posting only the non-zero legs; the self slot stays a memcpy. Its request vector moves into the Ticket, which now drains it in wait() the way PendingAlltoallv does -- resized once and indexed, never push_back'ed, because MPI holds those pointers until the wait. The branch is RANK-LOCAL, and that is a precondition rather than a proof: a rank choosing the collective waits forever on ranks that chose point-to-point. The default routing (linear, d = log2 R) gives fanout 1, so every rank's row holds at most one active leg and no row can straddle the budget. splitmix routing (monoprop_ROUTING=splitmix, or an explicit d) does not, and a layer whose per-rank partner counts land near num_ranks/4 can split the branch -- documented on flat_exchange_prefers_pairwise, not fixed here. layer_exchange_participates is unchanged for the same reason: the symmetric layout does let every rank agree on whether IT transfers anything, but the collective arm is still reachable, so skipping the round at local total 0 would strand it. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/Evolution.cpp | 2 + cpp/monoprop/detail/mpi/Exchange.h | 88 ++++++++++-- cpp/monoprop/detail/mpi/Pairwise.h | 5 +- cpp/tests/flat_exchange_tests.cpp | 212 +++++++++++++++++++++++++++++ 4 files changed, 298 insertions(+), 9 deletions(-) create mode 100644 cpp/tests/flat_exchange_tests.cpp diff --git a/cpp/monoprop/Evolution.cpp b/cpp/monoprop/Evolution.cpp index 296082cf..804fd6b3 100644 --- a/cpp/monoprop/Evolution.cpp +++ b/cpp/monoprop/Evolution.cpp @@ -73,6 +73,8 @@ auto &acquire_flat_exchange_buffers() { } // A property of the communicator, not the layer: all ranks participate even at local total_count 0. +// Still true with the pairwise arm: the transport branch is rank-local, so a collective may still be the +// one chosen and a rank that skipped the round strands it. auto layer_exchange_participates(const mpi::Comm &comm) -> bool { return mpi::size(comm) != 1; } diff --git a/cpp/monoprop/detail/mpi/Exchange.h b/cpp/monoprop/detail/mpi/Exchange.h index 9ec32530..449f361f 100644 --- a/cpp/monoprop/detail/mpi/Exchange.h +++ b/cpp/monoprop/detail/mpi/Exchange.h @@ -14,11 +14,16 @@ #pragma once +#include +#include #include #include #include #include "monoprop/detail/mpi/MPICompat.h" +#ifdef monoprop_ENABLE_MPI +#include "monoprop/detail/mpi/Pairwise.h" +#endif // Keeps #ifdef monoprop_ENABLE_MPI out of the consumers; non-MPI builds get self-copy stubs. @@ -28,11 +33,41 @@ namespace monoprop::mpi { // whatever the span holds, so a layout built for a differently sized communicator reads out of bounds. auto check_exchange_layout_width(std::span send_counts, const Comm &comm) -> void; +// Legs carrying a payload in either direction, which is what the pairwise path would post. +[[nodiscard]] inline auto active_leg_count(const int *send_counts, const int *recv_counts, int num_ranks) -> int { + int legs = 0; + for (int i = 0; i < num_ranks; ++i) { + legs += static_cast(send_counts[i] != 0 || recv_counts[i] != 0); + } + return legs; +} + +// A quarter of the fan-out: past that the pairwise post is >= N/2 requests against the collective's one +// tuned schedule, which wins there (a 54-peer sparse round measured 1.87x the collective). +inline constexpr int kSparseLegDivisor = 4; + +// Floored at 1, so an empty row and a one-peer row -- the two shapes linear routing produces -- land on +// the same side at every N; at N < 4 the bare quotient is 0 and would split them. +[[nodiscard]] inline auto sparse_leg_budget(int num_ranks) -> int { + return std::max(1, num_ranks / kSparseLegDivisor); +} + +// Which transport the Kind::Mpi arm takes. RANK-LOCAL, so it is a PRECONDITION that every rank lands on +// the same side: a rank choosing MPI_Ialltoallv waits forever on ranks that chose point-to-point. Linear +// routing (routing::Router, d >= 2 bits) gives that -- a generator reaches at most `ranks >> d` peers, +// so no row can exceed the budget -- while splitmix routing does not, and a layer where one rank has no +// cross-rank partners at all can then split the branch. +[[nodiscard]] inline auto flat_exchange_prefers_pairwise(const int *send_counts, const int *recv_counts, int num_ranks) + -> bool { + return active_leg_count(send_counts, recv_counts, num_ranks) <= sparse_leg_budget(num_ranks); +} + // Idempotent completion handle for a posted payload transfer; move-only, so a request is waited on -// exactly once. wait() is a no-op on the blocking path and in non-MPI builds. Owns its request: the -// destructor completes anything still in flight, because a dropped in-flight MPI_Ialltoallv -- what an +// exactly once. wait() is a no-op on the blocking path and in non-MPI builds. Owns its requests: the +// destructor completes anything still in flight, because a dropped in-flight transfer -- what an // exception between post and wait does -- keeps writing into a thread_local buffer the next exchange -// reallocates. +// reallocates. The pairwise arm's request vector lives here for the same reason: MPI reads it until the +// wait, and a vector move keeps its heap block, so the handle can travel. class [[nodiscard("call wait() on the Ticket to complete the posted transfer")]] Ticket { public: Ticket() = default; @@ -45,6 +80,8 @@ class [[nodiscard("call wait() on the Ticket to complete the posted transfer")]] wait(); // never drop a request this handle already owns request_ = other.request_; other.request_ = MPI_REQUEST_NULL; + requests_ = std::move(other.requests_); + posted_ = std::exchange(other.posted_, 0); } #endif (void)other; @@ -58,20 +95,37 @@ class [[nodiscard("call wait() on the Ticket to complete the posted transfer")]] MPI_Wait(&request_, MPI_STATUS_IGNORE); request_ = MPI_REQUEST_NULL; } + if (posted_ != 0) { + MPI_Waitall(posted_, requests_.data(), MPI_STATUSES_IGNORE); + posted_ = 0; + } +#endif + } + + // Requests wait() still has to drain: 1 for the collective, two per pairwise leg, 0 for nothing + // posted. The only handle on which transport a post took. + [[nodiscard]] auto in_flight() const -> int { +#ifdef monoprop_ENABLE_MPI + return static_cast(request_ != MPI_REQUEST_NULL) + posted_; +#else + return 0; #endif } #ifdef monoprop_ENABLE_MPI explicit Ticket(MPI_Request request) : request_(request) {} + Ticket(std::vector requests, int posted) : requests_(std::move(requests)), posted_(posted) {} private: - MPI_Request request_ = MPI_REQUEST_NULL; + MPI_Request request_ = MPI_REQUEST_NULL; // the dense collective + std::vector requests_; // the pairwise arm's pairs; `posted_` of them are live + int posted_ = 0; #endif }; -// Never skipped on zero total: all ranks must participate or the collective deadlocks. Non-blocking -// (MPI_Ialltoallv) in an MPI build (the Ticket completes it); non-MPI build does a per-rank self-copy -// (recv layout == send layout). +// Never skipped on zero total: the collective arm needs all ranks or it deadlocks. Non-blocking in an +// MPI build -- MPI_Ialltoallv, or Isend/Irecv over the active legs when the layout is sparse enough (the +// Ticket completes either); non-MPI build does a per-rank self-copy (recv layout == send layout). template inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, Comm comm) -> Ticket { // The in-process transports address the buffers as raw bytes; MPI_Ialltoallv below still takes the @@ -94,7 +148,25 @@ inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, comm.hyb->alltoallv(comm.shm_rank, args.bytes(), datatype::get()); return Ticket{}; } - (void)num_ranks; + if (flat_exchange_prefers_pairwise(args.send_counts, args.recv_counts, num_ranks)) { + // No plan and no count round: the count matrix is symmetric, so what this rank sends a peer IS + // that peer's recv count and both ends drop the same legs. A dense plan walks all N and posts + // only the non-zero ones, which is exactly that. + std::vector requests; + const int posted = sparse_pairwise(PeerPlan{}, + rank(comm), + num_ranks, + comm.mpi, + kFlatReplayTag, + datatype::get(), + sizeof(T), + reinterpret_cast(args.send), + PeerLayout{.counts = args.send_counts, .displs = args.send_displs}, + reinterpret_cast(args.recv), + PeerLayout{.counts = args.recv_counts, .displs = args.recv_displs}, + requests); + return Ticket(std::move(requests), posted); + } MPI_Request request = MPI_REQUEST_NULL; MPI_Ialltoallv(args.send, args.send_counts, diff --git a/cpp/monoprop/detail/mpi/Pairwise.h b/cpp/monoprop/detail/mpi/Pairwise.h index 970d4fb9..01cc6af0 100644 --- a/cpp/monoprop/detail/mpi/Pairwise.h +++ b/cpp/monoprop/detail/mpi/Pairwise.h @@ -25,7 +25,7 @@ namespace monoprop::mpi { -// One tag per (transport, verb), all four here so no two can collide unseen: one thread per rank calls +// One tag per (transport, verb), all five here so no two can collide unseen: one thread per rank calls // MPI, so the tag is all that keeps a count round in flight from being matched by a payload receive. // // Why Engine.h's run_exchange may post BOTH its begin_alltoallv rounds under kFlatPayloadTag on one @@ -37,6 +37,9 @@ inline constexpr int kHybridCountTag = 0x6D70; // 'mp' inline constexpr int kHybridPayloadTag = 0x6D71; inline constexpr int kFlatPayloadTag = 0x6D72; inline constexpr int kFlatCountTag = 0x6D73; +// Graph REPLAY payload (Exchange.h). Its own value, so a replay leg can match neither the build path's +// counts nor its payload even though both run over the same communicator. +inline constexpr int kFlatReplayTag = 0x6D74; // Per-peer element counts and offsets. Null `counts` is the fixed-block case: `block` each, at b*block. struct PeerLayout { diff --git a/cpp/tests/flat_exchange_tests.cpp b/cpp/tests/flat_exchange_tests.cpp new file mode 100644 index 00000000..def50037 --- /dev/null +++ b/cpp/tests/flat_exchange_tests.cpp @@ -0,0 +1,212 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// post_flat_alltoallv's transport choice (the graph REPLAY path): a dense layout keeps MPI_Ialltoallv, a +// sparse one goes point-to-point over the active legs. Ticket::in_flight() is what tells the two apart -- +// 1 for the collective, two requests per posted leg, 0 for a round with nothing to move. + +#include + +#include +#include +#include +#include + +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/Exchange.h" + +#ifdef monoprop_ENABLE_MPI +#include +#endif + +using monoprop::mpi::Comm; +using monoprop::mpi::flat_exchange_prefers_pairwise; +using monoprop::mpi::post_flat_alltoallv; +using monoprop::mpi::sparse_leg_budget; + +namespace { + +// The prefix sum post_flat_alltoallv takes on both sides of a symmetric layout. +auto displs_of(const std::vector &counts) -> std::vector { + std::vector displs(counts.size()); + int running = 0; + for (size_t i = 0; i < counts.size(); ++i) { + displs[i] = running; + running += counts[i]; + } + return displs; +} + +auto total_of(const std::vector &counts) -> int { + return std::accumulate(counts.begin(), counts.end(), 0); +} + +#ifdef monoprop_ENABLE_MPI +auto world_size() -> int { + int n = 0; + MPI_Comm_size(MPI_COMM_WORLD, &n); + return n; +} +auto world_rank() -> int { + int r = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &r); + return r; +} +#endif + +} // namespace + +// The boundary is <=, and the budget floors at 1 so an empty row and a one-peer row never split at N < 4 +// -- a split is a deadlock, not a slow round. +BOOST_AUTO_TEST_CASE(flat_exchange_pairwise_budget_boundary) { + for (const int n : {1, 2, 3, 4, 8, 16, 64}) { + const int budget = std::max(1, n / 4); + BOOST_REQUIRE_EQUAL(sparse_leg_budget(n), budget); + std::vector counts(static_cast(n), 0); + for (int legs = 0; legs <= n; ++legs) { + std::fill(counts.begin(), counts.end(), 0); + std::fill_n(counts.begin(), legs, 1); + BOOST_CHECK_EQUAL(flat_exchange_prefers_pairwise(counts.data(), counts.data(), n), legs <= budget); + } + } +} + +// A leg is active if EITHER side carries a payload: an asymmetric layout must not have one end post a +// receive the other never sends. +BOOST_AUTO_TEST_CASE(flat_exchange_active_legs_take_either_side) { + constexpr int n = 8; // budget 2 + std::vector send(static_cast(n), 0); + std::vector recv(static_cast(n), 0); + send[1] = 4; + recv[6] = 4; + BOOST_CHECK(flat_exchange_prefers_pairwise(send.data(), recv.data(), n)); + recv[3] = 4; + BOOST_CHECK(!flat_exchange_prefers_pairwise(send.data(), recv.data(), n)); +} + +// The self leg is a copy, not a message, so a one-rank world posts nothing at all and wait() is a no-op. +// Same on the non-MPI build, where the fallback self-copy runs instead. +BOOST_AUTO_TEST_CASE(flat_exchange_self_leg_is_copied_with_nothing_posted) { + Comm c{MPI_COMM_SELF}; + const std::vector counts{3}; + const auto displs = displs_of(counts); + const std::vector send{7, 8, 9}; + std::vector out(static_cast(total_of(counts)), -1); + auto ticket = post_flat_alltoallv({.send = send.data(), + .send_counts = counts.data(), + .send_displs = displs.data(), + .recv = out.data(), + .recv_counts = counts.data(), + .recv_displs = displs.data()}, + 1, + c); + BOOST_CHECK_EQUAL(ticket.in_flight(), 0); + ticket.wait(); + BOOST_CHECK(out == send); +} + +#ifdef monoprop_ENABLE_MPI + +// Every leg active: N legs is above the budget at every N >= 2, so the round stays on MPI_Ialltoallv. +BOOST_AUTO_TEST_CASE(flat_exchange_dense_layout_takes_the_collective) { + const int n = world_size(); + if (n < 2) { + return; + } + const int me = world_rank(); + Comm c{MPI_COMM_WORLD}; + const std::vector counts(static_cast(n), 1); + const auto displs = displs_of(counts); + std::vector send(static_cast(n)); + for (int d = 0; d < n; ++d) { + send[static_cast(d)] = (me * 1000) + d; + } + std::vector out(static_cast(total_of(counts)), -1); + auto ticket = post_flat_alltoallv({.send = send.data(), + .send_counts = counts.data(), + .send_displs = displs.data(), + .recv = out.data(), + .recv_counts = counts.data(), + .recv_displs = displs.data()}, + n, + c); + BOOST_CHECK_EQUAL(ticket.in_flight(), 1); // the collective, not 2N pairwise requests + ticket.wait(); + for (int src = 0; src < n; ++src) { + BOOST_CHECK_EQUAL(out[static_cast(src)], (src * 1000) + me); + } +} + +// One peer, me ^ 1 -- an involution, so the count matrix stays symmetric and both ends drop the same +// N - 1 legs. Two requests posted (one Irecv, one Isend) and the same bytes delivered. +BOOST_AUTO_TEST_CASE(flat_exchange_single_leg_takes_the_pairwise_path) { + const int n = world_size(); + if (n < 2 || (n % 2) != 0) { + return; + } + const int me = world_rank(); + const int peer = me ^ 1; + constexpr int len = 3; + Comm c{MPI_COMM_WORLD}; + std::vector counts(static_cast(n), 0); + counts[static_cast(peer)] = len; + const auto displs = displs_of(counts); + BOOST_REQUIRE_EQUAL(total_of(counts), len); + std::vector send(static_cast(len)); + for (int j = 0; j < len; ++j) { + send[static_cast(j)] = (me * 1000) + j; + } + std::vector out(static_cast(len), -1); + auto ticket = post_flat_alltoallv({.send = send.data(), + .send_counts = counts.data(), + .send_displs = displs.data(), + .recv = out.data(), + .recv_counts = counts.data(), + .recv_displs = displs.data()}, + n, + c); + BOOST_CHECK_EQUAL(ticket.in_flight(), 2); + ticket.wait(); + for (int j = 0; j < len; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(j)], (peer * 1000) + j); + } +} + +// No active leg anywhere: nothing is posted, wait() drains nothing, and the recv buffer is left as the +// caller sized it. The shape a layer with no cross-rank partners takes. +BOOST_AUTO_TEST_CASE(flat_exchange_empty_layout_posts_nothing) { + const int n = world_size(); + if (n < 2) { + return; + } + Comm c{MPI_COMM_WORLD}; + const std::vector counts(static_cast(n), 0); + const auto displs = displs_of(counts); + BOOST_REQUIRE_EQUAL(total_of(counts), 0); + const std::vector send(1, 0); + std::vector out(1, -1); // Evolution sizes an empty round to 1, not 0 + auto ticket = post_flat_alltoallv({.send = send.data(), + .send_counts = counts.data(), + .send_displs = displs.data(), + .recv = out.data(), + .recv_counts = counts.data(), + .recv_displs = displs.data()}, + n, + c); + BOOST_CHECK_EQUAL(ticket.in_flight(), 0); + ticket.wait(); + BOOST_CHECK_EQUAL(out[0], -1); +} + +#endif // monoprop_ENABLE_MPI From 25c00ce15ab66ef051cfd5b0ef6ff08b9f1b35c0 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 16:36:26 +0100 Subject: [PATCH 19/24] =?UTF-8?q?fix(mpi):=20=F0=9F=90=9B=20gate=20the=20p?= =?UTF-8?q?airwise=20replay=20on=20routing,=20not=20on=20a=20rank's=20row?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The leg-count budget was a rank-LOCAL predicate over a rank-varying quantity, so two ranks could land on opposite sides of it: one enters MPI_Ialltoallv and waits forever on the other, which posted point-to-point. The floor at 1 made the default routing safe by accident (fanout 1 gives every row 0 or 1 legs) and left splitmix and small-d configurations able to hang. Numeric tuning cannot fix that, so the budget is gone. The transport now keys on `wire_bits`, the resolved linear-bit count when the routing gives fanout 1, derived once in Evolution.cpp from routing::linear_bits_for -- the same number check_routing_agreement allreduces at construction and throws on. That makes it rank-uniform by construction, and it is also the actual reason the legs are empty. Any other geometry passes 0 and keeps today's collective. Exchange.h learns no routing: it takes an int. Kind::Hybrid gets it too, which is the layout that matters -- 8 ranks/node x 16 partitions went through the dense collective with no plan at all. The wire plan cannot come from the call site: only partition 0 reaches MPI, and its own row may be the empty one while a sibling holds the rank's only traffic. So partition 0 derives it in the B1->B2 window, where the published recv rows give the first view wider than one partition, and an empty rank resolves to the self peer rather than to dense -- keeping the branch a function of the gate alone. Asserted lossless against the send rows. Only the wire is narrowed; the serial O(R*S^2) staging sweeps still walk every rank, which needs the per-generator shift the recorded graph does not carry. sparse_pairwise takes an `active_legs` upper bound, so a dense plan over a one-leg layout no longer sizes its request vector at 2R (64 KB per exchange at R=4096, one malloc/free each). The Kind::Mpi arm keeps the DENSE plan on purpose: it walks all R and posts the non-zero legs, so no derived shift can drop a block there. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/Evolution.cpp | 19 +++- cpp/monoprop/detail/mpi/Exchange.h | 54 ++++----- cpp/monoprop/detail/mpi/HybridComm.h | 59 +++++++++- cpp/monoprop/detail/mpi/Pairwise.h | 13 ++- cpp/tests/flat_exchange_tests.cpp | 162 +++++++++++++++------------ 5 files changed, 192 insertions(+), 115 deletions(-) diff --git a/cpp/monoprop/Evolution.cpp b/cpp/monoprop/Evolution.cpp index 804fd6b3..bc0bb531 100644 --- a/cpp/monoprop/Evolution.cpp +++ b/cpp/monoprop/Evolution.cpp @@ -26,6 +26,7 @@ #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" #include "monoprop/detail/mpi/Exchange.h" #include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/Routing.h" namespace monoprop { namespace { @@ -73,12 +74,23 @@ auto &acquire_flat_exchange_buffers() { } // A property of the communicator, not the layer: all ranks participate even at local total_count 0. -// Still true with the pairwise arm: the transport branch is rank-local, so a collective may still be the -// one chosen and a rank that skipped the round strands it. +// Still true with the pairwise arm: `wire_bits` decides the transport for the whole communicator, so a +// rank that skipped the round strands the others whichever transport they are on. auto layer_exchange_participates(const mpi::Comm &comm) -> bool { return mpi::size(comm) != 1; } +// The transport gate for post_flat_alltoallv, and the ONLY thing allowed to choose it: rank-uniform by +// construction, because linear_bits_for reads the environment alone and check_routing_agreement +// allreduces exactly this number at construction and throws rather than proceed on a mismatch. Fanout 1 +// -- every generator's queries land on one destination rank -- is both why a layer's other legs are +// empty and why no rank can be on the other side of the branch. Any other geometry keeps the collective. +auto layer_exchange_wire_bits(const mpi::Comm &comm) -> int { + const auto ranks = static_cast(mpi::geometry(comm).ranks); + const size_t bits = routing::linear_bits_for(ranks); + return (ranks >> bits) == 1 ? static_cast(bits) : 0; +} + // Derives both sides at once: the count matrix is symmetric, so the recv layout is the send layout. auto derive_layer_exchange(const LayerTraversal &layer, const mpi::Comm &comm, int scale, LayerExchangeLayout &layout) -> void { @@ -117,7 +129,8 @@ inline auto begin_flat_exchange(FlatExchangeBuffers &buffers, const mpi::Comm &c .recv_counts = layout.counts.data(), .recv_displs = layout.displs.data()}, mpi::size(comm), - comm); + comm, + layer_exchange_wire_bits(comm)); return handle; } diff --git a/cpp/monoprop/detail/mpi/Exchange.h b/cpp/monoprop/detail/mpi/Exchange.h index 449f361f..2ecf306c 100644 --- a/cpp/monoprop/detail/mpi/Exchange.h +++ b/cpp/monoprop/detail/mpi/Exchange.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include @@ -33,7 +32,7 @@ namespace monoprop::mpi { // whatever the span holds, so a layout built for a differently sized communicator reads out of bounds. auto check_exchange_layout_width(std::span send_counts, const Comm &comm) -> void; -// Legs carrying a payload in either direction, which is what the pairwise path would post. +// Legs carrying a payload in either direction, i.e. an upper bound on what the pairwise path posts. [[nodiscard]] inline auto active_leg_count(const int *send_counts, const int *recv_counts, int num_ranks) -> int { int legs = 0; for (int i = 0; i < num_ranks; ++i) { @@ -42,26 +41,6 @@ auto check_exchange_layout_width(std::span send_counts, const Comm &c return legs; } -// A quarter of the fan-out: past that the pairwise post is >= N/2 requests against the collective's one -// tuned schedule, which wins there (a 54-peer sparse round measured 1.87x the collective). -inline constexpr int kSparseLegDivisor = 4; - -// Floored at 1, so an empty row and a one-peer row -- the two shapes linear routing produces -- land on -// the same side at every N; at N < 4 the bare quotient is 0 and would split them. -[[nodiscard]] inline auto sparse_leg_budget(int num_ranks) -> int { - return std::max(1, num_ranks / kSparseLegDivisor); -} - -// Which transport the Kind::Mpi arm takes. RANK-LOCAL, so it is a PRECONDITION that every rank lands on -// the same side: a rank choosing MPI_Ialltoallv waits forever on ranks that chose point-to-point. Linear -// routing (routing::Router, d >= 2 bits) gives that -- a generator reaches at most `ranks >> d` peers, -// so no row can exceed the budget -- while splitmix routing does not, and a layer where one rank has no -// cross-rank partners at all can then split the branch. -[[nodiscard]] inline auto flat_exchange_prefers_pairwise(const int *send_counts, const int *recv_counts, int num_ranks) - -> bool { - return active_leg_count(send_counts, recv_counts, num_ranks) <= sparse_leg_budget(num_ranks); -} - // Idempotent completion handle for a posted payload transfer; move-only, so a request is waited on // exactly once. wait() is a no-op on the blocking path and in non-MPI builds. Owns its requests: the // destructor completes anything still in flight, because a dropped in-flight transfer -- what an @@ -124,10 +103,17 @@ class [[nodiscard("call wait() on the Ticket to complete the posted transfer")]] }; // Never skipped on zero total: the collective arm needs all ranks or it deadlocks. Non-blocking in an -// MPI build -- MPI_Ialltoallv, or Isend/Irecv over the active legs when the layout is sparse enough (the -// Ticket completes either); non-MPI build does a per-rank self-copy (recv layout == send layout). +// MPI build -- MPI_Ialltoallv, or Isend/Irecv over the legs that carry a payload (the Ticket completes +// either); non-MPI build does a per-rank self-copy (recv layout == send layout). +// +// `wire_bits` picks the transport and MUST be RANK-UNIFORM: a rank choosing MPI_Ialltoallv waits forever +// on ranks that chose point-to-point, and no predicate over a rank's OWN row can promise that (rows vary, +// so any threshold on one straddles). It is the resolved linear-routing bit count when that routing gives +// fanout 1 -- one destination rank per generator, which is what empties the other legs -- and 0, today's +// collective, for every other geometry. The caller owns that derivation; see Evolution.cpp. template -inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, Comm comm) -> Ticket { +inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, Comm comm, int wire_bits = 0) + -> Ticket { // The in-process transports address the buffers as raw bytes; MPI_Ialltoallv below still takes the // typed pointers plus a datatype. Offsets stay in elements on both paths. if (comm.kind == Comm::Kind::Shm) { @@ -145,13 +131,18 @@ inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, } #ifdef monoprop_ENABLE_MPI if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoallv(comm.shm_rank, args.bytes(), datatype::get()); + // The wire is narrowed inside the verb, not here: only partition 0 reaches MPI, and it cannot + // name the rank's peer from its own row alone (its row may be the empty one). See HybridComm. + comm.hyb->alltoallv(comm.shm_rank, args.bytes(), datatype::get(), PeerPlan{}, wire_bits); return Ticket{}; } - if (flat_exchange_prefers_pairwise(args.send_counts, args.recv_counts, num_ranks)) { - // No plan and no count round: the count matrix is symmetric, so what this rank sends a peer IS - // that peer's recv count and both ends drop the same legs. A dense plan walks all N and posts - // only the non-zero ones, which is exactly that. + if (wire_bits > 0) { + // Which legs to drop needs no plan and no count round: the count matrix is symmetric, so what + // this rank sends a peer IS that peer's recv count and both ends drop the same legs on the same + // value. The plan stays DENSE -- it walks all N and posts only the non-zero legs, which is + // exactly that -- so a mis-derived shift cannot drop a block here; `wire_bits` only chooses the + // transport. `legs` sizes the request vector, which a dense plan would otherwise take to 2N. + const int legs = active_leg_count(args.send_counts, args.recv_counts, num_ranks); std::vector requests; const int posted = sparse_pairwise(PeerPlan{}, rank(comm), @@ -164,7 +155,8 @@ inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, PeerLayout{.counts = args.send_counts, .displs = args.send_displs}, reinterpret_cast(args.recv), PeerLayout{.counts = args.recv_counts, .displs = args.recv_displs}, - requests); + requests, + legs); return Ticket(std::move(requests), posted); } MPI_Request request = MPI_REQUEST_NULL; diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index fa6b1f34..3b95c7f4 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -114,9 +114,19 @@ class HybridComm { // See AlltoallvArgs for the send-buffer lifetime and the element-vs-byte convention; `dt` is the MPI // datatype whose extent is args.elem, and it stays a separate argument because the bundle is shared // with the non-MPI-capable transport. - auto alltoallv(int local_partition, const AlltoallvArgs &args, MPI_Datatype dt, PeerPlan plan = {}) -> void { - guard_partition0_(local_partition, "alltoallv", [this, local_partition, &args, dt, plan] { - alltoallv_impl_(local_partition, args, dt, plan); + // + // `derive_wire_bits` > 0 asks partition 0 to narrow the WIRE itself, to that many linear bits, from + // the destination ranks the whole rank actually uses. A caller cannot supply that plan: only + // partition 0 reaches MPI, and its own row may be the empty one while a sibling partition has the + // rank's only traffic. Legal only for a SYMMETRIC layout (recv counts are the send counts), which is + // what lets the peer set be read off the published recv rows; asserted against the send rows. + auto alltoallv(int local_partition, + const AlltoallvArgs &args, + MPI_Datatype dt, + PeerPlan plan = {}, + int derive_wire_bits = 0) -> void { + guard_partition0_(local_partition, "alltoallv", [this, local_partition, &args, dt, plan, derive_wire_bits] { + alltoallv_impl_(local_partition, args, dt, plan, derive_wire_bits); }); } @@ -208,7 +218,11 @@ class HybridComm { } // Flat variable all-to-all over caller-owned buffers; see AlltoallvArgs for the conventions. - auto alltoallv_impl_(int local_partition, const AlltoallvArgs &args, MPI_Datatype dt, PeerPlan plan) -> void { + auto alltoallv_impl_(int local_partition, + const AlltoallvArgs &args, + MPI_Datatype dt, + PeerPlan plan, + int derive_wire_bits = 0) -> void { const size_t u = static_cast(local_partition); Slot &me = slots_[u]; me.ptr = args.send; @@ -219,7 +233,14 @@ class HybridComm { // B2: partition 0 sizes/reallocates staging; must finish before any partition packs into stage_send_. if (local_partition == 0) { - fill_peers_(plan); + // Written here, read again in the B3->B4 window: partition 0 is this member's only toucher. + wire_plan_ = plan; + if (derive_wire_bits > 0) { + wire_plan_ = derived_wire_plan_(derive_wire_bits); + // The recv rows it was read off against the send rows: the symmetry the parameter needs. + assert(narrowing_is_lossless_(wire_plan_)); + } + fill_peers_(wire_plan_); size_staging_send_(args.elem); fill_recv_col_([this](int a, int t) { return row_recv_(t)[a]; }); size_staging_recv_(args.elem); @@ -232,7 +253,7 @@ class HybridComm { // B4: partition 0 moves the payload while peers park at the barrier. if (local_partition == 0) { - exchange_payload_(dt, args.elem, plan); + exchange_payload_(dt, args.elem, wire_plan_); } sync(); // B4 @@ -469,6 +490,30 @@ class HybridComm { } } + // The rank-level peer set, read off the recv rows every partition published before B1 -- the first + // point with a view wider than one partition's row. Under fanout-1 routing a layer's traffic is all + // on ONE rank, so this resolves to a shift; with nothing occupied it resolves to the self peer, whose + // legs are then all zero, and that keeps the collective-vs-pairwise branch a function of + // `derive_wire_bits` alone rather than of a rank's data (a data-dependent branch straddles and hangs). + // A set wider than one rank contradicts the caller's fanout claim: dense, so nothing is dropped. + auto derived_wire_plan_(int bits) -> PeerPlan { + int found = -1; + for (int u = 0; u < s_; ++u) { + const long long *rr = row_recv_(u); + for (int a = 0; a < r_; ++a) { + if (rr[a] != 0 && a != found) { + if (found >= 0) { + assert(false && "fanout claimed 1, but this rank's layer spans several peer ranks"); + return PeerPlan{}; + } + found = a; + } + } + } + const auto mask = static_cast((1U << static_cast(bits)) - 1U); + return PeerPlan{.bits = bits, .shift = found < 0 ? 0 : ((mpi_rank_ & mask) ^ (found & mask))}; + } + // Do the published rows put anything outside the plan's peers? If so the narrowing silently drops it. auto narrowing_is_lossless_(PeerPlan plan) const -> bool { for (int su = 0; su < s_; ++su) { @@ -766,6 +811,8 @@ class HybridComm { int count_posted_ = 0; // live requests in count_reqs_; always 0 on the dense (blocking) arm // This verb's peer ranks; see fill_peers_. std::vector peers_; + // alltoallv's wire plan, partition 0 only: written in B1->B2, read in B3->B4. See derived_wire_plan_. + PeerPlan wire_plan_; PartitionBarrier barrier_; }; diff --git a/cpp/monoprop/detail/mpi/Pairwise.h b/cpp/monoprop/detail/mpi/Pairwise.h index 01cc6af0..84b66293 100644 --- a/cpp/monoprop/detail/mpi/Pairwise.h +++ b/cpp/monoprop/detail/mpi/Pairwise.h @@ -61,6 +61,10 @@ struct PeerLayout { // POSTS ONLY, and returns how many of `reqs` are live. The caller waits, so `send`, `recv` and `reqs` // must all outlive that wait -- which is what lets a caller hold the round open (PendingAlltoallv) the // same way the dense branch holds an MPI_Ialltoallv. +// +// `active_legs` is an UPPER BOUND on the peers that will post, for a caller that already knows it (a +// dense plan over a mostly-empty layout sizes `reqs` at 2 * n_ranks otherwise); negative means "assume +// every peer posts". Too small an upper bound is caught by the assert below, not silently. [[nodiscard]] inline auto sparse_pairwise(PeerPlan plan, int me, int n_ranks, @@ -72,10 +76,12 @@ struct PeerLayout { PeerLayout send_lay, std::byte *recv, PeerLayout recv_lay, - std::vector &reqs) -> int { + std::vector &reqs, + int active_legs = -1) -> int { const int f = plan.count(n_ranks); - if (reqs.size() < static_cast(2 * f)) { - reqs.resize(static_cast(2 * f)); + const auto cap = static_cast(2 * (active_legs < 0 || active_legs > f ? f : active_legs)); + if (reqs.size() < cap) { + reqs.resize(cap); } int n_req = 0; for (int k = 0; k < f; ++k) { @@ -92,6 +98,7 @@ struct PeerLayout { } continue; } + assert(static_cast(n_req) + 2 <= reqs.size() || (rc == 0 && sc == 0)); // active_legs too small if (rc != 0) { MPI_Irecv(rbuf, rc, dt, b, tag, comm, &reqs[static_cast(n_req++)]); } diff --git a/cpp/tests/flat_exchange_tests.cpp b/cpp/tests/flat_exchange_tests.cpp index def50037..7621e315 100644 --- a/cpp/tests/flat_exchange_tests.cpp +++ b/cpp/tests/flat_exchange_tests.cpp @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// post_flat_alltoallv's transport choice (the graph REPLAY path): a dense layout keeps MPI_Ialltoallv, a -// sparse one goes point-to-point over the active legs. Ticket::in_flight() is what tells the two apart -- -// 1 for the collective, two requests per posted leg, 0 for a round with nothing to move. +// post_flat_alltoallv's transport choice (the graph REPLAY path). `wire_bits` alone picks it -- never the +// layout -- because a data-dependent branch straddles: a rank inside MPI_Ialltoallv waits forever on one +// that chose point-to-point. Ticket::in_flight() is what tells the two apart: 1 for the collective, two +// requests per posted leg, 0 for a round with nothing to move. #include @@ -27,13 +28,14 @@ #include "monoprop/detail/mpi/Exchange.h" #ifdef monoprop_ENABLE_MPI +#include + #include #endif +using monoprop::mpi::active_leg_count; using monoprop::mpi::Comm; -using monoprop::mpi::flat_exchange_prefers_pairwise; using monoprop::mpi::post_flat_alltoallv; -using monoprop::mpi::sparse_leg_budget; namespace { @@ -63,36 +65,27 @@ auto world_rank() -> int { MPI_Comm_rank(MPI_COMM_WORLD, &r); return r; } +// What Evolution's gate resolves to when routing gives fanout 1, which is the default at a power-of-two +// rank count. Tests must not derive it from their own layout -- that is the straddle. +auto wire_bits_for(int n) -> int { + return std::countr_zero(static_cast(n)); +} #endif } // namespace -// The boundary is <=, and the budget floors at 1 so an empty row and a one-peer row never split at N < 4 -// -- a split is a deadlock, not a slow round. -BOOST_AUTO_TEST_CASE(flat_exchange_pairwise_budget_boundary) { - for (const int n : {1, 2, 3, 4, 8, 16, 64}) { - const int budget = std::max(1, n / 4); - BOOST_REQUIRE_EQUAL(sparse_leg_budget(n), budget); - std::vector counts(static_cast(n), 0); - for (int legs = 0; legs <= n; ++legs) { - std::fill(counts.begin(), counts.end(), 0); - std::fill_n(counts.begin(), legs, 1); - BOOST_CHECK_EQUAL(flat_exchange_prefers_pairwise(counts.data(), counts.data(), n), legs <= budget); - } - } -} - -// A leg is active if EITHER side carries a payload: an asymmetric layout must not have one end post a -// receive the other never sends. +// The upper bound the request vector is sized from: a leg counts if EITHER side carries a payload, so a +// bound too small can never be handed to sparse_pairwise. BOOST_AUTO_TEST_CASE(flat_exchange_active_legs_take_either_side) { - constexpr int n = 8; // budget 2 + constexpr int n = 8; std::vector send(static_cast(n), 0); std::vector recv(static_cast(n), 0); + BOOST_CHECK_EQUAL(active_leg_count(send.data(), recv.data(), n), 0); send[1] = 4; recv[6] = 4; - BOOST_CHECK(flat_exchange_prefers_pairwise(send.data(), recv.data(), n)); - recv[3] = 4; - BOOST_CHECK(!flat_exchange_prefers_pairwise(send.data(), recv.data(), n)); + BOOST_CHECK_EQUAL(active_leg_count(send.data(), recv.data(), n), 2); + recv[1] = 4; // same leg, both sides + BOOST_CHECK_EQUAL(active_leg_count(send.data(), recv.data(), n), 2); } // The self leg is a copy, not a message, so a one-rank world posts nothing at all and wait() is a no-op. @@ -110,7 +103,8 @@ BOOST_AUTO_TEST_CASE(flat_exchange_self_leg_is_copied_with_nothing_posted) { .recv_counts = counts.data(), .recv_displs = displs.data()}, 1, - c); + c, + /*wire_bits=*/1); BOOST_CHECK_EQUAL(ticket.in_flight(), 0); ticket.wait(); BOOST_CHECK(out == send); @@ -118,21 +112,16 @@ BOOST_AUTO_TEST_CASE(flat_exchange_self_leg_is_copied_with_nothing_posted) { #ifdef monoprop_ENABLE_MPI -// Every leg active: N legs is above the budget at every N >= 2, so the round stays on MPI_Ialltoallv. -BOOST_AUTO_TEST_CASE(flat_exchange_dense_layout_takes_the_collective) { - const int n = world_size(); - if (n < 2) { - return; - } - const int me = world_rank(); +namespace { + +// One round over a symmetric layout, returning what wait() had to drain. `counts` is both sides. +auto run_round(int n, + int wire_bits, + const std::vector &counts, + const std::vector &send, + std::vector &out) -> int { Comm c{MPI_COMM_WORLD}; - const std::vector counts(static_cast(n), 1); const auto displs = displs_of(counts); - std::vector send(static_cast(n)); - for (int d = 0; d < n; ++d) { - send[static_cast(d)] = (me * 1000) + d; - } - std::vector out(static_cast(total_of(counts)), -1); auto ticket = post_flat_alltoallv({.send = send.data(), .send_counts = counts.data(), .send_displs = displs.data(), @@ -140,16 +129,46 @@ BOOST_AUTO_TEST_CASE(flat_exchange_dense_layout_takes_the_collective) { .recv_counts = counts.data(), .recv_displs = displs.data()}, n, - c); - BOOST_CHECK_EQUAL(ticket.in_flight(), 1); // the collective, not 2N pairwise requests + c, + wire_bits); + const int drained = ticket.in_flight(); ticket.wait(); - for (int src = 0; src < n; ++src) { - BOOST_CHECK_EQUAL(out[static_cast(src)], (src * 1000) + me); + return drained; +} + +} // namespace + +// wire_bits == 0 is today's collective whatever the layout holds -- including the sparse layout the +// pairwise arm exists for. The layout must not be able to move the branch. +BOOST_AUTO_TEST_CASE(flat_exchange_zero_wire_bits_always_takes_the_collective) { + const int n = world_size(); + if (n < 2 || (n % 2) != 0) { + return; + } + const int me = world_rank(); + const int peer = me ^ 1; + for (const bool dense : {true, false}) { + std::vector counts(static_cast(n), 0); + if (dense) { + std::fill(counts.begin(), counts.end(), 1); + } + else { + counts[static_cast(peer)] = 1; // one leg: the shape the pairwise arm is for + } + const auto displs = displs_of(counts); + std::vector send(static_cast(total_of(counts)), me); + std::vector out(static_cast(total_of(counts)), -1); + BOOST_CHECK_EQUAL(run_round(n, 0, counts, send, out), 1); + for (int i = 0; i < n; ++i) { + if (counts[static_cast(i)] != 0) { + BOOST_CHECK_EQUAL(out[static_cast(displs[static_cast(i)])], i); + } + } } } // One peer, me ^ 1 -- an involution, so the count matrix stays symmetric and both ends drop the same -// N - 1 legs. Two requests posted (one Irecv, one Isend) and the same bytes delivered. +// n - 1 legs. Two requests (one Irecv, one Isend) and the same bytes delivered. BOOST_AUTO_TEST_CASE(flat_exchange_single_leg_takes_the_pairwise_path) { const int n = world_size(); if (n < 2 || (n % 2) != 0) { @@ -158,54 +177,53 @@ BOOST_AUTO_TEST_CASE(flat_exchange_single_leg_takes_the_pairwise_path) { const int me = world_rank(); const int peer = me ^ 1; constexpr int len = 3; - Comm c{MPI_COMM_WORLD}; std::vector counts(static_cast(n), 0); counts[static_cast(peer)] = len; - const auto displs = displs_of(counts); - BOOST_REQUIRE_EQUAL(total_of(counts), len); std::vector send(static_cast(len)); for (int j = 0; j < len; ++j) { send[static_cast(j)] = (me * 1000) + j; } std::vector out(static_cast(len), -1); - auto ticket = post_flat_alltoallv({.send = send.data(), - .send_counts = counts.data(), - .send_displs = displs.data(), - .recv = out.data(), - .recv_counts = counts.data(), - .recv_displs = displs.data()}, - n, - c); - BOOST_CHECK_EQUAL(ticket.in_flight(), 2); - ticket.wait(); + BOOST_CHECK_EQUAL(run_round(n, wire_bits_for(n), counts, send, out), 2); for (int j = 0; j < len; ++j) { BOOST_CHECK_EQUAL(out[static_cast(j)], (peer * 1000) + j); } } +// Every leg active on the pairwise arm: it posts a pair per non-zero leg rather than falling back, so +// the transport really is the gate's choice and not the layout's. The self leg is a copy, not a pair. +BOOST_AUTO_TEST_CASE(flat_exchange_dense_layout_on_the_pairwise_arm) { + const int n = world_size(); + if (n < 2 || (n & (n - 1)) != 0) { + return; // off a power of two the gate resolves to 0 bits, i.e. the collective + } + const int me = world_rank(); + const std::vector counts(static_cast(n), 1); + const auto displs = displs_of(counts); + std::vector send(static_cast(n)); + for (int d = 0; d < n; ++d) { + send[static_cast(d)] = (me * 1000) + d; + } + std::vector out(static_cast(n), -1); + BOOST_CHECK_EQUAL(run_round(n, wire_bits_for(n), counts, send, out), 2 * (n - 1)); + for (int src = 0; src < n; ++src) { + BOOST_CHECK_EQUAL(out[static_cast(src)], (src * 1000) + me); + } +} + // No active leg anywhere: nothing is posted, wait() drains nothing, and the recv buffer is left as the -// caller sized it. The shape a layer with no cross-rank partners takes. +// caller sized it. A layer with no cross-rank partners takes this shape on EVERY rank at once -- the +// symmetric layout is what makes that true -- so the pairwise arm is still the agreed one. BOOST_AUTO_TEST_CASE(flat_exchange_empty_layout_posts_nothing) { const int n = world_size(); - if (n < 2) { + if (n < 2 || (n & (n - 1)) != 0) { return; } - Comm c{MPI_COMM_WORLD}; const std::vector counts(static_cast(n), 0); - const auto displs = displs_of(counts); BOOST_REQUIRE_EQUAL(total_of(counts), 0); const std::vector send(1, 0); std::vector out(1, -1); // Evolution sizes an empty round to 1, not 0 - auto ticket = post_flat_alltoallv({.send = send.data(), - .send_counts = counts.data(), - .send_displs = displs.data(), - .recv = out.data(), - .recv_counts = counts.data(), - .recv_displs = displs.data()}, - n, - c); - BOOST_CHECK_EQUAL(ticket.in_flight(), 0); - ticket.wait(); + BOOST_CHECK_EQUAL(run_round(n, wire_bits_for(n), counts, send, out), 0); BOOST_CHECK_EQUAL(out[0], -1); } From 3e6ab805a41f315f6dc1b80de456b3082f81b925 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 16:48:10 +0100 Subject: [PATCH 20/24] =?UTF-8?q?fix(mpi):=20=F0=9F=94=80=20reconcile=20th?= =?UTF-8?q?e=20boolean=20peer=20plan=20across=20the=20stack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four breaks a clean three-way merge did not surface, because each side edited a different line: - `PeerPlan{.bits=}` in `derived_wire_plan_` and three `hybrid_comm_tests` cases, written against the int dial the routing commit replaced with `.sparse`. - `routing::linear_bits_for`, deleted with the dial but still the replay transport's gate. Restored through `Router::bits_for`, which IS the private constructor, so the resolution and the non-power-of-two throw cannot drift. - `sparse_count`, dropped as the deleted fanout-2 case's only helper; the count round's new cases had since become a second user. - The fanout-2 sweep in `..._split_count_round_matches_the_dense_arm`: a sparse plan is fanout 1 by construction, so `f > 1` is no longer expressible. Co-Authored-By: Claude Opus 5 (1M context) --- cpp/monoprop/detail/mpi/HybridComm.h | 4 ++-- cpp/monoprop/detail/mpi/Routing.h | 11 +++++++++++ cpp/tests/hybrid_comm_tests.cpp | 23 +++++++++++------------ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 3b95c7f4..610dbd5b 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -510,8 +510,8 @@ class HybridComm { } } } - const auto mask = static_cast((1U << static_cast(bits)) - 1U); - return PeerPlan{.bits = bits, .shift = found < 0 ? 0 : ((mpi_rank_ & mask) ^ (found & mask))}; + // The plan is a boolean now, so every rank bit is a linear bit and the mask is the rank index. + return PeerPlan{.sparse = bits > 0, .shift = found < 0 ? 0 : (mpi_rank_ ^ found)}; } // Do the published rows put anything outside the plan's peers? If so the narrowing silently drops it. diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index 2cb565d8..c9a2ae04 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -192,6 +192,12 @@ class Router final { return linear_ ? static_cast(std::countr_zero(ranks_)) : 0; } + // The same number for a geometry alone, with no monomial width bound: it IS the constructor, so the + // resolution and the non-power-of-two throw cannot drift from the router's. + [[nodiscard]] static auto bits_for(size_t ranks, bool linear) -> size_t { + return Router{ranks, 1, linear}.linear_bits(); + } + // Flat destination slot in [0, flat_world). Branch is on a member, so it is perfectly predicted. template [[nodiscard]] [[gnu::always_inline]] inline auto dest(const Monomial &mono) const noexcept -> size_t { @@ -308,6 +314,11 @@ inline auto linear_requested() -> bool { return config::get().routing_mode.value_or(config::RoutingMode::Linear) == config::RoutingMode::Linear; } +// Resolved rank bits for a geometry, without a router: the replay transport gates on the number. +inline auto linear_bits_for(size_t ranks) -> size_t { + return Router::bits_for(ranks, linear_requested()); +} + template inline auto make_router(size_t ranks, size_t partitions) -> Router { return Router::for_modes(ranks, partitions, linear_requested()); diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index b2f3ce70..3ff7d5ba 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -686,6 +686,11 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_known_recv_counts_are_masked_through_the_plan) namespace { +// Varies along BOTH ends and hits 0, so a block landing on the wrong peer or the wrong partition +// changes a length, not just a value. +auto sparse_count(int src, int dst) -> int { + return ((src * 3) + (dst * 5)) % 4; +} auto sparse_tag(int src, int dst, int j) -> int { return (((src * 128) + dst) * 1000) + j; } @@ -924,16 +929,12 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_matches_the_dense_arm if (R < 2 || (R & (R - 1)) != 0) { return; // the XOR pairing needs a power-of-two rank count } - const int full = std::countr_zero(static_cast(R)); const int me = world_rank(); int cases = 0; - for (const int f : {1, 2}) { - const int bits = full - std::countr_zero(static_cast(f)); - if (bits < 1) { - continue; // bits == 0 IS the dense arm, which is the reference here - } - for (int shift = 0; shift < (1 << bits); ++shift) { - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + { + constexpr int f = 1; // a sparse plan is fanout 1 by construction; f > 1 is no longer expressible + for (int shift = 0; shift < R; ++shift) { + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; BOOST_REQUIRE_EQUAL(plan.count(R), f); ++cases; for (const int S : {1, 2, 3}) { @@ -995,9 +996,8 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_self_peer_only) { if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); const int me = world_rank(); - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = 0}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = 0}; BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); for (const int S : {1, 2, 3}) { const int P = R * S; @@ -1040,10 +1040,9 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_zero_count_peer) { if (R < 2 || (R & (R - 1)) != 0) { return; } - const int bits = std::countr_zero(static_cast(R)); const int me = world_rank(); for (int shift = 1; shift < R; ++shift) { // shift 0 is the self peer, covered above - const monoprop::mpi::PeerPlan plan{.bits = bits, .shift = shift}; + const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; const int peer = plan.peer(me, 0); BOOST_REQUIRE(peer != me); const int my_len = me < peer ? 0 : 4; // exactly one end of the pair is silent From 68a1907ba54475f5844e1774d97e3491b8706e69 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 17:32:02 +0100 Subject: [PATCH 21/24] =?UTF-8?q?feat(mpi):=20=E2=9C=A8=20name=20the=20slo?= =?UTF-8?q?t=20window=20a=20peer=20plan=20can=20reach?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under GF(2)-linear routing a generator's queries all land on one peer rank, so the reachable flat slots are that rank's S partitions -- one contiguous run of the P=R*S world instead of all of it. SlotWindow names the run, WindowIndex is its re-based index (a distinct type: a flat slot used as one would otherwise stay in bounds and address the wrong peer), and WindowVec is a vector over the run whose only flat-slot door asserts membership. PeerPlan::window derives it in one expression per field; dense is its count == P value, not a second case. No caller yet. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/mpi/Comm.h | 71 +++++++++++++++++++++++++ cpp/tests/routing_tests.cpp | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) diff --git a/cpp/monoprop/detail/mpi/Comm.h b/cpp/monoprop/detail/mpi/Comm.h index 3f68cf47..2a9d3055 100644 --- a/cpp/monoprop/detail/mpi/Comm.h +++ b/cpp/monoprop/detail/mpi/Comm.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -65,6 +66,68 @@ struct Comm { } }; +// A window-relative index. Distinct from a flat slot on purpose: the two are the same number only when +// the window starts at 0, so a swap addresses the wrong peer while staying in bounds. +struct WindowIndex { + size_t value = 0; + + constexpr WindowIndex() = default; + explicit constexpr WindowIndex(size_t v) noexcept : value(v) {} +}; + +// The contiguous run of flat destination slots a round can reach. Slots are rank-major +// (slot = rank * S + partition), so one rank's S partitions are contiguous and the single peer sparse +// routing leaves is exactly one such run; dense is the count == P value of the same run, not a second +// shape. See PeerPlan::window. +struct SlotWindow { + size_t base = 0; // first reachable flat slot + size_t count = 0; // slots in the run + + [[nodiscard]] constexpr auto stop() const -> size_t { return base + count; } + [[nodiscard]] constexpr auto contains(size_t slot) const -> bool { return slot >= base && slot < stop(); } + // The one flat-slot door: it asserts membership, so a slot from outside cannot become another's entry. + [[nodiscard]] constexpr auto index(size_t slot) const -> WindowIndex { + assert(contains(slot) && "flat slot outside the window it is being re-based into"); + return WindowIndex{slot - base}; + } + [[nodiscard]] constexpr auto slot(WindowIndex i) const -> size_t { + assert(i.value < count); + return base + i.value; + } +}; + +// A vector over a SlotWindow, addressed by flat slot through at_slot(); operator[] takes a WindowIndex, +// so a flat slot used as a raw index does not compile. Re-basing an array is only safe if every index +// site shifts together, and these two accessors are the only sites. +template +class WindowVec { +public: + WindowVec() = default; + explicit WindowVec(SlotWindow w) : win_(w), v_(w.count) {} + + auto reset(SlotWindow w) -> void { + win_ = w; + v_.assign(w.count, T{}); + } + + [[nodiscard]] auto window() const -> SlotWindow { return win_; } + [[nodiscard]] auto size() const -> size_t { return v_.size(); } + + [[nodiscard]] auto operator[](WindowIndex i) -> T & { return v_[i.value]; } + [[nodiscard]] auto operator[](WindowIndex i) const -> const T & { return v_[i.value]; } + [[nodiscard]] auto at_slot(size_t slot) -> T & { return v_[win_.index(slot).value]; } + [[nodiscard]] auto at_slot(size_t slot) const -> const T & { return v_[win_.index(slot).value]; } + + [[nodiscard]] auto begin() { return v_.begin(); } + [[nodiscard]] auto end() { return v_.end(); } + [[nodiscard]] auto begin() const { return v_.begin(); } + [[nodiscard]] auto end() const { return v_.end(); } + +private: + SlotWindow win_{}; + std::vector v_; +}; + // Which destination RANKS a round can touch, when the caller knows. Two states, matching // routing::Router: dense, or sparse over the single peer GF(2)-linear routing implies. // @@ -96,6 +159,14 @@ struct PeerPlan { // `k` indexes the peer set, which is a singleton when sparse. [[nodiscard]] constexpr auto peer(int me, int k) const -> int { return sparse ? (me ^ shift) : k; } [[nodiscard]] constexpr auto contains(int me, int b) const -> bool { return !sparse || b == (me ^ shift); } + // The flat slots reachable from `me_flat` over a `ranks` x `parts` world. One expression per field: + // sparse names the peer rank's `parts` slots, dense is the same with peer rank 0 and count(ranks) + // == ranks, i.e. the whole world. + [[nodiscard]] constexpr auto window(size_t me_flat, size_t ranks, size_t parts) const -> SlotWindow { + const size_t peer_rank = sparse ? ((me_flat / parts) ^ static_cast(shift)) : 0; + return SlotWindow{.base = peer_rank * parts, + .count = static_cast(count(static_cast(ranks))) * parts}; + } }; // Argument bundles for the variable all-to-all verbs, deliberately here rather than in HybridComm.h: diff --git a/cpp/tests/routing_tests.cpp b/cpp/tests/routing_tests.cpp index 452b9a88..f7a413d0 100644 --- a/cpp/tests/routing_tests.cpp +++ b/cpp/tests/routing_tests.cpp @@ -31,6 +31,7 @@ #include #include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/mpi/Routing.h" @@ -215,6 +216,99 @@ BOOST_AUTO_TEST_CASE(routing_fanout_is_one_under_linear_routing) { } } +// mpi::PeerPlan::window -- the slots a plan can reach, which is what lets the per-generator structures +// be S long instead of P. Dense is the count == P value of the same two expressions, so it is checked +// against the same formula rather than against a second one. +BOOST_AUTO_TEST_CASE(routing_slot_window_is_the_peer_ranks_partition_run) { + for (const auto [r, s] : {std::pair{8, 16}, {128, 16}, {16, 1}, {1, 1}, {4, 3}}) { + const size_t p = r * s; + for (size_t me = 0; me < p; ++me) { + const mpi::SlotWindow dense = mpi::PeerPlan{}.window(me, r, s); + BOOST_REQUIRE_EQUAL(dense.base, 0U); + BOOST_REQUIRE_EQUAL(dense.count, p); + BOOST_REQUIRE(dense.contains(me)); + + for (size_t shift = 0; shift < r; ++shift) { + const mpi::PeerPlan plan{.sparse = true, .shift = static_cast(shift)}; + const mpi::SlotWindow w = plan.window(me, r, s); + BOOST_REQUIRE_EQUAL(w.count, s); + BOOST_REQUIRE_EQUAL(w.base, ((me / s) ^ shift) * s); + // Every slot in the run belongs to the one peer rank the plan names, and no other. + for (size_t k = 0; k < w.count; ++k) { + const size_t slot = w.slot(mpi::WindowIndex{k}); + BOOST_REQUIRE(w.contains(slot)); + BOOST_REQUIRE_EQUAL(w.index(slot).value, k); + BOOST_REQUIRE_EQUAL(slot / s, ((me / s) ^ shift)); + BOOST_REQUIRE(plan.contains(static_cast(me / s), static_cast(slot / s))); + } + BOOST_REQUIRE(!w.contains(w.base + w.count)); + // Self is reachable only at shift 0; the engine's self-resolve leg turns on that. + BOOST_REQUIRE_EQUAL(w.contains(me), shift == 0); + } + } + } +} + +// The window must be symmetric, or the two ends of one exchange size different arrays: XOR is an +// involution, so the peer's own window points back at this rank's run. +BOOST_AUTO_TEST_CASE(routing_slot_window_pairing_is_symmetric) { + constexpr size_t kRanks = 32; + constexpr size_t kParts = 8; + for (size_t me = 0; me < kRanks * kParts; me += 3) { + for (size_t shift = 0; shift < kRanks; ++shift) { + const mpi::PeerPlan plan{.sparse = true, .shift = static_cast(shift)}; + const mpi::SlotWindow mine = plan.window(me, kRanks, kParts); + const mpi::SlotWindow theirs = plan.window(mine.base, kRanks, kParts); + BOOST_REQUIRE_EQUAL(theirs.base, (me / kParts) * kParts); + BOOST_REQUIRE_EQUAL(theirs.count, kParts); + } + } +} + +// The property the re-basing rests on: every destination the emit path can produce for one generator +// lies inside that generator's window, so `slot - base` is always a legal index. +BOOST_AUTO_TEST_CASE(routing_every_dest_lands_inside_the_generators_window) { + const auto terms = random_monomials(600, 6, 0x5107500DULL); + const auto gens = random_monomials(20, 4, 0x1CE0FF1CEULL); + const std::vector> geometries{{8, 16}, {16, 1}, {32, 4}, {1, 14}, {4, 3}}; + size_t checked = 0; + for (const auto &[r, s] : geometries) { + for (const bool linear : {false, true}) { + const auto router = Router::for_modes(r, s, linear); + for (const auto &g : gens) { + const size_t shift = router.rank_shift(g); + const mpi::PeerPlan plan{.sparse = router.is_linear(), .shift = static_cast(shift)}; + for (const auto &m : terms) { + const size_t me = router.dest(m); + const mpi::SlotWindow w = plan.window(me, r, s); + BOOST_REQUIRE(w.contains(router.dest_from_shift(m ^ g, me, shift))); + ++checked; + } + } + } + } + BOOST_TEST_MESSAGE("window containment checks: " << checked); + BOOST_TEST(checked >= 100000U); +} + +// WindowVec re-bases in exactly one place, so the flat slot the writer used is the flat slot the reader +// gets back -- including when the run does not start at 0. +BOOST_AUTO_TEST_CASE(routing_window_vec_round_trips_flat_slots) { + const mpi::SlotWindow w{.base = 48, .count = 16}; + mpi::WindowVec v(w); + BOOST_REQUIRE_EQUAL(v.size(), w.count); + for (size_t slot = w.base; slot < w.stop(); ++slot) { + v.at_slot(slot) = slot * 7; + } + for (size_t k = 0; k < w.count; ++k) { + BOOST_REQUIRE_EQUAL(v[mpi::WindowIndex{k}], (w.base + k) * 7); + BOOST_REQUIRE_EQUAL(v.at_slot(w.slot(mpi::WindowIndex{k})), (w.base + k) * 7); + } + v.reset(mpi::SlotWindow{.base = 0, .count = 4}); + BOOST_REQUIRE_EQUAL(v.size(), 4U); + BOOST_REQUIRE_EQUAL(v.at_slot(3), 0U); +} + // Without a power-of-two rank count there is no XOR structure to exploit, and there is no partial dial // to fall back to, so the geometry is rejected at construction rather than routed on a subspace. BOOST_AUTO_TEST_CASE(routing_non_power_of_two_ranks_throw_under_linear_routing) { From 18549e60cf4be026468d8ad7d7348085a05ede83 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 17:54:00 +0100 Subject: [PATCH 22/24] =?UTF-8?q?perf(evolution):=20=E2=9A=A1=20size=20the?= =?UTF-8?q?=20per-generator=20query=20path=20to=20the=20peer=20window?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under linear routing a generator's queries all land on one peer rank, so the reachable destinations are that rank's S slots, not the P=R*S world. Every per-generator per-slot structure from the scan to the wire was still allocated and swept over all P: at R=128, S=16 that is length-2048 arrays with 16 live entries, built twice per generator for ~416 non-identity generators over 29 layers. The six FusedScanResult arrays, the engine's queries_r / src_idx_r / src_val_r / combined_qv_, the probe's goff / sender ids, the resolver's responses, and begin_alltoallv's counts / pack / prefix / unpack sweeps are now window-length. begin_alltoallv derives the window from the plan alone, so dense is the count == P value of the same expression rather than a second arm; a caller may still hand it a whole [P] array, which the window then masks. Re-basing is safe by construction rather than by review: WindowVec::at_slot is the only place a flat slot becomes an index, and it asserts membership, while operator[] takes a WindowIndex so a bare flat slot will not compile. Self is inside the window only when the rank shift is zero, which resolve_self_queries now branches on and asserts against an empty self stage. GraphSink::acc stays flat [P] -- build_layer_storage_unified is P-shaped -- so the sink turns the window index back into a slot. MPICompat could not be deferred: the scan's arrays are moved into the engine and thence onto the wire with no seam that does not cost a P-allocation to bridge. Assisted-by: ClaudeCode:claude-opus-5 --- .../detail/evolution/layer_build/Engine.h | 149 +++++++++------- .../detail/evolution/layer_build/Resolve.h | 92 ++++++---- .../detail/evolution/layer_build/Scan.h | 63 ++++--- cpp/monoprop/detail/mpi/Comm.h | 2 + cpp/monoprop/detail/mpi/MPICompat.h | 167 +++++++++++++----- cpp/tests/evolution_detail_tests.cpp | 2 +- cpp/tests/mpi_utils_tests.cpp | 41 +++-- cpp/tests/sparse_resolve_tests.cpp | 44 +++-- 8 files changed, 364 insertions(+), 196 deletions(-) diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index d13e7759..3c93ecfa 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -84,7 +84,8 @@ struct GraphSink { std::vector acc; size_t def_in_base_ = 0; // deferred self-miss bases into acc[my_rank] size_t def_out_base_ = 0; - std::vector in_base_; // cross-rank per-rank base into acc[s].in_entries (set in prepare) + // Cross-rank base into acc[slot].in_entries, over the query window (set in prepare). + mpi::WindowVec in_base_; GraphSink(size_t R_, size_t my_rank_) : R(R_), my_rank(my_rank_), acc(R_) {} @@ -105,33 +106,35 @@ struct GraphSink { // Cross-rank (R>1). Send buffer = the plain query stream (no value fusion). The exchange is positional: // responses[s][q] must answer incoming[s][q], one resolution per query. - auto send_buffer(std::vector &queries, - std::vector> & /*vals*/, - std::vector & /*scratch*/) -> std::vector & { + auto send_buffer(mpi::WindowVec &queries, + mpi::WindowVec> & /*vals*/, + mpi::WindowVec & /*scratch*/) -> mpi::WindowVec & { return queries; } + // `acc` stays flat [P] -- finalize hands it to build_layer_storage_unified, which is P-shaped -- so + // the window index is turned back into a slot here rather than re-basing it. auto prepare(const IncomingProbe & /*pr*/, - size_t rank_count, MPOperator & /*op*/, - const std::vector> &responses) -> void { - in_base_.assign(rank_count, 0); - for (size_t s = 0; s < rank_count; ++s) { - in_base_[s] = acc[s].in_entries.size(); - acc[s].in_entries.resize(in_base_[s] + responses[s].size()); + const mpi::WindowVec> &responses) -> void { + const mpi::SlotWindow w = responses.window(); + in_base_.reset(w); + for (size_t k = 0; k < w.count; ++k) { + const mpi::WindowIndex wi{k}; + PartnerAcc &a = acc[w.slot(wi)]; + in_base_[wi] = a.in_entries.size(); + a.in_entries.resize(in_base_[wi] + responses[wi].size()); } } auto on_resolved(size_t g, - size_t s, + mpi::WindowIndex s, size_t q, size_t ip, const IncomingProbe &pr, - const std::vector & /*incoming*/) -> Response { - acc[s].in_entries[in_base_[s] + q] = {ip, pr.phase_of[g]}; + const mpi::WindowVec & /*incoming*/) -> Response { + acc[pr.window.slot(s)].in_entries[in_base_[s] + q] = {ip, pr.phase_of[g]}; return static_cast(ip); } - auto process_reserve(const std::vector> & /*inc_r*/, - size_t /*rank_count*/, - size_t /*my_rank*/) -> void {} + auto process_reserve(const mpi::WindowVec> & /*inc_r*/, size_t /*my_rank*/) -> void {} auto on_response_block(size_t r, const std::vector &resp, const std::vector &srcs, @@ -236,28 +239,30 @@ struct ContractSink { // Cross-rank (R>1). Send buffer = queries interleaved with their v_src stream into `scratch` // (combined_qv_), so one alltoallv carries query + value. - auto send_buffer(std::vector &queries, std::vector> &vals, std::vector &scratch) - -> std::vector & { - scratch.resize(queries.size()); - for (size_t r = 0; r < queries.size(); ++r) { - QueryCodec::build_fused(queries[r], vals[r], scratch[r]); + auto send_buffer(mpi::WindowVec &queries, + mpi::WindowVec> &vals, + mpi::WindowVec &scratch) -> mpi::WindowVec & { + const mpi::SlotWindow w = queries.window(); + scratch.reset(w); + for (size_t k = 0; k < w.count; ++k) { + const mpi::WindowIndex wi{k}; + QueryCodec::build_fused(queries[wi], vals[wi], scratch[wi]); } return scratch; } auto prepare(const IncomingProbe &pr, - size_t /*rank_count*/, MPOperator &op, - const std::vector> & /*responses*/) -> void { + const mpi::WindowVec> & /*responses*/) -> void { state_mask_ = schrodinger ? initial_state_mask(op.initial_state) : Monomial{}; cross_base_ = fc.cross_half.size(); fc.cross_half.resize(cross_base_ + pr.nq_total); } auto on_resolved(size_t g, - size_t s, + mpi::WindowIndex s, size_t /*q*/, size_t ip, const IncomingProbe &pr, - const std::vector &incoming) -> Response { + const mpi::WindowVec &incoming) -> Response { double v_tgt; if (ip < pr.base) { v_tgt = fused_scale ? op_coeffs[ip] * inv_cos : op_coeffs[ip]; @@ -278,11 +283,13 @@ struct ContractSink { /*is_insert=*/ip >= pr.base}; return v_tgt; } - auto process_reserve(const std::vector> &inc_r, size_t rank_count, size_t my_rank_) -> void { + auto process_reserve(const mpi::WindowVec> &inc_r, size_t my_rank_) -> void { + const mpi::SlotWindow w = inc_r.window(); size_t incoming = 0; - for (size_t r = 0; r < rank_count; ++r) { - if (r != my_rank_) { - incoming += inc_r[r].size(); + for (size_t k = 0; k < w.count; ++k) { + const mpi::WindowIndex wi{k}; + if (w.slot(wi) != my_rank_) { + incoming += inc_r[wi].size(); } } fc.cross_half.reserve(fc.cross_half.size() + incoming); @@ -338,18 +345,21 @@ struct LayerBuildEngine { // → distinct found, so each slot is marked once. MatchedEpochSet &matched; size_t combined_size; - std::vector queries_r; - std::vector> src_idx_r; + // The destination slots this generator can reach: `plan`'s window for my_rank. Every per-slot array + // below is sized to it, so a flat slot only ever enters through WindowVec::at_slot. + mpi::SlotWindow window; + mpi::WindowVec queries_r; + mpi::WindowVec> src_idx_r; std::vector deferred_self_misses; // Deferred-miss positions, concatenated in miss order; parallel to deferred_self_misses. std::vector deferred_pos_flat_; // This pass's self-owned queries as positions, straight from the scan: never encoded, so the resolve - // below has nothing to decode. Parallel to src_idx_r[my_rank]. + // below has nothing to decode. Parallel to src_idx_r's self slot. SelfQueryStage self_stage_; // Scan-captured v_src per query (ContractSink only via Sink::wants_values; empty for GraphSink). - std::vector> src_val_r; + mpi::WindowVec> src_val_r; // Fused query+value send scratch (ContractSink, R>1): shared by a gate's two exchange passes. - std::vector combined_qv_; + mpi::WindowVec combined_qv_; // Which destination ranks this gate's queries can reach. Dense unless the router is GF(2)-linear; // see mpi::PeerPlan. Derived once per layer in build_layer, never per query. mpi::PeerPlan plan; @@ -369,28 +379,38 @@ struct LayerBuildEngine { my_rank(my_rank_), matched(matched_scratch), combined_size(combined_size_), - queries_r(R_), - src_idx_r(R_), plan(plan_), sink(std::move(sink_)) { + const auto geom = mpi::geometry(comm); + window = plan.window(my_rank, static_cast(geom.ranks), static_cast(geom.partitions)); + assert(window.stop() <= R && window.count != 0); + queries_r.reset(window); + src_idx_r.reset(window); matched.begin_gate(combined_size); } // Resolve this rank's own query stream inline, then clear it so the alltoallv never sends to self. + // Self is inside the window only when this generator's rank shift is zero; otherwise the window names + // another rank outright and the scan cannot have staged a self-owned partner. auto resolve_self_queries(bool is_leader_pass) -> void { - std::vector &ls = src_idx_r[my_rank]; + if (!window.contains(my_rank)) { + assert(self_stage_.size() == 0 && "a self-owned partner outside this generator's peer window"); + self_stage_.clear(); + return; + } + std::vector &ls = src_idx_r.at_slot(my_rank); std::vector *lv = nullptr; if constexpr (Sink::wants_values) { - lv = &src_val_r[my_rank]; + lv = &src_val_r.at_slot(my_rank); } // The scan routes a self-owned partner to the stage, never to the wire buffer. - assert(queries_r[my_rank].empty() && "a self-owned query was encoded instead of staged"); + assert(queries_r.at_slot(my_rank).empty() && "a self-owned query was encoded instead of staged"); assert(ls.size() == self_stage_.size() && "the self stage does not hold exactly one query per source"); resolve_range_(ls, lv, is_leader_pass); self_stage_.clear(); ls.clear(); if constexpr (Sink::wants_values) { - src_val_r[my_rank].clear(); + src_val_r.at_slot(my_rank).clear(); } } @@ -401,10 +421,14 @@ struct LayerBuildEngine { // pass must also drop the queries a leader already matched, and that only holds once the leader pass // has run. auto run_exchange(bool is_leader_pass, - std::vector &&queries, - std::vector> &&src_idx, - std::vector> &&src_val, + mpi::WindowVec &&queries, + mpi::WindowVec> &&src_idx, + mpi::WindowVec> &&src_val, SelfQueryStage &&self_stage) -> void { + // The scan sized its arrays to the same plan, so the two windows must agree exactly -- a mismatch + // would re-base every slot against the wrong base. + assert(queries.window().base == window.base && queries.window().count == window.count); + assert(src_idx.window().base == window.base && src_idx.window().count == window.count); queries_r = std::move(queries); src_idx_r = std::move(src_idx); self_stage_ = std::move(self_stage); @@ -417,31 +441,32 @@ struct LayerBuildEngine { if (R <= 1) { return; } - std::vector &send = sink.send_buffer(queries_r, src_val_r, combined_qv_); - std::vector> inc_q; + mpi::WindowVec &send = sink.send_buffer(queries_r, src_val_r, combined_qv_); + mpi::WindowVec inc_q; mpi::begin_alltoallv(send, comm, /*skip_self=*/false, /*known_recv_counts=*/nullptr, plan).wait_into(inc_q); - auto resp = resolve_incoming(inc_q, local_op, R, is_leader_pass, matched, combined_size, sink); + auto resp = resolve_incoming(inc_q, local_op, is_leader_pass, matched, combined_size, sink); std::vector resp_recv = response_recv_counts(); - std::vector> inc_r; + mpi::WindowVec> inc_r; // The answers retrace the queries, and the pairing is an XOR involution, so the same plan holds. mpi::begin_alltoallv(resp, comm, /*skip_self=*/false, &resp_recv, plan).wait_into(inc_r); - process_responses(inc_r, src_idx_r, queries_r, R, my_rank, sink); + process_responses(inc_r, src_idx_r, queries_r, my_rank, sink); } // Followers a leader already matched must not be re-resolved over the wire, so compact them out. auto drop_matched_cross_rank_followers() -> void { using QC = QueryCodec; const QueryLayout layout = sink.querier_layout(); - for (size_t r = 0; r < R; ++r) { - if (r == my_rank) { + for (size_t k = 0; k < window.count; ++k) { + const mpi::WindowIndex wi{k}; + if (window.slot(wi) == my_rank) { continue; } - VecZ &q = queries_r[r]; - std::vector &s = src_idx_r[r]; + VecZ &q = queries_r[wi]; + std::vector &s = src_idx_r[wi]; // Fused: the v_src stream is parallel to the query/source streams, so compact it in lockstep. std::vector *v = nullptr; if constexpr (Sink::wants_values) { - v = &src_val_r[r]; + v = &src_val_r[wi]; } const size_t nq = s.size(); size_t kept = 0; @@ -501,13 +526,16 @@ struct LayerBuildEngine { private: // Response counts are the transpose of the query counts (one answer per query), so passing them as // known_recv_counts skips the response count-Alltoall round. + // FLAT [P], which is what begin_alltoallv's known_recv_counts is indexed by; only the window's slots + // can be non-zero, and the window is what masks the rest. auto response_recv_counts() const -> std::vector { - std::vector counts(R); - for (size_t r = 0; r < R; ++r) { - // One response per QUERY, and src_idx_r[r] holds one source per query: no walk, no division. - assert(src_idx_r[r].size() == QueryCodec::count_queries(queries_r[r], sink.querier_layout()) + std::vector counts(R, 0); + for (size_t k = 0; k < window.count; ++k) { + const mpi::WindowIndex wi{k}; + // One response per QUERY, and src_idx_r's block holds one source per query: no walk, no division. + assert(src_idx_r[wi].size() == QueryCodec::count_queries(queries_r[wi], sink.querier_layout()) && "a querier buffer does not hold exactly one query per source"); - counts[r] = static_cast(src_idx_r[r].size()); + counts[window.slot(wi)] = static_cast(src_idx_r[wi].size()); } return counts; } @@ -614,6 +642,9 @@ auto build_layer(MPOperator &local_op, // today's collective. const size_t gen_shift = router.rank_shift(gen); const auto plan = mpi::PeerPlan{.sparse = router.is_linear(), .shift = static_cast(gen_shift)}; + // The reachable slots, once per generator: S of the P=R*S world under linear routing, all P otherwise. + // Every per-slot structure from the scan to the wire is sized to this run. + const mpi::SlotWindow scan_window = plan.window(my_rank, router.ranks(), router.partitions()); // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); @@ -651,7 +682,7 @@ auto build_layer(MPOperator &local_op, cut_st, coeffs, only_rotate_len_k, - R, + scan_window, my_rank, router, gen_shift, diff --git a/cpp/monoprop/detail/evolution/layer_build/Resolve.h b/cpp/monoprop/detail/evolution/layer_build/Resolve.h index 0c49706f..7e6d5902 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Resolve.h +++ b/cpp/monoprop/detail/evolution/layer_build/Resolve.h @@ -24,6 +24,7 @@ #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" #include "monoprop/detail/evolution/layer_build/QueryCodec.h" +#include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" @@ -38,10 +39,12 @@ struct IncomingProbe { // The STORE's position width, not the wire's: these positions exist to become rows. using PosT = typename OperatorIndex::PosT; - std::vector goff; // rank_count+1 flat offsets: g = goff[s] + q - DefaultInitVector sender_of; // g → sender rank + // The slots `incoming` covers; senders are named by their index into it, never by a flat slot. + mpi::SlotWindow window; + std::vector goff; // window.count+1 flat offsets: g = goff[k] + q + DefaultInitVector sender_wi; // g → sender's WINDOW index (see sender_index/sender_slot) DefaultInitVector phase_of; // g → query phase - // g → WORD offset of that query inside incoming[sender_of[g]]; a query ordinal names no position. + // g → WORD offset of that query inside its sender's buffer; a query ordinal names no position. DefaultInitVector off_of; DefaultInitVector idx_of; // g → resolved index (hit: < base; miss: base+j) std::vector miss_g; // j → the g that became miss j (Phase 4 reads the key of miss_g[j]) @@ -55,6 +58,10 @@ struct IncomingProbe { // g → fold_hash of the query key, folded by the probe and reused by the insert. DefaultInitVector hash_of; + // The two ways to name query g's sender. sender_wi is re-based, so nothing else reads it. + [[nodiscard]] auto sender_index(size_t g) const -> mpi::WindowIndex { return mpi::WindowIndex{sender_wi[g]}; } + [[nodiscard]] auto sender_slot(size_t g) const -> size_t { return window.slot(sender_index(g)); } + // BUILDS a bitset, so cold consumers only -- the fully paired minority, never anything per-term. [[nodiscard]] auto mono_at(size_t g) const -> Monomial { Monomial m; @@ -77,28 +84,29 @@ struct IncomingProbe { // fused for the ContractSink resolver, plain for GraphSink. The caller runs Phase 3, then // insert_incoming_misses. Counts and offsets come from the decode walk; there is no record stride. template -auto probe_incoming_queries(const std::vector &incoming, // serialized, one VecZ per sender +auto probe_incoming_queries(const mpi::WindowVec &incoming, // serialized, one VecZ per sender slot MPOperator &op, - size_t rank_count, QueryLayout layout) -> IncomingProbe { using QC = QueryCodec; IncomingProbe pr; + pr.window = incoming.window(); + const size_t senders = pr.window.count; - pr.goff.assign(rank_count + 1, 0); - for (size_t s = 0; s < rank_count; ++s) { - const size_t nq = QC::count_queries(incoming[s], layout); - pr.goff[s + 1] = pr.goff[s] + nq; + pr.goff.assign(senders + 1, 0); + for (size_t k = 0; k < senders; ++k) { + const size_t nq = QC::count_queries(incoming[mpi::WindowIndex{k}], layout); + pr.goff[k + 1] = pr.goff[k] + nq; } - pr.nq_total = pr.goff[rank_count]; + pr.nq_total = pr.goff[senders]; if (pr.nq_total == 0) { return pr; } - pr.sender_of.resize(pr.nq_total); - for (size_t s = 0; s < rank_count; ++s) { - std::fill(pr.sender_of.begin() + static_cast(pr.goff[s]), - pr.sender_of.begin() + static_cast(pr.goff[s + 1]), - static_cast(s)); + pr.sender_wi.resize(pr.nq_total); + for (size_t k = 0; k < senders; ++k) { + std::fill(pr.sender_wi.begin() + static_cast(pr.goff[k]), + pr.sender_wi.begin() + static_cast(pr.goff[k + 1]), + static_cast(k)); } // Phase 1 (read-only): deserialize, then probe with the group-prefetch batch find. One walk per sender. @@ -111,20 +119,21 @@ auto probe_incoming_queries(const std::vector &incoming, // serialized, on pr.pos_flat.clear(); // A hint only: the measured mean is 5.33 positions, so this is one allocation but for an outlier. pr.pos_flat.reserve(pr.nq_total * QueryCodec::kReservePositionsPerQuery); - for (size_t s = 0; s < rank_count; ++s) { + for (size_t si = 0; si < senders; ++si) { + const VecZ &buf = incoming[mpi::WindowIndex{si}]; size_t off = 0; - for (size_t g = pr.goff[s]; g < pr.goff[s + 1]; ++g) { + for (size_t g = pr.goff[si]; g < pr.goff[si + 1]; ++g) { int ph = 0; - const size_t k = QC::k_at(incoming[s], off); + const size_t k = QC::k_at(buf, off); const size_t at = pr.pos_flat.size(); pr.pos_flat.resize(at + k); // default-init grow: read_positions writes every element pr.pos_off[g] = at; pr.k_of[g] = static_cast(k); pr.off_of[g] = off; - off = QC::read_positions(incoming[s], layout, off, pr.pos_flat.data() + at, ph); + off = QC::read_positions(buf, layout, off, pr.pos_flat.data() + at, ph); pr.phase_of[g] = ph; } - assert(off == incoming[s].size() && "the query walk did not consume the sender's whole buffer"); + assert(off == buf.size() && "the query walk did not consume the sender's whole buffer"); } { const size_t op_size = op.store->size(); @@ -179,19 +188,19 @@ auto insert_incoming_misses(MPOperator &op, const IncomingProbe -auto resolve_incoming(const std::vector &incoming, // serialized, one VecZ per sender +auto resolve_incoming(const mpi::WindowVec &incoming, // serialized, one VecZ per sender slot MPOperator &op, - size_t rank_count, bool is_leader_pass, MatchedEpochSet &matched, size_t combined_size, // pre-layer op size: bounds the matched set - Sink &sink) -> std::vector> { + Sink &sink) -> mpi::WindowVec> { using Resp = typename Sink::Response; - const IncomingProbe pr = - probe_incoming_queries(incoming, op, rank_count, sink.incoming_layout()); - std::vector> responses(rank_count); - for (size_t s = 0; s < rank_count; ++s) { - responses[s].assign(pr.goff[s + 1] - pr.goff[s], Sink::init_response()); + const IncomingProbe pr = probe_incoming_queries(incoming, op, sink.incoming_layout()); + // The response window is the query window: the pairing is an XOR involution, so a rank answers + // exactly the slots it queried. + mpi::WindowVec> responses(pr.window); + for (size_t k = 0; k < pr.window.count; ++k) { + responses[mpi::WindowIndex{k}].assign(pr.goff[k + 1] - pr.goff[k], Sink::init_response()); } if (pr.nq_total == 0) { return responses; @@ -199,10 +208,10 @@ auto resolve_incoming(const std::vector &incoming, // serialized, one VecZ // Phase 3 (scatter): responses + sink records + matched-follower marks. Freshly inserted partners // (ip ≥ combined_size) skip the mark. - sink.prepare(pr, rank_count, op, responses); + sink.prepare(pr, op, responses); for (size_t g = 0; g < pr.nq_total; ++g) { - const size_t s = pr.sender_of[g]; - const size_t q = g - pr.goff[s]; + const mpi::WindowIndex s = pr.sender_index(g); + const size_t q = g - pr.goff[s.value]; const size_t ip = pr.idx_of[g]; responses[s][q] = sink.on_resolved(g, s, q, ip, pr, incoming); if (is_leader_pass && ip < combined_size) { @@ -215,20 +224,25 @@ auto resolve_incoming(const std::vector &incoming, // serialized, one VecZ } // Querier rank (any cross-rank sink): fold each resolver response into a querier-side record. The self/ -// local rank was already resolved inline, so it is skipped here. inc_r[r][q] answers query q from rank r. +// local slot was already resolved inline, so it is skipped here (and is in the window only when this +// generator's rank shift is zero). inc_r[k][q] answers query q sent to the window's k-th slot. template -auto process_responses(const std::vector> &inc_r, - const std::vector> &src_idx, - const std::vector &queries, // serialized query buffers (for phase recovery) - size_t rank_count, +auto process_responses(const mpi::WindowVec> &inc_r, + const mpi::WindowVec> &src_idx, + const mpi::WindowVec &queries, // serialized query buffers (for phase recovery) size_t my_rank, Sink &sink) -> void { - sink.process_reserve(inc_r, rank_count, my_rank); - for (size_t r = 0; r < rank_count; ++r) { + const mpi::SlotWindow w = inc_r.window(); + assert(src_idx.window().base == w.base && src_idx.window().count == w.count); + assert(queries.window().base == w.base && queries.window().count == w.count); + sink.process_reserve(inc_r, my_rank); + for (size_t k = 0; k < w.count; ++k) { + const mpi::WindowIndex wi{k}; + const size_t r = w.slot(wi); if (r == my_rank) { continue; } - sink.on_response_block(r, inc_r[r], src_idx[r], queries[r]); + sink.on_response_block(r, inc_r[wi], src_idx[wi], queries[wi]); } } diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index 36d135e9..2cd41f95 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -33,6 +33,7 @@ #include "monoprop/detail/evolution/layer_build/PartnerMerge.h" #include "monoprop/detail/evolution/layer_build/QueryCodec.h" #include "monoprop/detail/graph_encoding/MPGraphEncodingTypes.h" +#include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/operator/InvertedIndex.h" #include "monoprop/detail/operator/MPOperator.h" @@ -220,18 +221,23 @@ template template struct FusedScanResult { - std::vector cos_blocks; // ascending, disjoint, chunk order - std::vector leader_queries; // size R: serialized leader queries per owner rank - std::vector> leader_src; // size R: parallel to leader_queries (source op idx) - std::vector follower_queries; // size R: serialized follower queries per owner rank - std::vector> follower_src; // size R: parallel to follower_queries + std::vector cos_blocks; // ascending, disjoint, chunk order + // The six arrays below are indexed by DESTINATION SLOT through WindowVec::at_slot, and cover only + // the slots this generator can reach: S of the P=R*S world under linear routing, all P under + // splitmix. See mpi::PeerPlan::window. + mpi::SlotWindow window; + mpi::WindowVec leader_queries; // serialized leader queries per owner slot + mpi::WindowVec> leader_src; // parallel to leader_queries (source op idx) + mpi::WindowVec follower_queries; // serialized follower queries per owner slot + mpi::WindowVec> follower_src; // parallel to follower_queries // Fused-contraction only (capture_values): signed pre-cos source coeff (v_src) parallel to // leader_src / follower_src. Empty when capture_values is false. - std::vector> leader_val; - std::vector> follower_val; - // Self-owned queries, staged as positions instead of encoded into leader_queries[my_rank]: they are - // resolved inline and never reach a wire, so the codec is not on this leg. Order matches - // leader_src[my_rank] / follower_src[my_rank], which is the accumulation order. + mpi::WindowVec> leader_val; + mpi::WindowVec> follower_val; + // Self-owned queries, staged as positions instead of encoded into the window's self slot: they are + // resolved inline and never reach a wire, so the codec is not on this leg. Order matches that slot's + // leader_src / follower_src, which is the accumulation order. Empty unless the window contains + // my_rank, i.e. unless this generator's rank shift is zero. SelfQueryStage leader_self; SelfQueryStage follower_self; }; @@ -245,6 +251,7 @@ struct FusedScanResult { // `gen_shift` is router.rank_shift(gen), and `op` must hold only terms `my_rank` owns -- then the owner of // M⊕G is rank(M) ^ gen_shift and the linear planes never run per term. Both are what mpi::PeerPlan // already assumes; a violation moves ownership silently, so the fast path asserts against dest(). +// `window` must be that plan's window for `my_rank`: it is what the six query arrays are sized to. template auto fused_find_and_collect(const MPOperator &op, const Monomial &gen, @@ -252,7 +259,7 @@ auto fused_find_and_collect(const MPOperator &op, const CutoffContext &cut_st, const VecD &coeffs, std::optional only_rotate_len_k, - size_t rank_count, + mpi::SlotWindow window, size_t my_rank, const routing::Router &router, size_t gen_shift, @@ -261,18 +268,21 @@ auto fused_find_and_collect(const MPOperator &op, double fused_scale_cos = 1.0) -> FusedScanResult { validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); const size_t gen_pop = gen.count(); + const size_t rank_count = router.flat_world(); const auto ectx = A::make_gen_context(gen); + assert(window.stop() <= rank_count && window.count != 0); FusedScanResult res; - res.leader_queries.assign(rank_count, VecZ{}); - res.leader_src.assign(rank_count, std::vector{}); - res.follower_queries.assign(rank_count, VecZ{}); - res.follower_src.assign(rank_count, std::vector{}); - // Sized to R even on the early-return paths below so the fused engine's per-rank src_val_r access - // is always in bounds (parallel to leader_src / follower_src). + res.window = window; + res.leader_queries.reset(window); + res.leader_src.reset(window); + res.follower_queries.reset(window); + res.follower_src.reset(window); + // Sized on the early-return paths below too, so the fused engine's per-slot src_val_r access is + // always in bounds (parallel to leader_src / follower_src). if (capture_values) { - res.leader_val.assign(rank_count, std::vector{}); - res.follower_val.assign(rank_count, std::vector{}); + res.leader_val.reset(window); + res.follower_val.reset(window); } { @@ -357,15 +367,20 @@ auto fused_find_and_collect(const MPOperator &op, r_prime = router.dest_from_shift(dense, my_rank, gen_shift); assert(r_prime == router.dest(dense)); // an identity, not an approximation } + // at_slot is the only re-basing door and asserts membership: a destination outside this + // generator's window means the shift is wrong, and would otherwise land on another peer. if (r_prime == my_rank) { (is_follower ? res.follower_self : res.leader_self).push(pos, k, phase); } else { - QueryCodec::push_positions(is_follower ? fq[r_prime] : lq[r_prime], pos, k, phase); + QueryCodec::push_positions(is_follower ? fq.at_slot(r_prime) : lq.at_slot(r_prime), + pos, + k, + phase); } - (is_follower ? fs[r_prime] : ls[r_prime]).push_back(i); + (is_follower ? fs : ls).at_slot(r_prime).push_back(i); if (capture_values) { - (is_follower ? fv[r_prime] : lv[r_prime]).push_back(v_src); + (is_follower ? fv : lv).at_slot(r_prime).push_back(v_src); } }; @@ -414,9 +429,9 @@ auto fused_find_and_collect(const MPOperator &op, // A hint only, off the measured mean of 5.33 positions; wider terms grow the buffer. const size_t pq = QueryCodec::kReservePositionsPerQuery; res.leader_self.reserve(n_anti - n_foll, pq); - ls[my_rank].reserve(n_anti - n_foll); + ls.at_slot(my_rank).reserve(n_anti - n_foll); res.follower_self.reserve(n_foll, pq); - fs[my_rank].reserve(n_foll); + fs.at_slot(my_rank).reserve(n_foll); } auto derive_coeff = [&](size_t i) -> std::pair { if (capture_values) { diff --git a/cpp/monoprop/detail/mpi/Comm.h b/cpp/monoprop/detail/mpi/Comm.h index 2a9d3055..4b45fff4 100644 --- a/cpp/monoprop/detail/mpi/Comm.h +++ b/cpp/monoprop/detail/mpi/Comm.h @@ -102,6 +102,8 @@ struct SlotWindow { template class WindowVec { public: + using value_type = T; + WindowVec() = default; explicit WindowVec(SlotWindow w) : win_(w), v_(w.count) {} diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index 82ca7b15..11aabb6e 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -124,12 +124,57 @@ auto allreduce_sum_inplace(VecD &values, Comm comm) -> void; // the default is dense, i.e. today's collective. auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, PeerPlan plan = {}) -> void; +// The per-slot block arrays begin_alltoallv and wait_into accept: a plain [P] vector-of-vectors, or a +// WindowVec over the slots a PeerPlan can reach. These four overload pairs are the whole difference -- +// the verbs below are one code path walking one SlotWindow. +template +using SlotBlockValue = typename Blocks::value_type::value_type; + +template +inline auto slot_window_of(const std::vector> &v) -> SlotWindow { + return SlotWindow{.base = 0, .count = v.size()}; +} +template +inline auto slot_window_of(const WindowVec> &v) -> SlotWindow { + return v.window(); +} + +template +inline auto slot_block(const std::vector> &v, size_t slot) -> const std::vector & { + return v[slot]; +} +template +inline auto slot_block(std::vector> &v, size_t slot) -> std::vector & { + return v[slot]; +} +template +inline auto slot_block(const WindowVec> &v, size_t slot) -> const std::vector & { + return v.at_slot(slot); +} +template +inline auto slot_block(WindowVec> &v, size_t slot) -> std::vector & { + return v.at_slot(slot); +} + +// A plain destination keeps the full-world shape (a non-peer's block is empty, not absent); a WindowVec +// takes the round's window. +template +inline auto reset_slots(std::vector> &v, SlotWindow /*w*/, size_t world) -> void { + v.assign(world, std::vector{}); +} +template +inline auto reset_slots(WindowVec> &v, SlotWindow w, size_t /*world*/) -> void { + v.reset(w); +} + // In-flight variable-size all-to-all owning its buffers + layout, so several can be in flight. // recv_counts is valid on return from begin_alltoallv; wait_into completes the payload transfer (a // no-op on the synchronous Shm / single-process paths) and unpacks by source. template struct PendingAlltoallv { int num_ranks = 0; + // The slots this round touches; counts/displs are zero outside it. Set by begin_alltoallv. + SlotWindow window; std::vector send_counts; std::vector send_displs; std::vector recv_counts; @@ -143,7 +188,8 @@ struct PendingAlltoallv { // and both move with the handle, so the pointers hold #endif - auto wait_into(std::vector> &recv_data) -> void { + template + auto wait_into(Dest &recv_data) -> void { #ifdef monoprop_ENABLE_MPI if (request != MPI_REQUEST_NULL) { MPI_Wait(&request, MPI_STATUS_IGNORE); @@ -154,61 +200,93 @@ struct PendingAlltoallv { posted = 0; } #endif - recv_data.resize(static_cast(num_ranks)); - for (int i = 0; i < num_ranks; ++i) { - const auto lo = recv_buffer.begin() + recv_displs[static_cast(i)]; - recv_data[static_cast(i)].assign(lo, lo + recv_counts[static_cast(i)]); + reset_slots(recv_data, window, static_cast(num_ranks)); + for (size_t k = 0; k < window.count; ++k) { + const size_t i = window.slot(WindowIndex{k}); + const auto lo = recv_buffer.begin() + recv_displs[i]; + slot_block(recv_data, i).assign(lo, lo + recv_counts[i]); } } }; +// Debug-only: a caller may supply more slots than the plan reaches, and anything it left outside the +// window is DROPPED rather than refused -- the silent failure mode a wrong-but-agreed shift produces. +template +inline auto assert_outside_window_is_empty_([[maybe_unused]] const Blocks &send_data, + [[maybe_unused]] SlotWindow supplied, + [[maybe_unused]] SlotWindow window) -> void { +#ifndef NDEBUG + for (size_t i = supplied.base; i < supplied.stop(); ++i) { + assert((window.contains(i) || slot_block(send_data, i).empty()) + && "a block outside the plan's peer window would be dropped in silence"); + } +#endif +} + // The count exchange runs eagerly (recv_counts known on return); the Kind::Mpi payload is non-blocking // (wait_into completes it), Shm / single-process transfer here. // skip_self: do not send the self slot (the caller handles self inline) — self send/recv = 0. // known_recv_counts: recv counts already known (e.g. the transpose of the query counts), so skip the // count exchange. The self slot is also zeroed when skip_self is set. -template -inline auto begin_alltoallv(const std::vector> &send_data, +template > +inline auto begin_alltoallv(const Blocks &send_data, Comm comm, bool skip_self = false, const std::vector *known_recv_counts = nullptr, PeerPlan plan = {}) -> PendingAlltoallv { const int num_ranks = size(comm); - if (static_cast(send_data.size()) != num_ranks) { + const int me = rank(comm); + const auto geom = geometry(comm); + PendingAlltoallv h; + h.num_ranks = num_ranks; + // The plan IS the mask, dense included -- it is the count == P value of the same window. A caller may + // hand a whole [P] array under a sparse plan (the tests do), so the supplied array only has to COVER + // the window; assert_outside_window_is_empty_ catches what it leaves outside, which is the silent + // drop a wrong-but-agreed shift produces. + h.window = + plan.window(static_cast(me), static_cast(geom.ranks), static_cast(geom.partitions)); + const SlotWindow supplied = slot_window_of(send_data); + if (h.window.stop() > static_cast(num_ranks) || supplied.base > h.window.base + || supplied.stop() < h.window.stop()) { throw CollectiveArgumentError( - std::format("begin_alltoallv: send_data size ({}) must equal number of ranks ({})", - send_data.size(), + std::format("begin_alltoallv: send_data covers slots [{}, {}), which does not cover the plan's " + "[{}, {}) in a {}-slot world", + supplied.base, + supplied.stop(), + h.window.base, + h.window.stop(), num_ranks)); } - PendingAlltoallv h; - h.num_ranks = num_ranks; - h.send_counts.resize(static_cast(num_ranks)); - h.send_displs.resize(static_cast(num_ranks)); - h.recv_displs.resize(static_cast(num_ranks)); + assert_outside_window_is_empty_(send_data, supplied, h.window); + h.send_counts.assign(static_cast(num_ranks), 0); + h.send_displs.assign(static_cast(num_ranks), 0); + h.recv_displs.assign(static_cast(num_ranks), 0); - const int self = skip_self ? rank(comm) : -1; - // Counts and their prefix in ONE sweep. Wide accumulator + checked narrowing: a wrapped count would - // size send_buffer short and then feed MPI a negative count/displacement. + const int self = skip_self ? me : -1; + // Counts and their prefix in ONE sweep over the window; the rest stay zero from the assign above. + // Wide accumulator + checked narrowing: a wrapped count would size send_buffer short and then feed + // MPI a negative count/displacement. long long running_send = 0; - for (int i = 0; i < num_ranks; ++i) { - const size_t n = (i == self) ? 0 : send_data[static_cast(i)].size(); + for (size_t k = 0; k < h.window.count; ++k) { + const size_t i = h.window.slot(WindowIndex{k}); + const size_t n = (static_cast(i) == self) ? 0 : slot_block(send_data, i).size(); const int c = checked_mpi_count(n, "Send count"); - h.send_counts[static_cast(i)] = c; - h.send_displs[static_cast(i)] = checked_mpi_count(running_send, "Send displacement"); + h.send_counts[i] = c; + h.send_displs[i] = checked_mpi_count(running_send, "Send displacement"); running_send += c; } h.send_buffer.resize(static_cast(checked_mpi_count(running_send, "Total send count"))); - for (int i = 0; i < num_ranks; ++i) { - const int c = h.send_counts[static_cast(i)]; + for (size_t k = 0; k < h.window.count; ++k) { + const size_t i = h.window.slot(WindowIndex{k}); + const int c = h.send_counts[i]; if (c == 0) { continue; } - std::copy(send_data[static_cast(i)].begin(), - send_data[static_cast(i)].begin() + c, - h.send_buffer.begin() + h.send_displs[static_cast(i)]); + const auto &block = slot_block(send_data, i); + std::copy(block.begin(), block.begin() + c, h.send_buffer.begin() + h.send_displs[i]); } - h.recv_counts.resize(static_cast(num_ranks)); + h.recv_counts.assign(static_cast(num_ranks), 0); const AlltoallvResolveArgs resolve_args{.send = h.send_buffer.data(), .send_counts = h.send_counts.data(), @@ -231,24 +309,14 @@ inline auto begin_alltoallv(const std::vector> &send_data, #endif if (known_recv_counts != nullptr) { - const auto avail = - static_cast(std::min(known_recv_counts->size(), static_cast(num_ranks))); - // Mask the caller's array through the plan, as alltoall_counts already does for the counts it - // exchanges: no receive is ever posted for a non-peer, so a non-zero count there sizes - // recv_buffer for bytes nothing writes and wait_into hands the caller uninitialised memory. Done - // by copying only the f peer blocks -- recv_counts is freshly zeroed, so the rest is the mask. - if (plan.dense()) { - std::copy_n(known_recv_counts->begin(), avail, h.recv_counts.begin()); - } - else { - const auto geom = geometry(comm); - const int me = rank(comm) / geom.partitions; - const int f = plan.count(geom.ranks); - for (int k = 0; k < f; ++k) { - const int base = plan.peer(me, k) * geom.partitions; - for (int t = 0; t < geom.partitions && base + t < avail; ++t) { - h.recv_counts[static_cast(base + t)] = (*known_recv_counts)[static_cast(base + t)]; - } + // The caller's array is FLAT [P]; the window is the mask, as alltoall_counts already masks the + // counts it exchanges. No receive is ever posted for a non-peer, so a non-zero count there sizes + // recv_buffer for bytes nothing writes and wait_into hands the caller uninitialised memory. + const size_t avail = known_recv_counts->size(); + for (size_t k = 0; k < h.window.count; ++k) { + const size_t i = h.window.slot(WindowIndex{k}); + if (i < avail) { + h.recv_counts[i] = (*known_recv_counts)[i]; } } if (self >= 0) { @@ -261,9 +329,10 @@ inline auto begin_alltoallv(const std::vector> &send_data, // Wide accumulator + checked narrowing: see checked_mpi_count. long long running = 0; - for (int i = 0; i < num_ranks; ++i) { - h.recv_displs[static_cast(i)] = checked_mpi_count(running, "Recv displacement"); - running += h.recv_counts[static_cast(i)]; + for (size_t k = 0; k < h.window.count; ++k) { + const size_t i = h.window.slot(WindowIndex{k}); + h.recv_displs[i] = checked_mpi_count(running, "Recv displacement"); + running += h.recv_counts[i]; } h.recv_buffer.resize(static_cast(checked_mpi_count(running, "Total recv count"))); diff --git a/cpp/tests/evolution_detail_tests.cpp b/cpp/tests/evolution_detail_tests.cpp index bade37f4..1cc0c90f 100644 --- a/cpp/tests/evolution_detail_tests.cpp +++ b/cpp/tests/evolution_detail_tests.cpp @@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(self_resolve_mark_bounded_by_combined_size) { }; stage_self(terms[1], 1); stage_self(terms[5], -1); - eng.src_idx_r[0] = {0, 2}; + eng.src_idx_r.at_slot(0) = {0, 2}; eng.resolve_self_queries(/*is_leader_pass=*/true); diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index a8684850..3fabc300 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -27,6 +27,7 @@ #include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Scan.h" +#include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/mpi/Routing.h" #include "monoprop/detail/operator/MPOperator.h" @@ -115,22 +116,27 @@ auto build_op(const std::vector> &terms) -> detail::MPOperator<32> return op; } -auto check_bucket_ownership(const std::vector &buckets, const routing::Router &router, size_t &checked) -> void { +auto check_bucket_ownership(const mpi::WindowVec &buckets, const routing::Router &router, size_t &checked) + -> void { // Every offset comes from the codec's walk: the record is VARIABLE WIDTH, so a hardcoded stride - // would compare a monomial decoded at the wrong offset against the wrong rank. + // would compare a monomial decoded at the wrong offset against the wrong rank. The bucket index is + // re-based, so the rank compared against is window.slot(k) -- a wrong window base shows up here. using QC = detail::QueryCodec<32>; const detail::QueryLayout layout{/*fused=*/false}; - for (size_t r = 0; r < buckets.size(); ++r) { + const mpi::SlotWindow w = buckets.window(); + for (size_t k = 0; k < w.count; ++k) { + const mpi::WindowIndex wi{k}; + const VecZ &bucket = buckets[wi]; size_t off = 0; - while (off < buckets[r].size()) { + while (off < bucket.size()) { Monomial<32> mono; int phase = 0; - QC::read_mono(buckets[r], off, mono, phase); - BOOST_REQUIRE_EQUAL(find_rank<32>(mono, router), r); - off = QC::next_off(buckets[r], layout, off); + QC::read_mono(bucket, off, mono, phase); + BOOST_REQUIRE_EQUAL(find_rank<32>(mono, router), w.slot(wi)); + off = QC::next_off(bucket, layout, off); ++checked; } - BOOST_REQUIRE_EQUAL(off, buckets[r].size()); + BOOST_REQUIRE_EQUAL(off, bucket.size()); } } @@ -185,6 +191,7 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { for (const bool linear : {false, true}) { const auto router = routing::Router::for_modes(ranks, /*partitions=*/1, linear); const size_t shift = router.rank_shift(gen); + const mpi::PeerPlan plan{.sparse = router.is_linear(), .shift = static_cast(shift)}; // Per router, not summed over them: the floors are what stops the loop passing on an empty // scan, and a sum lets one router carry the other. size_t checked = 0; @@ -198,6 +205,7 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { } BOOST_REQUIRE(!owned.empty()); auto op = build_op(owned); + const mpi::SlotWindow window = plan.window(my_rank, ranks, /*parts=*/1); VecD coeffs(op.store->size(), 1.0); const auto cut = detail::build_majorana_evolution_cutoff_state(std::nullopt, std::cref(coeffs), @@ -209,17 +217,24 @@ BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { cut, coeffs, std::nullopt, - ranks, + window, my_rank, router, shift, false, nullptr, 1.0); - BOOST_REQUIRE_EQUAL(res.leader_queries.size(), ranks); - // The scan routes a self-owned partner to the stage, so my own bucket must be empty here. - BOOST_REQUIRE(res.leader_queries[my_rank].empty()); - BOOST_REQUIRE(res.follower_queries[my_rank].empty()); + BOOST_REQUIRE_EQUAL(res.leader_queries.size(), window.count); + if (window.contains(my_rank)) { + // The scan routes a self-owned partner to the stage, so my own bucket must be empty. + BOOST_REQUIRE(res.leader_queries.at_slot(my_rank).empty()); + BOOST_REQUIRE(res.follower_queries.at_slot(my_rank).empty()); + } + else { + // A non-zero shift puts self outside the window entirely, so nothing may be staged. + BOOST_REQUIRE_EQUAL(res.leader_self.size(), 0U); + BOOST_REQUIRE_EQUAL(res.follower_self.size(), 0U); + } check_bucket_ownership(res.leader_queries, router, checked); check_bucket_ownership(res.follower_queries, router, checked); check_self_ownership(res.leader_self, router, my_rank, self_checked); diff --git a/cpp/tests/sparse_resolve_tests.cpp b/cpp/tests/sparse_resolve_tests.cpp index 37b70873..31161367 100644 --- a/cpp/tests/sparse_resolve_tests.cpp +++ b/cpp/tests/sparse_resolve_tests.cpp @@ -27,6 +27,7 @@ #include "monoprop/core/Monomial.h" #include "monoprop/detail/evolution/layer_build/QueryCodec.h" #include "monoprop/detail/evolution/layer_build/Resolve.h" +#include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/OperatorIndex.h" @@ -106,14 +107,16 @@ auto draw_distinct(std::mt19937_64 &rng, size_t n) -> std::vector -auto serialize(const std::vector>> &queries, bool fused) -> std::vector { - std::vector incoming(queries.size()); +auto serialize(const std::vector>> &queries, bool fused, mpi::SlotWindow window) + -> mpi::WindowVec { + mpi::WindowVec incoming(window); for (size_t s = 0; s < queries.size(); ++s) { + VecZ &buf = incoming[mpi::WindowIndex{s}]; for (size_t q = 0; q < queries[s].size(); ++q) { const int phase = ((q % 2) == 0) ? 1 : -1; - detail::QueryCodec::push(incoming[s], queries[s][q], phase); + detail::QueryCodec::push(buf, queries[s][q], phase); if (fused) { - detail::QueryCodec::push_value(incoming[s], 0.5 + static_cast(q)); + detail::QueryCodec::push_value(buf, 0.5 + static_cast(q)); } } } @@ -121,8 +124,14 @@ auto serialize(const std::vector>> &queries, bool } template -auto check_probe_matches_the_queries(std::mt19937_64 &rng, size_t n_seed, size_t n_query, size_t rank_count, bool fused) - -> void { +auto check_probe_matches_the_queries(std::mt19937_64 &rng, + size_t n_seed, + size_t n_query, + size_t rank_count, + bool fused, + size_t window_base = 0) -> void { + // A non-zero base is the case a re-basing bug survives: sender_slot must still name the flat slot. + const mpi::SlotWindow window{.base = window_base, .count = rank_count}; const auto seed_terms = draw_distinct(rng, n_seed); const auto fresh_terms = draw_distinct(rng, n_query); @@ -159,11 +168,13 @@ auto check_probe_matches_the_queries(std::mt19937_64 &rng, size_t n_seed, size_t } } - const auto incoming = serialize(queries, fused); + const auto incoming = serialize(queries, fused, window); const detail::QueryLayout layout{fused}; auto op = make_op(seed_terms); - const auto pr = detail::probe_incoming_queries(incoming, op, rank_count, layout); + const auto pr = detail::probe_incoming_queries(incoming, op, layout); + BOOST_REQUIRE_EQUAL(pr.window.base, window.base); + BOOST_REQUIRE_EQUAL(pr.window.count, window.count); BOOST_REQUIRE_EQUAL(pr.nq_total, expect_mono.size()); BOOST_REQUIRE(pr.nq_total > 0); @@ -186,7 +197,8 @@ auto check_probe_matches_the_queries(std::mt19937_64 &rng, size_t n_seed, size_t BOOST_TEST((pr.mono_at(g) == want)); BOOST_TEST(pr.k_of[g] == want.count()); BOOST_TEST(pr.phase_of[g] == expect_phase[g]); - BOOST_TEST(pr.sender_of[g] == expect_sender[g]); + BOOST_TEST(pr.sender_index(g).value == expect_sender[g]); + BOOST_TEST(pr.sender_slot(g) == window.base + expect_sender[g]); BOOST_TEST(pr.is_paired_at(g) == monoprop::is_paired(want)); std::vector key; @@ -263,12 +275,22 @@ BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_narrow_positions) { BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_wide_positions) { std::mt19937_64 rng(20260815); static_assert(sizeof(detail::OperatorIndex<250>::PosT) == 2, "this case exists to cover the wide store"); - check_probe_matches_the_queries<250>(rng, /*n_seed=*/60, /*n_query=*/140, /*rank_count=*/4, /*fused=*/false); + check_probe_matches_the_queries<250>(rng, + /*n_seed=*/60, + /*n_query=*/140, + /*rank_count=*/4, + /*fused=*/false, + /*window_base=*/16); } BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_fused_layout) { std::mt19937_64 rng(20260816); - check_probe_matches_the_queries<250>(rng, /*n_seed=*/50, /*n_query=*/120, /*rank_count=*/2, /*fused=*/true); + check_probe_matches_the_queries<250>(rng, + /*n_seed=*/50, + /*n_query=*/120, + /*rank_count=*/2, + /*fused=*/true, + /*window_base=*/6); } BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_single_sender) { From a38ad17c152eb10ae7dfdac2ae0456496d0f261d Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 17:57:51 +0100 Subject: [PATCH 23/24] =?UTF-8?q?perf(mpi):=20=E2=9A=A1=20narrow=20HybridC?= =?UTF-8?q?omm's=20serial=20staging=20sweeps=20to=20the=20peer=20ranks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sizing that partition 0 runs between B1 and B2 still swept the whole P=R*S world while S-1 partitions parked at the barrier, even though every sweep that writes these tables and every sweep that reads them walks the same peer set. col_sum_ (twice per send sizing) and recv_col_ are now zeroed over the peer slots only, and the two displacement prefixes walk peers_ instead of all R -- peers_ is ascending in both arms, so the prefix takes the same value at every peer as the full one, a non-peer contributing zero. publish_recv_rows_ runs on every partition, not just 0: it now zeroes the row and sums only the peers' blocks, replacing R*S adds with R stores and one block's worth. The row stays fully written because derived_wire_plan_ reads all of it. The [R] count and displacement arrays are still zeroed in full: MPI_Alltoallv reads every entry on the dense arm, and the zeros are what make the narrowed prefix exact. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/mpi/HybridComm.h | 38 ++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 610dbd5b..a373735f 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -432,6 +432,15 @@ class HybridComm { } } + // Zero only the peer ranks' slots of a [P] table. Every sweep that writes one of these tables and + // every sweep that reads it walks the same peer set, so the rest of the row is never looked at -- + // and these run SERIALLY on partition 0 while S-1 partitions park, so the width is the cost. + auto zero_peer_slots_(std::vector &table) -> void { + for (const int b : peers_) { + std::fill_n(table.begin() + (static_cast(b) * s_), s_, 0LL); + } + } + // What rank a's block of the count message holds for partition t, summed over source partitions. auto block_sum_(int a, int t) const -> long long { const int *blk = counts_recv_.data() + counts_idx_(a, t, 0); @@ -464,12 +473,14 @@ class HybridComm { // staging for a block no receive is posted for. auto publish_recv_rows_(int local_partition, const int *recv_counts, PeerPlan plan) -> void { long long *rr = row_recv_(local_partition); - for (int a = 0; a < r_; ++a) { + // Every entry is written, not just the peers': derived_wire_plan_ reads the whole row. + std::fill_n(rr, r_, 0LL); + const int f = plan.count(r_); + for (int k = 0; k < f; ++k) { + const int a = plan.peer(mpi_rank_, k); long long sum = 0; - if (plan.contains(mpi_rank_, a)) { - for (int su = 0; su < s_; ++su) { - sum += recv_counts[a * s_ + su]; - } + for (int su = 0; su < s_; ++su) { + sum += recv_counts[a * s_ + su]; } rr[a] = sum; } @@ -615,7 +626,7 @@ class HybridComm { // much as the message count does. auto size_staging_send_(size_t elem) -> void { // Pass A: the column sums W over source partitions, u outer so both sides sweep in address order. - std::ranges::fill(col_sum_, 0LL); + zero_peer_slots_(col_sum_); for (int u = 0; u < s_; ++u) { const int *row = counts_row_(u); for (const int b : peers_) { @@ -625,7 +636,10 @@ class HybridComm { } } } + // The [R] arrays stay fully zeroed -- MPI_Alltoallv reads every entry on the dense arm, and the + // zeros are what make the prefix below equal the full 0..R one at the peers' positions. std::ranges::fill(mpi_send_counts_, 0); + std::ranges::fill(mpi_send_displs_, 0); for (const int b : peers_) { const long long *col = col_sum_.data() + static_cast(b) * static_cast(s_); long long send_sum = 0; @@ -634,8 +648,10 @@ class HybridComm { } mpi_send_counts_[static_cast(b)] = checked_mpi_count(send_sum, "Per-rank send count"); } + // peers_ is ascending (dense is 0..R-1, sparse is a singleton), so a prefix over it takes the + // same value at every peer as a prefix over all R: a non-peer contributes zero. long long send_running = 0; - for (int b = 0; b < r_; ++b) { + for (const int b : peers_) { mpi_send_displs_[static_cast(b)] = checked_mpi_count(send_running, "Send displacement"); send_running += mpi_send_counts_[static_cast(b)]; } @@ -649,7 +665,7 @@ class HybridComm { } } // Pass B: the exclusive prefix over source partitions; col_sum_ is free to be reused for it here. - std::ranges::fill(col_sum_, 0LL); + zero_peer_slots_(col_sum_); for (int u = 0; u < s_; ++u) { const int *row = counts_row_(u); size_t *off = pack_off_.data() + pack_idx_(u, 0); @@ -670,7 +686,7 @@ class HybridComm { // published in Phase P0 (alltoallv) or from the count blocks just exchanged (the fused resolve). template auto fill_recv_col_(Value &&value) -> void { - std::ranges::fill(recv_col_, 0LL); + zero_peer_slots_(recv_col_); for (const int a : peers_) { for (int t = 0; t < s_; ++t) { recv_col_[static_cast(a) * static_cast(s_) + static_cast(t)] = value(a, t); @@ -682,6 +698,7 @@ class HybridComm { // post-B4 scatter re-derives the per-source offsets as it walks (a, su). auto size_staging_recv_(size_t elem) -> void { std::ranges::fill(mpi_recv_counts_, 0); + std::ranges::fill(mpi_recv_displs_, 0); for (const int a : peers_) { long long recv_sum = 0; for (int t = 0; t < s_; ++t) { @@ -689,8 +706,9 @@ class HybridComm { } mpi_recv_counts_[static_cast(a)] = checked_mpi_count(recv_sum, "Per-rank recv count"); } + // Same ascending-peers prefix as the send side. long long recv_running = 0; - for (int a = 0; a < r_; ++a) { + for (const int a : peers_) { mpi_recv_displs_[static_cast(a)] = checked_mpi_count(recv_running, "Recv displacement"); recv_running += mpi_recv_counts_[static_cast(a)]; } From 2f8d3dc25d8a18ac30c9a8a7595d9f65bca41de9 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 18:00:25 +0100 Subject: [PATCH 24/24] =?UTF-8?q?test(mpi):=20=E2=9C=85=20cover=20derived?= =?UTF-8?q?=5Fwire=5Fplan=5F=20with=20an=20empty=20partition-0=20row?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit alltoallv's derive_wire_bits path had no test and no library caller. It exists because only partition 0 reaches MPI while its own row may be the empty one, so the peer set has to be read off every partition's published recv rows; reading partition 0's alone resolves to the self peer, whose legs are all zero, and every block is dropped with no hang to show for it. The case puts the rank's only traffic on partition S-1 with partition 0 sending and receiving nothing, and checks the payload arrives carrying its source's global id -- so it fails on a plan that names the wrong peer as well as on one that names none. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/tests/hybrid_comm_tests.cpp | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp index 3ff7d5ba..f69ef203 100644 --- a/cpp/tests/hybrid_comm_tests.cpp +++ b/cpp/tests/hybrid_comm_tests.cpp @@ -1087,4 +1087,69 @@ BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_zero_count_peer) { } } +// derived_wire_plan_ is what lets alltoallv narrow its own wire when the caller cannot: only partition 0 +// reaches MPI, and ITS row may be the empty one while a sibling partition holds the rank's only traffic. +// Reading the peer set off partition 0's row alone resolves to the SELF peer, whose legs are all zero -- +// every block is then dropped with no hang to show for it. So this case puts the traffic where partition +// 0 cannot see it, and checks the payload arrives carrying its source's global id. +// +// alltoallv's derive_wire_bits has no library caller today, so this is its only exercise. +BOOST_AUTO_TEST_CASE(hybrid_comm_derived_wire_plan_reads_every_partitions_row) { + const int R = world_size(); + if (R < 2 || (R & (R - 1)) != 0) { + return; // the XOR pairing needs a power-of-two rank count + } + const int bits = std::countr_zero(static_cast(R)); + const int me = world_rank(); + const int peer = me ^ 1; // shift 1, so every rank derives the same pairing + constexpr int kLen = 5; + + for (const int S : {2, 4}) { + const int P = R * S; + const int carrier = S - 1; // never partition 0 -- that is the whole point of the case + std::vector> got(static_cast(S)); + auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { + const int g = (me * S) + u; + std::vector counts(static_cast(P), 0); + const std::vector displs(static_cast(P), 0); + std::vector send; + if (u == carrier) { + counts[static_cast((peer * S) + carrier)] = kLen; + for (int j = 0; j < kLen; ++j) { + send.push_back((g * 1000) + j); + } + } + // Symmetric layout: my peer's carrier partition sends me exactly what I send it, which is + // what derive_wire_bits requires and what lets the recv rows name the peer set. One block, so + // both displacements are zero. + std::vector recv(static_cast(u == carrier ? kLen : 0), -1); + const monoprop::mpi::AlltoallvArgs args{.send = reinterpret_cast(send.data()), + .send_counts = counts.data(), + .send_displs = displs.data(), + .recv = reinterpret_cast(recv.data()), + .recv_counts = counts.data(), + .recv_displs = displs.data(), + .elem = sizeof(int)}; + hyb.alltoallv(u, args, MPI_INT, monoprop::mpi::PeerPlan{}, /*derive_wire_bits=*/bits); + got[static_cast(u)] = recv; + }); + for (const auto &e : errs) { + BOOST_CHECK(e == nullptr); + } + for (int u = 0; u < S; ++u) { + if (u != carrier) { + BOOST_CHECK(got[static_cast(u)].empty()); + continue; + } + // The values name their sender, so this fails on a plan that named the wrong peer as well as + // on one that named none. + const int src = (peer * S) + carrier; + BOOST_REQUIRE_EQUAL(static_cast(got[static_cast(u)].size()), kLen); + for (int j = 0; j < kLen; ++j) { + BOOST_CHECK_EQUAL(got[static_cast(u)][static_cast(j)], (src * 1000) + j); + } + } + } +} + #endif // monoprop_ENABLE_MPI