From 5ac10d0d92bea825880de39b505407a6dcade860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Wed, 5 Aug 2026 07:32:26 +0000 Subject: [PATCH 01/80] refactor(c++): :art: split implementation into either source or inline files --- cpp/include/monoprop/MonomialPropagator.h | 12 +- cpp/monoprop/Validation.cpp | 4 + cpp/monoprop/Validation.h | 2 + .../detail/evolution/EvolutionHelpers.h | 5 - .../detail/evolution/layer_build/Engine.h | 7 +- .../detail/graph_encoding/CMakeLists.txt | 2 + .../detail/graph_encoding/MPGraphEncoding.cpp | 202 ++++++++++++++++++ .../graph_encoding/MPGraphEncodingStorage.h | 136 +----------- .../graph_encoding/MPGraphEncodingTypes.h | 50 +---- .../detail/monomial_propagator/CMakeLists.txt | 3 +- ...ropagatorImpl.h => MonomialPropagator.inl} | 50 ++--- .../MonomialPropagatorCommon.h | 15 +- .../MonomialPropagatorHelpers.h | 40 ---- cpp/monoprop/detail/mpi/CMakeLists.txt | 2 + cpp/monoprop/detail/mpi/Exchange.h | 34 +-- cpp/monoprop/detail/mpi/MPICompat.cpp | 153 +++++++++++++ cpp/monoprop/detail/mpi/MPICompat.h | 99 +-------- cpp/monoprop/detail/partition/CMakeLists.txt | 2 + cpp/monoprop/detail/partition/CpuTopology.cpp | 152 +++++++++++++ cpp/monoprop/detail/partition/CpuTopology.h | 129 +---------- 20 files changed, 584 insertions(+), 515 deletions(-) create mode 100644 cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp rename cpp/monoprop/detail/monomial_propagator/{MonomialPropagatorImpl.h => MonomialPropagator.inl} (96%) delete mode 100644 cpp/monoprop/detail/monomial_propagator/MonomialPropagatorHelpers.h create mode 100644 cpp/monoprop/detail/mpi/MPICompat.cpp create mode 100644 cpp/monoprop/detail/partition/CpuTopology.cpp diff --git a/cpp/include/monoprop/MonomialPropagator.h b/cpp/include/monoprop/MonomialPropagator.h index 1110a6f8..4ed7b42b 100644 --- a/cpp/include/monoprop/MonomialPropagator.h +++ b/cpp/include/monoprop/MonomialPropagator.h @@ -41,7 +41,6 @@ #include "monoprop/Validation.h" #include "monoprop/algebra/PauliAlgebra.h" #include "monoprop/detail/evolution/CosineRecompute.h" -#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" #include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/mpi/MPIUtils.h" @@ -286,12 +285,6 @@ class MonomialPropagator { return ev_and_grad(request, comm, cos); }; - static auto expected_num_params(const VecZ ¶meter_mapping) -> size_t; - - template > - static auto make_parameter_validated_functional(size_t expected_num_params, Fn func) - -> std::function; - /// Distribute op_dict across ranks and apply this rank's share; returns its new (terms, coeffs) /// so caches can refresh. auto apply_initial_operator_(const OperatorDict &op_dict) -> std::pair, VecD>; @@ -467,6 +460,5 @@ class MonomialPropagator { } // namespace monoprop -// These includes are here on purpose and should not be moved to the top -#include "monoprop/detail/monomial_propagator/MonomialPropagatorHelpers.h" -#include "monoprop/detail/monomial_propagator/MonomialPropagatorImpl.h" +// inline implementation +#include "monoprop/detail/monomial_propagator/MonomialPropagator.inl" diff --git a/cpp/monoprop/Validation.cpp b/cpp/monoprop/Validation.cpp index 29c8b504..a1a037f6 100644 --- a/cpp/monoprop/Validation.cpp +++ b/cpp/monoprop/Validation.cpp @@ -107,6 +107,10 @@ auto validate_expected_graph_layers(size_t current_layers, size_t expected_layer } } +auto expected_num_params(const VecZ ¶meter_mapping) -> size_t { + return parameter_mapping.empty() ? 0 : *std::ranges::max_element(parameter_mapping) + 1; +} + // NOLINTEND(misc-use-internal-linkage) } // namespace monoprop diff --git a/cpp/monoprop/Validation.h b/cpp/monoprop/Validation.h index 436c5444..5d464050 100644 --- a/cpp/monoprop/Validation.h +++ b/cpp/monoprop/Validation.h @@ -39,4 +39,6 @@ monoprop_EXPORT auto validate_functional_call(const VecD ¶meters, size_t exp // The graph must still have the layer count the functional was built against. monoprop_EXPORT auto validate_expected_graph_layers(size_t current_layers, size_t expected_layers) -> void; +monoprop_EXPORT auto expected_num_params(const VecZ ¶meter_mapping) -> size_t; + } // namespace monoprop diff --git a/cpp/monoprop/detail/evolution/EvolutionHelpers.h b/cpp/monoprop/detail/evolution/EvolutionHelpers.h index 04f93ba5..d7087da0 100644 --- a/cpp/monoprop/detail/evolution/EvolutionHelpers.h +++ b/cpp/monoprop/detail/evolution/EvolutionHelpers.h @@ -21,11 +21,6 @@ namespace monoprop::detail { inline constexpr size_t kMissingIndex = std::numeric_limits::max(); -inline auto empty_coeffs() -> const VecD & { - static const VecD coeffs; - return coeffs; -} - struct CutoffContext { bool check_atol = false; bool check_upper_atol = false; diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 63335a35..8a5a2b3b 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -512,6 +512,11 @@ struct LayerBuildEngine { } }; +static inline auto empty_coeffs() -> const VecD & { + static const VecD coeffs; + return coeffs; +} + // Primary-path layer builder: one fused scan, then two resolve passes into the chosen sink. See LayerBuilder.h. template auto build_layer(MPOperator &local_op, @@ -535,7 +540,7 @@ auto build_layer(MPOperator &local_op, // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); - const auto &coeffs = local_coeffs ? local_coeffs->get() : empty_coeffs(); + const auto &coeffs = local_coeffs.value_or(empty_coeffs()).get(); const CutoffEvaluator cut_eval{cutoff_fn}; // Fused cos sweep: fold the per-gate cosine scale into the scan's own coefficient pass. k==0 only (a diff --git a/cpp/monoprop/detail/graph_encoding/CMakeLists.txt b/cpp/monoprop/detail/graph_encoding/CMakeLists.txt index 378759f9..ebf1faea 100644 --- a/cpp/monoprop/detail/graph_encoding/CMakeLists.txt +++ b/cpp/monoprop/detail/graph_encoding/CMakeLists.txt @@ -7,3 +7,5 @@ target_sources( "MPGraphEncodingStorage.h" "MPGraphEncodingTypes.h" ) + +target_sources(monoprop-objs PRIVATE MPGraphEncoding.cpp) diff --git a/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp b/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp new file mode 100644 index 00000000..2e8da60f --- /dev/null +++ b/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp @@ -0,0 +1,202 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/graph_encoding/MPGraphEncodingStorage.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace monoprop::detail { + +auto checked_mpi_int(size_t value, const char *what) -> int { + if (value > static_cast(std::numeric_limits::max())) { + throw std::overflow_error( + std::format("{} {} exceeds the MPI int limit {}.", what, value, std::numeric_limits::max())); + } + return static_cast(value); +} + +auto build_layer_exchange_layout(const std::vector &send_counts, int scale, const char *what) + -> LayerExchangeLayout { + const std::string count_label = std::format("{} count", what); + const std::string displacement_label = std::format("{} displacement", what); + + LayerExchangeLayout layout; + layout.counts.resize(send_counts.size()); + layout.displs.resize(send_counts.size()); + size_t total = 0; + for (size_t r = 0; r < send_counts.size(); ++r) { + const size_t count = static_cast(scale) * send_counts[r]; + layout.counts[r] = checked_mpi_int(count, count_label.c_str()); + layout.displs[r] = checked_mpi_int(total, displacement_label.c_str()); + total += count; + } + layout.total_count = total; + return layout; +} + +auto build_derivative_exchange_layout(const LayerExchangeLayout &evolution) -> LayerExchangeLayout { + std::vector send_counts; + send_counts.reserve(evolution.counts.size()); + for (const int count : evolution.counts) { + send_counts.push_back(static_cast(count)); + } + return build_layer_exchange_layout(send_counts, 2, "Layer derivative exchange"); +} + +auto checked_term_index(size_t value, const char *what) -> TermIndex { + if (value > static_cast(std::numeric_limits::max())) { + throw std::overflow_error( + std::format("{} {} exceeds the TermIndex ceiling {}; rebuild with -Dmonoprop_WIDE_TERM_INDEX.", + what, + value, + std::numeric_limits::max())); + } + return static_cast(value); +} + +auto checked_packed_phase(int value, const char *what) -> int8_t { + if (value < static_cast(std::numeric_limits::min()) + || value > static_cast(std::numeric_limits::max())) { + throw std::overflow_error(std::format("{} {} exceeds the 8-bit phase limit.", what, value)); + } + return static_cast(value); +} + +auto make_packed_phase_storage(size_t count, bool use_binary_phases) -> PackedPhaseStorage { + PackedPhaseStorage storage; + storage.uses_binary_phases = use_binary_phases; + storage.total_count = count; + if (use_binary_phases) { + storage.phase_words.assign(packed_phase_word_count(count), 0); + } + else { + storage.phase_values.resize(count); + } + return storage; +} + +auto packed_phase_storage_bytes(const PackedPhaseStorage &storage) -> size_t { + return storage.uses_binary_phases ? storage.phase_words.capacity() * sizeof(uint64_t) + : storage.phase_values.capacity() * sizeof(int8_t); +} + +auto build_packed_cross_rank_storage(const std::vector &data) -> PackedCrossRankStorage { + PackedCrossRankStorage storage; + const size_t num_ranks = data.size(); + storage.ranges.resize(num_ranks); + + size_t total_b = 0; + size_t total_d = 0; + for (size_t rank = 0; rank < num_ranks; ++rank) { + const auto &partner = data[rank]; + auto &range = storage.ranges[rank]; + range.sin_send_offset = total_b; + range.sin_send_count = static_cast(partner.sin_send_indices.size()); + range.sin_recv_offset = total_d; + range.sin_recv_count = static_cast(partner.sin_recv_entries.size()); + range.in_count = static_cast(partner.in_count); + total_b += partner.sin_send_indices.size(); + total_d += partner.sin_recv_entries.size(); + } + + bool uses_binary_phases = true; + for (const auto &partner : data) { + bool non_binary_phase = false; + for (const auto &[recv_index, phase] : partner.sin_recv_entries) { + non_binary_phase = non_binary_phase || !is_binary_phase(phase); + } + uses_binary_phases = uses_binary_phases && !non_binary_phase; + } + + storage.sin_send_indices.resize(total_b); + storage.sin_recv_phases = make_packed_phase_storage(total_d, uses_binary_phases); + + for (size_t rank = 0; rank < num_ranks; ++rank) { + const auto &partner = data[rank]; + const size_t b_off = storage.ranges[rank].sin_send_offset; + const size_t d_off = storage.ranges[rank].sin_recv_offset; + + for (size_t k = 0; k < partner.sin_send_indices.size(); ++k) { + storage.sin_send_indices[b_off + k] = checked_term_index(partner.sin_send_indices[k], "Cross-rank B index"); + } + + for (size_t k = 0; k < partner.sin_recv_entries.size(); ++k) { + const auto &[i, phi] = partner.sin_recv_entries[k]; + (void)i; + store_packed_phase(storage.sin_recv_phases, d_off + k, phi, "Cross-rank D phase"); + } + } + + return storage; +} + +auto cross_rank_storage_bytes(const PackedCrossRankStorage &storage) -> size_t { + size_t bytes = + storage.ranges.capacity() * sizeof(CrossRankPartnerRange) + packed_phase_storage_bytes(storage.sin_recv_phases); + bytes += storage.sin_send_indices.capacity() * sizeof(TermIndex); + return bytes; +} + +auto layer_exchange_layout_storage_bytes(const LayerExchangeLayout &layout) -> size_t { + return layout.counts.capacity() * sizeof(int) + layout.displs.capacity() * sizeof(int); +} + +auto build_layer_storage_unified(std::vector all_partners, size_t my_rank) + -> std::shared_ptr { + auto storage = std::make_shared(); + + { + std::vector send_counts; + send_counts.reserve(all_partners.size()); + for (size_t r = 0; r < all_partners.size(); ++r) { + send_counts.push_back((r == my_rank) ? size_t{0} : all_partners[r].sin_send_indices.size()); + } + storage->evolution_exchange_layout = build_layer_exchange_layout(send_counts, 1); + + // The derivative layout (2x) is allocated lazily on first gradient read, but validated here: an + // overflow must throw during build_graph, not from inside the gradient collective window, where + // peers are already blocked in mpi::resolve_recv's count round -> a distributed hang, not an error. + static_cast(build_derivative_exchange_layout(storage->evolution_exchange_layout)); + } + + storage->cross_rank = build_packed_cross_rank_storage(std::move(all_partners)); + + // Both are indexed by the same rank space. + if (storage->evolution_exchange_layout.counts.size() != storage->cross_rank.rank_count()) { + throw ExchangeLayoutRankMismatch( + std::format("Layer exchange layout covers {} ranks but cross-rank storage has {}.", + storage->evolution_exchange_layout.counts.size(), + storage->cross_rank.rank_count())); + } + return storage; +} + +} // namespace monoprop::detail + +namespace monoprop { + +auto LayerCore::derivative_exchange_layout() const -> const LayerExchangeLayout & { + if (!derivative_exchange_layout_cache_) { + derivative_exchange_layout_cache_ = detail::build_derivative_exchange_layout(evolution_exchange_layout); + } + return *derivative_exchange_layout_cache_; +} + +} // namespace monoprop diff --git a/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h b/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h index fd15a77c..fe5f9e39 100644 --- a/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h +++ b/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h @@ -14,20 +14,16 @@ #pragma once -#include #include #include -#include #include #include #include -#include #include #include "monoprop/detail/graph_encoding/MPGraphEncodingTypes.h" namespace monoprop::detail { - // The layer exchange layout and the packed cross-rank storage disagree on the rank count. class ExchangeLayoutRankMismatch : public std::logic_error { public: @@ -35,24 +31,9 @@ class ExchangeLayoutRankMismatch : public std::logic_error { }; // The ceiling has to track the TermIndex width, not a fixed 32-bit limit. -inline auto checked_term_index(size_t value, const char *what) -> TermIndex { - if (value > static_cast(std::numeric_limits::max())) { - throw std::overflow_error( - std::format("{} {} exceeds the TermIndex ceiling {}; rebuild with -Dmonoprop_WIDE_TERM_INDEX.", - what, - value, - std::numeric_limits::max())); - } - return static_cast(value); -} +auto checked_term_index(size_t value, const char *what) -> TermIndex; -inline auto checked_packed_phase(int value, const char *what) -> int8_t { - if (value < static_cast(std::numeric_limits::min()) - || value > static_cast(std::numeric_limits::max())) { - throw std::overflow_error(std::format("{} {} exceeds the 8-bit phase limit.", what, value)); - } - return static_cast(value); -} +auto checked_packed_phase(int value, const char *what) -> int8_t; inline constexpr size_t kPackedPhaseWordBits = std::numeric_limits::digits; @@ -72,18 +53,9 @@ inline auto is_binary_phase(int value) -> bool { return value == -1 || value == 1; } -inline auto make_packed_phase_storage(size_t count, bool use_binary_phases) -> PackedPhaseStorage { - PackedPhaseStorage storage; - storage.uses_binary_phases = use_binary_phases; - storage.total_count = count; - if (use_binary_phases) { - storage.phase_words.assign(packed_phase_word_count(count), 0); - } - else { - storage.phase_values.resize(count); - } - return storage; -} +auto make_packed_phase_storage(size_t count, bool use_binary_phases) -> PackedPhaseStorage; + +auto packed_phase_storage_bytes(const PackedPhaseStorage &storage) -> size_t; inline auto packed_phase_at(const PackedPhaseStorage &storage, size_t idx) -> int { if (storage.uses_binary_phases) { @@ -103,60 +75,7 @@ inline auto store_packed_phase(PackedPhaseStorage &storage, size_t idx, int phas } } -inline auto packed_phase_storage_bytes(const PackedPhaseStorage &storage) -> size_t { - return storage.uses_binary_phases ? storage.phase_words.capacity() * sizeof(uint64_t) - : storage.phase_values.capacity() * sizeof(int8_t); -} - -inline auto build_packed_cross_rank_storage(std::vector data) -> PackedCrossRankStorage { - PackedCrossRankStorage storage; - const size_t num_ranks = data.size(); - storage.ranges.resize(num_ranks); - - size_t total_b = 0; - size_t total_d = 0; - for (size_t rank = 0; rank < num_ranks; ++rank) { - const auto &partner = data[rank]; - auto &range = storage.ranges[rank]; - range.sin_send_offset = total_b; - range.sin_send_count = static_cast(partner.sin_send_indices.size()); - range.sin_recv_offset = total_d; - range.sin_recv_count = static_cast(partner.sin_recv_entries.size()); - range.in_count = static_cast(partner.in_count); - total_b += partner.sin_send_indices.size(); - total_d += partner.sin_recv_entries.size(); - } - - bool uses_binary_phases = true; - for (const auto &partner : data) { - bool non_binary_phase = false; - for (const auto &[recv_index, phase] : partner.sin_recv_entries) { - non_binary_phase = non_binary_phase || !is_binary_phase(phase); - } - uses_binary_phases = uses_binary_phases && !non_binary_phase; - } - - storage.sin_send_indices.resize(total_b); - storage.sin_recv_phases = make_packed_phase_storage(total_d, uses_binary_phases); - - for (size_t rank = 0; rank < num_ranks; ++rank) { - const auto &partner = data[rank]; - const size_t b_off = storage.ranges[rank].sin_send_offset; - const size_t d_off = storage.ranges[rank].sin_recv_offset; - - for (size_t k = 0; k < partner.sin_send_indices.size(); ++k) { - storage.sin_send_indices[b_off + k] = checked_term_index(partner.sin_send_indices[k], "Cross-rank B index"); - } - - for (size_t k = 0; k < partner.sin_recv_entries.size(); ++k) { - const auto &[i, phi] = partner.sin_recv_entries[k]; - (void)i; - store_packed_phase(storage.sin_recv_phases, d_off + k, phi, "Cross-rank D phase"); - } - } - - return storage; -} +auto build_packed_cross_rank_storage(const std::vector &data) -> PackedCrossRankStorage; inline auto cross_rank_sin_send_index(const PackedCrossRankStorage &storage, size_t rank, size_t idx) -> size_t { const size_t offset = storage.ranges[rank].sin_send_offset + idx; @@ -177,47 +96,12 @@ inline auto cross_rank_sin_recv_phase(const PackedCrossRankStorage &storage, siz return packed_phase_at(storage.sin_recv_phases, storage.ranges[rank].sin_recv_offset + idx); } -inline auto cross_rank_storage_bytes(const PackedCrossRankStorage &storage) -> size_t { - size_t bytes = - storage.ranges.capacity() * sizeof(CrossRankPartnerRange) + packed_phase_storage_bytes(storage.sin_recv_phases); - bytes += storage.sin_send_indices.capacity() * sizeof(TermIndex); - return bytes; -} +auto cross_rank_storage_bytes(const PackedCrossRankStorage &storage) -> size_t; -inline auto layer_exchange_layout_storage_bytes(const LayerExchangeLayout &layout) -> size_t { - return layout.counts.capacity() * sizeof(int) + layout.displs.capacity() * sizeof(int); -} +auto layer_exchange_layout_storage_bytes(const LayerExchangeLayout &layout) -> size_t; // Local cycles fold into the self-rank slot (my_rank); the exchange layout zeroes counts[my_rank] so // MPI_Alltoallv skips it (replay does a local copy). -inline auto build_layer_storage_unified(std::vector all_partners, size_t my_rank) - -> std::shared_ptr { - auto storage = std::make_shared(); - - { - std::vector send_counts; - send_counts.reserve(all_partners.size()); - for (size_t r = 0; r < all_partners.size(); ++r) { - send_counts.push_back((r == my_rank) ? size_t{0} : all_partners[r].sin_send_indices.size()); - } - storage->evolution_exchange_layout = build_layer_exchange_layout(send_counts, 1); - - // The derivative layout (2x) is allocated lazily on first gradient read, but validated here: an - // overflow must throw during build_graph, not from inside the gradient collective window, where - // peers are already blocked in mpi::resolve_recv's count round -> a distributed hang, not an error. - static_cast(build_derivative_exchange_layout(storage->evolution_exchange_layout)); - } - - storage->cross_rank = build_packed_cross_rank_storage(std::move(all_partners)); - - // Both are indexed by the same rank space. - if (storage->evolution_exchange_layout.counts.size() != storage->cross_rank.rank_count()) { - throw ExchangeLayoutRankMismatch( - std::format("Layer exchange layout covers {} ranks but cross-rank storage has {}.", - storage->evolution_exchange_layout.counts.size(), - storage->cross_rank.rank_count())); - } - return storage; -} - +auto build_layer_storage_unified(std::vector all_partners, size_t my_rank) + -> std::shared_ptr; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/graph_encoding/MPGraphEncodingTypes.h b/cpp/monoprop/detail/graph_encoding/MPGraphEncodingTypes.h index e6c485c2..20be7ace 100644 --- a/cpp/monoprop/detail/graph_encoding/MPGraphEncodingTypes.h +++ b/cpp/monoprop/detail/graph_encoding/MPGraphEncodingTypes.h @@ -17,11 +17,8 @@ #include #include #include -#include #include #include -#include -#include #include #include @@ -43,46 +40,16 @@ struct LayerExchangeLayout final { namespace monoprop::detail { -inline auto checked_mpi_int(size_t value, const char *what) -> int { - if (value > static_cast(std::numeric_limits::max())) { - throw std::overflow_error( - std::format("{} {} exceeds the MPI int limit {}.", what, value, std::numeric_limits::max())); - } - return static_cast(value); -} +auto checked_mpi_int(size_t value, const char *what) -> int; // Per-rank MPI counts = send_counts[r] * scale, with prefix-sum displacements. send_counts is full-width // (size_t) so checked_mpi_int catches the narrowing to MPI's int. -inline auto build_layer_exchange_layout(const std::vector &send_counts, - int scale, - const char *what = "Layer exchange") -> LayerExchangeLayout { - const std::string count_label = std::format("{} count", what); - const std::string displacement_label = std::format("{} displacement", what); - - LayerExchangeLayout layout; - layout.counts.resize(send_counts.size()); - layout.displs.resize(send_counts.size()); - size_t total = 0; - for (size_t r = 0; r < send_counts.size(); ++r) { - const size_t count = static_cast(scale) * send_counts[r]; - layout.counts[r] = checked_mpi_int(count, count_label.c_str()); - layout.displs[r] = checked_mpi_int(total, displacement_label.c_str()); - total += count; - } - layout.total_count = total; - return layout; -} +auto build_layer_exchange_layout(const std::vector &send_counts, int scale, const char *what = "Layer exchange") + -> LayerExchangeLayout; // The derivative layout is the evolution layout at 2x (each rotation endpoint carries both the op and // state payload). -inline auto build_derivative_exchange_layout(const LayerExchangeLayout &evolution) -> LayerExchangeLayout { - std::vector send_counts; - send_counts.reserve(evolution.counts.size()); - for (const int count : evolution.counts) { - send_counts.push_back(static_cast(count)); - } - return build_layer_exchange_layout(send_counts, 2, "Layer derivative exchange"); -} +auto build_derivative_exchange_layout(const LayerExchangeLayout &evolution) -> LayerExchangeLayout; } // namespace monoprop::detail @@ -197,13 +164,4 @@ struct LayerCore final { mutable std::optional derivative_exchange_layout_cache_; }; -// Derived lazily (gradient path only), but the 2x overflow check is not deferred with it: -// build_layer_storage_unified validates it eagerly. -inline auto LayerCore::derivative_exchange_layout() const -> const LayerExchangeLayout & { - if (!derivative_exchange_layout_cache_) { - derivative_exchange_layout_cache_ = detail::build_derivative_exchange_layout(evolution_exchange_layout); - } - return *derivative_exchange_layout_cache_; -} - } // namespace monoprop diff --git a/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt b/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt index ded6cbbe..6afe6999 100644 --- a/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt +++ b/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt @@ -4,7 +4,6 @@ target_sources( FILE_SET headers TYPE HEADERS FILES - "MonomialPropagatorHelpers.h" "MonomialPropagatorCommon.h" - "MonomialPropagatorImpl.h" + "MonomialPropagator.inl" ) diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorImpl.h b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl similarity index 96% rename from cpp/monoprop/detail/monomial_propagator/MonomialPropagatorImpl.h rename to cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index d00e128c..8cedafe4 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorImpl.h +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -30,13 +30,14 @@ #include #include -#include "monoprop/MonomialPropagator.h" +#include "monoprop/Validation.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/EnvConfig.h" #include "monoprop/detail/evolution/CosineRecompute.h" #include "monoprop/detail/evolution/LayerBuilder.h" #include "monoprop/detail/evolution/layer_build/FusedApply.h" -#include "monoprop/detail/partition/PartitionGroup.h" // needs the complete type +#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" +#include "monoprop/detail/partition/PartitionGroup.h" namespace monoprop { @@ -963,28 +964,29 @@ auto MonomialPropagator::make_functional_(Fn &&func, std::optional(inverted_index, graph->replay_view(), basis_); - return make_parameter_validated_functional(num_params, - [func = std::move(func), - core_term, - state = std::move(state), - op = std::move(op), - graph = std::move(graph), - parameter_mapping, - gen_coeffs, - expected_layers, - cos = std::move(cos), - comm](const VecD ¶ms) -> R { - validate_expected_graph_layers(graph->layers(), expected_layers); - return func(EvalRequest{.e_core = core_term, - .state = state, - .op = op, - .parameter_mapping = parameter_mapping, - .gen_coeffs = gen_coeffs, - .graph = graph->replay_view(), - .params = params}, - comm, - cos); - }); + return [func = std::move(func), + core_term, + state = std::move(state), + op = std::move(op), + graph = std::move(graph), + parameter_mapping, + gen_coeffs, + num_params, + expected_layers, + cos = std::move(cos), + comm](const VecD ¶ms) -> R { + validate_functional_call(params, num_params); + validate_expected_graph_layers(graph->layers(), expected_layers); + return func(EvalRequest{.e_core = core_term, + .state = state, + .op = op, + .parameter_mapping = parameter_mapping, + .gen_coeffs = gen_coeffs, + .graph = graph->replay_view(), + .params = params}, + comm, + cos); + }; } template diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h b/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h index 103782e5..de221f30 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h @@ -14,19 +14,17 @@ #pragma once -#include #include -#include "monoprop/Evolution.h" #include "monoprop/TypeAliases.h" +#include "monoprop/algebra/AlgebraCommon.h" #include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/core/Monomial.h" namespace monoprop::detail { - -// A CutoffType enumerator neither cutoff factory knows. -class UnknownCutoffType : public std::runtime_error { +class UnknownCutoffTypeError : public std::runtime_error { public: - using std::runtime_error::runtime_error; + UnknownCutoffTypeError() : std::runtime_error("Unknown cutoff type") {} }; template @@ -38,7 +36,7 @@ auto cutoff_function(CutoffType cutoff_type, unsigned int cutoff, size_t logical case CutoffType::Support: return detail::SupportCutoff{cutoff, logical_num_modes}; default: - throw UnknownCutoffType("Unknown cutoff type"); + throw UnknownCutoffTypeError(); } } @@ -59,8 +57,7 @@ auto cutoff_function_basis_change(CutoffType cutoff_type, return support_cutoff(mapped_mono, cutoff, logical_num_modes); }; default: - throw UnknownCutoffType("Unknown cutoff type"); + throw UnknownCutoffTypeError(); } } - } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorHelpers.h b/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorHelpers.h deleted file mode 100644 index 97b5a8e0..00000000 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorHelpers.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include "monoprop/MonomialPropagator.h" -#include "monoprop/detail/evolution/EvolutionHelpers.h" -#include "monoprop/detail/evolution/LayerBuilder.h" - -namespace monoprop { - -template -auto MonomialPropagator::expected_num_params(const VecZ ¶meter_mapping) -> size_t { - return parameter_mapping.empty() ? 0 : *std::max_element(parameter_mapping.begin(), parameter_mapping.end()) + 1; -} - -template -template -auto MonomialPropagator::make_parameter_validated_functional(size_t expected_num_params, Fn func) - -> std::function { - return [expected_num_params, func = std::move(func)](const VecD ¶ms) -> R { - validate_functional_call(params, expected_num_params); - return func(params); - }; -} - -} // namespace monoprop diff --git a/cpp/monoprop/detail/mpi/CMakeLists.txt b/cpp/monoprop/detail/mpi/CMakeLists.txt index b4f5a136..c8226a41 100644 --- a/cpp/monoprop/detail/mpi/CMakeLists.txt +++ b/cpp/monoprop/detail/mpi/CMakeLists.txt @@ -15,3 +15,5 @@ target_sources( "RecvLayout.h" "ShmComm.h" ) + +target_sources(monoprop-objs PRIVATE MPICompat.cpp) diff --git a/cpp/monoprop/detail/mpi/Exchange.h b/cpp/monoprop/detail/mpi/Exchange.h index 2fb41922..6e230efd 100644 --- a/cpp/monoprop/detail/mpi/Exchange.h +++ b/cpp/monoprop/detail/mpi/Exchange.h @@ -14,8 +14,6 @@ #pragma once -#include -#include #include #include #include @@ -30,37 +28,7 @@ namespace monoprop::mpi { // Resolve the recv side of a send-count vector, reusing `cache` when comm size is unchanged: a // replayed graph's send pattern is fixed, so a hit removes one blocking count round-trip per layer // per evaluation. -inline auto resolve_recv(std::span send_counts, const Comm &comm, RecvLayoutCache &cache) - -> const RecvLayout & { - const auto n = static_cast(send_counts.size()); - const int comm_size = mpi::size(comm); - // alltoall_counts moves comm_size ints each way regardless of `n`, so a send vector that is not - // exactly one entry per rank reads and writes out of bounds — reachable because layouts outlive - // propagator copies and pare rebuilds. - if (n != comm_size) { - throw CollectiveArgumentError( - std::format("Exchange layout has {} send counts but the communicator has {} ranks — a graph built for one " - "communicator cannot be replayed on another of a different size.", - n, - comm_size)); - } - if (cache.comm_size == comm_size && static_cast(cache.layout.counts.size()) == n) { - return cache.layout; - } - - RecvLayout &out = cache.layout; - out.counts.resize(static_cast(n)); - alltoall_counts(send_counts.data(), out.counts.data(), n, comm); - out.displs.resize(static_cast(n)); - long long total = 0; - for (int i = 0; i < n; ++i) { - out.displs[static_cast(i)] = checked_mpi_count(total); - total += out.counts[static_cast(i)]; - } - out.total = checked_mpi_count(total); - cache.comm_size = comm_size; - return out; -} +auto resolve_recv(std::span send_counts, const Comm &comm, RecvLayoutCache &cache) -> const RecvLayout &; // Idempotent completion handle for a posted payload transfer; move-only, so a request is waited on // exactly once. wait() is a no-op on the blocking path and in non-MPI builds. Owns its request: the diff --git a/cpp/monoprop/detail/mpi/MPICompat.cpp b/cpp/monoprop/detail/mpi/MPICompat.cpp new file mode 100644 index 00000000..30141d4b --- /dev/null +++ b/cpp/monoprop/detail/mpi/MPICompat.cpp @@ -0,0 +1,153 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/mpi/Exchange.h" + +#include +#include +#include + +namespace monoprop::mpi { + +#ifdef monoprop_ENABLE_MPI +auto init(int *argc, char ***argv) -> void { + auto initialized = 0; + MPI_Initialized(&initialized); + if (!initialized) { + // serialized (not funneled): under the hybrid the one-at-a-time MPI calls come from each rank's + // partition-0 master, not the main thread. mpi4py already requests >= serialized. + auto required = MPI_THREAD_SERIALIZED; + auto provided = 0; + MPI_Init_thread(argc, argv, required, &provided); + if (provided < required) { + auto comm = MPI_COMM_WORLD; + std::print("Sorry, the MPI library does not provide MPI_THREAD_SERIALIZED support, which is required " + "by the partition/MPI hybrid transport.\n"); + MPI_Abort(comm, 1); + } + } +} + +auto finalize() -> void { + int finalized = 0; + MPI_Finalized(&finalized); + if (!finalized) { + MPI_Finalize(); + } +} +#endif // monoprop_ENABLE_MPI + +auto rank(const Comm &comm) -> int { + if (comm.kind == Comm::Kind::Shm) { + return comm.shm_rank; + } +#ifdef monoprop_ENABLE_MPI + if (comm.kind == Comm::Kind::Hybrid) { + return comm.hyb->global_rank(comm.shm_rank); + } + int r = 0; + if (MPI_Comm_rank(comm.mpi, &r) != MPI_SUCCESS) { + throw CollectiveArgumentError("MPI_Comm_rank failed"); + } + return r; +#else + return 0; +#endif +} + +auto size(const Comm &comm) -> int { + if (comm.kind == Comm::Kind::Shm) { + return comm.shm->size(); + } +#ifdef monoprop_ENABLE_MPI + if (comm.kind == Comm::Kind::Hybrid) { + return comm.hyb->size(); + } + int s = 0; + if (MPI_Comm_size(comm.mpi, &s) != MPI_SUCCESS) { + throw CollectiveArgumentError("MPI_Comm_size failed"); + } + return s; +#else + return 1; +#endif +} + +auto allreduce_sum_inplace(VecD &values, Comm comm) -> void { + if (comm.kind == Comm::Kind::Shm) { + comm.shm->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); + return; + } +#ifdef monoprop_ENABLE_MPI + if (comm.kind == Comm::Kind::Hybrid) { + comm.hyb->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); + return; + } + MPI_Allreduce(MPI_IN_PLACE, values.data(), static_cast(values.size()), MPI_DOUBLE, MPI_SUM, comm.mpi); +#else + (void)values; // single participant: identity +#endif +} + +auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm) -> void { + if (comm.kind == Comm::Kind::Shm) { + comm.shm->alltoall_counts(comm.shm_rank, send_counts, recv_counts); + return; + } +#ifdef monoprop_ENABLE_MPI + if (comm.kind == Comm::Kind::Hybrid) { + comm.hyb->alltoall_counts(comm.shm_rank, send_counts, recv_counts); + return; + } + (void)n; + MPI_Alltoall(send_counts, 1, MPI_INT, recv_counts, 1, MPI_INT, comm.mpi); +#else + for (int i = 0; i < n; ++i) { + recv_counts[i] = send_counts[i]; + } +#endif +} + +auto resolve_recv(std::span send_counts, const Comm &comm, RecvLayoutCache &cache) -> const RecvLayout & { + const auto n = static_cast(send_counts.size()); + const int comm_size = mpi::size(comm); + // alltoall_counts moves comm_size ints each way regardless of `n`, so a send vector that is not + // exactly one entry per rank reads and writes out of bounds — reachable because layouts outlive + // propagator copies and pare rebuilds. + if (n != comm_size) { + throw CollectiveArgumentError( + std::format("Exchange layout has {} send counts but the communicator has {} ranks — a graph built for one " + "communicator cannot be replayed on another of a different size.", + n, + comm_size)); + } + if (cache.comm_size == comm_size && static_cast(cache.layout.counts.size()) == n) { + return cache.layout; + } + + RecvLayout &out = cache.layout; + out.counts.resize(static_cast(n)); + alltoall_counts(send_counts.data(), out.counts.data(), n, comm); + out.displs.resize(static_cast(n)); + long long total = 0; + for (int i = 0; i < n; ++i) { + out.displs[static_cast(i)] = checked_mpi_count(total); + total += out.counts[static_cast(i)]; + } + out.total = checked_mpi_count(total); + cache.comm_size = comm_size; + return out; +} + +} // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index 485cd047..fb5c7fd3 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -40,31 +40,8 @@ namespace monoprop::mpi { #ifdef monoprop_ENABLE_MPI -inline auto init(int *argc = nullptr, char ***argv = nullptr) -> void { - auto initialized = 0; - MPI_Initialized(&initialized); - if (!initialized) { - // serialized (not funneled): under the hybrid the one-at-a-time MPI calls come from each rank's - // partition-0 master, not the main thread. mpi4py already requests >= serialized. - auto required = MPI_THREAD_SERIALIZED; - auto provided = 0; - MPI_Init_thread(argc, argv, required, &provided); - if (provided < required) { - auto comm = MPI_COMM_WORLD; - std::print("Sorry, the MPI library does not provide MPI_THREAD_SERIALIZED support, which is required " - "by the partition/MPI hybrid transport.\n"); - MPI_Abort(comm, 1); - } - } -} - -inline auto finalize() -> void { - int finalized = 0; - MPI_Finalized(&finalized); - if (!finalized) { - MPI_Finalize(); - } -} +auto init(int *argc = nullptr, char ***argv = nullptr) -> void; +auto finalize() -> void; namespace detail { template @@ -110,41 +87,8 @@ inline auto init(int * /*argc*/ = nullptr, char *** /*argv*/ = nullptr) -> void inline auto finalize() -> void {} #endif // monoprop_ENABLE_MPI -inline auto rank(const Comm &comm) -> int { - if (comm.kind == Comm::Kind::Shm) { - return comm.shm_rank; - } -#ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return comm.hyb->global_rank(comm.shm_rank); - } - int r = 0; - if (MPI_Comm_rank(comm.mpi, &r) != MPI_SUCCESS) { - throw CollectiveArgumentError("MPI_Comm_rank failed"); - } - return r; -#else - return 0; -#endif -} - -inline auto size(const Comm &comm) -> int { - if (comm.kind == Comm::Kind::Shm) { - return comm.shm->size(); - } -#ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return comm.hyb->size(); - } - int s = 0; - if (MPI_Comm_size(comm.mpi, &s) != MPI_SUCCESS) { - throw CollectiveArgumentError("MPI_Comm_size failed"); - } - return s; -#else - return 1; -#endif -} +auto rank(const Comm &comm) -> int; +auto size(const Comm &comm) -> int; template inline auto allreduce_sum(T local_val, Comm comm) -> T { @@ -163,41 +107,10 @@ inline auto allreduce_sum(T local_val, Comm comm) -> T { #endif } -inline auto allreduce_sum_inplace(VecD &values, Comm comm) -> void { - if (comm.kind == Comm::Kind::Shm) { - comm.shm->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); - return; - } -#ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); - return; - } - MPI_Allreduce(MPI_IN_PLACE, values.data(), static_cast(values.size()), MPI_DOUBLE, MPI_SUM, comm.mpi); -#else - (void)values; // single participant: identity -#endif -} +auto allreduce_sum_inplace(VecD &values, Comm comm) -> void; // `n` is the comm size. -inline auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm) -> void { - if (comm.kind == Comm::Kind::Shm) { - comm.shm->alltoall_counts(comm.shm_rank, send_counts, recv_counts); - return; - } -#ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoall_counts(comm.shm_rank, send_counts, recv_counts); - return; - } - (void)n; - MPI_Alltoall(send_counts, 1, MPI_INT, recv_counts, 1, MPI_INT, comm.mpi); -#else - for (int i = 0; i < n; ++i) { - recv_counts[i] = send_counts[i]; - } -#endif -} +auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm) -> void; // In-flight variable-size all-to-all owning its buffers + layout, so several can be in flight. // recv_counts is valid on return from begin_alltoallv; wait_into completes the payload transfer (a diff --git a/cpp/monoprop/detail/partition/CMakeLists.txt b/cpp/monoprop/detail/partition/CMakeLists.txt index d50fcfbb..19d4b69b 100644 --- a/cpp/monoprop/detail/partition/CMakeLists.txt +++ b/cpp/monoprop/detail/partition/CMakeLists.txt @@ -7,3 +7,5 @@ target_sources( "CpuTopology.h" "PartitionGroup.h" ) + +target_sources(monoprop-objs PRIVATE CpuTopology.cpp) diff --git a/cpp/monoprop/detail/partition/CpuTopology.cpp b/cpp/monoprop/detail/partition/CpuTopology.cpp new file mode 100644 index 00000000..5be85e90 --- /dev/null +++ b/cpp/monoprop/detail/partition/CpuTopology.cpp @@ -0,0 +1,152 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/partition/CpuTopology.h" + +#if defined(__linux__) + +#include +#include + +namespace monoprop::detail::partition { + +auto enumerate_physical_cores() -> std::vector { + const std::set allowed = topo_detail::allowed_cpus(); + const bool filter = !allowed.empty(); // no mask readable ⇒ accept every CPU + const auto is_allowed = [&](int cpu) { return !filter || allowed.contains(cpu); }; + + std::vector cores; + std::set seen_cores; // sibling-group key (min sibling) already recorded + std::vector> l3_members; // cpu-set per distinct L3 domain, in discovery order + + // Scan a bounded id range rather than stopping at the first gap: online CPU ids are not contiguous + // (offlined or hot-plugged CPUs leave holes), and breaking on the first unreadable id truncates the + // core list to whatever preceded the hole, silently under-partitioning and crowding the low CPUs. + const int scan_limit = filter ? *allowed.rbegin() + 1 : CPU_SETSIZE; + for (int cpu = 0; cpu < scan_limit; ++cpu) { + const std::string base = "/sys/devices/system/cpu/cpu" + std::to_string(cpu); + const std::string sib = topo_detail::read_line(base + "/topology/thread_siblings_list"); + if (sib.empty()) { + continue; + } + const auto siblings = topo_detail::parse_cpulist(sib); + const int group_key = siblings.empty() ? cpu : *std::min_element(siblings.begin(), siblings.end()); + if (seen_cores.contains(group_key)) { + continue; + } + seen_cores.insert(group_key); + + int rep = -1; + if (siblings.empty()) { + rep = is_allowed(cpu) ? cpu : -1; + } + else { + for (int s : siblings) { // parse_cpulist yields ascending order + if (is_allowed(s)) { + rep = s; + break; + } + } + } + if (rep < 0) { + continue; + } + + const auto l3 = topo_detail::parse_cpulist(topo_detail::read_line(base + "/cache/index3/shared_cpu_list")); + int domain = -1; + for (size_t d = 0; d < l3_members.size(); ++d) { + if (std::find(l3_members[d].begin(), l3_members[d].end(), group_key) != l3_members[d].end()) { + domain = static_cast(d); + break; + } + } + if (domain < 0) { + domain = static_cast(l3_members.size()); + l3_members.push_back(l3.empty() ? std::vector{group_key} : l3); + } + cores.push_back(PhysicalCore{rep, domain}); + } + return cores; +} + +auto partition_cpusets(size_t n, size_t group_index, size_t group_count) -> std::vector { + if (!config::get().partition_pinning) { + return {}; + } + const auto cores = enumerate_physical_cores(); + if (cores.empty() || group_count * n > cores.size()) { + return {}; + } + int max_domain = 0; + for (const auto &c : cores) { + max_domain = std::max(max_domain, c.l3_domain); + } + // Ordering: interleaved for a lone process, contiguous blocks for co-located ranks. + std::vector> by_domain(static_cast(max_domain) + 1); + for (const auto &c : cores) { + by_domain[static_cast(c.l3_domain)].push_back(c.cpu); + } + // Interleave `buckets` depth-first: bucket0[0], bucket1[0], …, bucket0[1], bucket1[1], … + const auto interleave = [](const std::vector> &buckets) { + std::vector out; + for (size_t depth = 0;; ++depth) { + bool any = false; + for (const auto &bucket : buckets) { + if (depth < bucket.size()) { + out.push_back(bucket[depth]); + any = true; + } + } + if (!any) { + return out; + } + } + }; + + std::vector order; + size_t offset = 0; + if (group_count <= by_domain.size()) { + // Domains dealt to this rank: group_index, +group_count, … (group_count == 1 ⇒ all of them). + std::vector> mine; + for (size_t d = group_index; d < by_domain.size(); d += group_count) { + mine.push_back(by_domain[d]); + } + order = interleave(mine); + } + else { + // More co-located ranks than L3 domains: flat domain-major order, one contiguous slice each. + for (const auto &bucket : by_domain) { + order.insert(order.end(), bucket.begin(), bucket.end()); + } + offset = group_index * n; + } + if (offset + n > order.size()) { + return {}; + } + + std::vector sets(n); + for (size_t i = 0; i < n; ++i) { + CPU_ZERO(&sets[i]); + CPU_SET(order[offset + i], &sets[i]); + } + return sets; +} + +auto pin_this_thread(const CpuSet &set) -> void { + pthread_setaffinity_np(pthread_self(), sizeof(CpuSet), &set); +} + +} // namespace monoprop::detail::partition + +#endif // __linux__ diff --git a/cpp/monoprop/detail/partition/CpuTopology.h b/cpp/monoprop/detail/partition/CpuTopology.h index 8eb146dd..e5820194 100644 --- a/cpp/monoprop/detail/partition/CpuTopology.h +++ b/cpp/monoprop/detail/partition/CpuTopology.h @@ -20,10 +20,7 @@ #include "monoprop/detail/EnvConfig.h" #if defined(__linux__) -#include #include -#include -#include #include #include #include @@ -103,136 +100,16 @@ inline auto allowed_cpus() -> std::set { // Enumerate physical cores (one per smt sibling group) the process is allowed to use, tagged with their // L3 domain. A core is included iff a sibling is in the allowed mask, with the smallest allowed sibling // as representative, so a partial allocation never pins outside the mask. Empty if /sys cannot be read. -inline auto enumerate_physical_cores() -> std::vector { - const std::set allowed = topo_detail::allowed_cpus(); - const bool filter = !allowed.empty(); // no mask readable ⇒ accept every CPU - const auto is_allowed = [&](int cpu) { return !filter || allowed.contains(cpu); }; - - std::vector cores; - std::set seen_cores; // sibling-group key (min sibling) already recorded - std::vector> l3_members; // cpu-set per distinct L3 domain, in discovery order - - // Scan a bounded id range rather than stopping at the first gap: online CPU ids are not contiguous - // (offlined or hot-plugged CPUs leave holes), and breaking on the first unreadable id truncates the - // core list to whatever preceded the hole, silently under-partitioning and crowding the low CPUs. - const int scan_limit = filter ? *allowed.rbegin() + 1 : CPU_SETSIZE; - for (int cpu = 0; cpu < scan_limit; ++cpu) { - const std::string base = "/sys/devices/system/cpu/cpu" + std::to_string(cpu); - const std::string sib = topo_detail::read_line(base + "/topology/thread_siblings_list"); - if (sib.empty()) { - continue; - } - const auto siblings = topo_detail::parse_cpulist(sib); - const int group_key = siblings.empty() ? cpu : *std::min_element(siblings.begin(), siblings.end()); - if (seen_cores.contains(group_key)) { - continue; - } - seen_cores.insert(group_key); - - int rep = -1; - if (siblings.empty()) { - rep = is_allowed(cpu) ? cpu : -1; - } - else { - for (int s : siblings) { // parse_cpulist yields ascending order - if (is_allowed(s)) { - rep = s; - break; - } - } - } - if (rep < 0) { - continue; - } - - const auto l3 = topo_detail::parse_cpulist(topo_detail::read_line(base + "/cache/index3/shared_cpu_list")); - int domain = -1; - for (size_t d = 0; d < l3_members.size(); ++d) { - if (std::find(l3_members[d].begin(), l3_members[d].end(), group_key) != l3_members[d].end()) { - domain = static_cast(d); - break; - } - } - if (domain < 0) { - domain = static_cast(l3_members.size()); - l3_members.push_back(l3.empty() ? std::vector{group_key} : l3); - } - cores.push_back(PhysicalCore{rep, domain}); - } - return cores; -} +auto enumerate_physical_cores() -> std::vector; // Build `n` partition cpusets, one physical core each. `group_index`/`group_count` place one MPI rank's // partitions among the ranks sharing this host, spread across L3 domains and disjoint from the other ranks' // — two ranks must never share a core (one rank's busy-polling collectives would starve the other's // barrier spins). Empty (⇒ unpinned) if the host lacks group_count*n cores. -inline auto partition_cpusets(size_t n, size_t group_index = 0, size_t group_count = 1) -> std::vector { - if (!config::get().partition_pinning) { - return {}; - } - const auto cores = enumerate_physical_cores(); - if (cores.empty() || group_count * n > cores.size()) { - return {}; - } - int max_domain = 0; - for (const auto &c : cores) { - max_domain = std::max(max_domain, c.l3_domain); - } - // Ordering: interleaved for a lone process, contiguous blocks for co-located ranks. - std::vector> by_domain(static_cast(max_domain) + 1); - for (const auto &c : cores) { - by_domain[static_cast(c.l3_domain)].push_back(c.cpu); - } - // Interleave `buckets` depth-first: bucket0[0], bucket1[0], …, bucket0[1], bucket1[1], … - const auto interleave = [](const std::vector> &buckets) { - std::vector out; - for (size_t depth = 0;; ++depth) { - bool any = false; - for (const auto &bucket : buckets) { - if (depth < bucket.size()) { - out.push_back(bucket[depth]); - any = true; - } - } - if (!any) { - return out; - } - } - }; - - std::vector order; - size_t offset = 0; - if (group_count <= by_domain.size()) { - // Domains dealt to this rank: group_index, +group_count, … (group_count == 1 ⇒ all of them). - std::vector> mine; - for (size_t d = group_index; d < by_domain.size(); d += group_count) { - mine.push_back(by_domain[d]); - } - order = interleave(mine); - } - else { - // More co-located ranks than L3 domains: flat domain-major order, one contiguous slice each. - for (const auto &bucket : by_domain) { - order.insert(order.end(), bucket.begin(), bucket.end()); - } - offset = group_index * n; - } - if (offset + n > order.size()) { - return {}; - } - - std::vector sets(n); - for (size_t i = 0; i < n; ++i) { - CPU_ZERO(&sets[i]); - CPU_SET(order[offset + i], &sets[i]); - } - return sets; -} +auto partition_cpusets(size_t n, size_t group_index = 0, size_t group_count = 1) -> std::vector; // A failing pthread call is ignored: only performance depends on it. -inline auto pin_this_thread(const CpuSet &set) -> void { - pthread_setaffinity_np(pthread_self(), sizeof(CpuSet), &set); -} +auto pin_this_thread(const CpuSet &set) -> void; #else // portable fallback: no topology, no pinning From f5d5a97a24910f9371006d8cf78c1c7e8794ac5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Wed, 5 Aug 2026 13:24:59 +0000 Subject: [PATCH 02/80] chore(sonar): clear lint reported by sonarqube --- cpp/monoprop/detail/evolution/EvolutionHelpers.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/monoprop/detail/evolution/EvolutionHelpers.h b/cpp/monoprop/detail/evolution/EvolutionHelpers.h index d7087da0..ba771b2e 100644 --- a/cpp/monoprop/detail/evolution/EvolutionHelpers.h +++ b/cpp/monoprop/detail/evolution/EvolutionHelpers.h @@ -18,7 +18,6 @@ #include "monoprop/algebra/MajoranaAlgebra.h" namespace monoprop::detail { - inline constexpr size_t kMissingIndex = std::numeric_limits::max(); struct CutoffContext { @@ -33,7 +32,7 @@ struct CutoffContext { if (!use_coeff_checks) { return 0.0; } - // Out-of-range indices read as zero: callers scan past the end of a shorter coeff vector. + const double coeff = i < coeffs.size() ? coeffs[i] : 0.0; return std::abs(coeff); } @@ -44,5 +43,4 @@ struct CutoffContext { } auto is_below_sin(double abs_coeff) const -> bool { return check_atol && (abs_sin_val * abs_coeff <= atol_value); } }; - } // namespace monoprop::detail From 2af3733e47a1f1e0dbbff8056a2260ddd1fd0f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Wed, 5 Aug 2026 13:30:31 +0000 Subject: [PATCH 03/80] style(c++): remove unused include --- cpp/monoprop/detail/evolution/EvolutionHelpers.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/monoprop/detail/evolution/EvolutionHelpers.h b/cpp/monoprop/detail/evolution/EvolutionHelpers.h index ba771b2e..64c5e008 100644 --- a/cpp/monoprop/detail/evolution/EvolutionHelpers.h +++ b/cpp/monoprop/detail/evolution/EvolutionHelpers.h @@ -15,7 +15,6 @@ #pragma once #include "monoprop/TypeAliases.h" -#include "monoprop/algebra/MajoranaAlgebra.h" namespace monoprop::detail { inline constexpr size_t kMissingIndex = std::numeric_limits::max(); From 800dfc215ff644dc5e45fbf3d390d09746a364f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Wed, 5 Aug 2026 14:07:58 +0000 Subject: [PATCH 04/80] refactor: rename EvolutionHelpers to CutoffContext --- cpp/include/monoprop/Evolution.h | 22 ------------------- cpp/monoprop/Evolution.cpp | 1 + cpp/monoprop/MPFunctions.cpp | 1 + cpp/monoprop/TypeAliases.h | 3 ++- cpp/monoprop/detail/evolution/CMakeLists.txt | 2 +- .../{EvolutionHelpers.h => CutoffContext.h} | 2 -- .../detail/evolution/layer_build/Engine.h | 2 +- .../detail/evolution/layer_build/Resolve.h | 2 +- .../detail/evolution/layer_build/Scan.h | 2 +- cpp/tests/evolution_detail_tests.cpp | 2 +- 10 files changed, 9 insertions(+), 30 deletions(-) rename cpp/monoprop/detail/evolution/{EvolutionHelpers.h => CutoffContext.h} (95%) diff --git a/cpp/include/monoprop/Evolution.h b/cpp/include/monoprop/Evolution.h index 28fdf2fb..ed370d9c 100644 --- a/cpp/include/monoprop/Evolution.h +++ b/cpp/include/monoprop/Evolution.h @@ -14,35 +14,15 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "monoprop/MPFunctions.h" #include "monoprop/TypeAliases.h" -#include "monoprop/Utilities.h" #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" #include "monoprop/detail/mpi/MPICompat.h" -#include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/monopropExport.h" namespace monoprop { struct Layer; -class MPGraph; class MPGraphView; -struct LayerCore; - /// One layer's rotation angle in factored form: the layer rotates by 2·gen_coeff·param. struct LayerAngle { double gen_coeff = 1.0; ///< the generator's coefficient for this layer @@ -72,5 +52,3 @@ monoprop_EXPORT auto state_operator_derivative_local(VecD &state, mpi::Comm comm, const detail::LayerCosAccumulate &cos_acc) -> double; } // namespace monoprop - -#include "monoprop/detail/evolution/EvolutionHelpers.h" diff --git a/cpp/monoprop/Evolution.cpp b/cpp/monoprop/Evolution.cpp index 511db856..47bf3234 100644 --- a/cpp/monoprop/Evolution.cpp +++ b/cpp/monoprop/Evolution.cpp @@ -22,6 +22,7 @@ #include "monoprop/MPGraph.h" #include "monoprop/TypeAliases.h" +#include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" #include "monoprop/detail/mpi/Exchange.h" #include "monoprop/detail/mpi/MPICompat.h" diff --git a/cpp/monoprop/MPFunctions.cpp b/cpp/monoprop/MPFunctions.cpp index 6785c507..b716aad2 100644 --- a/cpp/monoprop/MPFunctions.cpp +++ b/cpp/monoprop/MPFunctions.cpp @@ -19,6 +19,7 @@ #include #include "monoprop/Evolution.h" +#include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" namespace monoprop { diff --git a/cpp/monoprop/TypeAliases.h b/cpp/monoprop/TypeAliases.h index 1a6966cc..160f4aed 100644 --- a/cpp/monoprop/TypeAliases.h +++ b/cpp/monoprop/TypeAliases.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -35,7 +36,6 @@ #include #include -#include #include "monoprop/Bitset.h" #include "monoprop/core/Monomial.h" @@ -47,6 +47,7 @@ class OperatorIndex; } namespace monoprop { +inline constexpr size_t kMissingIndex = std::numeric_limits::max(); // materialize_row() returns a const ref (dense backend, zero-copy) or a fresh value (packed backend), // so callers must bind with `const auto&` to extend the temporary's lifetime. diff --git a/cpp/monoprop/detail/evolution/CMakeLists.txt b/cpp/monoprop/detail/evolution/CMakeLists.txt index 00971ef2..637fdab4 100644 --- a/cpp/monoprop/detail/evolution/CMakeLists.txt +++ b/cpp/monoprop/detail/evolution/CMakeLists.txt @@ -6,7 +6,7 @@ target_sources( FILES "CosineRecompute.h" "CosineRecomputeCallbacks.h" - "EvolutionHelpers.h" + "CutoffContext.h" "LayerBuilder.h" ) diff --git a/cpp/monoprop/detail/evolution/EvolutionHelpers.h b/cpp/monoprop/detail/evolution/CutoffContext.h similarity index 95% rename from cpp/monoprop/detail/evolution/EvolutionHelpers.h rename to cpp/monoprop/detail/evolution/CutoffContext.h index 64c5e008..c4072cc4 100644 --- a/cpp/monoprop/detail/evolution/EvolutionHelpers.h +++ b/cpp/monoprop/detail/evolution/CutoffContext.h @@ -17,8 +17,6 @@ #include "monoprop/TypeAliases.h" namespace monoprop::detail { -inline constexpr size_t kMissingIndex = std::numeric_limits::max(); - struct CutoffContext { bool check_atol = false; bool check_upper_atol = false; diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 8a5a2b3b..69050a73 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -26,7 +26,7 @@ #include #include "monoprop/algebra/Algebra.h" -#include "monoprop/detail/evolution/EvolutionHelpers.h" +#include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" #include "monoprop/detail/evolution/layer_build/Resolve.h" #include "monoprop/detail/evolution/layer_build/Scan.h" diff --git a/cpp/monoprop/detail/evolution/layer_build/Resolve.h b/cpp/monoprop/detail/evolution/layer_build/Resolve.h index 7bb97c8b..508fd2af 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Resolve.h +++ b/cpp/monoprop/detail/evolution/layer_build/Resolve.h @@ -20,7 +20,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/algebra/Algebra.h" -#include "monoprop/detail/evolution/EvolutionHelpers.h" +#include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" #include "monoprop/detail/operator/MPOperator.h" diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index cd21e073..cd4b8a75 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -24,7 +24,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/algebra/Algebra.h" -#include "monoprop/detail/evolution/EvolutionHelpers.h" +#include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" #include "monoprop/detail/graph_encoding/MPGraphEncodingTypes.h" #include "monoprop/detail/mpi/MPIUtils.h" diff --git a/cpp/tests/evolution_detail_tests.cpp b/cpp/tests/evolution_detail_tests.cpp index cc506ec0..39555948 100644 --- a/cpp/tests/evolution_detail_tests.cpp +++ b/cpp/tests/evolution_detail_tests.cpp @@ -20,7 +20,7 @@ #include #include "monoprop/TypeAliases.h" -#include "monoprop/detail/evolution/EvolutionHelpers.h" +#include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" using namespace monoprop; From 8ba6c49a1109c05a2404f246d0702a044a9d75f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Thu, 6 Aug 2026 09:28:08 +0000 Subject: [PATCH 05/80] style: re-format one file --- cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h b/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h index fe5f9e39..c1ed79c9 100644 --- a/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h +++ b/cpp/monoprop/detail/graph_encoding/MPGraphEncodingStorage.h @@ -53,7 +53,7 @@ inline auto is_binary_phase(int value) -> bool { return value == -1 || value == 1; } -auto make_packed_phase_storage(size_t count, bool use_binary_phases) -> PackedPhaseStorage; +auto make_packed_phase_storage(size_t count, bool use_binary_phases) -> PackedPhaseStorage; auto packed_phase_storage_bytes(const PackedPhaseStorage &storage) -> size_t; From 6631b3d4bfa3f6296b7f997e1b6fd52efb6d8ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Wed, 5 Aug 2026 12:33:01 +0000 Subject: [PATCH 06/80] =?UTF-8?q?refactor:=20=F0=9F=A7=B9=20use=20hwloc=20?= =?UTF-8?q?instead=20of=20custom=20topology=20discovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: GitHub Copilot: gpt-5.6-sol (plan), claude-haiku-4.5/claude-sonnet-4.6 (execute) --- .devcontainer/Dockerfile | 1 + .github/workflows/copilot-setup-steps.yml | 2 +- .github/workflows/docpages.yml | 2 +- .github/workflows/qa-analysis.yml | 6 +- .github/workflows/test.yml | 4 +- AGENTS.md | 1 + cpp/monoprop/CMakeLists.txt | 11 + cpp/monoprop/detail/partition/CpuTopology.cpp | 239 +++++++++++++----- cpp/monoprop/detail/partition/CpuTopology.h | 186 ++++++-------- cpp/tests/CMakeLists.txt | 1 + cpp/tests/cpu_topology_tests.cpp | 180 +++++++------ docs/content/docs/features/parallelism.mdx | 2 +- pyproject.toml | 2 +- tools/install-deps.sh | 64 ++++- 14 files changed, 440 insertions(+), 261 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ebe3345d..b89ec823 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -20,6 +20,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ openmpi-bin \ libboost-dev \ libboost-test-dev \ + libhwloc-dev \ libmsgpack-cxx-dev \ libopenmpi-dev \ && apt-get autoremove -y \ diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 74da2ab9..30925853 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies from APT run: | sudo apt-get update - sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev + sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - name: Install the latest version of uv uses: astral-sh/setup-uv@v9.0.0 diff --git a/.github/workflows/docpages.yml b/.github/workflows/docpages.yml index 307a7f60..99996567 100644 --- a/.github/workflows/docpages.yml +++ b/.github/workflows/docpages.yml @@ -52,7 +52,7 @@ jobs: - name: Install dependencies from APT run: | sudo apt-get update - sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev + sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - name: Install the latest version of uv uses: astral-sh/setup-uv@v9.0.0 diff --git a/.github/workflows/qa-analysis.yml b/.github/workflows/qa-analysis.yml index c88e5c3e..2f581c96 100644 --- a/.github/workflows/qa-analysis.yml +++ b/.github/workflows/qa-analysis.yml @@ -70,7 +70,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev + sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - name: Install the latest version of uv uses: astral-sh/setup-uv@v9.0.0 @@ -157,7 +157,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev + sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libhwloc-dev - name: Install the latest version of uv uses: astral-sh/setup-uv@v9.0.0 @@ -205,7 +205,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev + sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libhwloc-dev - name: Install the latest version of uv uses: astral-sh/setup-uv@v9.0.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3a4b353..18f4cde2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,10 +66,10 @@ jobs: - name: Install dependencies run: | if [[ "${{ matrix.runner }}" == "macos-15" ]]; then - brew install boost open-mpi msgpack-cxx + brew install boost open-mpi msgpack-cxx hwloc else sudo apt-get update - packages="libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev" + packages="libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev" if [[ "${{ matrix.compiler }}" == "clang++-18" ]]; then packages="$packages clang-18" fi diff --git a/AGENTS.md b/AGENTS.md index d33c3dc5..5056a539 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,6 +113,7 @@ mp = MajoranaPropagator(operator, initial_state, cutoff=4) - **uv**: Package management - **Boost**: Used for various utilities (unordered_map, unit tests) - **msgpack**: Serialization of the test-data fixtures only (`tests/data/*.msgpack`); consumed by the Python test loaders and the C++ test suite, not by the shipped library +- **hwloc**: CPU topology discovery and thread binding for partition placement (`CpuTopology.cpp`). Required system library (`libhwloc-dev` on Debian/Ubuntu, `hwloc` on Homebrew). Bundled into wheels automatically by auditwheel/delocate. - **MPI**: For distributed parallelization ## Common Tasks diff --git a/cpp/monoprop/CMakeLists.txt b/cpp/monoprop/CMakeLists.txt index 34eca4ab..b18c8b33 100644 --- a/cpp/monoprop/CMakeLists.txt +++ b/cpp/monoprop/CMakeLists.txt @@ -1,6 +1,13 @@ find_package(Boost 1.85 CONFIG REQUIRED) message(STATUS "Using Boost: ${Boost_DIR} (version ${Boost_VERSION})") +find_package(PkgConfig REQUIRED QUIET) +pkg_check_modules(HWLOC REQUIRED QUIET IMPORTED_TARGET GLOBAL "hwloc>=2.9") +message( + STATUS + "Using hwloc: ${HWLOC_LINK_LIBRARIES} (version ${HWLOC_VERSION})" +) + target_sources( monoprop-objs PRIVATE @@ -43,6 +50,8 @@ target_link_libraries( Boost::boost Threads::Threads $<$:MPI::MPI_CXX> + PRIVATE + PkgConfig::HWLOC ) set_target_properties( @@ -97,6 +106,8 @@ target_link_libraries( Boost::boost Threads::Threads $<$:MPI::MPI_CXX> + PRIVATE + PkgConfig::HWLOC ) add_subdirectory(algebra) diff --git a/cpp/monoprop/detail/partition/CpuTopology.cpp b/cpp/monoprop/detail/partition/CpuTopology.cpp index 5be85e90..2b70c96e 100644 --- a/cpp/monoprop/detail/partition/CpuTopology.cpp +++ b/cpp/monoprop/detail/partition/CpuTopology.cpp @@ -14,90 +14,96 @@ #include "monoprop/detail/partition/CpuTopology.h" -#if defined(__linux__) +#include -#include #include +#include +#include namespace monoprop::detail::partition { -auto enumerate_physical_cores() -> std::vector { - const std::set allowed = topo_detail::allowed_cpus(); - const bool filter = !allowed.empty(); // no mask readable ⇒ accept every CPU - const auto is_allowed = [&](int cpu) { return !filter || allowed.contains(cpu); }; +namespace { - std::vector cores; - std::set seen_cores; // sibling-group key (min sibling) already recorded - std::vector> l3_members; // cpu-set per distinct L3 domain, in discovery order - - // Scan a bounded id range rather than stopping at the first gap: online CPU ids are not contiguous - // (offlined or hot-plugged CPUs leave holes), and breaking on the first unreadable id truncates the - // core list to whatever preceded the hole, silently under-partitioning and crowding the low CPUs. - const int scan_limit = filter ? *allowed.rbegin() + 1 : CPU_SETSIZE; - for (int cpu = 0; cpu < scan_limit; ++cpu) { - const std::string base = "/sys/devices/system/cpu/cpu" + std::to_string(cpu); - const std::string sib = topo_detail::read_line(base + "/topology/thread_siblings_list"); - if (sib.empty()) { - continue; - } - const auto siblings = topo_detail::parse_cpulist(sib); - const int group_key = siblings.empty() ? cpu : *std::min_element(siblings.begin(), siblings.end()); - if (seen_cores.contains(group_key)) { - continue; - } - seen_cores.insert(group_key); +/* ── Process-lifetime hwloc topology ──────────────────────────────────────── */ - int rep = -1; - if (siblings.empty()) { - rep = is_allowed(cpu) ? cpu : -1; - } - else { - for (int s : siblings) { // parse_cpulist yields ascending order - if (is_allowed(s)) { - rep = s; - break; - } - } +// hwloc_topology_t is safe for concurrent read-only access after hwloc_topology_load(). +struct TopologyHolder { + hwloc_topology_t topo = nullptr; + + TopologyHolder() noexcept { + if (hwloc_topology_init(&topo) < 0) { + topo = nullptr; + return; } - if (rep < 0) { - continue; + /* Keep all L3 cache objects so shared-L3 domains can always be identified, even on + * topologies where the L3 appears private and would otherwise be suppressed by the + * default HWLOC_TYPE_FILTER_KEEP_STRUCTURE filter. */ + hwloc_topology_set_type_filter(topo, HWLOC_OBJ_L3CACHE, HWLOC_TYPE_FILTER_KEEP_ALL); + if (hwloc_topology_load(topo) < 0) { + hwloc_topology_destroy(topo); + topo = nullptr; } + } - const auto l3 = topo_detail::parse_cpulist(topo_detail::read_line(base + "/cache/index3/shared_cpu_list")); - int domain = -1; - for (size_t d = 0; d < l3_members.size(); ++d) { - if (std::find(l3_members[d].begin(), l3_members[d].end(), group_key) != l3_members[d].end()) { - domain = static_cast(d); - break; - } - } - if (domain < 0) { - domain = static_cast(l3_members.size()); - l3_members.push_back(l3.empty() ? std::vector{group_key} : l3); + ~TopologyHolder() { + if (topo) { + hwloc_topology_destroy(topo); } - cores.push_back(PhysicalCore{rep, domain}); } - return cores; + + TopologyHolder(const TopologyHolder &) = delete; + auto operator=(const TopologyHolder &) -> TopologyHolder & = delete; +}; + +// Returns the loaded topology, or nullptr when initialization failed. +// The static local is initialized once; subsequent calls return the cached handle. +auto get_topology() -> hwloc_topology_t { + static TopologyHolder holder; + return holder.topo; } -auto partition_cpusets(size_t n, size_t group_index, size_t group_count) -> std::vector { - if (!config::get().partition_pinning) { - return {}; +/* ── Effective allowed cpuset for the calling thread ──────────────────────── */ + +// Queries the current thread's affinity to respect any launcher-imposed restriction (cgroup, MPI +// process binding) narrower than the topology's own allowed cpuset. Falls back to the topology +// allowed cpuset when the cpubind query is unsupported on this platform. Caller must free the bitmap. +auto effective_allowed_cpuset(hwloc_topology_t topo) -> hwloc_cpuset_t { + hwloc_cpuset_t set = hwloc_bitmap_alloc(); + if (!set) { + return nullptr; } - const auto cores = enumerate_physical_cores(); + if (hwloc_get_cpubind(topo, set, HWLOC_CPUBIND_THREAD) == 0) { + return set; + } + /* cpubind query not supported (e.g. macOS without OS X binding): fall back. */ + hwloc_bitmap_free(set); + return hwloc_bitmap_dup(hwloc_topology_get_allowed_cpuset(topo)); +} + +} // anonymous namespace + +/* ── topo_detail::placement_order ─────────────────────────────────────────── */ + +namespace topo_detail { + +auto placement_order(const std::vector &cores, size_t n, size_t group_index, size_t group_count) + -> std::vector { if (cores.empty() || group_count * n > cores.size()) { return {}; } + int max_domain = 0; for (const auto &c : cores) { max_domain = std::max(max_domain, c.l3_domain); } - // Ordering: interleaved for a lone process, contiguous blocks for co-located ranks. + + /* Bucket representative PU indices by L3 domain id. */ std::vector> by_domain(static_cast(max_domain) + 1); for (const auto &c : cores) { by_domain[static_cast(c.l3_domain)].push_back(c.cpu); } - // Interleave `buckets` depth-first: bucket0[0], bucket1[0], …, bucket0[1], bucket1[1], … + + /* Interleave buckets depth-first: b0[0], b1[0], …, b0[1], b1[1], … */ const auto interleave = [](const std::vector> &buckets) { std::vector out; for (size_t depth = 0;; ++depth) { @@ -117,7 +123,7 @@ auto partition_cpusets(size_t n, size_t group_index, size_t group_count) -> std: std::vector order; size_t offset = 0; if (group_count <= by_domain.size()) { - // Domains dealt to this rank: group_index, +group_count, … (group_count == 1 ⇒ all of them). + /* Interleave arm: deal domains round-robin across ranks. */ std::vector> mine; for (size_t d = group_index; d < by_domain.size(); d += group_count) { mine.push_back(by_domain[d]); @@ -125,28 +131,127 @@ auto partition_cpusets(size_t n, size_t group_index, size_t group_count) -> std: order = interleave(mine); } else { - // More co-located ranks than L3 domains: flat domain-major order, one contiguous slice each. + /* More co-located ranks than L3 domains: flat domain-major order, one contiguous slice each. */ for (const auto &bucket : by_domain) { order.insert(order.end(), bucket.begin(), bucket.end()); } offset = group_index * n; } + if (offset + n > order.size()) { return {}; } + return std::vector(order.begin() + static_cast(offset), + order.begin() + static_cast(offset + n)); +} + +} // namespace topo_detail + +/* ── enumerate_physical_cores ──────────────────────────────────────────────── */ + +auto enumerate_physical_cores() -> std::vector { + const auto topo = get_topology(); + if (!topo) { + return {}; + } - std::vector sets(n); - for (size_t i = 0; i < n; ++i) { - CPU_ZERO(&sets[i]); - CPU_SET(order[offset + i], &sets[i]); + const hwloc_cpuset_t allowed = effective_allowed_cpuset(topo); + if (!allowed) { + return {}; + } + + const int core_depth = hwloc_get_type_depth(topo, HWLOC_OBJ_CORE); + if (core_depth == HWLOC_TYPE_DEPTH_UNKNOWN || core_depth == HWLOC_TYPE_DEPTH_MULTIPLE) { + hwloc_bitmap_free(allowed); + return {}; + } + + std::vector cores; + std::map l3_domain_map; // l3->logical_index → domain id + int next_domain_id = 0; + + const unsigned num_cores = hwloc_get_nbobjs_by_depth(topo, core_depth); + for (unsigned i = 0; i < num_cores; ++i) { + const hwloc_obj_t core = hwloc_get_obj_by_depth(topo, core_depth, i); + if (!core || !core->cpuset) { + continue; + } + + /* Skip cores that have no PU in the calling thread's effective allowed mask. */ + if (!hwloc_bitmap_intersects(core->cpuset, allowed)) { + continue; + } + + /* Lowest allowed PU on this core is the representative OS index. */ + hwloc_cpuset_t core_allowed = hwloc_bitmap_alloc(); + if (!core_allowed) { + continue; + } + hwloc_bitmap_and(core_allowed, core->cpuset, allowed); + const int rep = hwloc_bitmap_first(core_allowed); + hwloc_bitmap_free(core_allowed); + if (rep < 0) { + continue; + } + + /* Find the L3 cache ancestor and assign a stable domain id. Cores sharing an L3 object + * (same logical_index) receive the same domain id. Cores without an L3 ancestor each + * receive their own singleton domain so the placement algorithm can still spread across + * whatever structure the topology does have. */ + int domain; + const hwloc_obj_t l3 = hwloc_get_ancestor_obj_by_type(topo, HWLOC_OBJ_L3CACHE, core); + if (l3) { + const auto [it, inserted] = l3_domain_map.emplace(l3->logical_index, next_domain_id); + if (inserted) { + ++next_domain_id; + } + domain = it->second; + } + else { + domain = next_domain_id++; + } + + cores.push_back(PhysicalCore{rep, domain}); + } + + hwloc_bitmap_free(allowed); + return cores; +} + +/* ── partition_cpusets ─────────────────────────────────────────────────────── */ + +auto partition_cpusets(size_t n, size_t group_index, size_t group_count) -> std::vector { + if (!config::get().partition_pinning) { + return {}; + } + const auto cores = enumerate_physical_cores(); + const auto order = topo_detail::placement_order(cores, n, group_index, group_count); + + std::vector sets(order.size()); + for (size_t i = 0; i < order.size(); ++i) { + sets[i] = CpuSet{order[i]}; } return sets; } +/* ── pin_this_thread ───────────────────────────────────────────────────────── */ + auto pin_this_thread(const CpuSet &set) -> void { - pthread_setaffinity_np(pthread_self(), sizeof(CpuSet), &set); + if (set.pu < 0) { + return; + } + const auto topo = get_topology(); + if (!topo) { + return; + } + hwloc_cpuset_t cpuset = hwloc_bitmap_alloc(); + if (!cpuset) { + return; + } + hwloc_bitmap_only(cpuset, static_cast(set.pu)); + /* Errors are intentionally ignored: pinning is performance-only, not a correctness requirement. */ + hwloc_set_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT); + hwloc_bitmap_free(cpuset); } } // namespace monoprop::detail::partition - -#endif // __linux__ diff --git a/cpp/monoprop/detail/partition/CpuTopology.h b/cpp/monoprop/detail/partition/CpuTopology.h index e5820194..1b60359e 100644 --- a/cpp/monoprop/detail/partition/CpuTopology.h +++ b/cpp/monoprop/detail/partition/CpuTopology.h @@ -12,6 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +/*! + * @file CpuTopology.h + * @brief CPU-topology helpers for partition placement. + * + * Uses hwloc for cross-platform topology discovery and thread binding. + * Policy: one partition per physical core, spread across L3/CCX domains, each worker thread pinned + * to its representative PU. Falls back to unpinned execution when hwloc cannot load the topology or + * when binding is unsupported — pinning is a performance optimisation, not a correctness requirement. + */ + #pragma once #include @@ -19,122 +29,90 @@ #include "monoprop/detail/EnvConfig.h" -#if defined(__linux__) -#include -#include -#include -#include -#include -#elif defined(__APPLE__) -#include -#endif - -// CPU-topology helpers for partition placement (the one platform-specific file). Policy: one partition per -// physical core, spread across L3/ccx domains. The Linux fast path parses /sys and pins each master, -// intersected with the process's allowed-CPU mask; elsewhere partitions run unpinned (still correct, no -// locality win). - namespace monoprop::detail::partition { +/*! + * @brief One physical CPU core the process may use, tagged with its L3 cache domain. + * + * Produced by enumerate_physical_cores(). The representative PU is the lowest OS index in the + * calling thread's allowed affinity mask for the core, so a cgroup or launcher-imposed restriction + * is always respected. + */ struct PhysicalCore { - int cpu = 0; // representative hardware thread (an allowed SMT sibling of the core) - int l3_domain = 0; + int cpu = 0; //!< OS index of the representative PU (lowest allowed SMT sibling of the core). + int l3_domain = 0; //!< Sequential L3-cache domain id assigned by enumerate_physical_cores(). }; -#if defined(__linux__) - -using CpuSet = cpu_set_t; +/*! + * @brief Lightweight placement token: identifies the single PU a partition worker thread is pinned to. + */ +struct CpuSet { + int pu = -1; //!< OS PU index. -1 ⇒ invalid / not placed. +}; namespace topo_detail { - -// Parse a Linux cpulist ("0-3,16-19") into the set of CPU ids it names. -inline auto parse_cpulist(const std::string &text) -> std::vector { - std::vector out; - std::stringstream ss(text); - std::string tok; - while (std::getline(ss, tok, ',')) { - const auto dash = tok.find('-'); - if (dash == std::string::npos) { - if (!tok.empty()) { - out.push_back(std::stoi(tok)); - } - } - else { - const int lo = std::stoi(tok.substr(0, dash)); - const int hi = std::stoi(tok.substr(dash + 1)); - for (int c = lo; c <= hi; ++c) { - out.push_back(c); - } - } - } - return out; -} - -inline auto read_line(const std::string &path) -> std::string { - std::ifstream f(path); - std::string line; - if (f) { - std::getline(f, line); - } - return line; -} - -// The CPUs this process is allowed to run on (the cgroup / cpuset the launcher gave us). Empty ⇒ the -// query failed; callers then treat every CPU as allowed. -inline auto allowed_cpus() -> std::set { - std::set allowed; - cpu_set_t mask; - CPU_ZERO(&mask); - if (sched_getaffinity(0, sizeof(mask), &mask) == 0) { - for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { - if (CPU_ISSET(cpu, &mask)) { - allowed.insert(cpu); - } - } - } - return allowed; -} - +/*! + * @brief Apply the L3-domain interleaving placement policy to a synthetic core list. + * + * Factored out of partition_cpusets() so the scheduling logic can be exercised without depending + * on hwloc or live hardware. Cores are bucketed by their l3_domain, then interleaved depth-first + * across the domains dealt to this rank. When there are more co-located ranks than L3 domains the + * algorithm falls back to flat domain-major order with one contiguous slice per rank. + * + * @param cores Physical cores available to allocate from, with L3 domain tags. + * @param n Number of partitions (PUs) requested for this rank. + * @param group_index This rank's 0-based index among the co-located ranks on the host. + * @param group_count Total number of co-located ranks on the host. + * @returns Ordered PU OS indices of length @p n for the slice assigned to this rank, + * or empty when the request cannot be filled (empty core list, oversubscription, + * or computed offset out of range). + */ +auto placement_order(const std::vector &cores, size_t n, size_t group_index, size_t group_count) + -> std::vector; } // namespace topo_detail -// Enumerate physical cores (one per smt sibling group) the process is allowed to use, tagged with their -// L3 domain. A core is included iff a sibling is in the allowed mask, with the smallest allowed sibling -// as representative, so a partial allocation never pins outside the mask. Empty if /sys cannot be read. +/*! + * @brief Enumerate physical cores (one per SMT sibling group) the process may use. + * + * Queries the calling thread's CPU affinity via hwloc to respect any cgroup or launcher-imposed + * restriction. For each core whose cpuset intersects that affinity mask, the lowest matching OS + * index is recorded as the representative PU and the core is labelled with the sequential id of + * its nearest L3 cache ancestor (or a unique singleton id when no L3 object is present). + * + * @returns Vector of PhysicalCore in hwloc logical-core order, or empty when hwloc cannot load + * the topology or when no core passes the affinity filter. + * + * @note This function deliberately ignores @c monoprop_PARTITION_PINNING so that the auto + * partition-count heuristic (one partition per physical core) works even when pinning is + * disabled by the user. + */ auto enumerate_physical_cores() -> std::vector; -// Build `n` partition cpusets, one physical core each. `group_index`/`group_count` place one MPI rank's -// partitions among the ranks sharing this host, spread across L3 domains and disjoint from the other ranks' -// — two ranks must never share a core (one rank's busy-polling collectives would starve the other's -// barrier spins). Empty (⇒ unpinned) if the host lacks group_count*n cores. +/*! + * @brief Build placement tokens for one MPI rank's partitions. + * + * Calls enumerate_physical_cores() and applies topo_detail::placement_order() to select @p n + * distinct physical cores for this rank. Two co-located ranks must never share a core: one rank's + * busy-polling collectives would starve the other's barrier spins. + * + * @param n Number of partitions to place. + * @param group_index This rank's 0-based index among the co-located ranks on the host. + * @param group_count Total number of co-located ranks on the host. + * @returns Vector of @p n CpuSet tokens, or empty when @c monoprop_PARTITION_PINNING is disabled, + * hwloc is unavailable, or the host cannot provide @p group_count × @p n distinct cores. + */ auto partition_cpusets(size_t n, size_t group_index = 0, size_t group_count = 1) -> std::vector; -// A failing pthread call is ignored: only performance depends on it. +/*! + * @brief Bind the calling thread to the PU identified by @p set. + * + * Allocates a temporary hwloc bitmap, sets the single bit for @c set.pu, and calls + * @c hwloc_set_cpubind with @c HWLOC_CPUBIND_THREAD | @c HWLOC_CPUBIND_STRICT. The call is + * best-effort: hwloc errors are silently ignored because only performance, not correctness, + * depends on successful pinning. + * + * @param set Placement token as returned by partition_cpusets(). A token with @c pu == -1 + * is a no-op. + */ auto pin_this_thread(const CpuSet &set) -> void; - -#else // portable fallback: no topology, no pinning - -// A placeholder cpuset type so PartitionGroup's member/signatures are platform-independent. -struct CpuSet {}; - -// No /sys to parse. macOS reports its physical-core count so the partition-count policy stays accurate -// (threads still can't be pinned); other platforms return empty ⇒ hardware_concurrency()/2. -inline auto enumerate_physical_cores() -> std::vector { -#if defined(__APPLE__) - int n = 0; - size_t sz = sizeof(n); - if (sysctlbyname("hw.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) { - return std::vector(static_cast(n)); - } -#endif - return {}; -} - -inline auto partition_cpusets(size_t /*n*/, size_t /*group_index*/ = 0, size_t /*group_count*/ = 1) - -> std::vector { - return {}; -} -inline auto pin_this_thread(const CpuSet & /*set*/) -> void {} - -#endif // __linux__ - } // namespace monoprop::detail::partition diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 450a90a3..fe5c7c0b 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -46,6 +46,7 @@ target_link_libraries( monoprop-objs Boost::unit_test_framework msgpack-cxx + PkgConfig::HWLOC ) include(${CMAKE_CURRENT_LIST_DIR}/boost-test.cmake) diff --git a/cpp/tests/cpu_topology_tests.cpp b/cpp/tests/cpu_topology_tests.cpp index 55816317..7634852c 100644 --- a/cpp/tests/cpu_topology_tests.cpp +++ b/cpp/tests/cpu_topology_tests.cpp @@ -12,40 +12,61 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Coverage of CpuTopology.h (Linux /sys parsing + affinity pinning; count-only fallback elsewhere). +// Coverage of CpuTopology (hwloc-based topology discovery + thread affinity pinning). +// +// Tests are split into two layers: +// 1. Live smoke tests — exercise enumerate_physical_cores / partition_cpusets / pin_this_thread +// on the actual host topology; these validate end-to-end hwloc integration. +// 2. Policy unit tests — call topo_detail::placement_order with synthetic PhysicalCore vectors +// so the L3-domain interleaving and MPI-rank slicing logic can be checked deterministically +// without depending on live hardware or hwloc. #include #include #include #include -#include #include +#if defined(__linux__) +#include +#endif + #include "monoprop/detail/partition/CpuTopology.h" namespace partition = monoprop::detail::partition; +using partition::topo_detail::placement_order; + +/* RAII helper: save and restore the calling thread's CPU affinity around pin_this_thread() calls + * so that CTest is not left pinned to a single PU after the test completes. */ +struct AffinityGuard { +#if defined(__linux__) + cpu_set_t saved_{}; + AffinityGuard() { sched_getaffinity(0, sizeof(saved_), &saved_); } + ~AffinityGuard() { sched_setaffinity(0, sizeof(saved_), &saved_); } +#endif +}; + +/* ── Live smoke tests ─────────────────────────────────────────────────────── */ -// The enumerate/place/pin surface exists on every platform; exercise it regardless of OS. BOOST_AUTO_TEST_CASE(cpu_topology_enumerate_and_place) { const auto cores = partition::enumerate_physical_cores(); + AffinityGuard guard; // save affinity before any potential pin const auto one = partition::partition_cpusets(/*n=*/1); BOOST_CHECK(one.size() <= 1u); if (!one.empty()) { - // A placement only comes back where the engine can pin (Linux /sys), which implies cores were - // found. Pinning itself is best-effort and no-op-safe; drive it. + // A placement only comes back when topology discovery succeeded and pinning is enabled. BOOST_CHECK(!cores.empty()); partition::pin_this_thread(one.front()); + // guard restores affinity on scope exit } -#if defined(__linux__) - // With a readable /sys and pinning enabled, a non-empty core list must yield a placement. + // When topology discovery succeeds, a non-empty core list must produce a non-empty placement. if (!cores.empty()) { BOOST_CHECK_EQUAL(one.size(), 1u); } -#endif - // Asking for more physical cores than exist disables pinning (empty), never oversubscribes. + // Oversubscription must always return empty regardless of topology state. const auto too_many = partition::partition_cpusets(/*n=*/1'000'000); BOOST_CHECK(too_many.empty()); } @@ -53,88 +74,97 @@ BOOST_AUTO_TEST_CASE(cpu_topology_enumerate_and_place) { BOOST_AUTO_TEST_CASE(cpu_topology_place_co_located_ranks) { const auto cores = partition::enumerate_physical_cores(); if (cores.size() < 2) { - return; // need at least two cores to deal one to each of two co-located ranks + return; // need at least two cores for the disjoint-placement check } - // Two co-located ranks, one partition each. Covers both placement arms -- interleave across the - // dealt domains (group_count <= #L3), and the flat domain-major slice otherwise. + + // Two co-located ranks each requesting one partition. This exercises both placement arms: + // - interleave (group_count ≤ #L3 domains) + // - domain-major slice (group_count > #L3 domains) const auto rank0 = partition::partition_cpusets(/*n=*/1, /*group_index=*/0, /*group_count=*/2); const auto rank1 = partition::partition_cpusets(/*n=*/1, /*group_index=*/1, /*group_count=*/2); - // Off Linux there is no pinning, so both come back empty (unpinned, still disjoint by the scheduler). -#if defined(__linux__) - BOOST_CHECK_EQUAL(rank0.size(), 1u); - BOOST_CHECK_EQUAL(rank1.size(), 1u); -#else - BOOST_CHECK(rank0.empty()); - BOOST_CHECK(rank1.empty()); -#endif + BOOST_REQUIRE_EQUAL(rank0.size(), 1u); + BOOST_REQUIRE_EQUAL(rank1.size(), 1u); + // The two placements must be on distinct PUs; sharing would violate the MPI no-starvation + // invariant (one rank's busy-polling collectives cannot starve the other's barrier spins). + BOOST_CHECK(rank0.front().pu != rank1.front().pu); + // Oversubscription: 2 ranks × cores.size() partitions > total physical cores. const auto past_end = partition::partition_cpusets(/*n=*/cores.size(), /*group_index=*/1, /*group_count=*/2); BOOST_CHECK(past_end.empty()); } -#if defined(__linux__) - -using partition::topo_detail::parse_cpulist; -using partition::topo_detail::read_line; - -// Enumeration must span holes in the CPU id space (offline or hot-plugged CPUs leave unreadable ids). -// Oracle: the sibling groups re-derived straight from /sys over the whole allowed range. -BOOST_AUTO_TEST_CASE(cpu_topology_enumeration_spans_gaps_in_the_id_space) { - const auto allowed = partition::topo_detail::allowed_cpus(); - if (allowed.empty()) { - return; // affinity unreadable; enumeration accepts every CPU and there is nothing to compare - } +/* ── Policy unit tests (deterministic, no hwloc or live hardware) ─────────── */ + +BOOST_AUTO_TEST_CASE(cpu_topology_policy_interleave_across_l3) { + // 4 cores across 2 L3 domains; single rank receives all. + // by_domain[0] = {0, 4}, by_domain[1] = {2, 6} + // depth-first interleave: 0, 2, 4, 6 + const std::vector cores = {{0, 0}, {2, 1}, {4, 0}, {6, 1}}; + const auto order = placement_order(cores, 4, 0, 1); + BOOST_REQUIRE_EQUAL(order.size(), 4u); + BOOST_CHECK_EQUAL(order[0], 0); + BOOST_CHECK_EQUAL(order[1], 2); + BOOST_CHECK_EQUAL(order[2], 4); + BOOST_CHECK_EQUAL(order[3], 6); +} - std::set expected_groups; // one key (min sibling) per physical core with an allowed sibling - for (int cpu = 0; cpu <= *allowed.rbegin(); ++cpu) { - const std::string sib = - read_line("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/thread_siblings_list"); - if (sib.empty()) { - continue; - } - const auto siblings = parse_cpulist(sib); - if (siblings.empty()) { - continue; - } - if (std::any_of(siblings.begin(), siblings.end(), [&](int s) { return allowed.contains(s); })) { - expected_groups.insert(*std::min_element(siblings.begin(), siblings.end())); - } - } - if (expected_groups.empty()) { - return; // /sys unreadable on this host - } +BOOST_AUTO_TEST_CASE(cpu_topology_policy_disjoint_mpi_ranks) { + // 4 cores across 2 L3 domains; 2 co-located ranks each get 1 partition. + // rank0 is dealt domain 0, rank1 is dealt domain 1 ⇒ no shared PU. + const std::vector cores = {{0, 0}, {2, 1}, {4, 0}, {6, 1}}; + const auto r0 = placement_order(cores, 1, 0, 2); + const auto r1 = placement_order(cores, 1, 1, 2); + BOOST_REQUIRE_EQUAL(r0.size(), 1u); + BOOST_REQUIRE_EQUAL(r1.size(), 1u); + BOOST_CHECK(r0.front() != r1.front()); +} - const auto cores = partition::enumerate_physical_cores(); - BOOST_CHECK_EQUAL(cores.size(), expected_groups.size()); - for (const auto &core : cores) { - BOOST_TEST(allowed.contains(core.cpu)); +BOOST_AUTO_TEST_CASE(cpu_topology_policy_domain_major_more_ranks_than_l3) { + // 4 cores in 1 L3 domain; 2 ranks each get 2 partitions (flat domain-major arm). + // order = [0, 2, 4, 6]; rank0 offset=0 → {0,2}, rank1 offset=2 → {4,6}. + const std::vector cores = {{0, 0}, {2, 0}, {4, 0}, {6, 0}}; + const auto r0 = placement_order(cores, 2, 0, 2); + const auto r1 = placement_order(cores, 2, 1, 2); + BOOST_REQUIRE_EQUAL(r0.size(), 2u); + BOOST_REQUIRE_EQUAL(r1.size(), 2u); + + const std::set s0(r0.begin(), r0.end()); + const std::set s1(r1.begin(), r1.end()); + for (const auto cpu : s1) { + BOOST_CHECK(!s0.contains(cpu)); } } -BOOST_AUTO_TEST_CASE(cpu_topology_parse_cpulist_shapes) { - BOOST_TEST(parse_cpulist("5") == (std::vector{5}), boost::test_tools::per_element()); - BOOST_TEST(parse_cpulist("0-3") == (std::vector{0, 1, 2, 3}), boost::test_tools::per_element()); - BOOST_TEST(parse_cpulist("0-3,16-17") == (std::vector{0, 1, 2, 3, 16, 17}), boost::test_tools::per_element()); - BOOST_TEST(parse_cpulist("2,4,6") == (std::vector{2, 4, 6}), boost::test_tools::per_element()); +BOOST_AUTO_TEST_CASE(cpu_topology_policy_insufficient_cores_returns_empty) { + // 2 cores total; 2 ranks × 2 partitions = 4 > 2 ⇒ oversubscription. + const std::vector cores = {{0, 0}, {2, 0}}; + BOOST_CHECK(placement_order(cores, 2, 0, 2).empty()); + BOOST_CHECK(placement_order(cores, 2, 1, 2).empty()); - // Empty input and empty tokens contribute nothing (the !tok.empty() guard). - BOOST_CHECK(parse_cpulist("").empty()); - BOOST_TEST(parse_cpulist("1,,3") == (std::vector{1, 3}), boost::test_tools::per_element()); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_read_line_present_and_absent) { - BOOST_CHECK(read_line("/nonexistent/monoprop/topology/does_not_exist").empty()); + // Single rank requesting more cores than exist. + BOOST_CHECK(placement_order(cores, 3, 0, 1).empty()); - // cpu0 always exists on Linux, and its thread_siblings_list is a non-empty cpulist. - const std::string line = read_line("/sys/devices/system/cpu/cpu0/topology/thread_siblings_list"); - BOOST_CHECK(!line.empty()); - BOOST_CHECK(!parse_cpulist(line).empty()); + // Empty core list. + BOOST_CHECK(placement_order({}, 1, 0, 1).empty()); } -BOOST_AUTO_TEST_CASE(cpu_topology_allowed_cpus_nonempty_on_ci) { - // sched_getaffinity succeeds on Linux CI, so the process's allowed set is non-empty. - const auto allowed = partition::topo_detail::allowed_cpus(); - BOOST_CHECK(!allowed.empty()); +BOOST_AUTO_TEST_CASE(cpu_topology_policy_singleton_l3_domains) { + // 2 cores each in its own singleton domain (no shared L3). + // by_domain[0] = {0}, by_domain[1] = {4}; interleaved: 0, 4. + const std::vector cores = {{0, 0}, {4, 1}}; + const auto order = placement_order(cores, 2, 0, 1); + BOOST_REQUIRE_EQUAL(order.size(), 2u); + BOOST_CHECK_EQUAL(order[0], 0); + BOOST_CHECK_EQUAL(order[1], 4); } -#endif // __linux__ +BOOST_AUTO_TEST_CASE(cpu_topology_policy_uneven_domains) { + // 3 cores: 2 in domain 0, 1 in domain 1; single rank, 3 partitions. + // by_domain[0] = {0, 4}, by_domain[1] = {2}; interleaved: 0, 2, 4. + const std::vector cores = {{0, 0}, {2, 1}, {4, 0}}; + const auto order = placement_order(cores, 3, 0, 1); + BOOST_REQUIRE_EQUAL(order.size(), 3u); + BOOST_CHECK_EQUAL(order[0], 0); + BOOST_CHECK_EQUAL(order[1], 2); + BOOST_CHECK_EQUAL(order[2], 4); +} diff --git a/docs/content/docs/features/parallelism.mdx b/docs/content/docs/features/parallelism.mdx index 8ebacb7b..adb14a8c 100644 --- a/docs/content/docs/features/parallelism.mdx +++ b/docs/content/docs/features/parallelism.mdx @@ -29,7 +29,7 @@ whose partner lives in another partition are resolved through a per-gate exchang | --- | --- | --- | | `monoprop_NUM_THREADS` | one partition per physical core | Caps the number of partitions. Set it to run fewer partitions than cores. | | `monoprop_PARTITIONS` | `auto` | `auto` = one partition per core (capped by `monoprop_NUM_THREADS`); an integer `N` = exactly `N` partitions; `off` = one partition holding the whole operator. | -| `monoprop_PARTITION_PINNING` | `on` | `0`/`false`/`no` disables pinning each partition to a core. Has an effect only on Linux. | +| `monoprop_PARTITION_PINNING` | `on` | `0`/`false`/`no` disables pinning each partition to a core. Supported on platforms where hwloc can bind threads (Linux, macOS). | ```bash # Run 8 partitions instead of one-per-core: diff --git a/pyproject.toml b/pyproject.toml index 49b1c4b5..5731e903 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -353,5 +353,5 @@ before-build = ["./tools/install-deps.sh --skip-boost-test --skip-msgpack"] environment = { SKBUILD_CMAKE_ARGS = "-Dmonoprop_ENABLE_ARCH_FLAGS=OFF;-Dmonoprop_ENABLE_CXX_UNIT_TESTS=OFF;-Dmonoprop_ENABLE_MPI=OFF" } [tool.cibuildwheel.macos] -before-build = "brew install boost" +before-build = "brew install boost hwloc" environment = { MACOSX_DEPLOYMENT_TARGET = "15.0", SKBUILD_CMAKE_ARGS = "-Dmonoprop_ENABLE_ARCH_FLAGS=OFF;-Dmonoprop_ENABLE_CXX_UNIT_TESTS=OFF;-Dmonoprop_ENABLE_MPI=OFF" } diff --git a/tools/install-deps.sh b/tools/install-deps.sh index bdc9be53..027c6d01 100755 --- a/tools/install-deps.sh +++ b/tools/install-deps.sh @@ -10,7 +10,7 @@ Usage: $0 [INSTALL_PREFIX] [OPTIONS] Install C++ dependencies for monoprop project. -This script can install Boost Unordered, Boost Test, and msgpack-cxx. +This script can install Boost Unordered, Boost Test, msgpack-cxx, and hwloc. Each component can be skipped with the corresponding option. The default installation prefix is /usr/local. @@ -21,6 +21,7 @@ Options: --skip-boost-unordered Skip installing Boost unordered --skip-boost-test Skip installing Boost Test library (only install unordered) --skip-msgpack Skip installing msgpack-cxx library + --skip-hwloc Skip installing hwloc library --help, -h Show this help message Examples: @@ -28,7 +29,7 @@ Examples: $0 \$HOME/Software # Install all deps to \$HOME/Software $0 --skip-boost-test # Skip Boost Test, install rest to default location $0 /opt --skip-msgpack # Install to /opt, skip msgpack - $0 --skip-boost-test --skip-msgpack # Minimal install + $0 --skip-boost-test --skip-msgpack # Minimal install (Boost unordered + hwloc) EOF } @@ -38,6 +39,7 @@ INSTALL_PREFIX="$DEFAULT_PREFIX" INSTALL_BOOST_UNORDERED=true INSTALL_BOOST_TEST=true INSTALL_MSGPACK=true +INSTALL_HWLOC=true # Parse arguments while [[ $# -gt 0 ]]; do @@ -54,6 +56,10 @@ while [[ $# -gt 0 ]]; do INSTALL_MSGPACK=false shift ;; + --skip-hwloc) + INSTALL_HWLOC=false + shift + ;; --help|-h) show_help exit 0 @@ -81,6 +87,7 @@ echo "Installing C++ dependencies to: $INSTALL_PREFIX" echo "Boost unordered: $([ "$INSTALL_BOOST_UNORDERED" = true ] && echo "YES" || echo "SKIP")" echo "Boost Test: $([ "$INSTALL_BOOST_TEST" = true ] && echo "YES" || echo "SKIP")" echo "msgpack-cxx: $([ "$INSTALL_MSGPACK" = true ] && echo "YES" || echo "SKIP")" +echo "hwloc: $([ "$INSTALL_HWLOC" = true ] && echo "YES" || echo "SKIP")" echo # Create install directory if it doesn't exist @@ -152,14 +159,58 @@ install_msgpack() { fi } -# check that we're running on Ubuntu -. /etc/os-release -echo "Detected OS: $PRETTY_NAME" -install_boost +install_hwloc() { + if [ "$INSTALL_HWLOC" != true ]; then + echo "Skipping hwloc installation" + return 0 + fi + + local hwloc_version="2.13.0" + local major_minor + major_minor="$(echo "$hwloc_version" | cut -d. -f1-2)" + local tarball="hwloc-${hwloc_version}.tar.gz" + local src_dir="hwloc-${hwloc_version}" + + echo "Installing hwloc $hwloc_version..." + curl -fsSL "https://download.open-mpi.org/release/hwloc/v${major_minor}/${tarball}" -o "$tarball" + tar xzf "$tarball" + cd "$src_dir" + + local nproc_count + nproc_count="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" + + ./configure \ + --prefix="$INSTALL_PREFIX" \ + --enable-shared \ + --disable-static \ + --disable-doxygen \ + --disable-man-pages \ + --without-x + make -j"$nproc_count" + make install + + echo "Cleaning up $src_dir..." + cd - + rm -rf "$src_dir" "$tarball" +} + +# Detect the OS (non-fatal: macOS does not have /etc/os-release). +if [ -f /etc/os-release ]; then + . /etc/os-release + echo "Detected OS: $PRETTY_NAME" +elif command -v sw_vers &>/dev/null; then + echo "Detected OS: macOS $(sw_vers -productVersion)" +else + echo "Detected OS: unknown" +fi + +install_boost install_msgpack +install_hwloc + echo echo "Dependencies installation completed successfully!" echo "Install location: $INSTALL_PREFIX" @@ -171,3 +222,4 @@ echo "Installed components:" [ "$INSTALL_BOOST_UNORDERED" = true ] && echo " ✓ Boost unordered" || echo " ✗ Boost unordered (skipped)" [ "$INSTALL_BOOST_TEST" = true ] && echo " ✓ Boost Test" || echo " ✗ Boost Test (skipped)" [ "$INSTALL_MSGPACK" = true ] && echo " ✓ msgpack-cxx" || echo " ✗ msgpack-cxx (skipped)" +[ "$INSTALL_HWLOC" = true ] && echo " ✓ hwloc" || echo " ✗ hwloc (skipped)" From f252b7479959149a793aa2520cb9d9f16896f819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Thu, 6 Aug 2026 14:01:36 +0000 Subject: [PATCH 07/80] build(cmake): RPATH handling, to fix the stubgen --- src/monoprop/bindings/CMakeLists.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/monoprop/bindings/CMakeLists.txt b/src/monoprop/bindings/CMakeLists.txt index e8f2a4c1..e18ede35 100644 --- a/src/monoprop/bindings/CMakeLists.txt +++ b/src/monoprop/bindings/CMakeLists.txt @@ -133,6 +133,27 @@ COMMAND_ERROR_IS_FATAL ANY )" ) +if(APPLE) + set(_rpath "@loader_path/${CMAKE_INSTALL_LIBDIR}") +else() + set(_rpath "\$ORIGIN/${CMAKE_INSTALL_LIBDIR}") +endif() + +set_target_properties( + _core + PROPERTIES + MACOSX_RPATH + ON + SKIP_BUILD_RPATH + OFF + BUILD_WITH_INSTALL_RPATH + OFF + INSTALL_RPATH + "${_rpath}" + INSTALL_RPATH_USE_LINK_PATH + ON +) + install(TARGETS _core LIBRARY DESTINATION ${PROJECT_NAME}) # generation of Python typing stubs From c8650f94068bda96519d53e20b38be60d02dee32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Thu, 6 Aug 2026 14:02:31 +0000 Subject: [PATCH 08/80] ci(wheel): revert to before-all with before-build we re-ran the same installation script before every build. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5731e903..925a95bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -349,9 +349,9 @@ test-groups = ["test"] test-extras = ["qiskit"] [tool.cibuildwheel.linux] -before-build = ["./tools/install-deps.sh --skip-boost-test --skip-msgpack"] +before-all = ["./tools/install-deps.sh --skip-boost-test --skip-msgpack"] environment = { SKBUILD_CMAKE_ARGS = "-Dmonoprop_ENABLE_ARCH_FLAGS=OFF;-Dmonoprop_ENABLE_CXX_UNIT_TESTS=OFF;-Dmonoprop_ENABLE_MPI=OFF" } [tool.cibuildwheel.macos] -before-build = "brew install boost hwloc" +before-all = "brew install boost hwloc" environment = { MACOSX_DEPLOYMENT_TARGET = "15.0", SKBUILD_CMAKE_ARGS = "-Dmonoprop_ENABLE_ARCH_FLAGS=OFF;-Dmonoprop_ENABLE_CXX_UNIT_TESTS=OFF;-Dmonoprop_ENABLE_MPI=OFF" } From ea764dc7648ccecb8d63241f7f3faaefcb283045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 09:45:24 +0000 Subject: [PATCH 09/80] docs: document need for pkg-config to find hwloc --- AGENTS.md | 2 +- README.md | 2 ++ docs/content/docs/building.mdx | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 5056a539..3bc25f40 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,7 +113,7 @@ mp = MajoranaPropagator(operator, initial_state, cutoff=4) - **uv**: Package management - **Boost**: Used for various utilities (unordered_map, unit tests) - **msgpack**: Serialization of the test-data fixtures only (`tests/data/*.msgpack`); consumed by the Python test loaders and the C++ test suite, not by the shipped library -- **hwloc**: CPU topology discovery and thread binding for partition placement (`CpuTopology.cpp`). Required system library (`libhwloc-dev` on Debian/Ubuntu, `hwloc` on Homebrew). Bundled into wheels automatically by auditwheel/delocate. +- **hwloc**: CPU topology discovery and thread binding for partition placement (`CpuTopology.cpp`). Required system library (`libhwloc-dev` on Debian/Ubuntu, `hwloc` on Homebrew). Requires `pkg-config` so CMake can locate `hwloc`. Bundled into wheels automatically by auditwheel/delocate. - **MPI**: For distributed parallelization ## Common Tasks diff --git a/README.md b/README.md index cb1633fd..0ff2cc1a 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ ctest --test-dir build/editable/Release Full instructions — prerequisites, MPI options, and running the example executable — are in the [building guide](https://docs.algorithmiq.fi/monoprop/docs/building). +In particular, from-source builds require `hwloc` and `pkg-config` so CMake can +locate `hwloc`. ## Running the tests diff --git a/docs/content/docs/building.mdx b/docs/content/docs/building.mdx index 98ae8395..aa0fe949 100644 --- a/docs/content/docs/building.mdx +++ b/docs/content/docs/building.mdx @@ -25,6 +25,7 @@ without MPI, so a from-source build is required for multi-rank runs. - CMake and Ninja - Python 3.11 or newer and the `uv` package manager (for the bindings) - an MPI implementation such as Open MPI (only for MPI builds) +- `hwloc` (version 2.9+) and `pkg-config` (required so CMake can locate `hwloc`) The repository ships a [DevContainer](https://containers.dev/) with all of the above pre-configured; opening the folder in VS Code and rebuilding the container is From b5c21cd666094f5f4d3bdba57e9a722c6c3eba09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Thu, 6 Aug 2026 19:35:25 +0000 Subject: [PATCH 10/80] build(c++): report machine flags used --- .github/workflows/test.yml | 2 +- benches/conftest.py | 2 + cmake/compiler_flags/CXXFlags.cmake | 74 ++++++++++++++++++-- cpp/include/monoprop/CMakeLists.txt | 5 +- cpp/include/monoprop/Info.h.in | 4 +- cpp/include/monoprop/Variants.h.in | 63 +++++++++++++++++ src/monoprop/__init__.py | 2 + src/monoprop/bindings/bindings.cpp.in | 2 + tests/test_gcc_target_help_clean.py | 98 +++++++++++++++++++++++++++ tools/gcc-target-help-clean.py | 80 ++++++++++++++++++++++ 10 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 cpp/include/monoprop/Variants.h.in create mode 100644 tests/test_gcc_target_help_clean.py create mode 100644 tools/gcc-target-help-clean.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18f4cde2..ddb17a80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,7 +101,7 @@ jobs: - name: Get monoprop version run: | - uv run python -c "import monoprop as mp; print(mp.__version__)" + uv run python -c "import monoprop as mp; print(mp.__version__); print(mp.__variant__); print(mp.__compiler_flags__)" - name: Verify that find_package(monoprop) works run: | diff --git a/benches/conftest.py b/benches/conftest.py index 194ab191..a6eaf9ef 100644 --- a/benches/conftest.py +++ b/benches/conftest.py @@ -166,6 +166,8 @@ def _meta() -> dict[str, Any]: "cpu_count_physical": psutil.cpu_count(logical=False), "hostname": socket.gethostname(), "monoprop_version": monoprop.__version__, + "monoprop_variant": monoprop.__variant__, + "monoprop_compiler_flags": monoprop.__compiler_flags__, } diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index d5ebdc10..65174cf0 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -48,10 +48,7 @@ set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) # do not use compiler extensions to the C++ standard set(CMAKE_CXX_EXTENSIONS FALSE) -# CMP0155 has CMake scan every C++20-or-later source for `import`s. There are no modules here, so -# the scan is pure build overhead, and it is not portable: Clang needs the separate clang-scan-deps -# binary (packaged apart from the compiler), whose absence surfaces as a build failure rather than a -# configure error. Must be set before any target is created. +# disable scanning for C++20 modules (unused) set(CMAKE_CXX_SCAN_FOR_MODULES OFF) # generate a JSON database of compiler commands (useful for LSP IDEs) set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) @@ -74,6 +71,75 @@ if(monoprop_ENABLE_ARCH_FLAGS AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug") endif() endif() +# Query the machine-dependent flags for a given -march value and store the +# cleaned, space-separated string in the variable named by OUTPUT_VARIABLE. A +# MARCH of "default" queries the default target (no -march flag). +# +# Usage: +# _monoprop_query_machine_flags(MARCH OUTPUT_VARIABLE ) +function(_monoprop_query_machine_flags) + set( + _one_value_args + MARCH + OUTPUT_VARIABLE + ) + cmake_parse_arguments(PARSE_ARGV 0 _arg "" "${_one_value_args}" "") + + if(NOT _arg_OUTPUT_VARIABLE) + message( + FATAL_ERROR + "_monoprop_query_machine_flags: OUTPUT_VARIABLE is required" + ) + endif() + if(NOT _arg_MARCH) + message(FATAL_ERROR "_monoprop_query_machine_flags: MARCH is required") + endif() + + if(_arg_MARCH STREQUAL "default") + set(_march_args "") + else() + set(_march_args "-march=${_arg_MARCH}") + endif() + + # Report the machine-dependent flags GCC uses for each target variant by + # querying `gcc -march= -Q --help=target` and cleaning the output with + # tools/gcc-target-help-clean.py. + execute_process( + COMMAND + ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target + COMMAND + ${Python_EXECUTABLE} + "${PROJECT_SOURCE_DIR}/tools/gcc-target-help-clean.py" + OUTPUT_VARIABLE _flags + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _result + ) + if(NOT _result EQUAL 0) + message( + FATAL_ERROR + "Failed to query machine-dependent flags for '${_arg_MARCH}' (exit code ${_result})" + ) + endif() + set(${_arg_OUTPUT_VARIABLE} "${_flags}" PARENT_SCOPE) +endfunction() + +set(monoprop_DEFAULT_VARIANT_FLAGS "") +if(monoprop_ENABLE_ARCH_FLAGS) + _monoprop_query_machine_flags(MARCH native OUTPUT_VARIABLE monoprop_DEFAULT_VARIANT_FLAGS) +else() + _monoprop_query_machine_flags(MARCH default OUTPUT_VARIABLE monoprop_DEFAULT_VARIANT_FLAGS) +endif() + +set(monoprop_VARIANTS "") +set(monoprop_VARIANT_FLAGS "") + +# generate a header file with the macros needed to describe the variant +configure_file( + ${PROJECT_SOURCE_DIR}/cpp/include/monoprop/Variants.h.in + ${PROJECT_BINARY_DIR}/include/monoprop/Variants.h + @ONLY +) + set(monoprop_CXX_FLAGS "") include(${CMAKE_CURRENT_LIST_DIR}/GNU.CXX.cmake) include(${CMAKE_CURRENT_LIST_DIR}/Intel.CXX.cmake) diff --git a/cpp/include/monoprop/CMakeLists.txt b/cpp/include/monoprop/CMakeLists.txt index 99c964fa..4e9d7afa 100644 --- a/cpp/include/monoprop/CMakeLists.txt +++ b/cpp/include/monoprop/CMakeLists.txt @@ -12,10 +12,11 @@ target_sources( ${PROJECT_SOURCE_DIR}/cpp/include ${PROJECT_BINARY_DIR}/include FILES + "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/${PROJECT_NAME}Export.h" + "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Info.h" + "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/VariantMacros.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/Evolution.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MPFunctions.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MPGraph.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MonomialPropagator.h" - "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/${PROJECT_NAME}Export.h" - "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Info.h" ) diff --git a/cpp/include/monoprop/Info.h.in b/cpp/include/monoprop/Info.h.in index 8c50e5c6..f3027536 100644 --- a/cpp/include/monoprop/Info.h.in +++ b/cpp/include/monoprop/Info.h.in @@ -18,6 +18,8 @@ #include #include +#include "monoprop/Variants.h" + namespace monoprop { static constexpr auto build_type() noexcept -> std::string_view { return "@CMAKE_BUILD_TYPE@"; @@ -27,7 +29,7 @@ static auto compiler_flags() noexcept -> std::map { return { {"from-environment", "@CMAKE_CXX_FLAGS@"}, {"build-type-flags", "@_cmake_build_type_specific_flags@"}, - {"vectorization", "@ARCH_FLAG@"}, + {"machine-flags", std::string(variant_flags())}, {"project-defaults", "@CMAKE_CXX23_STANDARD_COMPILE_OPTION@ @monoprop_CXX_FLAGS@"}, {"user-appended", "@EXTRA_CXXFLAGS@"}, }; diff --git a/cpp/include/monoprop/Variants.h.in b/cpp/include/monoprop/Variants.h.in new file mode 100644 index 00000000..54e13fb0 --- /dev/null +++ b/cpp/include/monoprop/Variants.h.in @@ -0,0 +1,63 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +/** + * @brief Declares a compile-time function that reports the active FMV variant. + * + * Expands to a `consteval` function named `variant()` with a GNU + * `target("arch=...")` attribute bound to the provided architecture string. + * + * @param archstr Architecture suffix used in `arch=`. + */ +#define monoprop_VARIANT(archstr) \ + [[using gnu: target("arch=" archstr)]] consteval auto variant() noexcept -> std::string_view { \ + return "arch=" archstr; \ + } + +/** + * @brief Declares a compile-time function that reports the machine-dependent + * flags GCC applies for the given FMV variant. + * + * Expands to a `consteval` function named `machine_flags()` with a GNU + * `target("arch=...")` attribute bound to the provided architecture string. The + * returned value is the cleaned, space-separated list of machine flags GCC uses + * for that architecture, as reported by `gcc -march= -Q --help=target`. + * + * @param archstr Architecture suffix used in `arch=`. + * @param flagsstr Machine-dependent flags reported for the architecture. + */ +#define monoprop_VARIANT_FLAGS(archstr, flagsstr) \ + [[using gnu: target("arch=" archstr)]] consteval auto variant_flags() noexcept -> std::string_view { \ + return flagsstr; \ + } + +namespace monoprop { +[[using gnu: target("default")]] consteval auto variant() noexcept -> std::string_view { + return "default"; +} + +[[using gnu: target("default")]] consteval auto variant_flags() noexcept -> std::string_view { + return "@monoprop_DEFAULT_VARIANT_FLAGS@"; +} + +// clang-format off +@monoprop_VARIANTS@ + +@monoprop_VARIANT_FLAGS@ +// clang-format on +} // namespace monoprop diff --git a/src/monoprop/__init__.py b/src/monoprop/__init__.py index 33640365..323a4478 100644 --- a/src/monoprop/__init__.py +++ b/src/monoprop/__init__.py @@ -22,6 +22,7 @@ MAX_NUM_MODES, __build_type__, __compiler_flags__, + __variant__, antihermitian_generator_correction, has_mpi, is_antihermitian, @@ -57,6 +58,7 @@ "PauliPropagator", "__build_type__", "__compiler_flags__", + "__variant__", "__version__", "antihermitian_generator_correction", "expand_monomials", diff --git a/src/monoprop/bindings/bindings.cpp.in b/src/monoprop/bindings/bindings.cpp.in index 44770819..0fdc30ac 100644 --- a/src/monoprop/bindings/bindings.cpp.in +++ b/src/monoprop/bindings/bindings.cpp.in @@ -112,6 +112,8 @@ NB_MODULE(_core, m) { // clang-format on m.attr("__build_type__") = std::string(build_type()); m.attr("__compiler_flags__") = compiler_flags(); + m.attr("__variant__") = std::string(variant()); + #ifdef monoprop_ENABLE_MPI m.attr("has_mpi") = true; #else diff --git a/tests/test_gcc_target_help_clean.py b/tests/test_gcc_target_help_clean.py new file mode 100644 index 00000000..20a34020 --- /dev/null +++ b/tests/test_gcc_target_help_clean.py @@ -0,0 +1,98 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the ``gcc -Q --help=target`` output cleaner.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +_MODULE_PATH = Path(__file__).parents[1] / "tools" / "gcc-target-help-clean.py" +_spec = importlib.util.spec_from_file_location("gcc_target_help_clean", _MODULE_PATH) +assert _spec is not None +assert _spec.loader is not None +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) +clean_target_help = _module.clean_target_help + + +def _wrap(body: str) -> str: + return ( + "The following options are target specific:\n" + f"{body}\n" + "\n" + " Known assembler dialects (for use with the -masm= option):\n" + " att intel\n" + ) + + +def test_enabled_keeps_only_name() -> None: + text = _wrap(" -m64 \t\t[enabled]") + assert clean_target_help(text) == "-m64" + + +def test_disabled_is_dropped() -> None: + text = _wrap(" -m16 \t\t[disabled]") + assert clean_target_help(text) == "" + + +def test_equals_joins_with_value() -> None: + text = _wrap(" -mabi= \t\tsysv") + assert clean_target_help(text) == "-mabi=sysv" + + +def test_equals_empty_value_is_dropped() -> None: + text = _wrap(" -mcpu= \t\t") + assert clean_target_help(text) == "" + + +def test_equals_default_value_is_dropped() -> None: + text = _wrap(" -mcmodel= \t\t[default]") + assert clean_target_help(text) == "" + + +def test_alias_line_keeps_both_fields() -> None: + text = _wrap(" -msse5 \t\t-mavx") + assert clean_target_help(text) == "-msse5 -mavx" + + +def test_range_hint_is_stripped() -> None: + text = _wrap(" -mbranch-cost=<0,5> \t\t3") + assert clean_target_help(text) == "-mbranch-cost=3" + + +def test_only_section_between_markers_is_used() -> None: + text = ( + "-mignored-before \t\t[enabled]\n" + "The following options are target specific:\n" + " -m64 \t\t[enabled]\n" + " Known assembler dialects (for use with the -masm= option):\n" + " -mignored-after \t\t[enabled]\n" + ) + assert clean_target_help(text) == "-m64" + + +def test_missing_start_marker_returns_empty() -> None: + assert clean_target_help("nothing relevant here") == "" + + +def test_multiple_entries_joined_by_space() -> None: + text = _wrap( + " -m64 \t\t[enabled]\n" + " -m16 \t\t[disabled]\n" + " -mabi= \t\tsysv\n" + " -msse5 \t\t-mavx" + ) + assert clean_target_help(text) == "-m64 -mabi=sysv -msse5 -mavx" diff --git a/tools/gcc-target-help-clean.py b/tools/gcc-target-help-clean.py new file mode 100644 index 00000000..71d2d843 --- /dev/null +++ b/tools/gcc-target-help-clean.py @@ -0,0 +1,80 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa: INP001 + +"""Clean up the output of ``gcc -Q --help=target``. + +Reads the command's output from stdin, extracts the target-specific options +section, normalizes each line, and prints a single space-separated string. +""" + +from __future__ import annotations + +import re +import sys + +_START_MARKER = "The following options are target specific:" +_END_MARKER = "Known assembler dialects (for use with the -masm= option):" +_MULTISPACE = re.compile(r"\s{2,}") +_HINT = re.compile(r"<[^>]*>") + + +def clean_target_help(text: str) -> str: + """Normalize ``gcc -Q --help=target`` output into a single string. + + Args: + text: The full stdout of ``gcc -Q --help=target``. + + Returns: + A single space-separated string of the cleaned options. Rules: + lines containing ``[disabled]`` are dropped; lines whose value is + ``[enabled]`` keep only the option name; options ending in ``=`` are + joined to their value unless the value is empty or ``[default]`` (in + which case the line is dropped); any remaining line keeps both fields + joined by a single space. + """ + start = text.find(_START_MARKER) + if start == -1: + return "" + end = text.find(_END_MARKER, start) + section = text[start + len(_START_MARKER) : end if end != -1 else None] + + entries: list[str] = [] + for raw in section.splitlines(): + line = raw.strip() + if not line or "[disabled]" in line: + continue + fields = _MULTISPACE.split(line, maxsplit=1) + name = _HINT.sub("", fields[0]) + value = fields[1].strip() if len(fields) > 1 else "" + + if "[enabled]" in value: + entries.append(name) + elif "=" in name: + if value and value != "[default]": + entries.append(name + value) + else: + entries.append(f"{name} {value}".strip()) + + return " ".join(entries) + + +def main() -> None: + """Read stdin and print the cleaned target options.""" + sys.stdout.write(clean_target_help(sys.stdin.read()) + "\n") + + +if __name__ == "__main__": + main() From 6eb8f4af6a462a1f65da59bb0d9933174ebc6fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 10:20:55 +0200 Subject: [PATCH 11/80] fix: name of generated header file --- cpp/include/monoprop/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/monoprop/CMakeLists.txt b/cpp/include/monoprop/CMakeLists.txt index 4e9d7afa..f658be51 100644 --- a/cpp/include/monoprop/CMakeLists.txt +++ b/cpp/include/monoprop/CMakeLists.txt @@ -14,7 +14,7 @@ target_sources( FILES "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/${PROJECT_NAME}Export.h" "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Info.h" - "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/VariantMacros.h" + "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Variants.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/Evolution.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MPFunctions.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MPGraph.h" From 29e1bc8e235820a87d55f9332e5d2430c5e1340b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 10:26:33 +0000 Subject: [PATCH 12/80] fix: extend variant features extraction to AppleClang Assisted-by: GitHub Copilot, gpt-5.3-codex --- cmake/compiler_flags/CXXFlags.cmake | 73 ++++++++++++++++------ tests/test_clang_target_help_clean.py | 85 ++++++++++++++++++++++++++ tools/clang-target-help-clean.py | 87 +++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 17 deletions(-) create mode 100644 tests/test_clang_target_help_clean.py create mode 100644 tools/clang-target-help-clean.py diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index 65174cf0..2c5a3750 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -101,24 +101,63 @@ function(_monoprop_query_machine_flags) set(_march_args "-march=${_arg_MARCH}") endif() - # Report the machine-dependent flags GCC uses for each target variant by - # querying `gcc -march= -Q --help=target` and cleaning the output with - # tools/gcc-target-help-clean.py. - execute_process( - COMMAND - ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target - COMMAND - ${Python_EXECUTABLE} - "${PROJECT_SOURCE_DIR}/tools/gcc-target-help-clean.py" - OUTPUT_VARIABLE _flags - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE _result - ) - if(NOT _result EQUAL 0) - message( - FATAL_ERROR - "Failed to query machine-dependent flags for '${_arg_MARCH}' (exit code ${_result})" + if(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang) + # AppleClang does not support `-Q --help=target`. Query the driver with + # `-###` and normalize CPU/march flags from the reported invocation. + execute_process( + COMMAND + # gersemi: off + ${CMAKE_CXX_COMPILER} ${_march_args} -### -x c++ -c /dev/null + # gersemi: on + ERROR_VARIABLE _query_output + ERROR_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _query_result + ) + if(NOT _query_result EQUAL 0) + message( + WARNING + "Failed to query machine-dependent flags for '${_arg_MARCH}' with AppleClang (exit code ${_query_result}). Continuing with empty machine flags." + ) + set(_flags "") + else() + execute_process( + COMMAND + ${CMAKE_COMMAND} -E echo "${_query_output}" + COMMAND + ${Python_EXECUTABLE} + "${PROJECT_SOURCE_DIR}/tools/clang-target-help-clean.py" + OUTPUT_VARIABLE _flags + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _parse_result + ) + if(NOT _parse_result EQUAL 0) + message( + WARNING + "Failed to parse AppleClang machine-dependent flags for '${_arg_MARCH}' (exit code ${_parse_result}). Continuing with empty machine flags." + ) + set(_flags "") + endif() + endif() + else() + # Report the machine-dependent flags GCC uses for each target variant by + # querying `gcc -march= -Q --help=target` and cleaning the output + # with tools/gcc-target-help-clean.py. + execute_process( + COMMAND + ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target + COMMAND + ${Python_EXECUTABLE} + "${PROJECT_SOURCE_DIR}/tools/gcc-target-help-clean.py" + OUTPUT_VARIABLE _flags + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _result ) + if(NOT _result EQUAL 0) + message( + FATAL_ERROR + "Failed to query machine-dependent flags for '${_arg_MARCH}' (exit code ${_result})" + ) + endif() endif() set(${_arg_OUTPUT_VARIABLE} "${_flags}" PARENT_SCOPE) endfunction() diff --git a/tests/test_clang_target_help_clean.py b/tests/test_clang_target_help_clean.py new file mode 100644 index 00000000..6425ddeb --- /dev/null +++ b/tests/test_clang_target_help_clean.py @@ -0,0 +1,85 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the ``clang -###`` output cleaner.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +_MODULE_PATH = Path(__file__).parents[1] / "tools" / "clang-target-help-clean.py" +_spec = importlib.util.spec_from_file_location("clang_target_help_clean", _MODULE_PATH) +assert _spec is not None +assert _spec.loader is not None +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) +clean_target_help = _module.clean_target_help + + +def test_target_cpu_and_feature_are_emitted() -> None: + text = ( + "Apple clang version 16.0.0\n" + "Target: arm64-apple-darwin\n" + ' "/usr/bin/clang++" "-cc1" "-triple" "arm64-apple-macosx14.0.0" ' + '"-target-cpu" "apple-m4" "-target-feature" "+neon"\n' + ) + assert clean_target_help(text) == "-target-cpu=apple-m4 -target-feature=+neon" + + +def test_only_selected_m_flags_are_kept() -> None: + text = ( + ' "/usr/bin/clang++" "-cc1" "-mframe-pointer=non-leaf" ' + '"-march=armv8.6-a" "-mtune=apple-m4" "-mllvm" "-something"\n' + ) + assert clean_target_help(text) == "-march=armv8.6-a -mtune=apple-m4" + + +def test_spaced_flag_forms_are_normalized() -> None: + text = "-mcpu apple-m3 -mtune generic -march native -target-feature +crc" + assert ( + clean_target_help(text) + == "-mcpu=apple-m3 -mtune=generic -march=native -target-feature=+crc" + ) + + +def test_duplicates_are_removed_preserving_order() -> None: + text = ( + "-march=native -target-cpu apple-m3 -target-feature +neon " + "-target-feature +neon -march=native" + ) + assert ( + clean_target_help(text) + == "-march=native -target-cpu=apple-m3 -target-feature=+neon" + ) + + +def test_realistic_appleclang_output_shape() -> None: + text = ( + "Apple clang version 21.0.0 (clang-2100.0.123.102)\n" + "Target: arm64-apple-darwin25.3.0\n" + '"/Library/Developer/CommandLineTools/usr/bin/clang" "-cc1" ' + '"-target-cpu" "apple-m1" ' + '"-target-feature" "+v8.5a" ' + '"-target-feature" "+dotprod" ' + '"-target-feature" "+neon"\n' + ) + assert ( + clean_target_help(text) == "-target-cpu=apple-m1 -target-feature=+v8.5a " + "-target-feature=+dotprod -target-feature=+neon" + ) + + +def test_empty_input_returns_empty() -> None: + assert clean_target_help("") == "" diff --git a/tools/clang-target-help-clean.py b/tools/clang-target-help-clean.py new file mode 100644 index 00000000..46594418 --- /dev/null +++ b/tools/clang-target-help-clean.py @@ -0,0 +1,87 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa: INP001 + +"""Normalize machine flags from ``clang -###`` output. + +Reads the command output from stdin and extracts a stable, space-separated list +of machine-relevant flags. For AppleClang we keep only ``-march=``, ``-mcpu=``, +``-mtune=``, ``-target-cpu=``, and ``-target-feature=`` forms. +""" + +from __future__ import annotations + +import shlex +import sys + +_ALLOWED_PREFIXES = ( + "-march=", + "-mcpu=", + "-mtune=", + "-target-cpu=", + "-target-feature=", +) + + +def _append_unique(entries: list[str], seen: set[str], value: str) -> None: + """Append a flag only once, preserving first-seen order.""" + if value not in seen: + seen.add(value) + entries.append(value) + + +def _normalize_pair_flag(flag: str, value: str) -> str: + """Normalize pair-style machine flags into ``-key=value`` form.""" + return f"{flag}={value}" + + +def clean_target_help(text: str) -> str: + """Normalize ``clang -###`` output into a single machine-flag string. + + Args: + text: The full output generated by ``clang -###``. + + Returns: + A space-separated string containing unique normalized machine flags in + first-seen order. + """ + tokens = shlex.split(text.replace("\n", " ")) + entries: list[str] = [] + seen: set[str] = set() + pair_flags = {"-march", "-mcpu", "-mtune", "-target-cpu", "-target-feature"} + + idx = 0 + while idx < len(tokens): + tok = tokens[idx] + + if tok.startswith(_ALLOWED_PREFIXES): + _append_unique(entries, seen, tok) + elif tok in pair_flags and idx + 1 < len(tokens): + normalized = _normalize_pair_flag(tok, tokens[idx + 1]) + _append_unique(entries, seen, normalized) + idx += 1 + + idx += 1 + + return " ".join(entries) + + +def main() -> None: + """Read stdin and print normalized machine flags.""" + sys.stdout.write(clean_target_help(sys.stdin.read()) + "\n") + + +if __name__ == "__main__": + main() From d566d643e006e030651d74c8875e27dd2651d196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 10:38:43 +0000 Subject: [PATCH 13/80] fix: remove the attribute --- cpp/include/monoprop/Variants.h.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/monoprop/Variants.h.in b/cpp/include/monoprop/Variants.h.in index 54e13fb0..7cfe51a9 100644 --- a/cpp/include/monoprop/Variants.h.in +++ b/cpp/include/monoprop/Variants.h.in @@ -47,11 +47,11 @@ } namespace monoprop { -[[using gnu: target("default")]] consteval auto variant() noexcept -> std::string_view { +consteval auto variant() noexcept -> std::string_view { return "default"; } -[[using gnu: target("default")]] consteval auto variant_flags() noexcept -> std::string_view { +consteval auto variant_flags() noexcept -> std::string_view { return "@monoprop_DEFAULT_VARIANT_FLAGS@"; } From 94257a2899d08f3a66e8f7c1158701056a390a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 11:28:34 +0000 Subject: [PATCH 14/80] ci: print messages --- cmake/compiler_flags/CXXFlags.cmake | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index 2c5a3750..2f39bb70 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -102,8 +102,6 @@ function(_monoprop_query_machine_flags) endif() if(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang) - # AppleClang does not support `-Q --help=target`. Query the driver with - # `-###` and normalize CPU/march flags from the reported invocation. execute_process( COMMAND # gersemi: off @@ -113,6 +111,10 @@ function(_monoprop_query_machine_flags) ERROR_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _query_result ) + message( + STATUS + "_query_output : ${_query_output}\n_query_result : ${_query_result}" + ) if(NOT _query_result EQUAL 0) message( WARNING @@ -139,9 +141,14 @@ function(_monoprop_query_machine_flags) endif() endif() else() - # Report the machine-dependent flags GCC uses for each target variant by - # querying `gcc -march= -Q --help=target` and cleaning the output - # with tools/gcc-target-help-clean.py. + execute_process( + COMMAND + ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target + OUTPUT_VARIABLE _foo + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _result + ) + message(STATUS "_foo : ${_foo}\n_result : ${_result}") execute_process( COMMAND ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target From f08159562d9f73f5be66f044f2edc0b518be9638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 12:09:40 +0000 Subject: [PATCH 15/80] chore: figure out clang behavior --- cmake/compiler_flags/CXXFlags.cmake | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index 2f39bb70..1ae51015 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -105,11 +105,12 @@ function(_monoprop_query_machine_flags) execute_process( COMMAND # gersemi: off - ${CMAKE_CXX_COMPILER} ${_march_args} -### -x c++ -c /dev/null + ${CMAKE_CXX_COMPILER} ${_march_args} -\#\#\# -x c++ -c /dev/null # gersemi: on ERROR_VARIABLE _query_output ERROR_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _query_result + COMMAND_ECHO STDERR ) message( STATUS @@ -127,10 +128,11 @@ function(_monoprop_query_machine_flags) ${CMAKE_COMMAND} -E echo "${_query_output}" COMMAND ${Python_EXECUTABLE} - "${PROJECT_SOURCE_DIR}/tools/clang-target-help-clean.py" + "${PROJECT_SOURCE_DIR}/tools/target-help-clean.py" --mode clang OUTPUT_VARIABLE _flags OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _parse_result + COMMAND_ECHO STDERR ) if(NOT _parse_result EQUAL 0) message( @@ -147,17 +149,19 @@ function(_monoprop_query_machine_flags) OUTPUT_VARIABLE _foo OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _result + COMMAND_ECHO STDERR ) message(STATUS "_foo : ${_foo}\n_result : ${_result}") execute_process( COMMAND ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target COMMAND - ${Python_EXECUTABLE} - "${PROJECT_SOURCE_DIR}/tools/gcc-target-help-clean.py" + ${Python_EXECUTABLE} "${PROJECT_SOURCE_DIR}/tools/target-help-clean.py" + --mode gcc OUTPUT_VARIABLE _flags OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _result + COMMAND_ECHO STDERR ) if(NOT _result EQUAL 0) message( From 64f17bfe1552479ff1a1db4db1ce90511b61be04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 12:14:03 +0000 Subject: [PATCH 16/80] refactor: cleaner scripts for flags --- tests/test_clang_target_help_clean.py | 85 --------- tests/test_gcc_target_help_clean.py | 98 ----------- tests/test_target_help_clean.py | 241 ++++++++++++++++++++++++++ tools/clang-target-help-clean.py | 87 ---------- tools/gcc-target-help-clean.py | 80 --------- tools/target-help-clean.py | 142 +++++++++++++++ 6 files changed, 383 insertions(+), 350 deletions(-) delete mode 100644 tests/test_clang_target_help_clean.py delete mode 100644 tests/test_gcc_target_help_clean.py create mode 100644 tests/test_target_help_clean.py delete mode 100644 tools/clang-target-help-clean.py delete mode 100644 tools/gcc-target-help-clean.py create mode 100644 tools/target-help-clean.py diff --git a/tests/test_clang_target_help_clean.py b/tests/test_clang_target_help_clean.py deleted file mode 100644 index 6425ddeb..00000000 --- a/tests/test_clang_target_help_clean.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2026 Algorithmiq -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for the ``clang -###`` output cleaner.""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path - -_MODULE_PATH = Path(__file__).parents[1] / "tools" / "clang-target-help-clean.py" -_spec = importlib.util.spec_from_file_location("clang_target_help_clean", _MODULE_PATH) -assert _spec is not None -assert _spec.loader is not None -_module = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(_module) -clean_target_help = _module.clean_target_help - - -def test_target_cpu_and_feature_are_emitted() -> None: - text = ( - "Apple clang version 16.0.0\n" - "Target: arm64-apple-darwin\n" - ' "/usr/bin/clang++" "-cc1" "-triple" "arm64-apple-macosx14.0.0" ' - '"-target-cpu" "apple-m4" "-target-feature" "+neon"\n' - ) - assert clean_target_help(text) == "-target-cpu=apple-m4 -target-feature=+neon" - - -def test_only_selected_m_flags_are_kept() -> None: - text = ( - ' "/usr/bin/clang++" "-cc1" "-mframe-pointer=non-leaf" ' - '"-march=armv8.6-a" "-mtune=apple-m4" "-mllvm" "-something"\n' - ) - assert clean_target_help(text) == "-march=armv8.6-a -mtune=apple-m4" - - -def test_spaced_flag_forms_are_normalized() -> None: - text = "-mcpu apple-m3 -mtune generic -march native -target-feature +crc" - assert ( - clean_target_help(text) - == "-mcpu=apple-m3 -mtune=generic -march=native -target-feature=+crc" - ) - - -def test_duplicates_are_removed_preserving_order() -> None: - text = ( - "-march=native -target-cpu apple-m3 -target-feature +neon " - "-target-feature +neon -march=native" - ) - assert ( - clean_target_help(text) - == "-march=native -target-cpu=apple-m3 -target-feature=+neon" - ) - - -def test_realistic_appleclang_output_shape() -> None: - text = ( - "Apple clang version 21.0.0 (clang-2100.0.123.102)\n" - "Target: arm64-apple-darwin25.3.0\n" - '"/Library/Developer/CommandLineTools/usr/bin/clang" "-cc1" ' - '"-target-cpu" "apple-m1" ' - '"-target-feature" "+v8.5a" ' - '"-target-feature" "+dotprod" ' - '"-target-feature" "+neon"\n' - ) - assert ( - clean_target_help(text) == "-target-cpu=apple-m1 -target-feature=+v8.5a " - "-target-feature=+dotprod -target-feature=+neon" - ) - - -def test_empty_input_returns_empty() -> None: - assert clean_target_help("") == "" diff --git a/tests/test_gcc_target_help_clean.py b/tests/test_gcc_target_help_clean.py deleted file mode 100644 index 20a34020..00000000 --- a/tests/test_gcc_target_help_clean.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2026 Algorithmiq -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for the ``gcc -Q --help=target`` output cleaner.""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path - -_MODULE_PATH = Path(__file__).parents[1] / "tools" / "gcc-target-help-clean.py" -_spec = importlib.util.spec_from_file_location("gcc_target_help_clean", _MODULE_PATH) -assert _spec is not None -assert _spec.loader is not None -_module = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(_module) -clean_target_help = _module.clean_target_help - - -def _wrap(body: str) -> str: - return ( - "The following options are target specific:\n" - f"{body}\n" - "\n" - " Known assembler dialects (for use with the -masm= option):\n" - " att intel\n" - ) - - -def test_enabled_keeps_only_name() -> None: - text = _wrap(" -m64 \t\t[enabled]") - assert clean_target_help(text) == "-m64" - - -def test_disabled_is_dropped() -> None: - text = _wrap(" -m16 \t\t[disabled]") - assert clean_target_help(text) == "" - - -def test_equals_joins_with_value() -> None: - text = _wrap(" -mabi= \t\tsysv") - assert clean_target_help(text) == "-mabi=sysv" - - -def test_equals_empty_value_is_dropped() -> None: - text = _wrap(" -mcpu= \t\t") - assert clean_target_help(text) == "" - - -def test_equals_default_value_is_dropped() -> None: - text = _wrap(" -mcmodel= \t\t[default]") - assert clean_target_help(text) == "" - - -def test_alias_line_keeps_both_fields() -> None: - text = _wrap(" -msse5 \t\t-mavx") - assert clean_target_help(text) == "-msse5 -mavx" - - -def test_range_hint_is_stripped() -> None: - text = _wrap(" -mbranch-cost=<0,5> \t\t3") - assert clean_target_help(text) == "-mbranch-cost=3" - - -def test_only_section_between_markers_is_used() -> None: - text = ( - "-mignored-before \t\t[enabled]\n" - "The following options are target specific:\n" - " -m64 \t\t[enabled]\n" - " Known assembler dialects (for use with the -masm= option):\n" - " -mignored-after \t\t[enabled]\n" - ) - assert clean_target_help(text) == "-m64" - - -def test_missing_start_marker_returns_empty() -> None: - assert clean_target_help("nothing relevant here") == "" - - -def test_multiple_entries_joined_by_space() -> None: - text = _wrap( - " -m64 \t\t[enabled]\n" - " -m16 \t\t[disabled]\n" - " -mabi= \t\tsysv\n" - " -msse5 \t\t-mavx" - ) - assert clean_target_help(text) == "-m64 -mabi=sysv -msse5 -mavx" diff --git a/tests/test_target_help_clean.py b/tests/test_target_help_clean.py new file mode 100644 index 00000000..553a293e --- /dev/null +++ b/tests/test_target_help_clean.py @@ -0,0 +1,241 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the unified compiler target-help cleaner.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +_MODULE_PATH = Path(__file__).parents[1] / "tools" / "target-help-clean.py" +_spec = importlib.util.spec_from_file_location("target_help_clean", _MODULE_PATH) +assert _spec is not None +assert _spec.loader is not None +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) + +clean_clang_target_help = _module.clean_clang_target_help +clean_gcc_target_help = _module.clean_gcc_target_help +clean_target_help = _module.clean_target_help + + +def _wrap_gcc( + body: str, + trailer: str = "Known assembler dialects (for use with the -masm= option):", +) -> str: + return ( + "The following options are target specific:\n" + f"{body}\n" + "\n" + f" {trailer}\n" + " att intel\n" + ) + + +def test_dispatches_gcc_mode() -> None: + text = _wrap_gcc(" -m64 \t\t[enabled]") + assert clean_target_help(text, mode="gcc") == "-m64" + + +def test_dispatches_clang_mode() -> None: + text = ( + "Apple clang version 16.0.0\n" + "Target: arm64-apple-darwin\n" + ' "/usr/bin/clang++" "-cc1" "-target-cpu" "apple-m4" ' + '"-target-feature" "+neon"\n' + ) + assert ( + clean_target_help(text, mode="clang") + == "-target-cpu=apple-m4 -target-feature=+neon" + ) + + +def test_gcc_enabled_keeps_only_name() -> None: + text = _wrap_gcc(" -m64 \t\t[enabled]") + assert clean_gcc_target_help(text) == "-m64" + + +def test_gcc_disabled_is_dropped() -> None: + text = _wrap_gcc(" -m16 \t\t[disabled]") + assert clean_gcc_target_help(text) == "" + + +def test_gcc_equals_joins_with_value() -> None: + text = _wrap_gcc(" -mabi= \t\tsysv") + assert clean_gcc_target_help(text) == "-mabi=sysv" + + +def test_gcc_equals_empty_value_is_dropped() -> None: + text = _wrap_gcc(" -mcpu= \t\t") + assert clean_gcc_target_help(text) == "" + + +def test_gcc_equals_default_value_is_dropped() -> None: + text = _wrap_gcc(" -mcmodel= \t\t[default]") + assert clean_gcc_target_help(text) == "" + + +def test_gcc_alias_line_keeps_both_fields() -> None: + text = _wrap_gcc(" -msse5 \t\t-mavx") + assert clean_gcc_target_help(text) == "-msse5 -mavx" + + +def test_gcc_range_hint_is_stripped() -> None: + text = _wrap_gcc(" -mbranch-cost=<0,5> \t\t3") + assert clean_gcc_target_help(text) == "-mbranch-cost=3" + + +def test_gcc_only_target_section_is_used() -> None: + text = ( + "-mignored-before \t\t[enabled]\n" + "The following options are target specific:\n" + " -m64 \t\t[enabled]\n" + "\n" + " Known assembler dialects (for use with the -masm= option):\n" + " -mignored-after \t\t[enabled]\n" + ) + assert clean_gcc_target_help(text) == "-m64" + + +def test_gcc_blank_line_terminates_section_for_aarch64_trailer() -> None: + text = _wrap_gcc( + " -mabi= \t\tilp32\n" + " -mstrict-align \t\t[enabled]", + trailer="Known AArch64 ABIs (for use with the -mabi= option):", + ) + assert clean_gcc_target_help(text) == "-mabi=ilp32 -mstrict-align" + + +def test_gcc_trailer_content_after_blank_line_is_ignored() -> None: + text = ( + "The following options are target specific:\n" + " -m64 \t\t[enabled]\n" + "\n" + " Known AArch64 ABIs (for use with the -mabi= option):\n" + " -mabi= \t\tilp32\n" + ) + assert clean_gcc_target_help(text) == "-m64" + + +def test_gcc_missing_start_marker_returns_empty() -> None: + assert clean_gcc_target_help("nothing relevant here") == "" + + +def test_gcc_multiple_entries_joined_by_space() -> None: + text = _wrap_gcc( + " -m64 \t\t[enabled]\n" + " -m16 \t\t[disabled]\n" + " -mabi= \t\tsysv\n" + " -msse5 \t\t-mavx" + ) + assert clean_gcc_target_help(text) == "-m64 -mabi=sysv -msse5 -mavx" + + +def test_gcc_x86_64_reference_output_shapes() -> None: + text = _wrap_gcc( + " -m128bit-long-double \t\t[enabled]\n" + " -m16 \t\t[disabled]\n" + " -mabi= \t\tsysv\n" + " -mavx10.1-512 \t\t-mavx10.1\n" + " -mbranch-cost=<0,5> \t\t3\n" + " -mcmodel= \t\t[default]\n" + " -mcpu=\n" + " -mlarge-data-threshold= \t65536" + ) + assert clean_gcc_target_help(text) == ( + "-m128bit-long-double -mabi=sysv -mavx10.1-512 -mavx10.1 " + "-mbranch-cost=3 -mlarge-data-threshold=65536" + ) + + +def test_gcc_aarch64_reference_output_shapes() -> None: + text = _wrap_gcc( + " -mabi= lp64\n" + " -mbranch-protection=\n" + " -mearly-ldp-fusion [enabled]\n" + " -moverride=\n" + " -mstrict-align [disabled]\n" + " -msve-vector-bits= scalable", + trailer="Known AArch64 ABIs (for use with the -mabi= option):", + ) + assert clean_gcc_target_help(text) == ( + "-mabi=lp64 -mearly-ldp-fusion -msve-vector-bits=scalable" + ) + + +def test_clang_target_cpu_and_feature_are_emitted() -> None: + text = ( + "Apple clang version 16.0.0\n" + "Target: arm64-apple-darwin\n" + ' "/usr/bin/clang++" "-cc1" "-triple" "arm64-apple-macosx14.0.0" ' + '"-target-cpu" "apple-m4" "-target-feature" "+neon"\n' + ) + assert clean_clang_target_help(text) == "-target-cpu=apple-m4 -target-feature=+neon" + + +def test_clang_only_selected_m_flags_are_kept() -> None: + text = ( + ' "/usr/bin/clang++" "-cc1" "-mframe-pointer=non-leaf" ' + '"-march=armv8.6-a" "-mtune=apple-m4" "-mllvm" "-something"\n' + ) + assert clean_clang_target_help(text) == "-march=armv8.6-a -mtune=apple-m4" + + +def test_clang_spaced_flag_forms_are_normalized() -> None: + text = "-mcpu apple-m3 -mtune generic -march native -target-feature +crc" + assert ( + clean_clang_target_help(text) + == "-mcpu=apple-m3 -mtune=generic -march=native -target-feature=+crc" + ) + + +def test_clang_duplicates_are_removed_preserving_order() -> None: + text = ( + "-march=native -target-cpu apple-m3 -target-feature +neon " + "-target-feature +neon -march=native" + ) + assert ( + clean_clang_target_help(text) + == "-march=native -target-cpu=apple-m3 -target-feature=+neon" + ) + + +def test_clang_realistic_appleclang_output_shape() -> None: + text = ( + "Apple clang version 21.0.0 (clang-2100.0.123.102)\n" + "Target: arm64-apple-darwin25.3.0\n" + "Thread model: posix\n" + '\N{NO-BREAK SPACE}"/Library/Developer/CommandLineTools/usr/bin/clang" ' + '"-cc1" "-mframe-pointer=non-leaf" ' + '"-target-cpu" "apple-m1" ' + '"-target-feature" "+v8.5a" ' + '"-target-feature" "+dotprod" ' + '"-target-feature" "+neon"\n' + '"-target-feature" "+sb" "-target-abi" "darwinpcs"\n' + ) + assert ( + clean_clang_target_help(text) == "-target-cpu=apple-m1 -target-feature=+v8.5a " + "-target-feature=+dotprod -target-feature=+neon -target-feature=+sb" + ) + + +def test_clang_trailing_pair_flag_without_value_is_ignored() -> None: + assert clean_clang_target_help("-target-cpu apple-m1 -target-feature") == ( + "-target-cpu=apple-m1" + ) + + +def test_clang_empty_input_returns_empty() -> None: + assert clean_clang_target_help("") == "" diff --git a/tools/clang-target-help-clean.py b/tools/clang-target-help-clean.py deleted file mode 100644 index 46594418..00000000 --- a/tools/clang-target-help-clean.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2026 Algorithmiq -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ruff: noqa: INP001 - -"""Normalize machine flags from ``clang -###`` output. - -Reads the command output from stdin and extracts a stable, space-separated list -of machine-relevant flags. For AppleClang we keep only ``-march=``, ``-mcpu=``, -``-mtune=``, ``-target-cpu=``, and ``-target-feature=`` forms. -""" - -from __future__ import annotations - -import shlex -import sys - -_ALLOWED_PREFIXES = ( - "-march=", - "-mcpu=", - "-mtune=", - "-target-cpu=", - "-target-feature=", -) - - -def _append_unique(entries: list[str], seen: set[str], value: str) -> None: - """Append a flag only once, preserving first-seen order.""" - if value not in seen: - seen.add(value) - entries.append(value) - - -def _normalize_pair_flag(flag: str, value: str) -> str: - """Normalize pair-style machine flags into ``-key=value`` form.""" - return f"{flag}={value}" - - -def clean_target_help(text: str) -> str: - """Normalize ``clang -###`` output into a single machine-flag string. - - Args: - text: The full output generated by ``clang -###``. - - Returns: - A space-separated string containing unique normalized machine flags in - first-seen order. - """ - tokens = shlex.split(text.replace("\n", " ")) - entries: list[str] = [] - seen: set[str] = set() - pair_flags = {"-march", "-mcpu", "-mtune", "-target-cpu", "-target-feature"} - - idx = 0 - while idx < len(tokens): - tok = tokens[idx] - - if tok.startswith(_ALLOWED_PREFIXES): - _append_unique(entries, seen, tok) - elif tok in pair_flags and idx + 1 < len(tokens): - normalized = _normalize_pair_flag(tok, tokens[idx + 1]) - _append_unique(entries, seen, normalized) - idx += 1 - - idx += 1 - - return " ".join(entries) - - -def main() -> None: - """Read stdin and print normalized machine flags.""" - sys.stdout.write(clean_target_help(sys.stdin.read()) + "\n") - - -if __name__ == "__main__": - main() diff --git a/tools/gcc-target-help-clean.py b/tools/gcc-target-help-clean.py deleted file mode 100644 index 71d2d843..00000000 --- a/tools/gcc-target-help-clean.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2026 Algorithmiq -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ruff: noqa: INP001 - -"""Clean up the output of ``gcc -Q --help=target``. - -Reads the command's output from stdin, extracts the target-specific options -section, normalizes each line, and prints a single space-separated string. -""" - -from __future__ import annotations - -import re -import sys - -_START_MARKER = "The following options are target specific:" -_END_MARKER = "Known assembler dialects (for use with the -masm= option):" -_MULTISPACE = re.compile(r"\s{2,}") -_HINT = re.compile(r"<[^>]*>") - - -def clean_target_help(text: str) -> str: - """Normalize ``gcc -Q --help=target`` output into a single string. - - Args: - text: The full stdout of ``gcc -Q --help=target``. - - Returns: - A single space-separated string of the cleaned options. Rules: - lines containing ``[disabled]`` are dropped; lines whose value is - ``[enabled]`` keep only the option name; options ending in ``=`` are - joined to their value unless the value is empty or ``[default]`` (in - which case the line is dropped); any remaining line keeps both fields - joined by a single space. - """ - start = text.find(_START_MARKER) - if start == -1: - return "" - end = text.find(_END_MARKER, start) - section = text[start + len(_START_MARKER) : end if end != -1 else None] - - entries: list[str] = [] - for raw in section.splitlines(): - line = raw.strip() - if not line or "[disabled]" in line: - continue - fields = _MULTISPACE.split(line, maxsplit=1) - name = _HINT.sub("", fields[0]) - value = fields[1].strip() if len(fields) > 1 else "" - - if "[enabled]" in value: - entries.append(name) - elif "=" in name: - if value and value != "[default]": - entries.append(name + value) - else: - entries.append(f"{name} {value}".strip()) - - return " ".join(entries) - - -def main() -> None: - """Read stdin and print the cleaned target options.""" - sys.stdout.write(clean_target_help(sys.stdin.read()) + "\n") - - -if __name__ == "__main__": - main() diff --git a/tools/target-help-clean.py b/tools/target-help-clean.py new file mode 100644 index 00000000..577b9bed --- /dev/null +++ b/tools/target-help-clean.py @@ -0,0 +1,142 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa: INP001 + +"""Normalize compiler target-help output into stable machine-flag strings. + +The script reads compiler output from stdin and cleans it according to the +selected mode: + +- ``gcc`` for ``gcc -Q --help=target`` output +- ``clang`` for ``clang -###`` output +""" + +from __future__ import annotations + +import argparse +import re +import shlex +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + +_PAIR_FLAGS = ( + "-march", + "-mcpu", + "-mtune", + "-target-cpu", + "-target-feature", +) +_ALLOWED_PREFIXES = tuple(f"{flag}=" for flag in _PAIR_FLAGS) + +_START_MARKER = "The following options are target specific:" +_MULTISPACE = re.compile(r"\s{2,}") +_HINT = re.compile(r"<[^>]*>") + + +def _gcc_option_lines(text: str) -> Iterator[str]: + """Yield option lines from GCC's target-specific section.""" + _, marker, section = text.partition(_START_MARKER) + if not marker: + return + + options_started = False + for raw_line in section.splitlines(): + line = raw_line.strip() + if line.startswith("-"): + options_started = True + yield line + elif options_started: + return + + +def _normalize_gcc_option(line: str) -> str | None: + """Normalize one GCC target option, omitting inactive values.""" + if "[disabled]" in line: + return None + + fields = _MULTISPACE.split(line, maxsplit=1) + name = _HINT.sub("", fields[0]) + value = fields[1].strip() if len(fields) > 1 else "" + + if "[enabled]" in value: + return name + if "=" not in name: + return f"{name} {value}".strip() + if not value or value == "[default]": + return None + return name + value + + +def _clang_machine_flags(text: str) -> Iterator[str]: + """Yield normalized machine flags from Clang's command-line trace.""" + tokens = iter(shlex.split(text.replace("\n", " "))) + for token in tokens: + if token.startswith(_ALLOWED_PREFIXES): + yield token + elif token in _PAIR_FLAGS and (value := next(tokens, None)) is not None: + yield f"{token}={value}" + + +def clean_clang_target_help(text: str) -> str: + """Normalize ``clang -###`` output into a machine-flag string.""" + return " ".join(dict.fromkeys(_clang_machine_flags(text))) + + +def clean_gcc_target_help(text: str) -> str: + """Normalize ``gcc -Q --help=target`` output into a machine-flag string. + + Parsing starts after the target-specific marker and stops at the first + blank separator line. This keeps section detection architecture-agnostic. + """ + entries = ( + normalized + for line in _gcc_option_lines(text) + if (normalized := _normalize_gcc_option(line)) is not None + ) + return " ".join(entries) + + +def clean_target_help(text: str, mode: str) -> str: + """Dispatch target-help normalization according to compiler mode.""" + if mode == "gcc": + return clean_gcc_target_help(text) + if mode == "clang": + return clean_clang_target_help(text) + + raise ValueError(f"Unsupported mode: {mode}") + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + required=True, + choices=("gcc", "clang"), + help="Parser mode matching the compiler output format.", + ) + return parser + + +def main() -> None: + """Read stdin and print normalized target flags for the selected mode.""" + args = _build_parser().parse_args() + sys.stdout.write(clean_target_help(sys.stdin.read(), mode=args.mode) + "\n") + + +if __name__ == "__main__": + main() From a976b0ea98a22426fc480870d35072f1bb96629e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 12:20:00 +0000 Subject: [PATCH 17/80] chore: remove now useless prints --- cmake/compiler_flags/CXXFlags.cmake | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index 1ae51015..fedf2f4b 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -112,10 +112,6 @@ function(_monoprop_query_machine_flags) RESULT_VARIABLE _query_result COMMAND_ECHO STDERR ) - message( - STATUS - "_query_output : ${_query_output}\n_query_result : ${_query_result}" - ) if(NOT _query_result EQUAL 0) message( WARNING @@ -143,15 +139,6 @@ function(_monoprop_query_machine_flags) endif() endif() else() - execute_process( - COMMAND - ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target - OUTPUT_VARIABLE _foo - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE _result - COMMAND_ECHO STDERR - ) - message(STATUS "_foo : ${_foo}\n_result : ${_result}") execute_process( COMMAND ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target From cfec89d3d91aaf9d33386a849f7d53484da5bd16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 12:22:14 +0000 Subject: [PATCH 18/80] ci: pretty-print version, variant, compiler flags --- .github/workflows/test.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ddb17a80..bd64a22b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,9 +99,21 @@ jobs: run: | uv tree - - name: Get monoprop version + - name: Get monoprop information run: | - uv run python -c "import monoprop as mp; print(mp.__version__); print(mp.__variant__); print(mp.__compiler_flags__)" + uv run python <<'EOF' + import pprint + + import monoprop as mp + + pprint.pprint( + { + "version": mp.__version__, + "variant": mp.__variant__, + "compiler_flags": mp.__compiler_flags__, + } + ) + EOF - name: Verify that find_package(monoprop) works run: | From 1d919262d056770f4db96f30b9e2c65849ff29b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Fri, 7 Aug 2026 12:25:20 +0000 Subject: [PATCH 19/80] chore: remove one more debug print in cmake --- cmake/compiler_flags/CXXFlags.cmake | 3 --- cpp/include/monoprop/CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index fedf2f4b..f56236bd 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -110,7 +110,6 @@ function(_monoprop_query_machine_flags) ERROR_VARIABLE _query_output ERROR_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _query_result - COMMAND_ECHO STDERR ) if(NOT _query_result EQUAL 0) message( @@ -128,7 +127,6 @@ function(_monoprop_query_machine_flags) OUTPUT_VARIABLE _flags OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _parse_result - COMMAND_ECHO STDERR ) if(NOT _parse_result EQUAL 0) message( @@ -148,7 +146,6 @@ function(_monoprop_query_machine_flags) OUTPUT_VARIABLE _flags OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _result - COMMAND_ECHO STDERR ) if(NOT _result EQUAL 0) message( diff --git a/cpp/include/monoprop/CMakeLists.txt b/cpp/include/monoprop/CMakeLists.txt index f658be51..98159824 100644 --- a/cpp/include/monoprop/CMakeLists.txt +++ b/cpp/include/monoprop/CMakeLists.txt @@ -14,7 +14,7 @@ target_sources( FILES "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/${PROJECT_NAME}Export.h" "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Info.h" - "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Variants.h" + "${PROJECT_BINARY_DIR}/include/${PROJECT_NAME}/Variants.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/Evolution.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MPFunctions.h" "${PROJECT_SOURCE_DIR}/cpp/include/${PROJECT_NAME}/MPGraph.h" From 6db4bfb4c4e82ab8d63fef7825b267893c5e5f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Mon, 10 Aug 2026 08:36:26 +0000 Subject: [PATCH 20/80] chore(toml): :art: add .h.in files to cache keys --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 925a95bb..327a48c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ keyring-provider = "subprocess" cache-keys = [ { file = "pyproject.toml" }, { file = "cpp/include/**/*.h" }, + { file = "cpp/include/**/*.h.in" }, { file = "cpp/monoprop/**/*.{h,cpp}" }, { file = "**/CMakeLists.txt" }, { file = "cmake/**/*" }, From 45738c6284638512289171b87c9cebcbff4d8193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Mon, 10 Aug 2026 08:37:05 +0000 Subject: [PATCH 21/80] chore(c++): :lipstick: clean up doxygen docstrings, add ifdef for x86 --- cpp/include/monoprop/Variants.h.in | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/include/monoprop/Variants.h.in b/cpp/include/monoprop/Variants.h.in index 7cfe51a9..45ea84e2 100644 --- a/cpp/include/monoprop/Variants.h.in +++ b/cpp/include/monoprop/Variants.h.in @@ -17,7 +17,7 @@ #include /** - * @brief Declares a compile-time function that reports the active FMV variant. + * @brief Declares a compile-time function that reports the active variant. * * Expands to a `consteval` function named `variant()` with a GNU * `target("arch=...")` attribute bound to the provided architecture string. @@ -31,7 +31,7 @@ /** * @brief Declares a compile-time function that reports the machine-dependent - * flags GCC applies for the given FMV variant. + * flags GCC applies for the given variant. * * Expands to a `consteval` function named `machine_flags()` with a GNU * `target("arch=...")` attribute bound to the provided architecture string. The @@ -47,10 +47,16 @@ } namespace monoprop { +#if defined(__x86_64__) || defined(_M_X64) +[[using gnu: target("default")]] +#endif consteval auto variant() noexcept -> std::string_view { return "default"; } +#if defined(__x86_64__) || defined(_M_X64) +[[using gnu: target("default")]] +#endif consteval auto variant_flags() noexcept -> std::string_view { return "@monoprop_DEFAULT_VARIANT_FLAGS@"; } From 8729620b40bfc2b1358256308b36525cac8b8ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Mon, 10 Aug 2026 09:05:44 +0000 Subject: [PATCH 22/80] build(c++): :ambulance: ensure the clang flag extraction also works on linux --- cmake/compiler_flags/CXXFlags.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index f56236bd..2fa94331 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -101,7 +101,7 @@ function(_monoprop_query_machine_flags) set(_march_args "-march=${_arg_MARCH}") endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang) + if(CMAKE_CXX_COMPILER_ID MATCHES Clang) execute_process( COMMAND # gersemi: off From 77104774ee2ab6645ad07e32b113ac928c5381f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Di=20Remigio=20Eik=C3=A5s?= Date: Mon, 10 Aug 2026 10:20:27 +0000 Subject: [PATCH 23/80] chore: :white_check_mark: add baseline-capture tooling for the NumModes-NTTP removal Stage 0 of the NumModes-NTTP-removal plan: the regression instrument every later stage's "Verify" step relies on. - tools/capture-baseline.py: propagates every tests/data/*.msgpack fixture through MajoranaPropagator across a few cutoffs/cutoff_types, plus a couple of native PauliOperator smoke problems through PauliPropagator (Basis::Pauli coverage -- there is no Majorana -> Pauli operator converter in the public API to press a fermionic fixture into a qubit circuit, so this is hand-picked instead), and dumps term counts, the full (indices, coefficient) set in engine-native order, and the expectation value. Probe cutoffs are capped at 4 -- support cutoff grows combinatorially and 6 on the largest fixture (28 modes) produced 13.7M surviving terms. - justfile: `just capture-baseline [LABEL]` and `just diff-baseline [AGAINST]`. - .gitignore: exclude the capture output directory. Verified reproducible: two independent serial captures are byte-identical, and the three "exact" fixtures' captured energies agree with their fixture's actual_energy to ~1e-13. Assisted-by: ClaudeCode:claude-sonnet-5 --- .gitignore | 5 + justfile | 19 +++ tools/capture-baseline.py | 309 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 333 insertions(+) create mode 100755 tools/capture-baseline.py diff --git a/.gitignore b/.gitignore index e830d9ef..36d1458d 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,8 @@ benches/results/** # devcontainer files .devcontainer/devcontainer-lock.json + +notes/** + +# `just capture-baseline` / `just diff-baseline` output (tools/capture-baseline.py) +.baseline-capture/** diff --git a/justfile b/justfile index 0120d0e8..19012c2d 100644 --- a/justfile +++ b/justfile @@ -33,6 +33,25 @@ test: uv run python -m pytest -m "not mpi" ctest --test-dir build/editable/Release --output-on-failure +# Output directory for `capture-baseline` / `diff-baseline` (gitignored). +baseline_dir := ".baseline-capture" + +# Capture a golden baseline snapshot into `.baseline-capture/LABEL` (default "golden") via +# tools/capture-baseline.py -- the regression instrument for the NumModes-NTTP removal refactor +# (see the plan's Stage 0). Run once on an unmodified tree to seed the golden baseline, then +# `just diff-baseline` after every later-stage change. +capture-baseline LABEL='golden': + uv run --no-sync python tools/capture-baseline.py --out "{{ baseline_dir }}/{{ LABEL }}" + +# Rebuild monoprop and diff a fresh capture against a stored one (default "golden"). Byte-identical +# is the bar through the plan's Stages 1-2 and 4-5; Stage 6's sparse-row re-baseline is the one +# deliberate, documented exception (see the plan's "Verification strategy" section). +diff-baseline AGAINST='golden': + uv sync --all-extras --group test --reinstall-package monoprop --no-cache -v + rm -rf "{{ baseline_dir }}/candidate" + just capture-baseline candidate + diff -rq "{{ baseline_dir }}/{{ AGAINST }}" "{{ baseline_dir }}/candidate" + # MPI is off by default in source builds, so build an MPI-enabled editable install # first, then run the suite under mpiexec with --no-sync (avoids a per-rank resync). # Pass RANKS as either a single integer or a semicolon-separated list (e.g. "1;2;4"). diff --git a/tools/capture-baseline.py b/tools/capture-baseline.py new file mode 100755 index 00000000..1cfd7209 --- /dev/null +++ b/tools/capture-baseline.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Capture golden baselines for the NumModes-NTTP removal refactor (see Stage 0 of the plan). + +For every fixture in ``tests/data/*.msgpack``, propagates through +:class:`~monoprop.MajoranaPropagator` (``Basis::Majorana``) at a handful of cutoffs and cutoff +types, and for a small set of hand-written qubit smoke problems through +:class:`~monoprop.PauliPropagator` (``Basis::Pauli``). For each run this dumps: the term count, the +full ``(monomial indices, coefficient)`` set (in the engine's own iteration order -- ordering is +itself a regression signal, see the module docstring on diffing below), and the expectation value. + +Dual-basis note: the msgpack fixtures are all fermionic (Majorana). The public API only converts +Pauli -> Majorana (``PauliOperator.get_majorana_operator()``, the Jordan-Wigner image); there is no +Majorana -> Pauli operator converter to press a fixture into a qubit circuit, and hand-rolling that +transform for a test-only tool risks baking a silently-wrong transform into the "golden" baseline. +So ``Basis::Pauli`` coverage instead comes from ``_PAULI_SMOKE_CASES`` below: a few hardcoded native +``PauliOperator``/``Circuit`` problems, sized like the ones in ``tests/test_basis.py`` / +``tests/test_pauli.py``. Their values are not independently re-derived here -- as with the msgpack +fixtures' *non*-``_EXACT_FIXTURES``, they are simply frozen as-is for regression diffing. + +Ordering and diffing: terms are dumped in the engine's own returned order, not sorted -- through +Stage 5 of the plan this order is itself load-bearing (``SplitmixHash`` -> probe order -> MPI owner +routing -> insertion order -> floating-point accumulation order), so a silent reorder is exactly the +kind of regression this tool exists to catch. Stage 6 is a deliberate, documented exception: the +sparse-row hash changes on purpose, so ``just diff-baseline`` gets a ``--sorted`` / ``--tol`` mode +for that one re-baseline (see the plan's Stage 6 and "Verification strategy" sections). + +Usage (needs the `test` dependency group synced -- this reuses tests/cases.py, which imports +pytest-cases): + uv sync --all-extras --group test + uv run --no-sync python tools/capture-baseline.py --out .baseline-capture/