diff --git a/cpp/monoprop/detail/CMakeLists.txt b/cpp/monoprop/detail/CMakeLists.txt index f75f51c5..e9b76b93 100644 --- a/cpp/monoprop/detail/CMakeLists.txt +++ b/cpp/monoprop/detail/CMakeLists.txt @@ -5,8 +5,12 @@ target_sources( TYPE HEADERS FILES "EnvConfig.h" + "MemoryBytes.h" + "ProcessMemory.h" ) +target_sources(monoprop-objs PRIVATE ProcessMemory.cpp) + add_subdirectory(evolution) add_subdirectory(graph) add_subdirectory(graph_encoding) diff --git a/cpp/monoprop/detail/MemoryBytes.h b/cpp/monoprop/detail/MemoryBytes.h new file mode 100644 index 00000000..fa495851 --- /dev/null +++ b/cpp/monoprop/detail/MemoryBytes.h @@ -0,0 +1,33 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace monoprop::detail { + +/// Bytes a container has taken from the allocator. Variadic so a roll-up over many members is one call. +template +[[nodiscard]] inline auto capacity_bytes(const Vecs &...vecs) -> size_t { + return (0uz + ... + (vecs.capacity() * sizeof(typename Vecs::value_type))); +} + +/// Of capacity_bytes(): reserved and never written. Unfaulted, but not free -- a growth holds old+new at once. +template +[[nodiscard]] inline auto capacity_slack_bytes(const Vecs &...vecs) -> size_t { + return (0uz + ... + ((vecs.capacity() - vecs.size()) * sizeof(typename Vecs::value_type))); +} + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/ProcessMemory.cpp b/cpp/monoprop/detail/ProcessMemory.cpp new file mode 100644 index 00000000..21badab9 --- /dev/null +++ b/cpp/monoprop/detail/ProcessMemory.cpp @@ -0,0 +1,124 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/ProcessMemory.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) +#include +#endif + +namespace monoprop::detail { +namespace { + +auto slurp(const char *path) -> std::string { + std::ifstream in(path); + return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; +} + +auto digits_at(std::string_view text, size_t at) -> size_t { + size_t value = 0; + for (; at < text.size() && text[at] >= '0' && text[at] <= '9'; ++at) { + value = (value * 10) + static_cast(text[at] - '0'); + } + return value; +} + +// /proc reports these in kB. +auto status_field(std::string_view text, std::string_view key) -> size_t { + const auto at = text.find(key); + if (at == std::string_view::npos) { + return 0uz; + } + return digits_at(text, text.find_first_of("0123456789", at + key.size())) * 1024; +} + +constexpr std::string_view kSizeAttr{R"(size=")"}; +constexpr std::string_view kHeapTag{R"( size_t { + const auto at = xml.rfind(tag); + if (at == std::string_view::npos) { + return 0uz; + } + const auto size_at = xml.find(kSizeAttr, at); + return size_at == std::string_view::npos ? 0uz : digits_at(xml, size_at + kSizeAttr.size()); +} + +auto malloc_info_xml() -> std::string { +#if defined(__GLIBC__) + // open_memstream itself allocates, so this reads a few KiB above the state it describes. + char *buf = nullptr; + size_t len = 0; + FILE *stream = ::open_memstream(&buf, &len); + if (stream == nullptr) { + return {}; + } + const int rc = ::malloc_info(0, stream); + (void)std::fclose(stream); + // Owns the buffer before the copy below: constructing the string can throw, and free() must still run. + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc) -- open_memstream's buffer is malloc'd + const std::unique_ptr owned(buf, &std::free); + return (rc == 0 && buf != nullptr) ? std::string(buf, len) : std::string{}; +#else + return {}; +#endif +} + +auto read_process_memory() -> ProcessMemory { + ProcessMemory out; +#if defined(__linux__) + const std::string status = slurp("/proc/self/status"); + out.rss_bytes = status_field(status, "VmRSS:"); + out.peak_rss_bytes = status_field(status, "VmHWM:"); +#endif + const std::string xml = malloc_info_xml(); + if (xml.empty()) { + return out; + } + // `system current` is arenas only; mmap'd chunks are separate and never free (free() unmaps them). + const size_t mmapped = last_size_attr(xml, R"( ProcessMemory { + try { + return read_process_memory(); + } + catch (...) { + return {}; + } +} + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/ProcessMemory.h b/cpp/monoprop/detail/ProcessMemory.h new file mode 100644 index 00000000..6719b20d --- /dev/null +++ b/cpp/monoprop/detail/ProcessMemory.h @@ -0,0 +1,37 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace monoprop::detail { + +/// What the kernel and the allocator report, as opposed to what a byte ledger estimates. +/// +/// Every field is PER PROCESS: report once, never summed over partitions; over RANKS it is a job total. +/// `alloc_retained_bytes` is what a ledger cannot reach -- freed chunks stay faulted and stay resident. +struct ProcessMemory { + size_t rss_bytes{0uz}; ///< /proc/self/status VmRSS: pages faulted in right now. + size_t peak_rss_bytes{0uz}; ///< VmHWM: the kernel's peak over the process's life. + size_t alloc_in_use_bytes{0uz}; ///< malloc(3) chunks handed out and not yet freed. + size_t alloc_retained_bytes{0uz}; ///< Freed chunks the allocator still holds. + size_t alloc_system_bytes{0uz}; ///< What the allocator has taken from the kernel. + size_t alloc_arenas{0uz}; ///< Arena count (MALLOC_ARENA_MAX bounds it). +}; + +/// Zero-filled where the platform cannot answer, and never throws: a diagnostic must not fail its caller. +auto process_memory() noexcept -> ProcessMemory; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index 34814906..24554ab1 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -334,7 +334,12 @@ auto MonomialPropagator::partitioned_core_term_() const -> double { template auto MonomialPropagator::partitioned_operator_memory_usage_() const -> detail::MPOperatorMemoryBreakdown { - return sum_partitions_([](const MonomialPropagator &s) { return s.operator_memory_usage(); }); + auto out = sum_partitions_([](const MonomialPropagator &s) { return s.operator_memory_usage(); }); + // The transport belongs to the group, not to a partition, so it is added ONCE after the sum. + const auto [transport, staging] = partition_group_->transport_memory_bytes(); + out.transport_bytes = transport; + out.transport_staging_bytes = staging; + return out; } template diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 47a79880..1453519d 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -30,6 +30,7 @@ #include +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/mpi/CheckedCount.h" #include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/PartitionBarrier.h" @@ -97,6 +98,28 @@ class HybridComm { auto operator=(const HybridComm &) -> HybridComm & = delete; auto size() const -> int { return r_ * s_; } + + // Per PROCESS, not per partition; capacities, so staging reads at the largest exchange's high-water mark. + [[nodiscard]] auto staging_bytes() const -> size_t { + return monoprop::detail::capacity_bytes(stage_send_, stage_recv_, red_vec_); + } + [[nodiscard]] auto memory_bytes() const -> size_t { + return sizeof(HybridComm) + staging_bytes() + + monoprop::detail::capacity_bytes(slots_, + counts_send_, + counts_recv_, + mpi_send_counts_, + mpi_send_displs_, + mpi_recv_counts_, + mpi_recv_displs_, + pack_off_, + base_send_, + base_recv_, + col_sum_, + recv_col_, + counts_matrix_store_, + rows_store_); + } auto global_rank(int local_partition) const -> int { return mpi_rank_ * s_ + local_partition; } auto alltoall_counts(int local_partition, const int *send_counts /*[P]*/, int *recv_counts /*[P]*/) -> void { diff --git a/cpp/monoprop/detail/mpi/ShmComm.h b/cpp/monoprop/detail/mpi/ShmComm.h index b845383d..b41fb82f 100644 --- a/cpp/monoprop/detail/mpi/ShmComm.h +++ b/cpp/monoprop/detail/mpi/ShmComm.h @@ -24,6 +24,7 @@ #include #include +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/mpi/CheckedCount.h" #include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/PartitionBarrier.h" @@ -45,6 +46,12 @@ class ShmComm { auto size() const -> int { return n_; } + // Per PROCESS, not per partition. R == 1 has no funnel: alltoallv memcpys out of published pointers. + [[nodiscard]] auto memory_bytes() const -> size_t { + return sizeof(ShmComm) + monoprop::detail::capacity_bytes(slots_); + } + [[nodiscard]] auto staging_bytes() const -> size_t { return 0uz; } + // recv_counts[s] = what rank s sends to me (the transpose of the send-count matrix). auto alltoall_counts(int rank, const int *send_counts, int *recv_counts) -> void { slots_[static_cast(rank)].counts = send_counts; diff --git a/cpp/monoprop/detail/operator/InvertedIndex.h b/cpp/monoprop/detail/operator/InvertedIndex.h index f5a77116..4be58dd6 100644 --- a/cpp/monoprop/detail/operator/InvertedIndex.h +++ b/cpp/monoprop/detail/operator/InvertedIndex.h @@ -25,6 +25,7 @@ #include #include "monoprop/TypeAliases.h" +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/operator/RowAccess.h" namespace monoprop::detail { @@ -203,6 +204,15 @@ struct InvertedIndex { return total; } + // Diagnostic: the part of memory_bytes() no resize ever wrote. Unfaulted, but see reserved_bytes. + auto slack_bytes() const -> size_t { + size_t total = capacity_slack_bytes(row_parity_); + for (const auto &col : cols) { + total += capacity_slack_bytes(col.words, col.set_rows); + } + return total; + } + // Diagnostic tier split of memory_bytes(): {dense_bytes, sparse_bytes, dense_columns}. auto tier_memory_bytes() const -> std::array { std::array out{0, 0, 0}; diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index 774baf6c..15073332 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -29,6 +29,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/Utilities.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/operator/InvertedIndex.h" #include "monoprop/detail/operator/OperatorIndex.h" @@ -61,18 +62,13 @@ class OperatorTermNotFound : public std::runtime_error { template struct MPOperator { - // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr (keeping MPOperator - // itself cheaply movable). Always non-null. + // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr. Always non-null. std::unique_ptr> store{std::make_unique>()}; VecD op_coeffs; - // Only fully-paired terms score nonzero (see score_new_state_rows_), which on production models is - // ~0.07% of the rows -- a dense vector here is 99.9% zeros. state_rows_ is strictly ascending: rows are - // scored in ascending order and the set is only ever appended to. + // Sparse: only fully-paired terms score nonzero, ~0.07% of rows on production models. Strictly ascending. std::vector state_rows_; VecD state_vals_; // parallel to state_rows_; every entry is a unit phase (+-1), never 0 size_t state_scored_rows_{0uz}; // rows [0, state_scored_rows_) have been scored into state_rows_/state_vals_ - // The dense state: empty in Heisenberg unless a caller asks dense_state() to cache one; in Schrödinger - // it is the live coefficient vector evolution mutates in place. VecD state_coeffs; MonomialMap init_op_map{}; VecZ initial_state; @@ -260,9 +256,8 @@ struct MPOperator { } }; -// Callers must pass pairwise-distinct, currently-absent keys: bulk_insert then skips duplicate probes and -// slot k deterministically lands at base+k. Call after any pass that reads pre-insert op state -// (op.size() must equal the returned base). +// Keys must be pairwise-distinct AND currently absent: bulk_insert then skips duplicate probes, so slot k +// lands at base+k. per_slot must write row base+k before returning; op.size() must equal the returned base. template inline auto insert_absent_terms(MPOperator &op, size_t n, KeyAt &&key_at, PerSlot &&per_slot) -> size_t { const size_t base = op.store->grow_rows_geometric(n); @@ -288,23 +283,31 @@ struct MPOperatorMemoryBreakdown final { size_t init_operator_bytes{0uz}; size_t initial_state_bytes{0uz}; size_t inverted_index_bytes{0uz}; - // The MatchedEpochSet stamp array. Propagator-owned, so 0 unless MonomialPropagator fills it in. - size_t matched_scratch_bytes{0uz}; - - // Diagnostics: breakdowns of the fields above, deliberately excluded from total_bytes() so they can - // never double-count. - size_t inverted_index_dense_bytes{0uz}; // of inverted_index_bytes: full-height bitmap columns - size_t inverted_index_sparse_bytes{0uz}; // of inverted_index_bytes: ascending set-row lists + size_t matched_scratch_bytes{0uz}; // MatchedEpochSet stamps; propagator-owned, so 0 unless it fills them + size_t transport_bytes{0uz}; + // Diagnostics: breakdowns of the fields above, outside total_bytes() so they can never double-count. + size_t inverted_index_dense_bytes{0uz}; // full-height bitmap columns + size_t inverted_index_sparse_bytes{0uz}; // ascending set-row lists size_t inverted_index_dense_columns{0uz}; - size_t operator_terms_slack_bytes{0uz}; // of operator_terms_bytes: unused geometric-growth capacity - // of state_coeffs_bytes: entries of the state that are not exactly 0.0 + size_t operator_terms_slack_bytes{0uz}; // unused geometric-growth capacity size_t state_coeffs_nonzero{0uz}; - // Live entries behind init_operator_bytes, which is bucket_count(): bytes with no entries are dead buckets. size_t init_operator_entries{0uz}; + size_t transport_staging_bytes{0uz}; // of transport_bytes: the payload funnel, at its high-water mark + size_t inverted_index_slack_bytes{0uz}; // of inverted_index_bytes: capacity no resize ever wrote + size_t coeff_slack_bytes{0uz}; // of op_coeffs_bytes + state_coeffs_bytes: likewise + size_t operator_terms_peak_bytes{0uz}; // peak over TIME, so a sum over partitions is an upper bound + size_t indexing_peak_bytes{0uz}; // likewise + + // Derived, not stored: reserved and never written, but not free. Peak RSS tracks capacity rather than + // used bytes, because each growth holds the old fully-written buffer and the new one at once -- so + // shrink_to_fit cannot recover this and must cost another copy. Only fewer or earlier growths help. + auto reserved_bytes() const -> size_t { + return operator_terms_slack_bytes + inverted_index_slack_bytes + coeff_slack_bytes; + } auto total_bytes() const -> size_t { return operator_terms_bytes + op_coeffs_bytes + state_coeffs_bytes + indexing_bytes + init_operator_bytes - + initial_state_bytes + inverted_index_bytes + matched_scratch_bytes; + + initial_state_bytes + inverted_index_bytes + matched_scratch_bytes + transport_bytes; } auto operator+=(const MPOperatorMemoryBreakdown &o) -> MPOperatorMemoryBreakdown & { @@ -316,12 +319,18 @@ struct MPOperatorMemoryBreakdown final { initial_state_bytes += o.initial_state_bytes; inverted_index_bytes += o.inverted_index_bytes; matched_scratch_bytes += o.matched_scratch_bytes; + transport_bytes += o.transport_bytes; inverted_index_dense_bytes += o.inverted_index_dense_bytes; inverted_index_sparse_bytes += o.inverted_index_sparse_bytes; inverted_index_dense_columns += o.inverted_index_dense_columns; operator_terms_slack_bytes += o.operator_terms_slack_bytes; state_coeffs_nonzero += o.state_coeffs_nonzero; init_operator_entries += o.init_operator_entries; + transport_staging_bytes += o.transport_staging_bytes; + inverted_index_slack_bytes += o.inverted_index_slack_bytes; + coeff_slack_bytes += o.coeff_slack_bytes; + operator_terms_peak_bytes += o.operator_terms_peak_bytes; + indexing_peak_bytes += o.indexing_peak_bytes; return *this; } }; @@ -345,8 +354,12 @@ inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorM breakdown.inverted_index_dense_bytes = tiers[0]; breakdown.inverted_index_sparse_bytes = tiers[1]; breakdown.inverted_index_dense_columns = tiers[2]; + breakdown.inverted_index_slack_bytes = op.inverted_index_->slack_bytes(); } breakdown.operator_terms_slack_bytes = op.store->slack_bytes(); + breakdown.operator_terms_peak_bytes = op.store->rows_peak_bytes(); + breakdown.indexing_peak_bytes = op.store->index_peak_bytes(); + breakdown.coeff_slack_bytes = capacity_slack_bytes(op.op_coeffs, op.state_coeffs, op.state_rows_, op.state_vals_); // State phases are unit-magnitude, so at rest the scored count IS the nonzero count; a live vector needs a scan. breakdown.state_coeffs_nonzero = op.state_coeffs.empty() diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 8a3c29b2..541593b5 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -29,6 +29,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/MemoryBytes.h" namespace monoprop::detail { @@ -170,11 +171,7 @@ class OperatorIndex { } return overflow_.at(i).count(); } - [[nodiscard]] auto memory_bytes() const -> size_t { - size_t total = rows_.capacity() * sizeof(PosT); - total += overflow_.size() * (sizeof(value_type) + sizeof(size_t) + 24); - return total; - } + [[nodiscard]] auto memory_bytes() const -> size_t { return capacity_bytes(rows_) + overflow_bytes_(); } auto find(const key_type &key) const -> std::optional { const uint32_t h = fold_hash(key); @@ -276,6 +273,16 @@ class OperatorIndex { return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); } + // Diagnostics: peak over TIME of memory_bytes() / index_estimated_memory_bytes(). A growth holds + // old+new at once; no capacity field can show that. Each carries its resting field's other terms + // (the overflow map, the fixed header) so a peak can never read below the resting bytes. + [[nodiscard]] auto rows_peak_bytes() const -> size_t { + return overflow_bytes_() + (std::max(rows_peak_elems_, rows_.capacity()) * sizeof(PosT)); + } + [[nodiscard]] auto index_peak_bytes() const -> size_t { + return sizeof(OperatorIndex) + (std::max(table_.peak_slots, table_.slots.capacity()) * sizeof(Slot)); + } + private: struct Slot { TermIndex idx = kEmptySlot; @@ -321,6 +328,7 @@ class OperatorIndex { std::vector slots = std::vector(kMinSlots, Slot{}); size_t mask = kMinSlots - 1; size_t count = 0; + size_t peak_slots = 0; // see index_peak_bytes() auto rehash_if_needed() -> void { if ((count + 1) * 10 >= slots.size() * 7) { @@ -332,6 +340,8 @@ class OperatorIndex { if (new_cap <= slots.size()) { return; } + // assign() allocates before `old` dies, so both tables are live for the whole re-probe loop. + peak_slots = std::max(peak_slots, slots.size() + new_cap); std::vector old = std::move(slots); slots.assign(new_cap, Slot{}); mask = new_cap - 1; @@ -351,8 +361,22 @@ class OperatorIndex { // Slot count for `n` entries at ≤0.7 load. static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } + // Node + key + value + the bucket pointer, for the rows whose popcount exceeded inline_width_. + [[nodiscard]] auto overflow_bytes_() const -> size_t { + return overflow_.size() * (sizeof(value_type) + sizeof(size_t) + 24); + } + [[nodiscard]] auto capacity() const -> size_t { return rows_.capacity() / stride_; } - auto reserve_rows(size_t n) -> void { rows_.reserve(n * stride_); } + auto reserve_rows(size_t n) -> void { + const size_t want = n * stride_; + if (want <= rows_.capacity()) { + return; // no realloc, so nothing is ever doubled + } + // reserve() may hand back more than asked, so the duplicate is read off the post-growth capacity. + const size_t old_cap = rows_.capacity(); + rows_.reserve(want); + rows_peak_elems_ = std::max(rows_peak_elems_, old_cap + rows_.capacity()); + } auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } // Insert (idx, h) into the table with no duplicate probe — callers on this path insert provably distinct @@ -400,6 +424,7 @@ class OperatorIndex { // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; Table table_ = {}; + size_t rows_peak_elems_ = 0; // PosT count, matching table_.peak_slots; see rows_peak_bytes() }; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/partition/PartitionGroup.h b/cpp/monoprop/detail/partition/PartitionGroup.h index 85000680..67d8bba2 100644 --- a/cpp/monoprop/detail/partition/PartitionGroup.h +++ b/cpp/monoprop/detail/partition/PartitionGroup.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "monoprop/detail/mpi/Comm.h" @@ -106,6 +107,16 @@ class PartitionGroup { ~PartitionGroup() { stop_and_join_(); } auto partition_count() const -> int { return n_; } + + // {total, staging} for the GROUP-owned transport: the facade adds it once, a partition reports 0. + [[nodiscard]] auto transport_memory_bytes() const -> std::pair { +#ifdef monoprop_ENABLE_MPI + if (hyb_) { + return {hyb_->memory_bytes(), hyb_->staging_bytes()}; + } +#endif + return shm_ ? std::pair{shm_->memory_bytes(), shm_->staging_bytes()} : std::pair{0uz, 0uz}; + } auto partition(int s) -> MonomialPropagator & { return *partitions_[static_cast(s)]; } auto partition(int s) const -> const MonomialPropagator & { return *partitions_[static_cast(s)]; } diff --git a/cpp/tests/mp_operator_tests.cpp b/cpp/tests/mp_operator_tests.cpp index 2c822c38..0fdef50b 100644 --- a/cpp/tests/mp_operator_tests.cpp +++ b/cpp/tests/mp_operator_tests.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include "monoprop/MonomialPropagator.h" #include "monoprop/algebra/Algebra.h" +#include "monoprop/detail/ProcessMemory.h" #include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" @@ -81,6 +83,29 @@ auto sparse_state_equals(const detail::MPOperator<8>::SparseState &sparse, const && std::ranges::equal(sparse.values, expected.second); } +// A distinct Monomial<8> per `bits`, so a caller can mint hundreds without enumerating index sets. +auto monomial_from_bits(size_t bits) -> Monomial<8> { + Monomial<8> mono; + for (size_t b = 0; b < 2 * 8; ++b) { // 2 * NumModes + if (((bits >> b) & 1UZ) != 0UZ) { + mono.set(b); + } + } + return mono; +} + +// One insert_absent_terms call per batch: a single bulk call reserves once from empty, duplicating nothing. +auto grow_in_batches(detail::MPOperator<8> &op, size_t batches, size_t per_batch) -> void { + for (size_t batch = 0; batch < batches; ++batch) { + const size_t first = (batch * per_batch) + 1; + detail::insert_absent_terms<8>( + op, + per_batch, + [&](size_t k) { return monomial_from_bits(first + k); }, + [&](size_t k, size_t base) { assign_row<8>(*op.store, base + k, monomial_from_bits(first + k)); }); + } +} + } // namespace BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_paired_terms_majorana_and_pauli) { @@ -400,6 +425,166 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_init_operator_entries_out_of_to BOOST_CHECK_EQUAL(acc.total_bytes(), 120U); } +// transport_bytes IS summed by total_bytes(); group-owned, so estimate_memory_usage leaves it 0. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_counts_transport_in_total_and_sum) { + detail::MPOperatorMemoryBreakdown<8> acc; + acc.op_coeffs_bytes = 100; + acc.transport_bytes = 9; + BOOST_CHECK_EQUAL(acc.total_bytes(), 109U); + + detail::MPOperatorMemoryBreakdown<8> other; + other.op_coeffs_bytes = 20; + other.transport_bytes = 4; + + acc += other; + BOOST_CHECK_EQUAL(acc.transport_bytes, 13U); + BOOST_CHECK_EQUAL(acc.total_bytes(), 133U); + + const auto bare = detail::estimate_memory_usage<8>(build_indexed_op({indices_to_bitset<8>({0, 1})})); + BOOST_CHECK_EQUAL(bare.transport_bytes, 0U); + BOOST_CHECK_GT(bare.total_bytes(), 0U); +} + +// All five are accumulated by operator+= and NONE reaches total_bytes(): summing them would double-count. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_new_diagnostics_out_of_total) { + detail::MPOperatorMemoryBreakdown<8> acc; + acc.op_coeffs_bytes = 100; + acc.transport_staging_bytes = 1; + acc.inverted_index_slack_bytes = 2; + acc.coeff_slack_bytes = 3; + acc.operator_terms_peak_bytes = 5; + acc.indexing_peak_bytes = 6; + BOOST_CHECK_EQUAL(acc.total_bytes(), 100U); + + detail::MPOperatorMemoryBreakdown<8> other; + other.op_coeffs_bytes = 20; + other.transport_staging_bytes = 10; + other.inverted_index_slack_bytes = 20; + other.coeff_slack_bytes = 30; + other.operator_terms_peak_bytes = 50; + other.indexing_peak_bytes = 60; + + acc += other; + BOOST_CHECK_EQUAL(acc.total_bytes(), 120U); + BOOST_CHECK_EQUAL(acc.transport_staging_bytes, 11U); + BOOST_CHECK_EQUAL(acc.inverted_index_slack_bytes, 22U); + BOOST_CHECK_EQUAL(acc.coeff_slack_bytes, 33U); + BOOST_CHECK_EQUAL(acc.operator_terms_peak_bytes, 55U); + BOOST_CHECK_EQUAL(acc.indexing_peak_bytes, 66U); +} + +// The counter must be shown to MOVE: strictly above the resting fields after growth, equal to them before. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_growth_peaks_exceed_resting_capacity) { + detail::MPOperator<8> op; + const auto fresh = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_EQUAL(fresh.operator_terms_peak_bytes, fresh.operator_terms_bytes); + BOOST_CHECK_EQUAL(fresh.indexing_peak_bytes, fresh.indexing_bytes); + + grow_in_batches(op, 40, 15); + BOOST_CHECK_EQUAL(op.size(), 600U); + + const auto grown = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_GT(grown.operator_terms_peak_bytes, grown.operator_terms_bytes); + BOOST_CHECK_GT(grown.indexing_peak_bytes, grown.indexing_bytes); + // The duplicate is the OLD buffer at a 1.5x growth, so the peak cannot reach 3x the resting bytes. + BOOST_CHECK_LT(grown.operator_terms_peak_bytes, 3U * grown.operator_terms_bytes); + BOOST_CHECK_LT(grown.indexing_peak_bytes, 3U * grown.indexing_bytes); +} + +// A peak must be a peak OF the resting field. Rows whose popcount exceeds the inline width spill to the +// overflow map, which operator_terms_bytes counts -- so a peak that omitted them could read BELOW resting +// and make the growth duplicate (peak - resting) negative. grow_in_batches cannot reach this: its keys +// come from bits <= 600, whose popcount is at most 9, so every row it makes stays inline. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_peak_covers_overflow_rows) { + constexpr size_t kBatches = 10; + constexpr size_t kPerBatch = 20; + // The default inline width is 11 of the 16 positions, so every key below spills losslessly. + std::vector wide; + for (size_t bits = 0; bits < (1UZ << 16) && wide.size() < kBatches * kPerBatch; ++bits) { + if (std::popcount(bits) > detail::OperatorIndex<8>::kDefaultInlinePositions) { + wide.push_back(bits); + } + } + BOOST_REQUIRE_EQUAL(wide.size(), kBatches * kPerBatch); + + detail::MPOperator<8> op; + for (size_t batch = 0; batch < kBatches; ++batch) { + const size_t first = batch * kPerBatch; + detail::insert_absent_terms<8>( + op, + kPerBatch, + [&](size_t k) { return monomial_from_bits(wide[first + k]); }, + [&](size_t k, size_t base) { assign_row<8>(*op.store, base + k, monomial_from_bits(wide[first + k])); }); + } + + const auto b = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_GT(b.operator_terms_peak_bytes, b.operator_terms_bytes); + BOOST_CHECK_GT(b.indexing_peak_bytes, b.indexing_bytes); +} + +// reserved_bytes rolls up the three slacks: reserved and unwritten, which is not the same as costing nothing. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_reserved_bytes_rolls_up_the_slacks) { + detail::MPOperator<8> op; + grow_in_batches(op, 40, 15); + op.op_coeffs.reserve(op.op_coeffs.size() + 64); + (void)op.inverted_index(); + + const auto b = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_EQUAL(b.reserved_bytes(), + b.operator_terms_slack_bytes + b.inverted_index_slack_bytes + b.coeff_slack_bytes); + BOOST_CHECK_GT(b.operator_terms_slack_bytes, 0U); // geometric growth always overshoots + BOOST_CHECK_GT(b.coeff_slack_bytes, 0U); // the reserve above + BOOST_CHECK_LE(b.inverted_index_slack_bytes, b.inverted_index_bytes); + BOOST_CHECK_LT(b.reserved_bytes(), b.total_bytes()); +} + +// ASan and TSan replace malloc, so glibc's arenas stay empty and malloc_info(3) reports zero bytes. +// Compile-time, never a runtime zero-check: that would let a regression on a normal build skip the checks. +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define monoprop_TEST_MALLOC_REPLACED 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) +#define monoprop_TEST_MALLOC_REPLACED 1 +#endif +#endif +#ifndef monoprop_TEST_MALLOC_REPLACED +#define monoprop_TEST_MALLOC_REPLACED 0 +#endif + +// The kernel's and the allocator's own numbers, so ledger coverage is derivable from the engine alone. +BOOST_AUTO_TEST_CASE(process_memory_reports_the_kernel_and_the_allocator) { + const auto before = detail::process_memory(); + // A live 32 MiB allocation: a diagnostic that cannot see one is not measuring. + std::vector hold(32UZ << 20, '\1'); + const auto after = detail::process_memory(); + BOOST_CHECK_EQUAL(hold.front(), '\1'); // and keeps `hold` alive across the second read +#if defined(__linux__) && defined(__GLIBC__) + // /proc is not the allocator's, so the kernel fields and the identity hold on both branches below. + BOOST_CHECK_GT(before.rss_bytes, 0U); + BOOST_CHECK_GE(before.peak_rss_bytes, before.rss_bytes); + BOOST_CHECK_GE(after.rss_bytes, before.rss_bytes); + BOOST_CHECK_EQUAL(before.alloc_in_use_bytes + before.alloc_retained_bytes, before.alloc_system_bytes); +#if monoprop_TEST_MALLOC_REPLACED + // Absent COHERENTLY: every byte field zero, before and after, not merely the one CI tripped over. + BOOST_CHECK_EQUAL(before.alloc_system_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_in_use_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_retained_bytes, 0U); + BOOST_CHECK_EQUAL(after.alloc_system_bytes, 0U); + BOOST_CHECK_EQUAL(after.alloc_in_use_bytes, 0U); +#else + BOOST_CHECK_GT(before.alloc_system_bytes, 0U); + BOOST_CHECK_GE(before.alloc_arenas, 1U); + BOOST_CHECK_GE(after.alloc_in_use_bytes, before.alloc_in_use_bytes + hold.size()); +#endif +#else + // The documented contract off Linux/glibc: zero-filled rather than wrong. + BOOST_CHECK_EQUAL(before.rss_bytes, 0U); + BOOST_CHECK_EQUAL(after.rss_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_system_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_arenas, 0U); +#endif +} + BOOST_AUTO_TEST_CASE(mp_operator_copy_constructor_clones_store_and_coeffs) { auto op = build_indexed_op({indices_to_bitset<8>({0, 1}), indices_to_bitset<8>({2, 3})}); op.initial_state = {0}; diff --git a/src/monoprop/bindings/binder.h b/src/monoprop/bindings/binder.h index 4076c7ff..929b4f4a 100644 --- a/src/monoprop/bindings/binder.h +++ b/src/monoprop/bindings/binder.h @@ -35,6 +35,7 @@ #include #include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/ProcessMemory.h" #include "monoprop/detail/mpi/MPICompat.h" namespace nb = nanobind; @@ -254,8 +255,11 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { "Total bytes held by the graph on this rank"); // total_bytes() alone cannot say whether the row store or the transposed inverted index dominates. + // The d_proc_* keys are the kernel's and the allocator's own numbers, so coverage needs no harness. + // They are PER PROCESS, and the C++ breakdown never carries them so operator+= cannot sum them. cls.def("operator_memory_breakdown", [](const MonomialPropagator &self) { const auto b = self.operator_memory_usage(); + const auto proc = monoprop::detail::process_memory(); return std::map{{"operator_terms_bytes", b.operator_terms_bytes}, {"op_coeffs_bytes", b.op_coeffs_bytes}, {"state_coeffs_bytes", b.state_coeffs_bytes}, @@ -264,6 +268,7 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { {"initial_state_bytes", b.initial_state_bytes}, {"inverted_index_bytes", b.inverted_index_bytes}, {"matched_scratch_bytes", b.matched_scratch_bytes}, + {"transport_bytes", b.transport_bytes}, {"total_bytes", b.total_bytes()}, // Diagnostics, outside total_bytes(). {"d_invidx_dense_bytes", b.inverted_index_dense_bytes}, @@ -271,7 +276,20 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { {"d_invidx_dense_columns", b.inverted_index_dense_columns}, {"d_terms_slack_bytes", b.operator_terms_slack_bytes}, {"d_state_coeffs_nonzero", b.state_coeffs_nonzero}, - {"d_init_operator_entries", b.init_operator_entries}}; + {"d_init_operator_entries", b.init_operator_entries}, + {"d_transport_staging_bytes", b.transport_staging_bytes}, + {"d_invidx_slack_bytes", b.inverted_index_slack_bytes}, + {"d_coeff_slack_bytes", b.coeff_slack_bytes}, + {"d_reserved_bytes", b.reserved_bytes()}, + {"d_terms_peak_bytes", b.operator_terms_peak_bytes}, + {"d_indexing_peak_bytes", b.indexing_peak_bytes}, + // Per PROCESS, not per partition. See the note above. + {"d_proc_rss_bytes", proc.rss_bytes}, + {"d_proc_peak_rss_bytes", proc.peak_rss_bytes}, + {"d_proc_alloc_in_use_bytes", proc.alloc_in_use_bytes}, + {"d_proc_alloc_retained_bytes", proc.alloc_retained_bytes}, + {"d_proc_alloc_system_bytes", proc.alloc_system_bytes}, + {"d_proc_alloc_arenas", proc.alloc_arenas}}; }); // The graph does not partition: its arrays are indexed by the flat world, so these grow with a P