From 938c103f9c17a97fc8285a8d691fd270086eb1a4 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Tue, 25 Aug 2026 23:59:44 +0100 Subject: [PATCH 1/8] =?UTF-8?q?feat(memory):=20=F0=9F=93=88=20name=20the?= =?UTF-8?q?=20resident=20bytes=20the=20operator=20ledger=20cannot=20see?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At a fixed 1,569,152,761 terms the ledger's own total is flat from 1 to 32 nodes while the kernel's peak RSS grows 2.15x, so coverage falls 91% -> 40% and the difference is silent. Measured on Deucalion x86, hubbard c10 / atol 2.6e-06, layout B (8 ranks x 16 partitions per node), jobs 1851543 (N=1) and 1851544 (N=32), _core.so md5 3b2113539aee3ea59a0f0488f990a304. The gap is three things, not one: * 79.5 GiB of a 215 GiB job at N=32 is glibc holding freed chunks it never returned. Those pages were faulted while the chunk was live and stay resident, so every large transient the engine allocates and frees is charged to the process forever while belonging to no structure a ledger can name. It is ~0.3 GiB per rank at every width, flat across a 32x range in terms per rank, so at 256 ranks it reaches 54.4 B/term against a ledger of 62.0. * 9.9 GiB of total_bytes() at N=32 is reserved capacity nothing ever wrote, which the kernel therefore never charges. At N=1 the ledger reads 94.2 GiB against 89.8 GiB resident: it over-claims, and the old coverage figure hid that by comparing an end-of-run capacity against a peak-over-time. * 2.2 GiB is the in-process transport, which no field covered at all. So the engine now reports: * transport_bytes, a real field inside total_bytes(), filled once by the partitioned facade from PartitionGroup::transport_memory_bytes(). A partition reports 0, so the sum over S cannot multiply it. d_transport_staging_bytes breaks out the payload funnel and stays outside total_bytes(). * reserved_bytes, rolling up the row-store, inverted-index and coefficient slacks. Reserved and never written -- but NOT free, and the comment says so: job 1851566 measured peak RSS against that slack directly and found it tracks capacity, not used bytes (flat to +/-0.073 GiB within a x1.5 branch, stepping -0.650 GiB at the capacity boundary), because each growth holds the old fully-written buffer and the new one at once. The corollary is that shrink_to_fit cannot recover it and must cost another copy; only fewer or earlier growths help. * d_terms_peak_bytes / d_indexing_peak_bytes, the growth duplicate at the two monolithic reallocation sites. Capacity at end cannot represent a peak over time. Both sites are cold paths, so no hot loop changes. * detail::process_memory(), reading VmRSS/VmHWM and malloc_info(3), bound as d_proc_*. Coverage is now derivable from one call instead of an external harness, and d_proc_peak_rss_bytes agrees with `/usr/bin/time -v` to 100.0% at 8 ranks and 99.9% at 256. It is zero-filled off Linux/glibc and cannot throw, and equally when something replaces malloc: under ASan or TSan glibc's arenas stay empty, so the byte fields read zero and the test asserts that absence is coherent rather than skipping the case. The five identity terms sum to the measured peak exactly at both rungs. What remains is 45 GiB at N=32 that is 256 Python interpreters, mpi4py and MPI's own buffers, which the engine cannot reach; it is now the named residue rather than an unexplained one. Assisted-by: claude-code:claude-opus-5 --- cpp/monoprop/detail/CMakeLists.txt | 3 + cpp/monoprop/detail/ProcessMemory.cpp | 116 +++++++++++++ cpp/monoprop/detail/ProcessMemory.h | 37 +++++ .../MonomialPropagator.inl | 7 +- cpp/monoprop/detail/mpi/HybridComm.h | 17 ++ cpp/monoprop/detail/mpi/ShmComm.h | 4 + cpp/monoprop/detail/operator/InvertedIndex.h | 10 ++ cpp/monoprop/detail/operator/MPOperator.h | 35 +++- cpp/monoprop/detail/operator/OperatorIndex.h | 20 ++- .../detail/partition/PartitionGroup.h | 11 ++ cpp/tests/mp_operator_tests.cpp | 157 ++++++++++++++++++ src/monoprop/bindings/binder.h | 20 ++- 12 files changed, 433 insertions(+), 4 deletions(-) create mode 100644 cpp/monoprop/detail/ProcessMemory.cpp create mode 100644 cpp/monoprop/detail/ProcessMemory.h diff --git a/cpp/monoprop/detail/CMakeLists.txt b/cpp/monoprop/detail/CMakeLists.txt index f75f51c5..faea724a 100644 --- a/cpp/monoprop/detail/CMakeLists.txt +++ b/cpp/monoprop/detail/CMakeLists.txt @@ -5,8 +5,11 @@ target_sources( TYPE HEADERS FILES "EnvConfig.h" + "ProcessMemory.h" ) +target_sources(monoprop-objs PRIVATE ProcessMemory.cpp) + add_subdirectory(evolution) add_subdirectory(graph) add_subdirectory(graph_encoding) diff --git a/cpp/monoprop/detail/ProcessMemory.cpp b/cpp/monoprop/detail/ProcessMemory.cpp new file mode 100644 index 00000000..af5e7858 --- /dev/null +++ b/cpp/monoprop/detail/ProcessMemory.cpp @@ -0,0 +1,116 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/ProcessMemory.h" + +#include +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) +#include +#endif + +namespace monoprop::detail { +namespace { + +auto slurp(const char *path) -> std::string { + std::string out; + FILE *file = std::fopen(path, "re"); + if (file == nullptr) { + return out; + } + std::array buf{}; + for (size_t n = 0; (n = std::fread(buf.data(), 1, buf.size(), file)) > 0;) { + out.append(buf.data(), n); + } + (void)std::fclose(file); + return out; +} + +auto digits_at(std::string_view text, size_t at) -> size_t { + size_t value = 0; + for (; at < text.size() && text[at] >= '0' && text[at] <= '9'; ++at) { + value = (value * 10) + static_cast(text[at] - '0'); + } + return value; +} + +// /proc reports these in kB. +auto status_field(std::string_view text, std::string_view key) -> size_t { + const auto at = text.find(key); + if (at == std::string_view::npos) { + return 0uz; + } + return digits_at(text, text.find_first_of("0123456789", at + key.size())) * 1024; +} + +// The per-arena element names repeat once more in the process-wide roll-up, which is LAST. +auto last_size_attr(std::string_view xml, std::string_view tag) -> size_t { + const auto at = xml.rfind(tag); + if (at == std::string_view::npos) { + return 0uz; + } + const auto size_at = xml.find("size=\"", at); + return size_at == std::string_view::npos ? 0uz : digits_at(xml, size_at + 6); +} + +auto malloc_info_xml() -> std::string { +#if defined(__GLIBC__) + // open_memstream itself allocates, so this reads a few KiB above the state it describes. + char *buf = nullptr; + size_t len = 0; + FILE *stream = ::open_memstream(&buf, &len); + if (stream == nullptr) { + return {}; + } + const int rc = ::malloc_info(0, stream); + (void)std::fclose(stream); + std::string out = (rc == 0 && buf != nullptr) ? std::string(buf, len) : std::string{}; + std::free(buf); // NOLINT(cppcoreguidelines-no-malloc) -- open_memstream's buffer is malloc'd + return out; +#else + return {}; +#endif +} + +} // namespace + +auto process_memory() -> ProcessMemory { + ProcessMemory out; +#if defined(__linux__) + const std::string status = slurp("/proc/self/status"); + out.rss_bytes = status_field(status, "VmRSS:"); + out.peak_rss_bytes = status_field(status, "VmHWM:"); +#endif + const std::string xml = malloc_info_xml(); + if (xml.empty()) { + return out; + } + // `system current` is arenas only; mmap'd chunks are separate and never free (free() unmaps them). + const size_t mmapped = last_size_attr(xml, " + +namespace monoprop::detail { + +/// What the kernel and the allocator report, as opposed to what a byte ledger estimates. +/// +/// Every field is PER PROCESS: report once, never summed over partitions; over RANKS it is a job total. +/// `alloc_retained_bytes` is what a ledger cannot reach -- freed chunks stay faulted and stay resident. +struct ProcessMemory { + size_t rss_bytes{0uz}; ///< /proc/self/status VmRSS: pages faulted in right now. + size_t peak_rss_bytes{0uz}; ///< VmHWM: the kernel's peak over the process's life. + size_t alloc_in_use_bytes{0uz}; ///< malloc(3) chunks handed out and not yet freed. + size_t alloc_retained_bytes{0uz}; ///< Freed chunks the allocator still holds. + size_t alloc_system_bytes{0uz}; ///< What the allocator has taken from the kernel. + size_t alloc_arenas{0uz}; ///< Arena count (MALLOC_ARENA_MAX bounds it). +}; + +/// Zero-filled where the platform cannot answer, and never throws: a diagnostic must not fail its caller. +auto process_memory() -> ProcessMemory; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index b2e0aae1..495c0d2c 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -334,7 +334,12 @@ auto MonomialPropagator::partitioned_core_term_() const -> double { template auto MonomialPropagator::partitioned_operator_memory_usage_() const -> detail::MPOperatorMemoryBreakdown { - return sum_partitions_([](const MonomialPropagator &s) { return s.operator_memory_usage(); }); + auto out = sum_partitions_([](const MonomialPropagator &s) { return s.operator_memory_usage(); }); + // The transport belongs to the group, not to a partition, so it is added ONCE after the sum. + const auto [transport, staging] = partition_group_->transport_memory_bytes(); + out.transport_bytes = transport; + out.transport_staging_bytes = staging; + return out; } template diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 47a79880..937ed588 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -97,6 +97,18 @@ class HybridComm { auto operator=(const HybridComm &) -> HybridComm & = delete; auto size() const -> int { return r_ * s_; } + + // Per PROCESS, not per partition; capacities, so staging reads at the largest exchange's high-water mark. + [[nodiscard]] auto staging_bytes() const -> size_t { + return stage_send_.capacity() + stage_recv_.capacity() + (red_vec_.capacity() * sizeof(double)); + } + [[nodiscard]] auto memory_bytes() const -> size_t { + return sizeof(HybridComm) + staging_bytes() + cap_bytes_(slots_) + cap_bytes_(counts_send_) + + cap_bytes_(counts_recv_) + cap_bytes_(mpi_send_counts_) + cap_bytes_(mpi_send_displs_) + + cap_bytes_(mpi_recv_counts_) + cap_bytes_(mpi_recv_displs_) + cap_bytes_(pack_off_) + + cap_bytes_(base_send_) + cap_bytes_(base_recv_) + cap_bytes_(col_sum_) + cap_bytes_(recv_col_) + + cap_bytes_(counts_matrix_store_) + cap_bytes_(rows_store_); + } auto global_rank(int local_partition) const -> int { return mpi_rank_ * s_ + local_partition; } auto alltoall_counts(int local_partition, const int *send_counts /*[P]*/, int *recv_counts /*[P]*/) -> void { @@ -588,6 +600,11 @@ class HybridComm { auto sync() -> void { barrier_.sync(); } + template + static auto cap_bytes_(const std::vector &v) -> size_t { + return v.capacity() * sizeof(T); + } + MPI_Comm parent_; int s_; int r_ = 1; diff --git a/cpp/monoprop/detail/mpi/ShmComm.h b/cpp/monoprop/detail/mpi/ShmComm.h index b845383d..f6c1e8e7 100644 --- a/cpp/monoprop/detail/mpi/ShmComm.h +++ b/cpp/monoprop/detail/mpi/ShmComm.h @@ -45,6 +45,10 @@ class ShmComm { auto size() const -> int { return n_; } + // Per PROCESS, not per partition. R == 1 has no funnel: alltoallv memcpys out of published pointers. + [[nodiscard]] auto memory_bytes() const -> size_t { return sizeof(ShmComm) + (slots_.capacity() * sizeof(Slot)); } + [[nodiscard]] auto staging_bytes() const -> size_t { return 0uz; } + // recv_counts[s] = what rank s sends to me (the transpose of the send-count matrix). auto alltoall_counts(int rank, const int *send_counts, int *recv_counts) -> void { slots_[static_cast(rank)].counts = send_counts; diff --git a/cpp/monoprop/detail/operator/InvertedIndex.h b/cpp/monoprop/detail/operator/InvertedIndex.h index f5a77116..feecc107 100644 --- a/cpp/monoprop/detail/operator/InvertedIndex.h +++ b/cpp/monoprop/detail/operator/InvertedIndex.h @@ -203,6 +203,16 @@ struct InvertedIndex { return total; } + // Diagnostic: the part of memory_bytes() no resize ever wrote. Unfaulted, but see reserved_bytes. + auto slack_bytes() const -> size_t { + size_t total = 0; + for (const auto &col : cols) { + total += (col.words.capacity() - col.words.size()) * sizeof(uint64_t); + total += (col.set_rows.capacity() - col.set_rows.size()) * sizeof(TermIndex); + } + return total + ((row_parity_.capacity() - row_parity_.size()) * sizeof(uint64_t)); + } + // Diagnostic tier split of memory_bytes(): {dense_bytes, sparse_bytes, dense_columns}. auto tier_memory_bytes() const -> std::array { std::array out{0, 0, 0}; diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index 774baf6c..b7e9f350 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -274,6 +274,12 @@ inline auto insert_absent_terms(MPOperator &op, size_t n, KeyAt &&key_ return base; } +// Reserved-but-never-written capacity. Unfaulted -- but not free: see reserved_bytes. +template +inline auto capacity_slack_bytes(const Vec &v, size_t elem) -> size_t { + return (v.capacity() - v.size()) * elem; +} + template inline auto unordered_flat_map_storage_bytes(const FlatMap &map) -> size_t { return sizeof(FlatMap) + map.bucket_count() * (sizeof(typename FlatMap::value_type) + sizeof(unsigned char)); @@ -290,6 +296,8 @@ struct MPOperatorMemoryBreakdown final { size_t inverted_index_bytes{0uz}; // The MatchedEpochSet stamp array. Propagator-owned, so 0 unless MonomialPropagator fills it in. size_t matched_scratch_bytes{0uz}; + // The group's transport, per process: 0 unless the partitioned facade fills it in (see matched_scratch). + size_t transport_bytes{0uz}; // Diagnostics: breakdowns of the fields above, deliberately excluded from total_bytes() so they can // never double-count. @@ -301,10 +309,19 @@ struct MPOperatorMemoryBreakdown final { size_t state_coeffs_nonzero{0uz}; // Live entries behind init_operator_bytes, which is bucket_count(): bytes with no entries are dead buckets. size_t init_operator_entries{0uz}; + size_t transport_staging_bytes{0uz}; // of transport_bytes: the payload funnel, at its high-water mark + size_t inverted_index_slack_bytes{0uz}; // of inverted_index_bytes: capacity no resize ever wrote + size_t coeff_slack_bytes{0uz}; // of op_coeffs_bytes + state_coeffs_bytes: likewise + // Never written, yet peak RSS tracks CAPACITY not size (job 1851566): each x1.5 growth holds old+new at + // once, so this is deferred cost, not free space -- and shrink_to_fit adds a copy rather than removing one. + size_t reserved_bytes{0uz}; + // Peaks over TIME: a growth holds old+new at once. Summed, an upper bound -- peaks need not coincide. + size_t operator_terms_peak_bytes{0uz}; + size_t indexing_peak_bytes{0uz}; auto total_bytes() const -> size_t { return operator_terms_bytes + op_coeffs_bytes + state_coeffs_bytes + indexing_bytes + init_operator_bytes - + initial_state_bytes + inverted_index_bytes + matched_scratch_bytes; + + initial_state_bytes + inverted_index_bytes + matched_scratch_bytes + transport_bytes; } auto operator+=(const MPOperatorMemoryBreakdown &o) -> MPOperatorMemoryBreakdown & { @@ -316,12 +333,19 @@ struct MPOperatorMemoryBreakdown final { initial_state_bytes += o.initial_state_bytes; inverted_index_bytes += o.inverted_index_bytes; matched_scratch_bytes += o.matched_scratch_bytes; + transport_bytes += o.transport_bytes; inverted_index_dense_bytes += o.inverted_index_dense_bytes; inverted_index_sparse_bytes += o.inverted_index_sparse_bytes; inverted_index_dense_columns += o.inverted_index_dense_columns; operator_terms_slack_bytes += o.operator_terms_slack_bytes; state_coeffs_nonzero += o.state_coeffs_nonzero; init_operator_entries += o.init_operator_entries; + transport_staging_bytes += o.transport_staging_bytes; + inverted_index_slack_bytes += o.inverted_index_slack_bytes; + coeff_slack_bytes += o.coeff_slack_bytes; + reserved_bytes += o.reserved_bytes; + operator_terms_peak_bytes += o.operator_terms_peak_bytes; + indexing_peak_bytes += o.indexing_peak_bytes; return *this; } }; @@ -345,8 +369,17 @@ inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorM breakdown.inverted_index_dense_bytes = tiers[0]; breakdown.inverted_index_sparse_bytes = tiers[1]; breakdown.inverted_index_dense_columns = tiers[2]; + breakdown.inverted_index_slack_bytes = op.inverted_index_->slack_bytes(); } breakdown.operator_terms_slack_bytes = op.store->slack_bytes(); + breakdown.operator_terms_peak_bytes = op.store->rows_peak_bytes(); + breakdown.indexing_peak_bytes = op.store->index_peak_bytes(); + breakdown.coeff_slack_bytes = capacity_slack_bytes(op.op_coeffs, sizeof(double)) + + capacity_slack_bytes(op.state_coeffs, sizeof(double)) + + capacity_slack_bytes(op.state_rows_, sizeof(TermIndex)) + + capacity_slack_bytes(op.state_vals_, sizeof(double)); + breakdown.reserved_bytes = + breakdown.operator_terms_slack_bytes + breakdown.inverted_index_slack_bytes + breakdown.coeff_slack_bytes; // State phases are unit-magnitude, so at rest the scored count IS the nonzero count; a live vector needs a scan. breakdown.state_coeffs_nonzero = op.state_coeffs.empty() diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 8a3c29b2..1a220e87 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -276,6 +276,14 @@ class OperatorIndex { return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); } + // Diagnostics: peak over TIME. A growth holds old+new at once; no capacity field can show that. + [[nodiscard]] auto rows_peak_bytes() const -> size_t { + return std::max(rows_peak_bytes_, rows_.capacity() * sizeof(PosT)); + } + [[nodiscard]] auto index_peak_bytes() const -> size_t { + return std::max(table_.peak_bytes, index_estimated_memory_bytes()); + } + private: struct Slot { TermIndex idx = kEmptySlot; @@ -321,6 +329,7 @@ class OperatorIndex { std::vector slots = std::vector(kMinSlots, Slot{}); size_t mask = kMinSlots - 1; size_t count = 0; + size_t peak_bytes = 0; // see index_peak_bytes() auto rehash_if_needed() -> void { if ((count + 1) * 10 >= slots.size() * 7) { @@ -332,6 +341,8 @@ class OperatorIndex { if (new_cap <= slots.size()) { return; } + // assign() allocates before `old` dies, so both tables are live for the whole re-probe loop. + peak_bytes = std::max(peak_bytes, (slots.size() + new_cap) * sizeof(Slot)); std::vector old = std::move(slots); slots.assign(new_cap, Slot{}); mask = new_cap - 1; @@ -352,7 +363,13 @@ class OperatorIndex { static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } [[nodiscard]] auto capacity() const -> size_t { return rows_.capacity() / stride_; } - auto reserve_rows(size_t n) -> void { rows_.reserve(n * stride_); } + auto reserve_rows(size_t n) -> void { + const size_t want = n * stride_; + if (want > rows_.capacity()) { + rows_peak_bytes_ = std::max(rows_peak_bytes_, (rows_.capacity() + want) * sizeof(PosT)); + } + rows_.reserve(want); + } auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } // Insert (idx, h) into the table with no duplicate probe — callers on this path insert provably distinct @@ -400,6 +417,7 @@ class OperatorIndex { // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; Table table_ = {}; + size_t rows_peak_bytes_ = 0; // see rows_peak_bytes() }; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/partition/PartitionGroup.h b/cpp/monoprop/detail/partition/PartitionGroup.h index 85000680..67d8bba2 100644 --- a/cpp/monoprop/detail/partition/PartitionGroup.h +++ b/cpp/monoprop/detail/partition/PartitionGroup.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "monoprop/detail/mpi/Comm.h" @@ -106,6 +107,16 @@ class PartitionGroup { ~PartitionGroup() { stop_and_join_(); } auto partition_count() const -> int { return n_; } + + // {total, staging} for the GROUP-owned transport: the facade adds it once, a partition reports 0. + [[nodiscard]] auto transport_memory_bytes() const -> std::pair { +#ifdef monoprop_ENABLE_MPI + if (hyb_) { + return {hyb_->memory_bytes(), hyb_->staging_bytes()}; + } +#endif + return shm_ ? std::pair{shm_->memory_bytes(), shm_->staging_bytes()} : std::pair{0uz, 0uz}; + } auto partition(int s) -> MonomialPropagator & { return *partitions_[static_cast(s)]; } auto partition(int s) const -> const MonomialPropagator & { return *partitions_[static_cast(s)]; } diff --git a/cpp/tests/mp_operator_tests.cpp b/cpp/tests/mp_operator_tests.cpp index 2c822c38..cbb3e18c 100644 --- a/cpp/tests/mp_operator_tests.cpp +++ b/cpp/tests/mp_operator_tests.cpp @@ -27,6 +27,7 @@ #include "monoprop/MonomialPropagator.h" #include "monoprop/algebra/Algebra.h" +#include "monoprop/detail/ProcessMemory.h" #include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" @@ -81,6 +82,30 @@ auto sparse_state_equals(const detail::MPOperator<8>::SparseState &sparse, const && std::ranges::equal(sparse.values, expected.second); } +// A distinct Monomial<8> per `bits`, so a caller can mint hundreds without enumerating index sets. +auto monomial_from_bits(size_t bits) -> Monomial<8> { + Monomial<8> mono; + for (size_t b = 0; b < 16; ++b) { + if (((bits >> b) & 1UZ) != 0UZ) { + mono.set(b); + } + } + return mono; +} + +// One insert_absent_terms call per batch: a single bulk call reserves once from empty, duplicating nothing. +auto grow_in_batches(detail::MPOperator<8> &op, size_t batches, size_t per_batch) -> void { + for (size_t batch = 0; batch < batches; ++batch) { + detail::insert_absent_terms<8>( + op, + per_batch, + [&](size_t k) { return monomial_from_bits((batch * per_batch) + k + 1); }, + [&](size_t k, size_t base) { + assign_row<8>(*op.store, base + k, monomial_from_bits((batch * per_batch) + k + 1)); + }); + } +} + } // namespace BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_paired_terms_majorana_and_pauli) { @@ -400,6 +425,138 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_init_operator_entries_out_of_to BOOST_CHECK_EQUAL(acc.total_bytes(), 120U); } +// transport_bytes IS summed by total_bytes(); group-owned, so estimate_memory_usage leaves it 0. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_counts_transport_in_total_and_sum) { + detail::MPOperatorMemoryBreakdown<8> acc; + acc.op_coeffs_bytes = 100; + acc.transport_bytes = 9; + BOOST_CHECK_EQUAL(acc.total_bytes(), 109U); + + detail::MPOperatorMemoryBreakdown<8> other; + other.op_coeffs_bytes = 20; + other.transport_bytes = 4; + + acc += other; + BOOST_CHECK_EQUAL(acc.transport_bytes, 13U); + BOOST_CHECK_EQUAL(acc.total_bytes(), 133U); + + const auto bare = detail::estimate_memory_usage<8>(build_indexed_op({indices_to_bitset<8>({0, 1})})); + BOOST_CHECK_EQUAL(bare.transport_bytes, 0U); + BOOST_CHECK_GT(bare.total_bytes(), 0U); +} + +// All six are accumulated by operator+= and NONE reaches total_bytes(): summing them would double-count. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_new_diagnostics_out_of_total) { + detail::MPOperatorMemoryBreakdown<8> acc; + acc.op_coeffs_bytes = 100; + acc.transport_staging_bytes = 1; + acc.inverted_index_slack_bytes = 2; + acc.coeff_slack_bytes = 3; + acc.reserved_bytes = 4; + acc.operator_terms_peak_bytes = 5; + acc.indexing_peak_bytes = 6; + BOOST_CHECK_EQUAL(acc.total_bytes(), 100U); + + detail::MPOperatorMemoryBreakdown<8> other; + other.op_coeffs_bytes = 20; + other.transport_staging_bytes = 10; + other.inverted_index_slack_bytes = 20; + other.coeff_slack_bytes = 30; + other.reserved_bytes = 40; + other.operator_terms_peak_bytes = 50; + other.indexing_peak_bytes = 60; + + acc += other; + BOOST_CHECK_EQUAL(acc.total_bytes(), 120U); + BOOST_CHECK_EQUAL(acc.transport_staging_bytes, 11U); + BOOST_CHECK_EQUAL(acc.inverted_index_slack_bytes, 22U); + BOOST_CHECK_EQUAL(acc.coeff_slack_bytes, 33U); + BOOST_CHECK_EQUAL(acc.reserved_bytes, 44U); + BOOST_CHECK_EQUAL(acc.operator_terms_peak_bytes, 55U); + BOOST_CHECK_EQUAL(acc.indexing_peak_bytes, 66U); +} + +// The counter must be shown to MOVE: strictly above the resting fields after growth, equal to them before. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_growth_peaks_exceed_resting_capacity) { + detail::MPOperator<8> op; + const auto fresh = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_EQUAL(fresh.operator_terms_peak_bytes, fresh.operator_terms_bytes); + BOOST_CHECK_EQUAL(fresh.indexing_peak_bytes, fresh.indexing_bytes); + + grow_in_batches(op, 40, 15); + BOOST_CHECK_EQUAL(op.size(), 600U); + + const auto grown = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_GT(grown.operator_terms_peak_bytes, grown.operator_terms_bytes); + BOOST_CHECK_GT(grown.indexing_peak_bytes, grown.indexing_bytes); + // The duplicate is the OLD buffer at a 1.5x growth, so the peak cannot reach 3x the resting bytes. + BOOST_CHECK_LT(grown.operator_terms_peak_bytes, 3U * grown.operator_terms_bytes); + BOOST_CHECK_LT(grown.indexing_peak_bytes, 3U * grown.indexing_bytes); +} + +// reserved_bytes rolls up the three slacks: reserved and unwritten, which is not the same as costing nothing. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_reserved_bytes_rolls_up_the_slacks) { + detail::MPOperator<8> op; + grow_in_batches(op, 40, 15); + op.op_coeffs.reserve(op.op_coeffs.size() + 64); + (void)op.inverted_index(); + + const auto b = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_EQUAL(b.reserved_bytes, + b.operator_terms_slack_bytes + b.inverted_index_slack_bytes + b.coeff_slack_bytes); + BOOST_CHECK_GT(b.operator_terms_slack_bytes, 0U); // geometric growth always overshoots + BOOST_CHECK_GT(b.coeff_slack_bytes, 0U); // the reserve above + BOOST_CHECK_LE(b.inverted_index_slack_bytes, b.inverted_index_bytes); + BOOST_CHECK_LT(b.reserved_bytes, b.total_bytes()); +} + +// ASan and TSan replace malloc, so glibc's arenas stay empty and malloc_info(3) reports zero bytes. +// Compile-time, never a runtime zero-check: that would let a regression on a normal build skip the checks. +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define monoprop_TEST_MALLOC_REPLACED 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) +#define monoprop_TEST_MALLOC_REPLACED 1 +#endif +#endif +#ifndef monoprop_TEST_MALLOC_REPLACED +#define monoprop_TEST_MALLOC_REPLACED 0 +#endif + +// The kernel's and the allocator's own numbers, so ledger coverage is derivable from the engine alone. +BOOST_AUTO_TEST_CASE(process_memory_reports_the_kernel_and_the_allocator) { + const auto before = detail::process_memory(); + // A live 32 MiB allocation: a diagnostic that cannot see one is not measuring. + std::vector hold(32UZ << 20, '\1'); + const auto after = detail::process_memory(); + BOOST_CHECK_EQUAL(hold.front(), '\1'); // and keeps `hold` alive across the second read +#if defined(__linux__) && defined(__GLIBC__) + // /proc is not the allocator's, so the kernel fields and the identity hold on both branches below. + BOOST_CHECK_GT(before.rss_bytes, 0U); + BOOST_CHECK_GE(before.peak_rss_bytes, before.rss_bytes); + BOOST_CHECK_GE(after.rss_bytes, before.rss_bytes); + BOOST_CHECK_EQUAL(before.alloc_in_use_bytes + before.alloc_retained_bytes, before.alloc_system_bytes); +#if monoprop_TEST_MALLOC_REPLACED + // Absent COHERENTLY: every byte field zero, before and after, not merely the one CI tripped over. + BOOST_CHECK_EQUAL(before.alloc_system_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_in_use_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_retained_bytes, 0U); + BOOST_CHECK_EQUAL(after.alloc_system_bytes, 0U); + BOOST_CHECK_EQUAL(after.alloc_in_use_bytes, 0U); +#else + BOOST_CHECK_GT(before.alloc_system_bytes, 0U); + BOOST_CHECK_GE(before.alloc_arenas, 1U); + BOOST_CHECK_GE(after.alloc_in_use_bytes, before.alloc_in_use_bytes + hold.size()); +#endif +#else + // The documented contract off Linux/glibc: zero-filled rather than wrong. + BOOST_CHECK_EQUAL(before.rss_bytes, 0U); + BOOST_CHECK_EQUAL(after.rss_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_system_bytes, 0U); + BOOST_CHECK_EQUAL(before.alloc_arenas, 0U); +#endif +} + BOOST_AUTO_TEST_CASE(mp_operator_copy_constructor_clones_store_and_coeffs) { auto op = build_indexed_op({indices_to_bitset<8>({0, 1}), indices_to_bitset<8>({2, 3})}); op.initial_state = {0}; diff --git a/src/monoprop/bindings/binder.h b/src/monoprop/bindings/binder.h index 4076c7ff..caeeb279 100644 --- a/src/monoprop/bindings/binder.h +++ b/src/monoprop/bindings/binder.h @@ -35,6 +35,7 @@ #include #include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/ProcessMemory.h" #include "monoprop/detail/mpi/MPICompat.h" namespace nb = nanobind; @@ -254,8 +255,11 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { "Total bytes held by the graph on this rank"); // total_bytes() alone cannot say whether the row store or the transposed inverted index dominates. + // The d_proc_* keys are the kernel's and the allocator's own numbers, so coverage needs no harness. + // They are PER PROCESS, and the C++ breakdown never carries them so operator+= cannot sum them. cls.def("operator_memory_breakdown", [](const MonomialPropagator &self) { const auto b = self.operator_memory_usage(); + const auto proc = monoprop::detail::process_memory(); return std::map{{"operator_terms_bytes", b.operator_terms_bytes}, {"op_coeffs_bytes", b.op_coeffs_bytes}, {"state_coeffs_bytes", b.state_coeffs_bytes}, @@ -264,6 +268,7 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { {"initial_state_bytes", b.initial_state_bytes}, {"inverted_index_bytes", b.inverted_index_bytes}, {"matched_scratch_bytes", b.matched_scratch_bytes}, + {"transport_bytes", b.transport_bytes}, {"total_bytes", b.total_bytes()}, // Diagnostics, outside total_bytes(). {"d_invidx_dense_bytes", b.inverted_index_dense_bytes}, @@ -271,7 +276,20 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { {"d_invidx_dense_columns", b.inverted_index_dense_columns}, {"d_terms_slack_bytes", b.operator_terms_slack_bytes}, {"d_state_coeffs_nonzero", b.state_coeffs_nonzero}, - {"d_init_operator_entries", b.init_operator_entries}}; + {"d_init_operator_entries", b.init_operator_entries}, + {"d_transport_staging_bytes", b.transport_staging_bytes}, + {"d_invidx_slack_bytes", b.inverted_index_slack_bytes}, + {"d_coeff_slack_bytes", b.coeff_slack_bytes}, + {"d_reserved_bytes", b.reserved_bytes}, + {"d_terms_peak_bytes", b.operator_terms_peak_bytes}, + {"d_indexing_peak_bytes", b.indexing_peak_bytes}, + // Per PROCESS, not per partition. See the note above. + {"d_proc_rss_bytes", proc.rss_bytes}, + {"d_proc_peak_rss_bytes", proc.peak_rss_bytes}, + {"d_proc_alloc_in_use_bytes", proc.alloc_in_use_bytes}, + {"d_proc_alloc_retained_bytes", proc.alloc_retained_bytes}, + {"d_proc_alloc_system_bytes", proc.alloc_system_bytes}, + {"d_proc_alloc_arenas", proc.alloc_arenas}}; }); // The graph does not partition: its arrays are indexed by the flat world, so these grow with a P From 20bd33bacd665bdc81c4594ac07e6b00c48c29fc Mon Sep 17 00:00:00 2001 From: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:19:01 +0100 Subject: [PATCH 2/8] Refactor MPOperator comments and structure Signed-off-by: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> --- cpp/monoprop/detail/operator/MPOperator.h | 33 ++++++----------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index b7e9f350..3cc1024e 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -61,18 +61,12 @@ class OperatorTermNotFound : public std::runtime_error { template struct MPOperator { - // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr (keeping MPOperator - // itself cheaply movable). Always non-null. + // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr. Always non-null. std::unique_ptr> store{std::make_unique>()}; VecD op_coeffs; - // Only fully-paired terms score nonzero (see score_new_state_rows_), which on production models is - // ~0.07% of the rows -- a dense vector here is 99.9% zeros. state_rows_ is strictly ascending: rows are - // scored in ascending order and the set is only ever appended to. - std::vector state_rows_; + std::vector state_rows_; // sparse state VecD state_vals_; // parallel to state_rows_; every entry is a unit phase (+-1), never 0 - size_t state_scored_rows_{0uz}; // rows [0, state_scored_rows_) have been scored into state_rows_/state_vals_ - // The dense state: empty in Heisenberg unless a caller asks dense_state() to cache one; in Schrödinger - // it is the live coefficient vector evolution mutates in place. + size_t state_scored_rows_{0uz}; // cache of the state values (its super sparse so this is useful) VecD state_coeffs; MonomialMap init_op_map{}; VecZ initial_state; @@ -260,9 +254,7 @@ struct MPOperator { } }; -// Callers must pass pairwise-distinct, currently-absent keys: bulk_insert then skips duplicate probes and -// slot k deterministically lands at base+k. Call after any pass that reads pre-insert op state -// (op.size() must equal the returned base). +// Callers must pass pairwise-distinct template inline auto insert_absent_terms(MPOperator &op, size_t n, KeyAt &&key_at, PerSlot &&per_slot) -> size_t { const size_t base = op.store->grow_rows_geometric(n); @@ -294,28 +286,19 @@ struct MPOperatorMemoryBreakdown final { size_t init_operator_bytes{0uz}; size_t initial_state_bytes{0uz}; size_t inverted_index_bytes{0uz}; - // The MatchedEpochSet stamp array. Propagator-owned, so 0 unless MonomialPropagator fills it in. size_t matched_scratch_bytes{0uz}; - // The group's transport, per process: 0 unless the partitioned facade fills it in (see matched_scratch). size_t transport_bytes{0uz}; - - // Diagnostics: breakdowns of the fields above, deliberately excluded from total_bytes() so they can - // never double-count. - size_t inverted_index_dense_bytes{0uz}; // of inverted_index_bytes: full-height bitmap columns - size_t inverted_index_sparse_bytes{0uz}; // of inverted_index_bytes: ascending set-row lists + // Diagnostics: breakdowns of the fields above + size_t inverted_index_dense_bytes{0uz}; // full-height bitmap columns + size_t inverted_index_sparse_bytes{0uz}; // ascending set-row lists size_t inverted_index_dense_columns{0uz}; - size_t operator_terms_slack_bytes{0uz}; // of operator_terms_bytes: unused geometric-growth capacity - // of state_coeffs_bytes: entries of the state that are not exactly 0.0 + size_t operator_terms_slack_bytes{0uz}; // unused geometric-growth capacity size_t state_coeffs_nonzero{0uz}; - // Live entries behind init_operator_bytes, which is bucket_count(): bytes with no entries are dead buckets. size_t init_operator_entries{0uz}; size_t transport_staging_bytes{0uz}; // of transport_bytes: the payload funnel, at its high-water mark size_t inverted_index_slack_bytes{0uz}; // of inverted_index_bytes: capacity no resize ever wrote size_t coeff_slack_bytes{0uz}; // of op_coeffs_bytes + state_coeffs_bytes: likewise - // Never written, yet peak RSS tracks CAPACITY not size (job 1851566): each x1.5 growth holds old+new at - // once, so this is deferred cost, not free space -- and shrink_to_fit adds a copy rather than removing one. size_t reserved_bytes{0uz}; - // Peaks over TIME: a growth holds old+new at once. Summed, an upper bound -- peaks need not coincide. size_t operator_terms_peak_bytes{0uz}; size_t indexing_peak_bytes{0uz}; From d67182ffdd02dc487d6c9f70e681abd2507b9e5a Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 09:47:58 +0100 Subject: [PATCH 3/8] =?UTF-8?q?style(operator):=20=F0=9F=8E=A8=20clang-for?= =?UTF-8?q?mat=20MPOperator=20comment=20alignment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 --- cpp/monoprop/detail/operator/MPOperator.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index 3cc1024e..efe1cbd7 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -65,8 +65,8 @@ struct MPOperator { std::unique_ptr> store{std::make_unique>()}; VecD op_coeffs; std::vector state_rows_; // sparse state - VecD state_vals_; // parallel to state_rows_; every entry is a unit phase (+-1), never 0 - size_t state_scored_rows_{0uz}; // cache of the state values (its super sparse so this is useful) + VecD state_vals_; // parallel to state_rows_; every entry is a unit phase (+-1), never 0 + size_t state_scored_rows_{0uz}; // cache of the state values (its super sparse so this is useful) VecD state_coeffs; MonomialMap init_op_map{}; VecZ initial_state; From c828609f5cde0d0fb8cf523e1ec50a86e6bb4127 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 27 Aug 2026 11:19:33 +0100 Subject: [PATCH 4/8] =?UTF-8?q?fix(memory):=20=F0=9F=A9=B9=20address=20rev?= =?UTF-8?q?iew,=20and=20make=20each=20peak=20a=20peak=20of=20its=20resting?= =?UTF-8?q?=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review: - ProcessMemory reads /proc via ifstream, so fopen's "e" mode flag is no longer a portability question that could silently zero VmRSS/VmHWM. - reserve_rows reads the growth duplicate off the POST-reserve capacity; reserve() may hand back more than asked. - Restores the insert_absent_terms contract 20bd33ba truncated mid-sentence: currently-absent keys and per_slot-writes-before-bulk_insert are preconditions the code does not enforce. Restores the state_scored_rows_ comment, which the same commit replaced with a wrong one (it is a scored-row watermark, not a cache). Correctness of the peaks. rows_peak_bytes() omitted the overflow map while operator_terms_bytes counts it, so an operator with enough spilled rows reported a peak BELOW its resting bytes and a negative growth duplicate; index_peak_bytes() compared a table-only peak against a figure including sizeof(OperatorIndex). Both now carry their resting field's other terms, and both counters are stored in elements so the two accessors read the same way. Covered by a new test whose keys have popcount > the inline width -- grow_in_batches cannot reach the overflow path, its keys top out at popcount 9. /simplify: - capacity_bytes / capacity_slack_bytes move to detail/MemoryBytes.h, deducing the element size. Replaces HybridComm's private cap_bytes_, three hand-rolled copies in InvertedIndex::slack_bytes(), and the explicit `elem` argument at four sites. - reserved_bytes is derived from the three slacks it rolls up rather than stored, removing a fourth state to keep in sync and an operator+= line. - process_memory() holds its documented "never throws" in code: a noexcept wrapper round an inner reader, so a throw yields a zero-filled result rather than a partially-filled one. malloc_info's open_memstream buffer is now owned, so the string copy throwing cannot leak it. - Named constants for the malloc_info tags and raw string literals for the XML. Gated on Deucalion dev-x86 (job 1853765, _core.so ab0fc4055a03): ctest -L unit 247/247, -L serial 246/246, both including the new case. The prior revision passed the full 2-node sweep (job 1853743): 245 serial, 1/1 mpi, 593 Python x 10 layouts. Assisted-by: ClaudeCode:claude-opus-5 --- cpp/monoprop/detail/CMakeLists.txt | 1 + cpp/monoprop/detail/MemoryBytes.h | 33 ++++++++++++ cpp/monoprop/detail/ProcessMemory.cpp | 56 +++++++++++--------- cpp/monoprop/detail/ProcessMemory.h | 2 +- cpp/monoprop/detail/mpi/HybridComm.h | 28 ++++++---- cpp/monoprop/detail/mpi/ShmComm.h | 5 +- cpp/monoprop/detail/operator/InvertedIndex.h | 8 +-- cpp/monoprop/detail/operator/MPOperator.h | 43 ++++++++------- cpp/monoprop/detail/operator/OperatorIndex.h | 33 +++++++----- cpp/tests/mp_operator_tests.cpp | 50 +++++++++++++---- src/monoprop/bindings/binder.h | 2 +- 11 files changed, 173 insertions(+), 88 deletions(-) create mode 100644 cpp/monoprop/detail/MemoryBytes.h diff --git a/cpp/monoprop/detail/CMakeLists.txt b/cpp/monoprop/detail/CMakeLists.txt index faea724a..e9b76b93 100644 --- a/cpp/monoprop/detail/CMakeLists.txt +++ b/cpp/monoprop/detail/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources( TYPE HEADERS FILES "EnvConfig.h" + "MemoryBytes.h" "ProcessMemory.h" ) diff --git a/cpp/monoprop/detail/MemoryBytes.h b/cpp/monoprop/detail/MemoryBytes.h new file mode 100644 index 00000000..fa495851 --- /dev/null +++ b/cpp/monoprop/detail/MemoryBytes.h @@ -0,0 +1,33 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace monoprop::detail { + +/// Bytes a container has taken from the allocator. Variadic so a roll-up over many members is one call. +template +[[nodiscard]] inline auto capacity_bytes(const Vecs &...vecs) -> size_t { + return (0uz + ... + (vecs.capacity() * sizeof(typename Vecs::value_type))); +} + +/// Of capacity_bytes(): reserved and never written. Unfaulted, but not free -- a growth holds old+new at once. +template +[[nodiscard]] inline auto capacity_slack_bytes(const Vecs &...vecs) -> size_t { + return (0uz + ... + ((vecs.capacity() - vecs.size()) * sizeof(typename Vecs::value_type))); +} + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/ProcessMemory.cpp b/cpp/monoprop/detail/ProcessMemory.cpp index af5e7858..21badab9 100644 --- a/cpp/monoprop/detail/ProcessMemory.cpp +++ b/cpp/monoprop/detail/ProcessMemory.cpp @@ -15,9 +15,11 @@ #include "monoprop/detail/ProcessMemory.h" #include -#include #include #include +#include +#include +#include #include #include @@ -29,17 +31,8 @@ namespace monoprop::detail { namespace { auto slurp(const char *path) -> std::string { - std::string out; - FILE *file = std::fopen(path, "re"); - if (file == nullptr) { - return out; - } - std::array buf{}; - for (size_t n = 0; (n = std::fread(buf.data(), 1, buf.size(), file)) > 0;) { - out.append(buf.data(), n); - } - (void)std::fclose(file); - return out; + std::ifstream in(path); + return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; } auto digits_at(std::string_view text, size_t at) -> size_t { @@ -59,14 +52,17 @@ auto status_field(std::string_view text, std::string_view key) -> size_t { return digits_at(text, text.find_first_of("0123456789", at + key.size())) * 1024; } +constexpr std::string_view kSizeAttr{R"(size=")"}; +constexpr std::string_view kHeapTag{R"( size_t { const auto at = xml.rfind(tag); if (at == std::string_view::npos) { return 0uz; } - const auto size_at = xml.find("size=\"", at); - return size_at == std::string_view::npos ? 0uz : digits_at(xml, size_at + 6); + const auto size_at = xml.find(kSizeAttr, at); + return size_at == std::string_view::npos ? 0uz : digits_at(xml, size_at + kSizeAttr.size()); } auto malloc_info_xml() -> std::string { @@ -80,17 +76,16 @@ auto malloc_info_xml() -> std::string { } const int rc = ::malloc_info(0, stream); (void)std::fclose(stream); - std::string out = (rc == 0 && buf != nullptr) ? std::string(buf, len) : std::string{}; - std::free(buf); // NOLINT(cppcoreguidelines-no-malloc) -- open_memstream's buffer is malloc'd - return out; + // Owns the buffer before the copy below: constructing the string can throw, and free() must still run. + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc) -- open_memstream's buffer is malloc'd + const std::unique_ptr owned(buf, &std::free); + return (rc == 0 && buf != nullptr) ? std::string(buf, len) : std::string{}; #else return {}; #endif } -} // namespace - -auto process_memory() -> ProcessMemory { +auto read_process_memory() -> ProcessMemory { ProcessMemory out; #if defined(__linux__) const std::string status = slurp("/proc/self/status"); @@ -102,15 +97,28 @@ auto process_memory() -> ProcessMemory { return out; } // `system current` is arenas only; mmap'd chunks are separate and never free (free() unmaps them). - const size_t mmapped = last_size_attr(xml, " ProcessMemory { + try { + return read_process_memory(); + } + catch (...) { + return {}; + } +} + } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/ProcessMemory.h b/cpp/monoprop/detail/ProcessMemory.h index 41513a9f..6719b20d 100644 --- a/cpp/monoprop/detail/ProcessMemory.h +++ b/cpp/monoprop/detail/ProcessMemory.h @@ -32,6 +32,6 @@ struct ProcessMemory { }; /// Zero-filled where the platform cannot answer, and never throws: a diagnostic must not fail its caller. -auto process_memory() -> ProcessMemory; +auto process_memory() noexcept -> ProcessMemory; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h index 937ed588..1453519d 100644 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ b/cpp/monoprop/detail/mpi/HybridComm.h @@ -30,6 +30,7 @@ #include +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/mpi/CheckedCount.h" #include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/PartitionBarrier.h" @@ -100,14 +101,24 @@ class HybridComm { // Per PROCESS, not per partition; capacities, so staging reads at the largest exchange's high-water mark. [[nodiscard]] auto staging_bytes() const -> size_t { - return stage_send_.capacity() + stage_recv_.capacity() + (red_vec_.capacity() * sizeof(double)); + return monoprop::detail::capacity_bytes(stage_send_, stage_recv_, red_vec_); } [[nodiscard]] auto memory_bytes() const -> size_t { - return sizeof(HybridComm) + staging_bytes() + cap_bytes_(slots_) + cap_bytes_(counts_send_) - + cap_bytes_(counts_recv_) + cap_bytes_(mpi_send_counts_) + cap_bytes_(mpi_send_displs_) - + cap_bytes_(mpi_recv_counts_) + cap_bytes_(mpi_recv_displs_) + cap_bytes_(pack_off_) - + cap_bytes_(base_send_) + cap_bytes_(base_recv_) + cap_bytes_(col_sum_) + cap_bytes_(recv_col_) - + cap_bytes_(counts_matrix_store_) + cap_bytes_(rows_store_); + return sizeof(HybridComm) + staging_bytes() + + monoprop::detail::capacity_bytes(slots_, + counts_send_, + counts_recv_, + mpi_send_counts_, + mpi_send_displs_, + mpi_recv_counts_, + mpi_recv_displs_, + pack_off_, + base_send_, + base_recv_, + col_sum_, + recv_col_, + counts_matrix_store_, + rows_store_); } auto global_rank(int local_partition) const -> int { return mpi_rank_ * s_ + local_partition; } @@ -600,11 +611,6 @@ class HybridComm { auto sync() -> void { barrier_.sync(); } - template - static auto cap_bytes_(const std::vector &v) -> size_t { - return v.capacity() * sizeof(T); - } - MPI_Comm parent_; int s_; int r_ = 1; diff --git a/cpp/monoprop/detail/mpi/ShmComm.h b/cpp/monoprop/detail/mpi/ShmComm.h index f6c1e8e7..b41fb82f 100644 --- a/cpp/monoprop/detail/mpi/ShmComm.h +++ b/cpp/monoprop/detail/mpi/ShmComm.h @@ -24,6 +24,7 @@ #include #include +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/mpi/CheckedCount.h" #include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/PartitionBarrier.h" @@ -46,7 +47,9 @@ class ShmComm { auto size() const -> int { return n_; } // Per PROCESS, not per partition. R == 1 has no funnel: alltoallv memcpys out of published pointers. - [[nodiscard]] auto memory_bytes() const -> size_t { return sizeof(ShmComm) + (slots_.capacity() * sizeof(Slot)); } + [[nodiscard]] auto memory_bytes() const -> size_t { + return sizeof(ShmComm) + monoprop::detail::capacity_bytes(slots_); + } [[nodiscard]] auto staging_bytes() const -> size_t { return 0uz; } // recv_counts[s] = what rank s sends to me (the transpose of the send-count matrix). diff --git a/cpp/monoprop/detail/operator/InvertedIndex.h b/cpp/monoprop/detail/operator/InvertedIndex.h index feecc107..4be58dd6 100644 --- a/cpp/monoprop/detail/operator/InvertedIndex.h +++ b/cpp/monoprop/detail/operator/InvertedIndex.h @@ -25,6 +25,7 @@ #include #include "monoprop/TypeAliases.h" +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/operator/RowAccess.h" namespace monoprop::detail { @@ -205,12 +206,11 @@ struct InvertedIndex { // Diagnostic: the part of memory_bytes() no resize ever wrote. Unfaulted, but see reserved_bytes. auto slack_bytes() const -> size_t { - size_t total = 0; + size_t total = capacity_slack_bytes(row_parity_); for (const auto &col : cols) { - total += (col.words.capacity() - col.words.size()) * sizeof(uint64_t); - total += (col.set_rows.capacity() - col.set_rows.size()) * sizeof(TermIndex); + total += capacity_slack_bytes(col.words, col.set_rows); } - return total + ((row_parity_.capacity() - row_parity_.size()) * sizeof(uint64_t)); + return total; } // Diagnostic tier split of memory_bytes(): {dense_bytes, sparse_bytes, dense_columns}. diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index efe1cbd7..ac8c3032 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -29,6 +29,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/Utilities.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/MemoryBytes.h" #include "monoprop/detail/operator/InvertedIndex.h" #include "monoprop/detail/operator/OperatorIndex.h" @@ -64,9 +65,10 @@ struct MPOperator { // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr. Always non-null. std::unique_ptr> store{std::make_unique>()}; VecD op_coeffs; - std::vector state_rows_; // sparse state - VecD state_vals_; // parallel to state_rows_; every entry is a unit phase (+-1), never 0 - size_t state_scored_rows_{0uz}; // cache of the state values (its super sparse so this is useful) + // Sparse: only fully-paired terms score nonzero, ~0.07% of rows on production models. Strictly ascending. + std::vector state_rows_; + VecD state_vals_; // parallel to state_rows_; every entry is a unit phase (+-1), never 0 + size_t state_scored_rows_{0uz}; // rows [0, state_scored_rows_) have been scored into state_rows_/state_vals_ VecD state_coeffs; MonomialMap init_op_map{}; VecZ initial_state; @@ -254,7 +256,8 @@ struct MPOperator { } }; -// Callers must pass pairwise-distinct +// Keys must be pairwise-distinct AND currently absent: bulk_insert then skips duplicate probes, so slot k +// lands at base+k. per_slot must write row base+k before returning; op.size() must equal the returned base. template inline auto insert_absent_terms(MPOperator &op, size_t n, KeyAt &&key_at, PerSlot &&per_slot) -> size_t { const size_t base = op.store->grow_rows_geometric(n); @@ -266,12 +269,6 @@ inline auto insert_absent_terms(MPOperator &op, size_t n, KeyAt &&key_ return base; } -// Reserved-but-never-written capacity. Unfaulted -- but not free: see reserved_bytes. -template -inline auto capacity_slack_bytes(const Vec &v, size_t elem) -> size_t { - return (v.capacity() - v.size()) * elem; -} - template inline auto unordered_flat_map_storage_bytes(const FlatMap &map) -> size_t { return sizeof(FlatMap) + map.bucket_count() * (sizeof(typename FlatMap::value_type) + sizeof(unsigned char)); @@ -286,9 +283,11 @@ struct MPOperatorMemoryBreakdown final { size_t init_operator_bytes{0uz}; size_t initial_state_bytes{0uz}; size_t inverted_index_bytes{0uz}; - size_t matched_scratch_bytes{0uz}; + size_t matched_scratch_bytes{0uz}; // MatchedEpochSet stamps; propagator-owned, so 0 unless it fills them + // Group-owned, so the partitioned facade ASSIGNS it after sum_partitions_; the += below is for + // hand-built breakdowns only, and summing two already-partitioned breakdowns would double it. size_t transport_bytes{0uz}; - // Diagnostics: breakdowns of the fields above + // Diagnostics: breakdowns of the fields above, outside total_bytes() so they can never double-count. size_t inverted_index_dense_bytes{0uz}; // full-height bitmap columns size_t inverted_index_sparse_bytes{0uz}; // ascending set-row lists size_t inverted_index_dense_columns{0uz}; @@ -298,9 +297,15 @@ struct MPOperatorMemoryBreakdown final { size_t transport_staging_bytes{0uz}; // of transport_bytes: the payload funnel, at its high-water mark size_t inverted_index_slack_bytes{0uz}; // of inverted_index_bytes: capacity no resize ever wrote size_t coeff_slack_bytes{0uz}; // of op_coeffs_bytes + state_coeffs_bytes: likewise - size_t reserved_bytes{0uz}; - size_t operator_terms_peak_bytes{0uz}; - size_t indexing_peak_bytes{0uz}; + size_t operator_terms_peak_bytes{0uz}; // peak over TIME, so a sum over partitions is an upper bound + size_t indexing_peak_bytes{0uz}; // likewise + + // Derived, not stored: reserved and never written, but NOT free. Peak RSS tracks capacity rather than + // used bytes, because each growth holds the old fully-written buffer and the new one at once -- so + // shrink_to_fit cannot recover this and must cost another copy. Only fewer or earlier growths help. + auto reserved_bytes() const -> size_t { + return operator_terms_slack_bytes + inverted_index_slack_bytes + coeff_slack_bytes; + } auto total_bytes() const -> size_t { return operator_terms_bytes + op_coeffs_bytes + state_coeffs_bytes + indexing_bytes + init_operator_bytes @@ -326,7 +331,6 @@ struct MPOperatorMemoryBreakdown final { transport_staging_bytes += o.transport_staging_bytes; inverted_index_slack_bytes += o.inverted_index_slack_bytes; coeff_slack_bytes += o.coeff_slack_bytes; - reserved_bytes += o.reserved_bytes; operator_terms_peak_bytes += o.operator_terms_peak_bytes; indexing_peak_bytes += o.indexing_peak_bytes; return *this; @@ -357,12 +361,7 @@ inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorM breakdown.operator_terms_slack_bytes = op.store->slack_bytes(); breakdown.operator_terms_peak_bytes = op.store->rows_peak_bytes(); breakdown.indexing_peak_bytes = op.store->index_peak_bytes(); - breakdown.coeff_slack_bytes = capacity_slack_bytes(op.op_coeffs, sizeof(double)) - + capacity_slack_bytes(op.state_coeffs, sizeof(double)) - + capacity_slack_bytes(op.state_rows_, sizeof(TermIndex)) - + capacity_slack_bytes(op.state_vals_, sizeof(double)); - breakdown.reserved_bytes = - breakdown.operator_terms_slack_bytes + breakdown.inverted_index_slack_bytes + breakdown.coeff_slack_bytes; + breakdown.coeff_slack_bytes = capacity_slack_bytes(op.op_coeffs, op.state_coeffs, op.state_rows_, op.state_vals_); // State phases are unit-magnitude, so at rest the scored count IS the nonzero count; a live vector needs a scan. breakdown.state_coeffs_nonzero = op.state_coeffs.empty() diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 1a220e87..541593b5 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -29,6 +29,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/MemoryBytes.h" namespace monoprop::detail { @@ -170,11 +171,7 @@ class OperatorIndex { } return overflow_.at(i).count(); } - [[nodiscard]] auto memory_bytes() const -> size_t { - size_t total = rows_.capacity() * sizeof(PosT); - total += overflow_.size() * (sizeof(value_type) + sizeof(size_t) + 24); - return total; - } + [[nodiscard]] auto memory_bytes() const -> size_t { return capacity_bytes(rows_) + overflow_bytes_(); } auto find(const key_type &key) const -> std::optional { const uint32_t h = fold_hash(key); @@ -276,12 +273,14 @@ class OperatorIndex { return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); } - // Diagnostics: peak over TIME. A growth holds old+new at once; no capacity field can show that. + // Diagnostics: peak over TIME of memory_bytes() / index_estimated_memory_bytes(). A growth holds + // old+new at once; no capacity field can show that. Each carries its resting field's other terms + // (the overflow map, the fixed header) so a peak can never read below the resting bytes. [[nodiscard]] auto rows_peak_bytes() const -> size_t { - return std::max(rows_peak_bytes_, rows_.capacity() * sizeof(PosT)); + return overflow_bytes_() + (std::max(rows_peak_elems_, rows_.capacity()) * sizeof(PosT)); } [[nodiscard]] auto index_peak_bytes() const -> size_t { - return std::max(table_.peak_bytes, index_estimated_memory_bytes()); + return sizeof(OperatorIndex) + (std::max(table_.peak_slots, table_.slots.capacity()) * sizeof(Slot)); } private: @@ -329,7 +328,7 @@ class OperatorIndex { std::vector slots = std::vector(kMinSlots, Slot{}); size_t mask = kMinSlots - 1; size_t count = 0; - size_t peak_bytes = 0; // see index_peak_bytes() + size_t peak_slots = 0; // see index_peak_bytes() auto rehash_if_needed() -> void { if ((count + 1) * 10 >= slots.size() * 7) { @@ -342,7 +341,7 @@ class OperatorIndex { return; } // assign() allocates before `old` dies, so both tables are live for the whole re-probe loop. - peak_bytes = std::max(peak_bytes, (slots.size() + new_cap) * sizeof(Slot)); + peak_slots = std::max(peak_slots, slots.size() + new_cap); std::vector old = std::move(slots); slots.assign(new_cap, Slot{}); mask = new_cap - 1; @@ -362,13 +361,21 @@ class OperatorIndex { // Slot count for `n` entries at ≤0.7 load. static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } + // Node + key + value + the bucket pointer, for the rows whose popcount exceeded inline_width_. + [[nodiscard]] auto overflow_bytes_() const -> size_t { + return overflow_.size() * (sizeof(value_type) + sizeof(size_t) + 24); + } + [[nodiscard]] auto capacity() const -> size_t { return rows_.capacity() / stride_; } auto reserve_rows(size_t n) -> void { const size_t want = n * stride_; - if (want > rows_.capacity()) { - rows_peak_bytes_ = std::max(rows_peak_bytes_, (rows_.capacity() + want) * sizeof(PosT)); + if (want <= rows_.capacity()) { + return; // no realloc, so nothing is ever doubled } + // reserve() may hand back more than asked, so the duplicate is read off the post-growth capacity. + const size_t old_cap = rows_.capacity(); rows_.reserve(want); + rows_peak_elems_ = std::max(rows_peak_elems_, old_cap + rows_.capacity()); } auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } @@ -417,7 +424,7 @@ class OperatorIndex { // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; Table table_ = {}; - size_t rows_peak_bytes_ = 0; // see rows_peak_bytes() + size_t rows_peak_elems_ = 0; // PosT count, matching table_.peak_slots; see rows_peak_bytes() }; } // namespace monoprop::detail diff --git a/cpp/tests/mp_operator_tests.cpp b/cpp/tests/mp_operator_tests.cpp index cbb3e18c..0fdef50b 100644 --- a/cpp/tests/mp_operator_tests.cpp +++ b/cpp/tests/mp_operator_tests.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -85,7 +86,7 @@ auto sparse_state_equals(const detail::MPOperator<8>::SparseState &sparse, const // A distinct Monomial<8> per `bits`, so a caller can mint hundreds without enumerating index sets. auto monomial_from_bits(size_t bits) -> Monomial<8> { Monomial<8> mono; - for (size_t b = 0; b < 16; ++b) { + for (size_t b = 0; b < 2 * 8; ++b) { // 2 * NumModes if (((bits >> b) & 1UZ) != 0UZ) { mono.set(b); } @@ -96,13 +97,12 @@ auto monomial_from_bits(size_t bits) -> Monomial<8> { // One insert_absent_terms call per batch: a single bulk call reserves once from empty, duplicating nothing. auto grow_in_batches(detail::MPOperator<8> &op, size_t batches, size_t per_batch) -> void { for (size_t batch = 0; batch < batches; ++batch) { + const size_t first = (batch * per_batch) + 1; detail::insert_absent_terms<8>( op, per_batch, - [&](size_t k) { return monomial_from_bits((batch * per_batch) + k + 1); }, - [&](size_t k, size_t base) { - assign_row<8>(*op.store, base + k, monomial_from_bits((batch * per_batch) + k + 1)); - }); + [&](size_t k) { return monomial_from_bits(first + k); }, + [&](size_t k, size_t base) { assign_row<8>(*op.store, base + k, monomial_from_bits(first + k)); }); } } @@ -445,14 +445,13 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_counts_transport_in_total_and_sum) { BOOST_CHECK_GT(bare.total_bytes(), 0U); } -// All six are accumulated by operator+= and NONE reaches total_bytes(): summing them would double-count. +// All five are accumulated by operator+= and NONE reaches total_bytes(): summing them would double-count. BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_new_diagnostics_out_of_total) { detail::MPOperatorMemoryBreakdown<8> acc; acc.op_coeffs_bytes = 100; acc.transport_staging_bytes = 1; acc.inverted_index_slack_bytes = 2; acc.coeff_slack_bytes = 3; - acc.reserved_bytes = 4; acc.operator_terms_peak_bytes = 5; acc.indexing_peak_bytes = 6; BOOST_CHECK_EQUAL(acc.total_bytes(), 100U); @@ -462,7 +461,6 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_new_diagnostics_out_of_total) { other.transport_staging_bytes = 10; other.inverted_index_slack_bytes = 20; other.coeff_slack_bytes = 30; - other.reserved_bytes = 40; other.operator_terms_peak_bytes = 50; other.indexing_peak_bytes = 60; @@ -471,7 +469,6 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_new_diagnostics_out_of_total) { BOOST_CHECK_EQUAL(acc.transport_staging_bytes, 11U); BOOST_CHECK_EQUAL(acc.inverted_index_slack_bytes, 22U); BOOST_CHECK_EQUAL(acc.coeff_slack_bytes, 33U); - BOOST_CHECK_EQUAL(acc.reserved_bytes, 44U); BOOST_CHECK_EQUAL(acc.operator_terms_peak_bytes, 55U); BOOST_CHECK_EQUAL(acc.indexing_peak_bytes, 66U); } @@ -494,6 +491,37 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_growth_peaks_exceed_resting_capacity) BOOST_CHECK_LT(grown.indexing_peak_bytes, 3U * grown.indexing_bytes); } +// A peak must be a peak OF the resting field. Rows whose popcount exceeds the inline width spill to the +// overflow map, which operator_terms_bytes counts -- so a peak that omitted them could read BELOW resting +// and make the growth duplicate (peak - resting) negative. grow_in_batches cannot reach this: its keys +// come from bits <= 600, whose popcount is at most 9, so every row it makes stays inline. +BOOST_AUTO_TEST_CASE(mp_operator_breakdown_peak_covers_overflow_rows) { + constexpr size_t kBatches = 10; + constexpr size_t kPerBatch = 20; + // The default inline width is 11 of the 16 positions, so every key below spills losslessly. + std::vector wide; + for (size_t bits = 0; bits < (1UZ << 16) && wide.size() < kBatches * kPerBatch; ++bits) { + if (std::popcount(bits) > detail::OperatorIndex<8>::kDefaultInlinePositions) { + wide.push_back(bits); + } + } + BOOST_REQUIRE_EQUAL(wide.size(), kBatches * kPerBatch); + + detail::MPOperator<8> op; + for (size_t batch = 0; batch < kBatches; ++batch) { + const size_t first = batch * kPerBatch; + detail::insert_absent_terms<8>( + op, + kPerBatch, + [&](size_t k) { return monomial_from_bits(wide[first + k]); }, + [&](size_t k, size_t base) { assign_row<8>(*op.store, base + k, monomial_from_bits(wide[first + k])); }); + } + + const auto b = detail::estimate_memory_usage<8>(op); + BOOST_CHECK_GT(b.operator_terms_peak_bytes, b.operator_terms_bytes); + BOOST_CHECK_GT(b.indexing_peak_bytes, b.indexing_bytes); +} + // reserved_bytes rolls up the three slacks: reserved and unwritten, which is not the same as costing nothing. BOOST_AUTO_TEST_CASE(mp_operator_breakdown_reserved_bytes_rolls_up_the_slacks) { detail::MPOperator<8> op; @@ -502,12 +530,12 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_reserved_bytes_rolls_up_the_slacks) { (void)op.inverted_index(); const auto b = detail::estimate_memory_usage<8>(op); - BOOST_CHECK_EQUAL(b.reserved_bytes, + BOOST_CHECK_EQUAL(b.reserved_bytes(), b.operator_terms_slack_bytes + b.inverted_index_slack_bytes + b.coeff_slack_bytes); BOOST_CHECK_GT(b.operator_terms_slack_bytes, 0U); // geometric growth always overshoots BOOST_CHECK_GT(b.coeff_slack_bytes, 0U); // the reserve above BOOST_CHECK_LE(b.inverted_index_slack_bytes, b.inverted_index_bytes); - BOOST_CHECK_LT(b.reserved_bytes, b.total_bytes()); + BOOST_CHECK_LT(b.reserved_bytes(), b.total_bytes()); } // ASan and TSan replace malloc, so glibc's arenas stay empty and malloc_info(3) reports zero bytes. diff --git a/src/monoprop/bindings/binder.h b/src/monoprop/bindings/binder.h index caeeb279..929b4f4a 100644 --- a/src/monoprop/bindings/binder.h +++ b/src/monoprop/bindings/binder.h @@ -280,7 +280,7 @@ auto bind_monomial_propagator(nb::module_ &mod) -> void { {"d_transport_staging_bytes", b.transport_staging_bytes}, {"d_invidx_slack_bytes", b.inverted_index_slack_bytes}, {"d_coeff_slack_bytes", b.coeff_slack_bytes}, - {"d_reserved_bytes", b.reserved_bytes}, + {"d_reserved_bytes", b.reserved_bytes()}, {"d_terms_peak_bytes", b.operator_terms_peak_bytes}, {"d_indexing_peak_bytes", b.indexing_peak_bytes}, // Per PROCESS, not per partition. See the note above. From e88e2c4df3e13b3473e6e20e77b622e45d9b6e78 Mon Sep 17 00:00:00 2001 From: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:59:08 +0100 Subject: [PATCH 5/8] Clean up comments in MPOperator.h Removed comments explaining the transport_bytes variable. Signed-off-by: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> --- cpp/monoprop/detail/operator/MPOperator.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index ac8c3032..8a4a7178 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -284,8 +284,6 @@ struct MPOperatorMemoryBreakdown final { size_t initial_state_bytes{0uz}; size_t inverted_index_bytes{0uz}; size_t matched_scratch_bytes{0uz}; // MatchedEpochSet stamps; propagator-owned, so 0 unless it fills them - // Group-owned, so the partitioned facade ASSIGNS it after sum_partitions_; the += below is for - // hand-built breakdowns only, and summing two already-partitioned breakdowns would double it. size_t transport_bytes{0uz}; // Diagnostics: breakdowns of the fields above, outside total_bytes() so they can never double-count. size_t inverted_index_dense_bytes{0uz}; // full-height bitmap columns From f633c62a6c50b3743b308f4085a0c2858fddb337 Mon Sep 17 00:00:00 2001 From: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:00:08 +0100 Subject: [PATCH 6/8] Fix comment grammar in MPOperator.h Corrected grammatical error in comment regarding memory management. Signed-off-by: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> --- cpp/monoprop/detail/operator/MPOperator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index 8a4a7178..15073332 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -298,7 +298,7 @@ struct MPOperatorMemoryBreakdown final { size_t operator_terms_peak_bytes{0uz}; // peak over TIME, so a sum over partitions is an upper bound size_t indexing_peak_bytes{0uz}; // likewise - // Derived, not stored: reserved and never written, but NOT free. Peak RSS tracks capacity rather than + // Derived, not stored: reserved and never written, but not free. Peak RSS tracks capacity rather than // used bytes, because each growth holds the old fully-written buffer and the new one at once -- so // shrink_to_fit cannot recover this and must cost another copy. Only fewer or earlier growths help. auto reserved_bytes() const -> size_t { From 88cfebd1a55b6fd6ca7bfa55ea17925052f54733 Mon Sep 17 00:00:00 2001 From: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:01:36 +0100 Subject: [PATCH 7/8] Fix typo in OperatorIndex.h variable names Signed-off-by: Aaron Miller <61472721+diagonal-hamiltonian@users.noreply.github.com> --- cpp/monoprop/detail/operator/OperatorIndex.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 541593b5..1926e091 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -273,7 +273,7 @@ class OperatorIndex { return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); } - // Diagnostics: peak over TIME of memory_bytes() / index_estimated_memory_bytes(). A growth holds + // Diagnostics: over time of memory_bytes() / index_estimated_memory_bytes(). A growth holds // old+new at once; no capacity field can show that. Each carries its resting field's other terms // (the overflow map, the fixed header) so a peak can never read below the resting bytes. [[nodiscard]] auto rows_peak_bytes() const -> size_t { @@ -375,7 +375,7 @@ class OperatorIndex { // reserve() may hand back more than asked, so the duplicate is read off the post-growth capacity. const size_t old_cap = rows_.capacity(); rows_.reserve(want); - rows_peak_elems_ = std::max(rows_peak_elems_, old_cap + rows_.capacity()); + rows__elems_ = std::max(rows__elems_, old_cap + rows_.capacity()); } auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } @@ -424,7 +424,7 @@ class OperatorIndex { // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; Table table_ = {}; - size_t rows_peak_elems_ = 0; // PosT count, matching table_.peak_slots; see rows_peak_bytes() + size_t rows__elems_ = 0; // PosT count, matching table_._slots; see rows__bytes() }; } // namespace monoprop::detail From ac8203e2291d00bbdb8789dd3f5434a4f08fa77e Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Fri, 28 Aug 2026 11:08:29 +0100 Subject: [PATCH 8/8] Revert "Fix typo in OperatorIndex.h variable names" This reverts commit 88cfebd1a55b6fd6ca7bfa55ea17925052f54733. --- cpp/monoprop/detail/operator/OperatorIndex.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 1926e091..541593b5 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -273,7 +273,7 @@ class OperatorIndex { return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); } - // Diagnostics: over time of memory_bytes() / index_estimated_memory_bytes(). A growth holds + // Diagnostics: peak over TIME of memory_bytes() / index_estimated_memory_bytes(). A growth holds // old+new at once; no capacity field can show that. Each carries its resting field's other terms // (the overflow map, the fixed header) so a peak can never read below the resting bytes. [[nodiscard]] auto rows_peak_bytes() const -> size_t { @@ -375,7 +375,7 @@ class OperatorIndex { // reserve() may hand back more than asked, so the duplicate is read off the post-growth capacity. const size_t old_cap = rows_.capacity(); rows_.reserve(want); - rows__elems_ = std::max(rows__elems_, old_cap + rows_.capacity()); + rows_peak_elems_ = std::max(rows_peak_elems_, old_cap + rows_.capacity()); } auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } @@ -424,7 +424,7 @@ class OperatorIndex { // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; Table table_ = {}; - size_t rows__elems_ = 0; // PosT count, matching table_._slots; see rows__bytes() + size_t rows_peak_elems_ = 0; // PosT count, matching table_.peak_slots; see rows_peak_bytes() }; } // namespace monoprop::detail