diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 705b5695..956bf2f2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,6 +16,9 @@ "source=monoprop-docs-next,target=${containerWorkspaceFolder}/docs/.next,type=volume", "source=monoprop-docs-node-modules,target=${containerWorkspaceFolder}/docs/node_modules,type=volume" ], + "runArgs": [ + "--shm-size=1g" + ], "remoteUser": "vscode", "customizations": { "vscode": { diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b6e4023..c009c80e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -132,37 +132,37 @@ jobs: using namespace monoprop; auto main() -> int { - constexpr size_t kModes = 2; OperatorDict ham; ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; // Graph-building / Schrodinger path: detail/graph_encoding/MPGraphEncodingStorage.h. - MonomialPropagator graph_sim(ham, - 2 * kModes, - VecZ{0, 1}, - std::optional{4U}, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt); + MonomialPropagator graph_sim(ham, + 6, + VecZ{0, 1}, + 4, + std::optional{4U}, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt); const std::vector monos{{0}, {1}, {2}}; graph_sim.build_graph(monos, VecZ{0, 1, 2}, VecD{1.0, 1.0, 1.0}); graph_sim.graph_memory_usage(); graph_sim.expectation_value_and_gradient(VecD{0.1, 0.2, 0.3}); - MonomialPropagator partition_sim(ham, - 2 * kModes, - VecZ{0, 1}, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kModes, - Basis::Majorana, - 2); + MonomialPropagator partition_sim(ham, + 6, + VecZ{0, 1}, + 4, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt, + Basis::Majorana, + 2); partition_sim.size(); return 0; diff --git a/.gitignore b/.gitignore index 9b2e2a80..ea61186d 100644 --- a/.gitignore +++ b/.gitignore @@ -161,7 +161,6 @@ Thumbs.db external/upstream/_srcs/ tests/cpp/_srcs/ -_dispatch*.py _constants.py build*/ Testing/ @@ -178,6 +177,11 @@ benches/results/** # devcontainer files .devcontainer/devcontainer-lock.json +notes/** + +# `just capture-baseline` / `just diff-baseline` output (tools/capture-baseline.py) +.baseline-capture/** + # Useful when running in clusters logs/ diff --git a/AGENTS.md b/AGENTS.md index cf2f2fd0..c7c92020 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,8 +36,8 @@ monoprop is a high-performance C++/Python hybrid library implementing Majorana a - **Core C++ Engine**: Public headers in `cpp/include/monoprop/` and implementation in `cpp/monoprop/` - **Python Interface**: User-facing API in `src/monoprop/` with C++ bindings in `src/monoprop/bindings/` -- **Template-Based Design**: Heavily templated C++ code with compile-time mode limits (`monoprop_MAX_NUM_MODES`) -- **Generated Code**: Python dispatch and C++ bindings auto-generated via `tools/generate-*.py` +- **Runtime mode width**: there is no compile-time ceiling on the mode count. A propagator takes its + logical width as a constructor argument and sizes its monomial storage from it at runtime. - **uv workspace**: the repository root is the `monoprop` package; `packages/*` holds the sibling distributions. See "Workspace layout" below. @@ -68,13 +68,17 @@ Key files: - `src/monoprop/monomial_propagator.py`: abstract base `MonomialPropagator`; the concrete user-facing front-ends are `src/monoprop/majorana_propagator.py` (`MajoranaPropagator`) and `src/monoprop/pauli_propagator.py` (`PauliPropagator`). -- `cpp/include/monoprop/MonomialPropagator.h`: the single templated C++ engine `MonomialPropagator` - (the Majorana/Pauli choice is a runtime `Basis`, not a separate class). Its `only_rotate_len_k` - arguments use `std::optional`; `std::nullopt` means no gate-application length cap. -- `src/monoprop/bindings/binder.h`: hand-written binding template; `tools/generate-*.py` generate the - per-mode-width `bindings.cpp` and `_dispatch.py` from it (do not hand-edit the generated files). - Both generators take the 32-mode storage-block rule from `tools/_binding_layout.py` — they must - agree, or dispatch routes at a template the bindings never instantiated. +- `cpp/include/monoprop/MonomialPropagator.h`: the single C++ engine `MonomialPropagator` (the + Majorana/Pauli choice is a runtime `Basis`, not a separate class; so is the mode count). + The ctor takes one width, `num_modes`, the logical one. `detail::storage_modes_for()` derives the width + monomials are actually stored at from it — rounded up to a whole 32-mode block, never below one + block. + Test code that reads a propagator's monomials therefore has to take the width from the monomial + (`mono.size()`) or from `storage_num_modes()`, never from the logical width. +- `src/monoprop/bindings/bindings.cpp`: the binding for that one class, in one translation unit. + Nothing here is generated -- there is no `binder.h` and no `bindings.cpp.in` any more, so the nanobind + version the module reports arrives as `monoprop_NANOBIND_VERSION` from CMake rather than through a + configured template. - `CMakePresets.json`: the single source of truth for the supported C++ unit-test build/run entry points. The presets adopt the scikit-build-core trees generated by `uv sync`; regenerate the tree with `uv sync`, then use the matching `skbuild-*` preset to build or run CTest. @@ -86,21 +90,194 @@ Key files: series. - **`rounds > 1` overlaps two rounds' live memory** (`setup=` runs before the prior round's teardown) — pin `--bench-rounds=1`; `record_memory` measures that construction transient, - not per-op cost (use `op_memory`). -- **Peak memory is the kernel's `VmHWM` high-water mark** — exact, with no sampling. Under - MPI the ranks' peaks are summed, which errs high (disjoint transients, and shared pages - charged to every rank): an upper bound, good for regressions, not for provisioning. + not per-op cost (use `op_memory`). The overlap is not fixable from the benchmark's side — + pytest-benchmark holds the prior round's args across `setup=` — so anything that must be read + with nothing built is taken on the first round only, as `bench_models.py` does for its baseline + RSS. +- **Peak memory is `HighWaterMark`, not sampled PSS** — exact, unlike `/proc/self/smaps_rollup`. ### Core abstractions (the propagation backbone) -- **`Monomial`** (`cpp/monoprop/core/Monomial.h`) = `Bitset<2*N>`: ONE basis operator, two bits per - mode/qubit. Basis-agnostic — read as a Majorana product, or as a Pauli string (JW image). - Collections: `MonomialList` (no coeffs) and `MonomialMap` (monomial → real coeff). -- **Row access** (`cpp/monoprop/detail/operator/RowAccess.h`): the one backend-agnostic vocabulary - (`materialize_row`, `assign_row`, `row_popcount`, `for_each_row_position`) over the dense - `MonomialList` and the packed `detail::OperatorIndex`. Any template parameterized on the row - store must include that header — the `OperatorIndex` overloads live in `monoprop::`, so ADL cannot - find them from a `monoprop::detail` argument. +- **A monomial** (`cpp/monoprop/core/Monomial.h`): ONE basis operator, two bits per mode/qubit. + Basis-agnostic — read as a Majorana product, or as a Pauli string (JW image). Collections: + `MonomialList` (no coeffs) and `MonomialMap` (monomial → real coeff). + `Bitset` (`cpp/monoprop/Bitset.h`) *is* a monomial — there is no monomial type and no alias for one, + because the width is data, not a template parameter: the first 8 words are inline and wider bitsets + spill the whole word array to the heap. Per-word loops go through `detail::with_nwords`, which + dispatches a runtime word count to a compile-time-unrolled arm, in place of the `if constexpr` + branches a compile-time width allowed. A width therefore only exists per *value*: construct with one + (`Bitset(num_bits)`), never default-construct as a zero monomial (that is width 0 — copy-then-reset + instead), take a NumModes from `mono.size() / 2`, and recover a width with an instance call + (`x.size()`), never a qualified `decltype(x)::size()`. Above 8 words (256 modes) the words spill to the + heap, so memory accounting over a container of monomials must add `Bitset::heap_bytes()` per element; + `sizeof(Bitset)` per element looks right until someone runs a wide system. + Two consequences for per-term code, both measured: a `Bitset` is sized for the widest supported + width rather than its own, and constructing one is a real construction rather than a compile-time + constant. So in anything on the per-term path, write a word loop (`x.word(w) & m.word(w)`) instead + of an `a & b` / `^` / `>>` chain that materializes a temporary per step, and take masks from + `cached_even_bits` (`Utilities.h`) or a per-layer context rather than rebuilding them per call. + Within-word pair tricks like `(word >> 1) & even_mask` are safe because a mode's two bits are + `2m, 2m+1` and never straddle a word. + Only words `[0, num_words())` hold a value: the inline tail above them is deliberately left + indeterminate so a copy costs the operand's own width instead of the widest supported one. Every + reader — the word loops, `operator==`, `SplitmixHash`, the MPI readers — must therefore stop at + `num_words()`, and a new one that walks the whole inline array will read garbage rather than zeros. +- **The row-store seam**: a dense monomial is a transient, not the storage. `detail/operator/RowAccess.h` + declares four accessors — `materialize_row`, `assign_row`, `row_popcount`, `for_each_row_position` — and + three backends answer them: `std::vector`, `detail::OperatorIndex` (packed position lists) and + `detail::SparseRowStore` (fixed-width mode lanes plus one 2-bit-per-slot `codes` word per row). Reach + rows through the accessors, never through a backend's own API, and add any fourth backend to + `cpp/tests/row_accessor_tests.cpp`, which asserts that all of them agree through every accessor. + A row slot is sized at runtime from the width: `OperatorIndex` keeps one `uint8_t` array and one + `uint16_t` array with exactly one non-empty, bound per call by its private `with_rows`, because the + rows are the operator's largest array and one fixed `uint16_t` doubles the whole footprint at or below + 128 modes — where both shipping models sit. Rows are payload, never a hash input and never serialized, + so a widening there changes no term and no energy: `just diff-baseline` cannot see it, and the gate is + instead the `operator_terms_bytes` assertions in `cpp/tests/operator_index_tests.cpp` and + `tests/test_mode_width.py`. + `SparseRowStore` sizes its *codes* array the same way and for the same reason, over three arrays + (`with_codes`): a codes word is two bits per slot, so a store sized from a cutoff bound only ever sets + 12 of 64 bits at cutoff 6 and 16 at cutoff 8 — the two shipping models. Storing 2 bytes there instead of + 8 is worth **-30% and -25% of `operator_terms_bytes`** on the benchmark Hubbard model (20 → 14 and + 24 → 18 live bytes per row), which is the whole per-row gap to `OperatorIndex`'s `(1 + inline_width)` + payloads: at the 256-mode crossover the two backends' row arrays are now byte-for-byte equal. Only the + array narrows — every reader still sees a `CodesT`, zero-extended on load — so `CodesAlgebra.h`, + `sparse_row_hash` and `SparseRow` are untouched and both baselines stay byte-identical. Its own gate is + `codes_width_follows_the_slot_count` in `cpp/tests/sparse_row_store_tests.cpp`, since the Python one is + a dense-row property and skips under `monoprop_ROW_STORE=sparse`. + It is also not a time cost, and the way it is not is the lesson: the dispatch and the narrowing casts + are worth **+2.2% user instructions**, but `cycles:u` falls 1-2% and task-clock with it, because four + times as many rows fit each line of the array the cutoff walk reads. Measured paired over alternating + builds, `monoprop_ROW_STORE=sparse` forced, pinned single-threaded; instructions reproduce to five + significant figures, cycles to about 1%. Peak RSS on that run fell 7.8%, and minor faults rose 5.7% -- + the codes array reallocates at different sizes, which is a growth-pattern effect, not a footprint one. + The mode lanes are the remaining gap and are *not* narrowable the same way: below 160 storage modes + `OperatorIndex` puts positions in a `uint8_t` where `RowMode` stays `uint16_t`, so sparse is still 2x + dense there — but `kPadLane`/`kOverflowLane` take the top two `RowMode` values, so a byte lane would + cap at 254 modes, under the 256 the crossover sits at. Moving those markers into spare codes bits is + the only route, and it buys the threshold width and nothing above it. +- **Which backend, and where it is bound**: a propagator uses one of the two, chosen once from its + storage width by `SparseRowStore::preferred_for_modes()` — a build-time constant + (`monoprop_SPARSE_ROW_MIN_MODES`, derived in `CMakeLists.txt` from whether `ARCH_FLAG` is actually + emitted rather than from the option that asks for it, and deliberately not a cache entry) because what + moves the crossover is the target ISA. `monoprop_ROW_STORE=dense|sparse` forces it process-wide; an + unrecognized value throws rather than falling back, since the point of setting it is to know which + backend ran. `MPOperator` holds one pointer per backend with exactly one non-null and binds the live + one via `with_store` — **once per layer, inside `build_layer`**, never per term: the scan asks the + store for a row per anticommuting term, so everything downstream is templated on the store + (`LayerBuildEngine`, `fused_find_and_collect`, `probe_incoming_queries`). Off that path, + use `MPOperator`'s forwarding accessors; there is no accessor handing out a store, because there is no + one type to hand out. Every C++ case runs a second time under `monoprop_ROW_STORE=sparse` (the + `sparse-rows` ctest label) — every fixture is below the crossover, so without that the sparse backend + would ship untested. The two backends agree on term sets and values but not on term *order*, so + compare them with `just diff-baseline-sparse` (tolerance), never `just diff-baseline` (byte-wise). + A benchmark run records the backend it resolved to in its artifact's `meta` (`monoprop_row_store` as + asked, `row_store_effective` as run) and `REPORT.md` shows both: under the default `auto` the setting + alone does not identify the backend, and the two differ in footprint and in accumulation order. + `algebra/CodesAlgebra.h` is the structural algebra on a sparse row, one function per dense + counterpart, reading the `codes` word instead of looping over storage words, plus `sparse_toggle` -- + the product `M ⊕ G` as one merge over two ascending lane arrays, which is the per-term operation the + representation exists for. It is exact, not an approximation: `cpp/tests/codes_algebra_tests.cpp` and + `codes_product_tests.cpp` assert agreement with each dense version over the fixtures and randomized + rows, and those tests are the gate on making the codes form the default. Change one side and you must + change the other. A product can occupy more modes than either input, so a scratch row is sized + `CutoffEvaluator::max_mode_bound()` **plus the generator's locality**; past its capacity + `sparse_toggle` reports `overflowed` and the caller must fall back to the dense product — never + truncate, because a truncated mode list still carries a plausible-looking `codes` word. +- **The per-term kernel seam**: everything the scan asks about one anticommuting term — the product, + the overlap, the rotation sign, the structural cutoff, the owner rank and the query record — goes + through the per-gate object `TermProductsFor` selects (`layer_build/TermProduct.h`), so the + scan itself names no representation. `SparseTermProducts` answers the first four off the `codes` word + and falls back to `DenseTermProducts` per term when there is no row to read (a spilled store row, a + product past the scratch capacity) or no codes form of the cutoff (`CutoffEvaluator` recovered neither + concrete functor, e.g. under a basis change). `cpp/tests/term_product_tests.cpp` compares the two + kernels answer for answer: extend it with any new answer, or that answer ships untested. +- **The third thing bound once per layer**: the storage word count, beside the algebra and the backend. + `with_kernel_width` (`layer_build/TermProduct.h`, over `detail::with_nwords`) turns + `gen.num_words()` into a template parameter `W` at the same seam in `build_layer`, and + `fused_find_and_collect` and the `TermProductsFor` specialization + `DenseTermProductsW` are templated on it, so every per-term word loop has a compile-time trip + count and every operand's storage pointer is resolved once per gate — which is what a `Bitset` + used to give for free. Measured worth ~10% at two and four storage words and nothing at seven or eight, + so `kNarrowKernelWords` (`TermProduct.h`, 4) caps which widths get an instantiation; the cost is ~11% + of `.text`. Not a build option, unlike `monoprop_SPARSE_ROW_MIN_MODES`: the cap is a *storage word* + count, so it names the same width regime on every machine, where the sparse crossover has to follow + the target ISA. Two conditions on that number, both measured. It is the **Majorana** path: + the Pauli rotation sign already loops over the generator's non-zero words only, so `W` binds no trip + count there and the 127-qubit kicked-Ising model gains ~1%. And it scales with how much of a run is in + the per-term product at all, so a loose `lower_atol` — which rejects a term on its coefficient before + the product is computed — sees about a third of it. Three consequences for the code. + `DenseTermProductsW` specializes the cutoff only for a **length** cutoff over the **whole register**; a + support cutoff or a narrower active window keeps going through `CutoffEvaluator`, and + `uses_word_cutoff()` is how a test tells those apart. (A support arm was tried and measured: ~1% worse + everywhere, and no gain even on the Pauli models that use it, because their per-term time is not in the + cutoff.) `WordKernel` (`Bitset.h`) is the four word ops with `W` fixed that stand in for a `Bitset` + *method* — standing in for one is the membership rule, which is why the scan's fifth bound-width pass, + `fully_paired_words`, lives in `algebra/AlgebraCommon.h` beside the `cutoff_sums` it answers for and + the even-bit literal it shares with `CutoffMasks::make`. Two of the four — the fused XOR and the + AND-fold behind `parity_and` — are the *same* definitions `Bitset`'s own inline arms use + (`detail::fused_xor_words` / `and_fold_words`, declared ahead of both), because one of them decides + emitted term signs and neither may drift from the method it stands in for. `splitmix` is deliberately + a second implementation instead: that value is `monomial_hash`, so it routes MPI ownership and must + stay bit-identical, and `cpp/tests/word_kernel_tests.cpp` asserts it equal to `SplitmixHash` at every + `W` rather than by construction. `term_product_tests.cpp` compares the whole kernel against + `DenseTermProducts` over the whole inline regime, not just the capped widths — both files sweep the + regime through the one `test_utils::for_each_inline_width` in `cpp/tests/InlineWidths.h`, so the range + cannot be narrowed in one of them alone. And the kernel's precondition is that every operand is + inline, so `W` is never bound above `Bitset::kInlineWords` — which `kNarrowKernelWords` is + `static_assert`ed against in the same header, so lowering `kInlineWords` fails the compile rather than + silently specializing a spilled width. `DenseTermProductsW` is non-copyable because its three word + pointers point into its own bitsets; a copy would read and write the original's storage. + Two further bindings were tried past this seam and both measured at nothing — under 0.05% of the + instruction count on either shipping model, pinned single-threaded — because the optimizer already + hoists them out of the inlined scan loop: resolving the algebra's per-term sign inputs into a + per-gate struct the kernel holds (`A::SignContext`), and writing the query record with the word count + bound (`query_push_words`, one capacity check instead of one per word). Measure any third one the + same way before adding it; wall clock cannot see this range, and neither can an instruction count taken + with the thread pool live, which spins hard enough to inflate the total ~14x. +- **The query record**: a store is queried in the form it keys its rows by, so a resolve never converts — + `QueryKeysFor` (`layer_build/Common.h`) picks the batch, and `query_payload_words_for(store, + capacity)` the width. A buffer is `[nq][record 0]…[record nq-1][dense escape tail]`: the header, + because a tail means `size/stride` is no longer the record count; the tail, because a query is `M ⊕ G` + and a fully paired product escapes the cutoff, so no fixed-stride sparse record can hold every one. An + escaped record keeps its place and its stride, marks lane 0 with `SparseRowStore::kOverflowLane` and + carries its tail *index* where the codes word would go — an index into the tail, never an offset into + the buffer, which is what lets the fused sink widen every record without renumbering anything. Push + records through `TermProducts::push(QueryOut{records, escapes}, phase)` and finish a stream with + `append_escape_tail`; never append a record after the tail has started. + A batch's *retained* keys — the ones the deferred self-miss list reads after the slots have been + refilled — are a flat word arena at the batch's own width in both forms, never a container of + monomials: a `Bitset` is sized for the widest inline width whatever its own is, and one key is + retained per term a layer inserts, so a `MonomialList` there carried 72 bytes where a 128-bit + monomial needs 16. It is worth -1.2% user instructions and cycles on the + benchmark Hubbard model, pinned single-threaded; the gain roughly triples at a looser `lower_atol`, + which inserts fewer terms per layer. `retained()` hands back a scratch view, so at most one retained + key may be read at a time. Two neighbouring changes were tried and both measured *worse* than the + arena alone, so do not re-derive them: lowering `Bitset::kInlineWords` to 4 on top of it (wider + by-value monomials then spill, once the bulk site is already packed), and keeping the batch across + layers so its arena retains capacity (the resting footprint costs more than the reallocation saves). + Force-inlining the leaves the runtime-width build calls out of line — `OperatorIndex::popcount`, 4.4% + of samples — measured at nothing, and that is the general lesson here: the per-term instruction count + runs ~26% above the compile-time-width engine but only about a fifth of that reaches cycles, so this + path is not instruction-bound. Object size is the lever, not instruction count. + Two things to know before measuring any of this again. `benches/bench_models.py` cannot resolve a + change below ~3%: rounds within one run agree to under 1%, but the same binary re-run drifts up to + 8%, and across sessions -4% to +10% — so a real 1% shows up there as nothing. And a multi-threaded + reading inverts: the partition pool spins, so `cycles:u` *rises* on a change that lowers wall time. + Pin `monoprop_NUM_THREADS=1` and count instructions. + `owner()` is still the dense `monomial_hash` on both sides, because owner routing is that hash + everywhere including `find_rank` — so a multi-rank run still materializes one monomial per surviving + term, and moving that means changing `find_rank` too. +- **The anticommutation fold** (`detail::InvertedIndex`): the transpose of the row store, one column per + *bit position* — not per mode. That keying is settled and measured: a mode-keyed column cannot answer + a generator slot that names one Majorana of a mode, which is 66% of the Hubbard generators' slots and + 31% of the kicked-Ising ones, so anything mode-shaped is an + additional derived tier, never a re-keying. Two facts to keep in mind before touching it. The fold is + `O(rows)` per *gate* regardless of how few terms anticommute, so it is 15% of a many-cheap-gates Pauli + run and under 3% of a few-expensive-gates Majorana one — measure on the right workload. And its memory + is the dense tier (>90% on both shipping models) until the operator gets sparse enough that the + `Column` vector itself takes over, which happens below roughly `terms < 5 × modes`; + `d_invidx_columns_bytes` in `operator_memory_breakdown()` is that term. - **`Basis` / the `Algebra` policy** (`cpp/monoprop/algebra/`): the two algebras are sibling models (`MajoranaAlgebra`, `PauliAlgebra` in `algebra/Algebra.h`) over shared structural primitives (`algebra/AlgebraCommon.h`). The propagation backbone (the scan/fold in `detail/evolution/...`) is @@ -126,30 +303,31 @@ just build-docs # Build documentation ``` -### Template Metaprogramming +### Mode width -C++ code uses extensive compile-time templates with `NumModes` parameter: -```cpp -template -class MonomialPropagator { /* ... */ }; -``` - -### Mode-Based Dispatching - -Python automatically dispatches to the appropriate C++ template based on the operator's mode count. -`MonomialPropagator` is an abstract base; construct a concrete front-end (which reads the mode count -off the operator — there is no `num_modes` argument): +There is one compiled `MonomialPropagator`, whatever the mode count. `MonomialPropagator` in Python is +an abstract base; construct a concrete front-end, which reads the mode count off the operator — there +is no `num_modes` argument: ```python -# Routes to MonomialPropagator<4> in C++ (Basis::Majorana here; PauliPropagator uses Basis::Pauli) +# Basis::Majorana here; PauliPropagator passes Basis::Pauli to the same C++ class mp = MajoranaPropagator(operator, initial_state, cutoff=4) ``` - +Above 250 modes (8 words, `Bitset::kInlineWords`) a monomial's words spill to the heap, so wide +systems work but pay an allocation per by-value monomial. ### Testing Structure - `tests/cases.py`: Parametrized test cases using `pytest-cases`; `load_problem()` loads a `tests/data/*.msgpack` fixture directly into the public API (`MonomialCircuit` + `MonomialOperator`). C++ tests use the equivalent `test_utils::load_case()` in `cpp/tests/TestData.h` - Fixture msgpack schema is documented in `tests/data/README.md` - Tests validate against exact solutions for small systems - Heavy use of `@parametrize_with_cases` decorators +- Nothing on disk is wider than 28 modes, i.e. one 32-mode storage block. A case that needs the + wide-system regime (260 logical / 288 storage modes: nine words per monomial, so past `Bitset`'s eight + inline ones *and* above the crossover a wheel selects sparse rows at) is + *derived*: `ModeEmbedding`/`WIDE_EMBEDDING` in `tests/cases.py` and `test_utils::embed_case` in + `cpp/tests/TestData.h` relabel a fixture's modes into a wider system. The map is monotone, so the + fixture's exact energy and gradient still apply and the wide run owes the narrow run's evolved + operator term for term (`tests/test_wide_system.py`, `cpp/tests/wide_system_tests.cpp`). Add a case + that way rather than checking in a blob that differs from an existing one by a permutation. - **pytest's fd-level capture hides C++ stderr** (e.g. `COMMPROF`) — rerun with `-s` to see it. - **A slow CTest run on an MPI build is `MPI_Init` fabric probing, not slow tests** — see `monoprop_TEST_EXCLUDE_MPI_FABRIC` in `cpp/tests/CMakeLists.txt`. @@ -173,9 +351,8 @@ mp = MajoranaPropagator(operator, initial_state, cutoff=4) 5. Add a one-line `///` summary if the declaration is in `cpp/include/monoprop/`; elsewhere add a plain `//` note only where the code does not already say it. 6. Implement in the corresponding `.cpp` under `cpp/monoprop/`. -7. Add Python bindings in `src/monoprop/bindings/binder.h` -8. Regenerate bindings with `tools/generate-binders.py` -9. Test with both C++ and Python tests +7. Add Python bindings in `src/monoprop/bindings/bindings.cpp` +8. Test with both C++ and Python tests ## Documentation Maintenance Policy @@ -190,8 +367,8 @@ When changing behavior, APIs, build/test workflows, paths, or developer conventi ### Debugging Build Issues - Check `build/*/compile_commands.json` for compilation flags - Use `rm -rf build` to clear environment-specific builds -- Verify `monoprop_MAX_NUM_MODES` matches your use case (default: 250) - **`uv sync` does not relink `bin/monoprop_unit_tests.x`** — check its mtime against the source's; recipe for a standalone C++ build tree in `docs/content/docs/building.mdx`. -This is a sophisticated scientific computing project requiring careful attention to template instantiation, build system configuration, and the C++/Python boundary. +This is a sophisticated scientific computing project requiring careful attention to build system +configuration and the C++/Python boundary. diff --git a/CMakeLists.txt b/CMakeLists.txt index c98b529e..edbfe7bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,12 +42,6 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release") endif() -set( - monoprop_MAX_NUM_MODES - "250" - CACHE STRING - "Maximum number of simulable Fermionic modes with Python bindings" -) option(monoprop_ENABLE_MPI "Enable MPI parallelization" OFF) option( monoprop_WIDE_TERM_INDEX @@ -77,6 +71,23 @@ endif() include(${PROJECT_SOURCE_DIR}/cmake/compiler_flags/Sanitizers.cmake) include(${PROJECT_SOURCE_DIR}/cmake/compiler_flags/CXXFlags.cmake) +# Sparse/dense crossover depends on vector popcount support. +# Dense scales with storage-word passes, while sparse is mostly width-flat. +# Without vector popcount, dense degrades earlier. +# +# This is intentionally not a cache variable: it should track the flags actually used +# for compilation. monoprop_ROW_STORE=dense|sparse already lets users force a backend +# at runtime without rebuilding. +# +# Thresholds are the first full 32-mode block where sparse is clearly faster than dense +# beyond run-to-run noise, measured end-to-end in the propagator. Expect about +/-1 block +# variation across machines due to cache and popcount throughput. +if(ARCH_FLAG) + set(monoprop_SPARSE_ROW_MIN_MODES 768) +else() + set(monoprop_SPARSE_ROW_MIN_MODES 256) +endif() + # report on compiler flags in use message(STATUS "Configuring a ${CMAKE_BUILD_TYPE} build") string(TOUPPER ${CMAKE_BUILD_TYPE} _cmake_build_type_upper) @@ -101,8 +112,11 @@ message(STATUS " Sanitizer profile : ${monoprop_SANITIZER}") message(STATUS " MPI parallelization : ${monoprop_ENABLE_MPI}") message(STATUS " Wide term index : ${monoprop_WIDE_TERM_INDEX}") -message(STATUS " Max simulable modes : ${monoprop_MAX_NUM_MODES}") message(STATUS " C++ unit tests : ${monoprop_ENABLE_CXX_UNIT_TESTS}") +message( + STATUS + " Sparse rows from : ${monoprop_SPARSE_ROW_MIN_MODES} modes" +) include(GNUInstallDirs) @@ -112,8 +126,13 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}) add_library(monoprop-objs OBJECT "") add_library(monoprop SHARED $) -# must run before add_subdirectory(cpp): CTest's enabled-ness does not propagate -# back up to a parent directory that has already been added as a subdirectory. +# Testing is enabled from the *top-level* list file, and must run before add_subdirectory(cpp), on +# purpose: CTest's root is wherever enable_testing() was called, so called from cpp/ it wrote no +# top-level CTestTestfile.cmake and every documented entry point (the CMakePresets test presets, +# `just test-mpi`, `just test-wide`) pointed ctest at a directory with no tests -- ctest reports "No +# tests were found" and exits 0 for that, so those commands were silently running nothing. And +# CTest's enabled-ness does not propagate back up to a parent directory that has already been added +# as a subdirectory, so this must precede add_subdirectory(cpp) below rather than follow it. if(monoprop_ENABLE_CXX_UNIT_TESTS) enable_testing() include(CTest) diff --git a/README.md b/README.md index 3f03bbbe..011eb90a 100644 --- a/README.md +++ b/README.md @@ -117,10 +117,13 @@ uv sync --all-groups --all-extras -v # installs the workspace, incl. the benc uv run python -m pytest -m "not mpi" # Python tests (serial) just test-mpi # Python + C++ tests under MPI just test-wide # Python + C++ unit tests with a 64-bit TermIndex +just test-sparse-rows # Python tests with the support-form row backend forced ``` -See the [testing guide](https://docs.monoprop.algorithmiq.tech/testing) -for the with/without-MPI details and the rank matrix. +`ctest` runs every C++ case twice, once per row backend — the second pass carries +the `sparse-rows` label. See the +[testing guide](https://docs.algorithmiq.fi/monoprop/docs/testing) for that, the +with/without-MPI details, and the rank matrix. ## Repository layout diff --git a/benches/bench_models.py b/benches/bench_models.py index 7a702335..bac6de12 100644 --- a/benches/bench_models.py +++ b/benches/bench_models.py @@ -51,6 +51,7 @@ def test_model( benchmark, bench_comm, model_configs, + model_rounds, model, record_model_config, record_model_stats, @@ -64,7 +65,12 @@ def test_model( state: dict[str, Any] = {} def setup(): - state["baseline_rss"] = resting_rss_bytes() + # First round only: only then does setup() run before any model is built, matching + # `Baseline RSS` (resting memory before construction). In later rounds, + # pytest-benchmark still holds the previous round's args during setup(), so the + # old propagator is still live and cannot be reclaimed. That would make the reading + # baseline + one full model and misstate the model's memory cost versus `Peak RSS`. + state.setdefault("baseline_rss", resting_rss_bytes()) state["built"] = build_fn(config, comm=bench_comm) return (state["built"], steps), {} @@ -74,10 +80,14 @@ def run(built, n_steps): propagator.propagate(circuit) return propagator.expectation_value() + # setup() runs before every round, so each round rebuilds the model and evolves a fresh + # propagator -- these simulations are in place, and replaying a mutated one would time the wrong + # thing. record_model_stats below then describes the last round, which is what any round would + # produce: the term counts and memory are deterministic. result = benchmark.pedantic( barriered(run, bench_comm), setup=barrier_setup(bench_comm, setup), - rounds=1, + rounds=model_rounds, iterations=1, ) assert isinstance(result, float) diff --git a/benches/conftest.py b/benches/conftest.py index e6284a69..dc0e7982 100644 --- a/benches/conftest.py +++ b/benches/conftest.py @@ -178,6 +178,16 @@ def pytest_addoption(parser: pytest.Parser) -> None: group.addoption(f"--{name}", type=int, default=default, help=help_text) models = parser.getgroup("monoprop-models", "monoprop fixed-model overrides") + # One round is enough for the memory and term-count stats, which are deterministic, but it yields + # no spread for the timing -- and these models are expensive enough that a single sample can sit + # well off the median. Raise this when a timing difference is the point of the run. + models.addoption( + "--model-rounds", + type=int, + default=1, + help="Rounds per fixed model; each rebuilds the model first. >1 gives a median and stddev " + "(default: 1).", + ) for model, (config_cls, _builder, _steps) in MODELS.items(): for field in fields(config_cls): models.addoption( @@ -214,6 +224,7 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]: "nodes": nodes, "ranks_per_node": ranks_per_node, "monoprop_threads": os.environ.get("monoprop_NUM_THREADS", "default"), # noqa: SIM112 + "monoprop_row_store": os.environ.get("monoprop_ROW_STORE") or "auto", # noqa: SIM112 "cpu_count_logical": psutil.cpu_count(logical=True), "cpu_count_physical": psutil.cpu_count(logical=False), "hostname": socket.gethostname(), @@ -224,7 +235,6 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]: "python_version": platform.python_version(), "nanobind_version": monoprop.__nanobind_version__, "nanobind_backend_version": nanobind_backend_version, - "monoprop_max_num_modes": monoprop.MAX_NUM_MODES, "malloc_arena_max": os.environ.get("MALLOC_ARENA_MAX", "default"), "omp_num_threads": os.environ.get("OMP_NUM_THREADS", "default"), # Filled by _record_placement: the threads exist only once a propagator does. @@ -238,6 +248,27 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]: return meta +def _record_row_store(propagator: Any) -> None: + """Fold one propagator's resolved row backend into this run's metadata. + + ``monoprop_ROW_STORE`` says what was asked for, not what ran: unset lets the storage width pick, + and the crossover it picks against is a build-time constant. The two backends accumulate a term + sum in different orders and have different footprints, so a report has to name the one that ran. + Widths differ within a run, hence so can the backend: a disagreement records as ``"mixed"`` + rather than letting the last propagator speak for the others. + """ + if _rank() != 0: + return + # Read straight off the binding, with no getattr fallback: a benchmark whose whole job is to name + # the backend that ran must fail loudly against an extension that cannot say, not quietly record + # nothing. + resolved = "sparse" if propagator._simulator.rows_are_sparse else "dense" + seen = _RESULTS["meta"].get("row_store_effective") + _RESULTS["meta"]["row_store_effective"] = ( + resolved if seen in (None, resolved) else "mixed" + ) + + def _params(config: pytest.Config) -> dict[str, Any]: """Return the resolved random-problem hyperparameters (defaults included).""" return { @@ -295,6 +326,12 @@ def bench_rounds(request: pytest.FixtureRequest) -> int: return int(request.config.getoption("--bench-rounds")) +@pytest.fixture(scope="session") +def model_rounds(request: pytest.FixtureRequest) -> int: + """Return the round count for the fixed-model benchmarks.""" + return int(request.config.getoption("--model-rounds")) + + @pytest.fixture(scope="session") def model_configs(request: pytest.FixtureRequest) -> dict[str, Any]: """Return each fixed model's config, every field resolved from the CLI. @@ -353,6 +390,7 @@ def _record_model_stats( ) -> None: """Record term count, operator memory breakdown and footprint under ``key``.""" _record("opsize", key, {"terms": _reduce_sum(comm, propagator.size())}) + _record_row_store(propagator) # Placement is only observable while the propagator's threads are alive. _record_placement(comm) @@ -553,6 +591,7 @@ def built_graph( # Under MPI the operator is partitioned, so sum the partitions. _record("opsize", picture, {"terms": _reduce_sum(bench_comm, mp.size())}) + _record_row_store(mp) # Settled RSS once the build's transients are released -- the persistent # footprint the per-operation peak cannot see. diff --git a/cmake/compiler_flags/CXXFlags.cmake b/cmake/compiler_flags/CXXFlags.cmake index 5d12b9e1..a6805df6 100644 --- a/cmake/compiler_flags/CXXFlags.cmake +++ b/cmake/compiler_flags/CXXFlags.cmake @@ -81,19 +81,13 @@ if(monoprop_ENABLE_ARCH_FLAGS) endif() endif() -# Query the machine-dependent flags for a given -march value and store the -# cleaned, space-separated string in the variable named by OUTPUT_VARIABLE. A -# MARCH of "default" queries the default target (no -march flag). +# Query the machine-dependent flags the compiler applies under a given architecture selection and +# store the cleaned, space-separated string in the variable named by OUTPUT_VARIABLE. # # Usage: -# _monoprop_query_machine_flags(MARCH OUTPUT_VARIABLE ) +# _monoprop_query_machine_flags(ARCH_FLAGS OUTPUT_VARIABLE ) function(_monoprop_query_machine_flags) - set( - _one_value_args - MARCH - OUTPUT_VARIABLE - ) - cmake_parse_arguments(PARSE_ARGV 0 _arg "" "${_one_value_args}" "") + cmake_parse_arguments(PARSE_ARGV 0 _arg "" "OUTPUT_VARIABLE" "ARCH_FLAGS") if(NOT _arg_OUTPUT_VARIABLE) message( @@ -101,21 +95,19 @@ function(_monoprop_query_machine_flags) "_monoprop_query_machine_flags: OUTPUT_VARIABLE is required" ) endif() - if(NOT _arg_MARCH) - message(FATAL_ERROR "_monoprop_query_machine_flags: MARCH is required") - endif() - if(_arg_MARCH STREQUAL "default") - set(_march_args "") + set(_arch_args ${_arg_ARCH_FLAGS}) + if(_arch_args) + string(JOIN " " _arch_label ${_arch_args}) else() - set(_march_args "-march=${_arg_MARCH}") + set(_arch_label "the default target") endif() if(CMAKE_CXX_COMPILER_ID MATCHES Clang) execute_process( COMMAND # gersemi: off - ${CMAKE_CXX_COMPILER} ${_march_args} -\#\#\# -x c++ -c /dev/null + ${CMAKE_CXX_COMPILER} ${_arch_args} -\#\#\# -x c++ -c /dev/null # gersemi: on ERROR_VARIABLE _query_output ERROR_STRIP_TRAILING_WHITESPACE @@ -124,7 +116,7 @@ function(_monoprop_query_machine_flags) if(NOT _query_result EQUAL 0) message( WARNING - "Failed to query machine-dependent flags for '${_arg_MARCH}' with AppleClang (exit code ${_query_result}). Continuing with empty machine flags." + "Failed to query machine-dependent flags for ${_arch_label} with AppleClang (exit code ${_query_result}). Continuing with empty machine flags." ) set(_flags "") else() @@ -141,7 +133,7 @@ function(_monoprop_query_machine_flags) if(NOT _parse_result EQUAL 0) message( WARNING - "Failed to parse AppleClang machine-dependent flags for '${_arg_MARCH}' (exit code ${_parse_result}). Continuing with empty machine flags." + "Failed to parse AppleClang machine-dependent flags for ${_arch_label} (exit code ${_parse_result}). Continuing with empty machine flags." ) set(_flags "") endif() @@ -149,7 +141,7 @@ function(_monoprop_query_machine_flags) else() execute_process( COMMAND - ${CMAKE_CXX_COMPILER} ${_march_args} -Q --help=target + ${CMAKE_CXX_COMPILER} ${_arch_args} -Q --help=target COMMAND ${Python_EXECUTABLE} "${PROJECT_SOURCE_DIR}/tools/target-help-clean.py" --mode gcc @@ -160,19 +152,19 @@ function(_monoprop_query_machine_flags) if(NOT _result EQUAL 0) message( FATAL_ERROR - "Failed to query machine-dependent flags for '${_arg_MARCH}' (exit code ${_result})" + "Failed to query machine-dependent flags for ${_arch_label} (exit code ${_result})" ) endif() endif() set(${_arg_OUTPUT_VARIABLE} "${_flags}" PARENT_SCOPE) endfunction() +# Empty is the no-arch-flag build and queries the default target. set(monoprop_DEFAULT_VARIANT_FLAGS "") -if(monoprop_ENABLE_ARCH_FLAGS) - _monoprop_query_machine_flags(MARCH native OUTPUT_VARIABLE monoprop_DEFAULT_VARIANT_FLAGS) -else() - _monoprop_query_machine_flags(MARCH default OUTPUT_VARIABLE monoprop_DEFAULT_VARIANT_FLAGS) -endif() +_monoprop_query_machine_flags( + ARCH_FLAGS ${ARCH_FLAG} + OUTPUT_VARIABLE monoprop_DEFAULT_VARIANT_FLAGS +) set(monoprop_VARIANTS "") set(monoprop_VARIANT_FLAGS "") diff --git a/cpp/include/monoprop/MonomialPropagator.h b/cpp/include/monoprop/MonomialPropagator.h index b684ff58..8727b4dd 100644 --- a/cpp/include/monoprop/MonomialPropagator.h +++ b/cpp/include/monoprop/MonomialPropagator.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -45,19 +46,35 @@ #include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/partition/StagedCollect.h" +#include "monoprop/monopropExport.h" namespace monoprop { namespace detail { struct FusedContract; namespace partition { -template class PartitionGroup; } // namespace partition + +/*! \brief Compute the storage width for a given system width. + * + * Rounds \a num_modes up to the next whole 32-mode block and returns at least + * one full block. + * + * This keeps the hash index probe layout aligned across nearby system sizes + * while avoiding partially populated words for small systems. + * + * \param num_modes Number of modes in the system. + * \return Storage width rounded up to a whole 32-mode block. + */ +[[nodiscard]] inline auto storage_modes_for(size_t num_modes) -> size_t { + constexpr size_t kModesPerBlock = 32; + return std::max(kModesPerBlock, ((num_modes + kModesPerBlock - 1) / kModesPerBlock) * kModesPerBlock); +} } // namespace detail /// A propagator setting is out of range, or inconsistent with another setting. -// Covers a crossed atol pair and a logical width outside [1, NumModes]; also thrown from -// MonomialPropagatorImpl.h +// Covers a crossed atol pair and a zero mode count; also thrown from MonomialPropagator.cpp class PropagatorConfigError : public std::runtime_error { public: using std::runtime_error::runtime_error; @@ -69,21 +86,51 @@ class MultiPartitionUnsupported : public std::runtime_error { using std::runtime_error::runtime_error; }; -template -class MonomialPropagator { +/// The MPI ranks resolved different partition counts. +// The count comes from partitions= or the environment on every rank independently, so the fix is to the +// launch, and it may belong to a different rank. +class PartitionCountMismatch : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/// The requested operation does not agree with the graph this propagator currently holds. +// Either it requires no stored graph, or its parameter_mapping matches neither the stored layer nor gate +// count. The caller recovers by contracting or rebuilding the graph, not by fixing an isolated argument. +class GraphStateConflict : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/// The (basis, cutoff_type, basis_change) triple is inconsistent. +// A Pauli basis with a Length cutoff or a basis change, or a basis-change table that is not +// 2*num_modes rows. +class CutoffConfigError : public std::invalid_argument { +public: + using std::invalid_argument::invalid_argument; +}; + +/// A coefficient-informed build_graph() was given fewer parameter values than replaying the stored graph +/// as a seed needs. +class SeedParametersTooShort : public std::invalid_argument { +public: + using std::invalid_argument::invalid_argument; +}; + +class monoprop_EXPORT MonomialPropagator { public: - using PartitionChildFactory = std::function>(mpi::Comm)>; + using PartitionChildFactory = std::function(mpi::Comm)>; MonomialPropagator(const OperatorDict &initial_operator, unsigned int cutoff, const VecZ &initial_state, + size_t num_modes, std::optional schrodinger_cutoff, mpi::Comm comm, std::optional lower_atol = std::nullopt, std::optional upper_atol = std::nullopt, CutoffType cutoff_type = CutoffType::Length, std::optional> basis_change = std::nullopt, - size_t logical_num_modes = NumModes, Basis basis = Basis::Majorana, size_t partitions = 0, PartitionChildFactory child_factory = nullptr); @@ -96,10 +143,22 @@ class MonomialPropagator { MonomialPropagator(const MonomialPropagator &other); auto operator=(const MonomialPropagator &) -> MonomialPropagator & = delete; - static constexpr auto num_modes{NumModes}; - static constexpr auto storage_num_modes{NumModes}; - - auto logical_num_modes() const -> size_t { return logical_num_modes_; } + /// The system's width, as passed to the constructor. + auto num_modes() const -> size_t { return num_modes_; } + /// Monomials are stored at this width; >= num_modes(). See detail::storage_modes_for. + /// Read off the operator rather than kept as a member: it is the width that actually runs, and a + /// second copy of a derived value is a second thing the copy constructor has to keep in step. + auto storage_num_modes() const -> size_t { return mp_op_.num_bits() / 2; } + + /// Whether this propagator stores its terms as sparse rows rather than dense monomials — the choice + /// made from storage_num_modes() and `monoprop_ROW_STORE`. Both backends compute the same terms and + /// the same expectation value; they hash rows differently, so they differ in term order and hence in + /// floating-point accumulation order. Not partitioned: every partition decides from the same storage + /// width, and the facade's own operator holds no terms — so a facade answers from partition 0, whose + /// store is the one that runs. + [[nodiscard]] auto rows_are_sparse() const -> bool { + return partition_group_ ? first_partition_().rows_are_sparse() : mp_op_.rows_are_sparse(); + } /// Term count on this rank (allreduce for global). auto size() const -> size_t { return partition_group_ ? partitioned_size_() : mp_op_.size(); } @@ -116,11 +175,11 @@ class MonomialPropagator { } /// This rank's operator storage. Single-partition only — see require_single_partition_. - auto mp_op() -> detail::MPOperator & { + auto mp_op() -> detail::MPOperator & { require_single_partition_("mp_op()"); return mp_op_; } - auto mp_op() const -> const detail::MPOperator & { + auto mp_op() const -> const detail::MPOperator & { require_single_partition_("mp_op()"); return mp_op_; } @@ -133,7 +192,7 @@ class MonomialPropagator { return graph_.storage_memory_usage(); } - auto operator_memory_usage() const -> detail::MPOperatorMemoryBreakdown { + auto operator_memory_usage() const -> detail::MPOperatorMemoryBreakdown { if (partition_group_) { return partitioned_operator_memory_usage_(); } @@ -156,14 +215,19 @@ class MonomialPropagator { /// graph_layers(), optimizer order) or a per-gate one (length n_gates()); on a tie, per-layer wins. auto set_parameter_mapping(const VecZ ¶meter_mapping) -> void; - /// This rank's monomial → coefficient index. Single-partition only — see require_single_partition_. - auto indexing() -> detail::OperatorIndex & { - require_single_partition_("indexing()"); - return *mp_op_.store; + /// This rank's terms as fn(monomial, coefficient index), in the index's own slot order. + /// Single-partition only — see require_single_partition_. No accessor for the store itself: which + /// backend holds the rows is a runtime choice (see MPOperator::with_store), so there is no one type + /// to hand out. + template + auto for_each_term(Fn &&fn) const -> void { + require_single_partition_("for_each_term()"); + mp_op_.for_each_term(std::forward(fn)); } - auto indexing() const -> const detail::OperatorIndex & { - require_single_partition_("indexing()"); - return *mp_op_.store; + /// This rank's term count. Single-partition only. + auto num_local_terms() const -> size_t { + require_single_partition_("num_local_terms()"); + return mp_op_.size(); } /// Per-layer (cos_inds, local_cycles, cross_rank_sin_send, cross_rank_sin_recv) for this @@ -181,7 +245,7 @@ class MonomialPropagator { new_lower_atol.value(), upper_atol_.value())); } - update_setting_([&](MonomialPropagator &p) { p.lower_atol_ = new_lower_atol; }); + update_setting_([&new_lower_atol](MonomialPropagator &p) { p.lower_atol_ = new_lower_atol; }); } auto update_upper_atol(std::optional new_upper_atol) -> void { @@ -191,31 +255,37 @@ class MonomialPropagator { new_upper_atol.value(), lower_atol_.value())); } - update_setting_([&](MonomialPropagator &p) { p.upper_atol_ = new_upper_atol; }); + update_setting_([&new_upper_atol](MonomialPropagator &p) { p.upper_atol_ = new_upper_atol; }); } - /// Existing terms are not re-truncated. + /// Existing terms are not re-truncated by the cutoff function itself; the row store that holds them + /// is resized in place when the new cutoff moves its width bound, so a term admitted under the old + /// cutoff keeps its row (same index, same content) rather than paying the overflow-map cost for the + /// rest of the propagator's life. auto update_cutoff(unsigned int new_cutoff) -> void { - update_setting_([&](MonomialPropagator &p) { + update_setting_([&new_cutoff](MonomialPropagator &p) { p.cutoff_ = new_cutoff; p.regenerate_cutoff_fn_(); + p.resize_row_store_if_needed_(); }); } auto update_cutoff_type(CutoffType new_cutoff_type) -> void { validate_cutoff_config_(new_cutoff_type, basis_change_); - update_setting_([&](MonomialPropagator &p) { + update_setting_([&new_cutoff_type](MonomialPropagator &p) { p.cutoff_type_ = new_cutoff_type; p.regenerate_cutoff_fn_(); + p.resize_row_store_if_needed_(); }); } /// The basis the cutoff is measured in; nullopt ⇒ the native basis. auto update_basis_change(std::optional> new_basis_change) -> void { validate_cutoff_config_(cutoff_type_, new_basis_change); - update_setting_([&](MonomialPropagator &p) { + update_setting_([&new_basis_change](MonomialPropagator &p) { p.basis_change_ = new_basis_change; p.regenerate_cutoff_fn_(); + p.resize_row_store_if_needed_(); }); } @@ -272,7 +342,7 @@ class MonomialPropagator { /// Contract the graph into the operator (Heisenberg) or state (Schrodinger). `inplace` consumes the /// graph and updates internal state; otherwise nothing is mutated. Core term excluded either way. - /// Coefficients are positioned by the owning partition's indexing(), so on a facade the result is + /// Coefficients are positioned by the owning partition's own index, so on a facade the result is /// the per-partition blocks concatenated in partition order: the same multiset as an unpartitioned /// run, but not positionally stable across partition counts — and the count is auto-picked from the /// host's core count unless pinned. Use evolved_operator_terms() when positions must mean something. @@ -286,34 +356,48 @@ class MonomialPropagator { virtual auto update_initial_operator(const OperatorDict &op_dict) -> void { apply_initial_operator_(op_dict); } protected: - virtual auto clone_() const -> std::unique_ptr> { - return std::make_unique>(*this); + virtual auto clone_() const -> std::unique_ptr { + return std::make_unique(*this); } - static inline const auto ev_fn = [](const EvalRequest &request, - mpi::Comm comm, - const detail::CosCallbacks &cos) -> double { return ev(request, comm, cos); }; + static inline const auto ev_fn = [](const EvalRequest &request, mpi::Comm comm, const detail::CosCallbacks &cos) { + return ev(request, comm, cos); + }; static inline const auto ev_and_grad_fn = - [](const EvalRequest &request, mpi::Comm comm, const detail::CosCallbacks &cos) -> std::pair { - return ev_and_grad(request, comm, cos); - }; + [](const EvalRequest &request, mpi::Comm comm, const detail::CosCallbacks &cos) { + return ev_and_grad(request, comm, cos); + }; /// Distribute op_dict across ranks and apply this rank's share; returns its new (terms, coeffs) /// so caches can refresh. - auto apply_initial_operator_(const OperatorDict &op_dict) -> std::pair, VecD>; + auto apply_initial_operator_(const OperatorDict &op_dict) -> std::pair; bool schrodinger_; mpi::Comm comm_; // real MPI across nodes, or an in-process comm across partitions - CutoffFn cutoff_fn_; - detail::MPOperator mp_op_; + CutoffFn cutoff_fn_; + detail::MPOperator mp_op_; MPGraph graph_; // Per-gate layer-build scratch, reused across gates; carries no state between them. detail::MatchedEpochSet matched_scratch_; - // A perf hint, never a correctness constraint: overflow spills losslessly. Sized to the cutoff's - // structural position bound when it has one. - auto packed_inline_width_() const -> size_t; + // Row width bound in modes, shared by both row-store backends so a construction-time choice (in + // particular, Schrödinger's initial term set being wider than the post-first-layer cutoff would + // suggest) is made once rather than risking the two backends disagreeing on it. + auto row_width_bound_() const -> size_t; + + // The named backend's ideal row width for the current cutoff/basis-change configuration: one + // function for the branch the constructor's store setup and resize_row_store_if_needed_() would + // otherwise duplicate. A perf hint, never a correctness constraint -- an over-long row spills + // losslessly. The caller names the backend, since the constructor picks one before there is a store + // to read it off and every later caller has one. + auto target_row_width_(bool sparse) const -> size_t; + + // Resizes the live backend to its target row width when it has moved, migrating existing rows rather + // than dropping them. Must run after any setting change that can move the cutoff-derived bound + // (update_cutoff, update_cutoff_type, update_basis_change) -- see the row-width discussion on + // update_cutoff(). + auto resize_row_store_if_needed_() -> void; // `requested` 0 ⇒ env/auto. Returns 1 for the ordinary single-partition path. static auto resolve_partition_count_(size_t requested, mpi::Comm comm) -> size_t; @@ -328,9 +412,27 @@ class MonomialPropagator { auto for_each_partition_(const std::function &fn) -> void; + // for_each_partition_ with the partition rank. The two map_ helpers below are defined here rather + // than in the .cpp because a derived class in another translation unit instantiates them, and this + // type-erased primitive is what lets them see the partitions without seeing PartitionGroup. + auto for_each_partition_indexed_(const std::function &fn) -> void; + + auto partition_count_() const -> size_t; + // One result per partition, in partition order. template > - auto map_partitions_(Fn fn) -> std::vector; + auto map_partitions_(Fn fn) -> std::vector { + return map_partitions_indexed_([&fn](int, MonomialPropagator &p) -> R { return fn(p); }); + } + + // The slots are written from the owning master, so `fn` must not touch the vector itself -- see + // detail::staged_collect for what that rules out. + template > + auto map_partitions_indexed_(Fn fn) -> std::vector { + return detail::staged_collect(partition_count_(), [this, &fn](auto &&emit) { + for_each_partition_indexed_([&emit, &fn](int r, MonomialPropagator &p) { emit(r, fn(r, p)); }); + }); + } // Concatenated in partition order. The partitions are disjoint, so the result enumerates the whole // operator (deterministic for a fixed partition count). @@ -352,32 +454,38 @@ class MonomialPropagator { auto is_partition_facade() const -> bool { return static_cast(partition_group_); } - template > - auto map_partitions_indexed_(Fn fn) -> std::vector; + // The backend decision, in one place: monoprop_ROW_STORE if it forces one, else the measured + // crossover on the storage width. Throws if the variable holds something unrecognized -- see + // config::Settings::row_store for why an unrecognized value is not silently ignored. + auto use_sparse_rows_() const -> bool; private: + CutoffType cutoff_type_; + + // Immutable after construction. + Basis basis_{Basis::Majorana}; + unsigned int cutoff_; - std::optional lower_atol_, upper_atol_; double core_term_{0.0}; // Bumped by every initial-operator re-weight. A functional snapshots the operator coefficients, so // it captures this and rejects a later call once it moves, as it does for a rebuilt graph. size_t initial_operator_epoch_{0}; - size_t logical_num_modes_{NumModes}; - - CutoffType cutoff_type_; - std::optional> basis_change_; - - // Immutable after construction. - Basis basis_{Basis::Majorana}; + // The system's width + size_t num_modes_; // Intra-process partition runtime. Null ⇒ ordinary single-partition propagator; non-null ⇒ a partition facade // whose own mp_op_/graph_ are unused and every method fans out to the S partition propagators. - std::unique_ptr> partition_group_; + std::unique_ptr partition_group_; // PartitionGroup rebinds a cloned partition's comm_ to its own transport during a deep copy. - friend class detail::partition::PartitionGroup; + friend class detail::partition::PartitionGroup; + + std::optional lower_atol_; + std::optional upper_atol_; + + std::optional> basis_change_; // A facade's own graph_/mp_op_ are never populated, so handing them out would return plausible-looking // empty state; there is no meaningful merge either, since the callers want one partition's raw layout. @@ -396,7 +504,7 @@ class MonomialPropagator { auto partitioned_graph_size_() const -> std::pair; auto partitioned_graph_layers_() const -> size_t; auto partitioned_core_term_() const -> double; - auto partitioned_operator_memory_usage_() const -> detail::MPOperatorMemoryBreakdown; + auto partitioned_operator_memory_usage_() const -> detail::MPOperatorMemoryBreakdown; auto partitioned_graph_memory_usage_() const -> GraphMemoryBreakdown; auto cos_index_count_() const -> size_t; @@ -480,10 +588,7 @@ class MonomialPropagator { // Reconstruct the optimizer-order (parameter_mapping, gen_coeffs) arrays from the layers' gate info. auto graph_gate_arrays_() const -> std::pair; - auto evolve_operator_with_recompute_(VecD &&coeffs, const MPGraphView &graph, const VecD ¶ms) -> VecD; + auto evolve_operator_with_recompute_(VecD &&coeffs, const MPGraphView &graph, const VecD ¶ms) const -> VecD; }; } // namespace monoprop - -// inline implementation -#include "monoprop/detail/monomial_propagator/MonomialPropagator.inl" diff --git a/cpp/monoprop/Bitset.h b/cpp/monoprop/Bitset.h index 6ad135a3..a85e06c5 100644 --- a/cpp/monoprop/Bitset.h +++ b/cpp/monoprop/Bitset.h @@ -16,213 +16,596 @@ #include #include +#include #include #include #include #include +#include + +namespace monoprop::detail { + +// Dispatches a runtime word count in [0, 8] to a fully-unrolled arm: W is known at compile time inside +// `f`, so a per-word loop written against it has no back-edge and no trip-count prologue/tail. Callers +// gate on `n <= Bitset::kInlineWords` themselves and fall back to a plain runtime loop above that; +// `default` is an unreachable safety net, not a ninth arm. +template +[[gnu::always_inline]] inline auto with_nwords(size_t n, F &&f) -> decltype(auto) { + switch (n) { + case 0: + return f(std::integral_constant{}); + case 1: + return f(std::integral_constant{}); + case 2: + return f(std::integral_constant{}); + case 3: + return f(std::integral_constant{}); + case 4: + return f(std::integral_constant{}); + case 5: + return f(std::integral_constant{}); + case 6: + return f(std::integral_constant{}); + case 7: + return f(std::integral_constant{}); + default: + return f(std::integral_constant{}); + } +} + +// The two popcounts a fused XOR reports, without its result. Nested in Bitset as FusedCounts, and +// named here because the word pass that produces them is declared before the class. +struct FusedWordCounts { + size_t overlap; // popcount(a & b) + size_t result_count; // popcount(a ^ b) +}; + +// The word passes shared by Bitset's own inline arms and by the per-gate WordKernel below. They live +// here, ahead of both, so each is defined once: Bitset reaches them through with_nwords and the kernel +// through its bound W, and the two must not be able to drift -- one of them decides emitted term signs +// and the other feeds a cutoff. +// +// `nwords` is the exact word count of every operand, passed either as a std::integral_constant (the +// inline regime, where the trip count is then a compile-time one) or as a plain size_t (a spilled +// width). One definition serves both so the two cannot drift. Correct at 0 (an empty fold), which is +// the arm with_nwords hands a zero-width bitset. +template +[[gnu::always_inline]] inline auto fused_xor_words(const uint64_t *a, + const uint64_t *b, + uint64_t *out, + N nwords) noexcept -> FusedWordCounts { + size_t overlap = 0; + size_t result_count = 0; + for (size_t i = 0; i < nwords; ++i) { + const uint64_t n = a[i] ^ b[i]; + out[i] = n; + overlap += static_cast(std::popcount(a[i] & b[i])); + result_count += static_cast(std::popcount(n)); + } + return {overlap, result_count}; +} + +// XOR-fold of a & b into one word. Folding first and popcounting once is what makes the caller's +// parity that of the whole AND rather than of any per-word rounding. +template +[[gnu::always_inline]] inline auto and_fold_words(const uint64_t *a, const uint64_t *b, N nwords) noexcept -> uint64_t { + uint64_t folded = 0; + for (size_t i = 0; i < nwords; ++i) { + folded ^= a[i] & b[i]; + } + return folded; +} + +} // namespace monoprop::detail namespace monoprop { -// std::bitset replacement over contiguous uint64_t words: zero-copy MPI, word-wise hashing, -// portable std::countr_zero scanning, memcpy-safe. -template +// std::bitset replacement over contiguous uint64_t words, with a *runtime* width: zero-copy MPI (via +// data()/word()), word-wise hashing, portable std::countr_zero scanning. The first kInlineWords words +// live inline (250 modes -> 500 bits -> 8 words, which was the compile-time ceiling before it was +// removed and is still where the interesting models sit); above that a bitset spills the *entire* word +// array to the heap, keeping data()/word(i) a single contiguous view regardless of which storage is +// active -- so wider systems are correct, but pay an allocation per by-value monomial. Every +// per-word loop routes through with_words_, which binds the count as a compile-time W for +// n <= kInlineWords (the hot regime) and hands it over as a runtime count above that. +// +// Unlike the Bitset template this replaces, one object is sized for the *widest* supported +// bitset rather than exactly for its own width (72 bytes vs the old 8/16/32/64 for 32/64/128/250 +// modes). That is inherent to owning the bits inline at a runtime width, and it is paid per monomial +// wherever monomials are held by value in bulk. Keep hot paths off by-value temporaries because of +// it: prefer a word loop over `a & b` chains, and hoist masks out of per-term code. The plan's route +// out is Stage 6's arena, where a Bitset becomes a non-owning {pointer, width} view over +// exactly-sized storage and the size question disappears. class Bitset { - static_assert(NumBits > 0, "Bitset requires at least 1 bit"); - +public: + // The word vocabulary is public because callers reason in words: data() already hands out a + // word_type*, and a per-gate kernel that binds the word count needs the same three names to say + // what it binds (see detail::WordKernel). using word_type = uint64_t; static constexpr auto word_width = sizeof(word_type) * 8; + static constexpr size_t kInlineWords = 8; - static constexpr auto kNumWords = (NumBits + word_width - 1) / word_width; - static constexpr auto kTopBits = NumBits % word_width; - static constexpr auto kTopMask = kTopBits ? ((word_type{1} << kTopBits) - 1) : ~word_type{0}; + // The word count a bitset of `num_bits` occupies. Public because callers that size a per-monomial + // buffer need it without holding a monomial; it is the constructor's own arithmetic, so the 64 in + // it stays spelled once. + [[nodiscard]] static constexpr auto words_for(size_t num_bits) noexcept -> size_t { + return (num_bits + word_width - 1) / word_width; + } - std::array words_{}; +private: + // The inline words and the heap pointer are never both live -- nwords_ alone selects which -- so + // they share storage. A std::vector member instead costs 24 bytes on *every* monomial at *every* + // width, and monomials are stored by value in bulk (MonomialList, OperatorIndex::overflow_, + // IncomingProbe::mono), so that overhead is multiplied by the term count. Sizing the object for + // the widest inline width already costs enough; see the class comment above. + // + // Deliberately *not* initializing: a value-initialized inline_ zero-fills all kInlineWords words, + // and since a default member initializer also runs before a copy constructor's body, every copy + // paid that fill before overwriting it. Each constructor writes exactly the words it owns. + union Storage { + std::array inline_; + word_type *heap_; + Storage() noexcept {} + }; + + Storage s_; + uint32_t nwords_ = 0; + uint32_t top_bits_ = 0; // bits used in the last word; 0 means "all word_width bits used" + + // Invariant: only words [0, nwords_) hold a value. The inline tail above nwords_ is indeterminate, + // and no reader may touch it -- every word loop, operator==, and SplitmixHash run nwords_, and the + // MPI readers memcpy num_words() words. That is what lets a copy cost the operand's own width + // instead of the widest supported one (16 bytes at 64 modes rather than 64), which is paid per + // element wherever monomials are held by value in bulk. + + // Copies the live words only. Precondition: !spilled() -- with_nwords caps at kInlineWords, so a + // spilled width would silently copy the first 8 words and drop the rest; those paths memcpy. + auto copy_inline_from(const Bitset &o) noexcept -> void { + word_type *d = s_.inline_.data(); + const word_type *s = o.data(); + detail::with_nwords(nwords_, [d, s](std::integral_constant) { + for (size_t i = 0; i < W; ++i) { + d[i] = s[i]; + } + }); + } - constexpr auto sanitize_top() noexcept -> void { - if constexpr (kTopBits != 0) { - words_[kNumWords - 1] &= kTopMask; + // Zeroes the live words only, same precondition and same reason as copy_inline_from. An unrolled + // store loop rather than std::memset: the length is a runtime value, so memset would be an out-of-line + // call on a path that is one or two stores wide. + auto zero_inline() noexcept -> void { + word_type *d = s_.inline_.data(); + detail::with_nwords(nwords_, [d](std::integral_constant) { + for (size_t i = 0; i < W; ++i) { + d[i] = 0; + } + }); + } + + // The regime split every word loop below shares: bind the count as a compile-time W while the + // words are inline (the hot regime), hand it over as a runtime count once they have spilled. `f` + // takes it as `auto n`, so one body serves both arms -- writing the loop once per arm is how the + // two drift, and one of them (parity_and) decides emitted term signs. + template + [[gnu::always_inline]] auto with_words_(F &&f) const noexcept -> decltype(auto) { + if (nwords_ <= kInlineWords) { + return detail::with_nwords(nwords_, f); + } + return f(static_cast(nwords_)); + } + + // The three compound bitwise ops differ only in the word operation, so they share one loop. Width + // is asserted by the callers, which name the operator in the message. + template + auto apply_words_(const Bitset &rhs, Op op) noexcept -> Bitset & { + word_type *a = data(); + const word_type *b = rhs.data(); + with_words_([a, b, op](auto n) { + for (size_t i = 0; i < n; ++i) + a[i] = op(a[i], b[i]); + }); + return *this; + } + + // Selects the active union member. Must be consulted *before* nwords_ is overwritten by an + // assignment, and *after* it is set by a constructor. + [[nodiscard]] auto spilled() const noexcept -> bool { return nwords_ > kInlineWords; } + + [[nodiscard]] auto top_mask() const noexcept -> word_type { + return top_bits_ ? ((word_type{1} << top_bits_) - 1) : ~word_type{0}; + } + + auto sanitize_top() noexcept -> void { + if (nwords_ != 0) { + data()[nwords_ - 1] &= top_mask(); } } public: - constexpr Bitset() noexcept = default; + Bitset() noexcept = default; + + // num_bits: the logical width, zeroed. + explicit Bitset(size_t num_bits) noexcept + : nwords_(static_cast(words_for(num_bits))), + top_bits_(static_cast(num_bits % word_width)) { + if (spilled()) { + s_.heap_ = new word_type[nwords_]{}; + } + else { + zero_inline(); + } + } + + // num_bits plus a value packed into word 0. Width can no longer be implied by the type (unlike the + // old Bitset(uint64_t) implicit conversion), so this stays explicit and two-argument. + explicit Bitset(size_t num_bits, uint64_t val) noexcept : Bitset(num_bits) { + if (nwords_ != 0) { + data()[0] = val; + sanitize_top(); + } + } + + Bitset(const Bitset &o) noexcept : nwords_(o.nwords_), top_bits_(o.top_bits_) { + if (spilled()) { + s_.heap_ = new word_type[nwords_]; + std::memcpy(s_.heap_, o.s_.heap_, nwords_ * sizeof(word_type)); + } + else { + copy_inline_from(o); + } + } + + // Steals the pointer in the spilled case and copies the live words otherwise, not the union's whole + // object representation. Zeroing the source's nwords_ makes it inline-empty, so its destructor + // frees nothing. + Bitset(Bitset &&o) noexcept : nwords_(o.nwords_), top_bits_(o.top_bits_) { + if (spilled()) { + s_.heap_ = o.s_.heap_; + } + else { + copy_inline_from(o); + } + o.nwords_ = 0; + o.top_bits_ = 0; + } + + auto operator=(const Bitset &o) noexcept -> Bitset & { + if (this == &o) { + return *this; + } + // Same width is the overwhelmingly common case (a monomial assigned from another monomial of + // the same operator), and it needs no reallocation at all. + if (nwords_ == o.nwords_) { + top_bits_ = o.top_bits_; + if (spilled()) { + std::memcpy(s_.heap_, o.s_.heap_, nwords_ * sizeof(word_type)); + } + else { + copy_inline_from(o); + } + return *this; + } + if (spilled()) { + delete[] s_.heap_; + } + nwords_ = o.nwords_; + top_bits_ = o.top_bits_; + if (spilled()) { + s_.heap_ = new word_type[nwords_]; + std::memcpy(s_.heap_, o.s_.heap_, nwords_ * sizeof(word_type)); + } + else { + copy_inline_from(o); + } + return *this; + } - constexpr explicit(false) Bitset(uint64_t val) noexcept : words_{val} { sanitize_top(); } + auto operator=(Bitset &&o) noexcept -> Bitset & { + if (this == &o) { + return *this; + } + if (spilled()) { + delete[] s_.heap_; + } + // spilled() reads nwords_, so the two tests below straddle the assignment on purpose: the first + // frees against the old width, the second selects the union member for the new one. + nwords_ = o.nwords_; + top_bits_ = o.top_bits_; + if (spilled()) { + s_.heap_ = o.s_.heap_; + } + else { + copy_inline_from(o); + } + o.nwords_ = 0; + o.top_bits_ = 0; + return *this; + } - [[nodiscard]] constexpr auto count() const noexcept -> size_t { - size_t c = 0; - for (size_t i = 0; i < kNumWords; ++i) - c += static_cast(std::popcount(words_[i])); - return c; + ~Bitset() noexcept { + if (spilled()) { + delete[] s_.heap_; + } } - [[nodiscard]] constexpr auto test(size_t pos) const noexcept -> bool { - return (words_[pos / word_width] >> (pos % word_width)) & 1; + // Bytes this bitset owns *outside* its own object, so a container counting sizeof(Bitset) per element + // can add what a spilled element points at. Zero for an inline width, which is why a container that + // omits it looks correct until someone runs past 8 words. + [[nodiscard]] auto heap_bytes() const noexcept -> size_t { return spilled() ? nwords_ * sizeof(word_type) : 0; } + + [[nodiscard]] auto data() const noexcept -> const word_type * { return spilled() ? s_.heap_ : s_.inline_.data(); } + [[nodiscard]] auto data() noexcept -> word_type * { return spilled() ? s_.heap_ : s_.inline_.data(); } + [[nodiscard]] auto word(size_t i) const noexcept -> uint64_t { return data()[i]; } + + [[nodiscard]] auto num_words() const noexcept -> size_t { return nwords_; } + [[nodiscard]] auto size() const noexcept -> size_t { + if (nwords_ == 0) { + return 0; + } + return top_bits_ != 0 ? (static_cast(nwords_ - 1) * word_width + top_bits_) + : static_cast(nwords_) * word_width; } - [[nodiscard]] constexpr auto any() const noexcept -> bool { - for (size_t i = 0; i < kNumWords; ++i) - if (words_[i]) - return true; - return false; + [[nodiscard]] auto count() const noexcept -> size_t { + const word_type *w = data(); + return with_words_([w](auto n) { + size_t c = 0; + for (size_t i = 0; i < n; ++i) + c += static_cast(std::popcount(w[i])); + return c; + }); } - [[nodiscard]] constexpr auto none() const noexcept -> bool { return !any(); } + [[nodiscard]] auto test(size_t pos) const noexcept -> bool { + return (data()[pos / word_width] >> (pos % word_width)) & 1; + } - [[nodiscard]] static constexpr auto size() noexcept -> size_t { return NumBits; } + [[nodiscard]] auto any() const noexcept -> bool { + const word_type *w = data(); + return with_words_([w](auto n) { + for (size_t i = 0; i < n; ++i) + if (w[i]) + return true; + return false; + }); + } + + [[nodiscard]] auto none() const noexcept -> bool { return !any(); } + // Every binary op below loops *this*'s word count and indexes the other operand unchecked, so a + // narrower operand is read past its own width. That is not merely wrong-but-harmless: it only + // reads zeros from the inline array while the *result* fits inline, and once *this* is spilled + // (> kInlineWords) it reads off the end of the narrower operand's array. Widths must match at + // every call site, so this is asserted rather than handled -- Release keeps the loops bare. + // // popcount(*this & other) without materializing the temporary. - [[nodiscard]] constexpr auto count_and(const Bitset &o) const noexcept -> size_t { - size_t c = 0; - for (size_t i = 0; i < kNumWords; ++i) - c += static_cast(std::popcount(words_[i] & o.words_[i])); - return c; + [[nodiscard]] auto count_and(const Bitset &o) const noexcept -> size_t { + assert(nwords_ == o.nwords_ && "Bitset::count_and width mismatch"); + const word_type *a = data(); + const word_type *b = o.data(); + return with_words_([a, b](auto n) { + size_t c = 0; + for (size_t i = 0; i < n; ++i) + c += static_cast(std::popcount(a[i] & b[i])); + return c; + }); } - [[nodiscard]] constexpr auto parity_and(const Bitset &o) const noexcept -> bool { - word_type parity_word = 0; - for (size_t i = 0; i < kNumWords; ++i) - parity_word ^= words_[i] & o.words_[i]; + [[nodiscard]] auto parity_and(const Bitset &o) const noexcept -> bool { + assert(nwords_ == o.nwords_ && "Bitset::parity_and width mismatch"); + const word_type *a = data(); + const word_type *b = o.data(); + const word_type parity_word = with_words_([a, b](auto n) { return detail::and_fold_words(a, b, n); }); return (std::popcount(parity_word) & 1U) != 0; } - constexpr auto set(size_t pos) noexcept -> Bitset & { - words_[pos / word_width] |= uint64_t(1) << (pos % word_width); + // Every quantity a caller building `*this ^ gen` typically also needs alongside it: the XORed + // result, popcount(*this & gen) (overlap), and popcount(result). Composing this from operator^ + // and count_and() costs two full passes over the words; once the width is a runtime value + // rather than a compile-time width, each pass also pays + // its own loop prologue/tail, so the separate-ops cost keeps growing where this stays one pass. + // Kept alongside the existing composable ops -- a cold path that only needs one of the three + // should keep using them. result needs no sanitize_top(): XOR of two already-sanitized operands + // never sets a bit above NumBits. + // + // Forward-declared here, defined below: a member holding Bitset by value can't be nested inside + // Bitset's own (still-incomplete) definition -- unlike the old Bitset, a template, this + // is no longer a template instantiated as one unit, so the usual incomplete-type rule applies. + struct FusedXor; + + // Returned by value so the pass that computes the two counts can write the XOR straight into a + // caller-owned destination. + using FusedCounts = detail::FusedWordCounts; + + // fused_xor's one word pass, writing the XOR into `out` instead of into a fresh Bitset. The hot + // path wants this form: `out` is a scratch monomial that lives for the whole gate, so a term + // costs the word pass alone -- no width recomputation, no spill test, no allocation, and none of + // the copies a by-value result goes through on its way to the caller's variable. + auto fused_xor_into(const Bitset &gen, Bitset &out) const noexcept -> FusedCounts; + + [[nodiscard]] auto fused_xor(const Bitset &gen) const noexcept -> FusedXor; + + auto set(size_t pos) noexcept -> Bitset & { + data()[pos / word_width] |= uint64_t(1) << (pos % word_width); return *this; } - constexpr auto operator&=(const Bitset &rhs) noexcept -> Bitset & { - for (auto i = 0uz; i < kNumWords; ++i) - words_[i] &= rhs.words_[i]; + // Clear every bit, keeping the width. Not the same as assigning a default-constructed Bitset, + // which drops the width to 0 -- the distinction matters wherever code needs "a zero monomial the + // same shape as this one", which is copy-then-reset and nothing shorter (see change_basis). + auto reset() noexcept -> Bitset & { + std::memset(data(), 0, nwords_ * sizeof(word_type)); return *this; } - constexpr auto operator|=(const Bitset &rhs) noexcept -> Bitset & { - for (auto i = 0uz; i < kNumWords; ++i) - words_[i] |= rhs.words_[i]; - return *this; + auto operator&=(const Bitset &rhs) noexcept -> Bitset & { + assert(nwords_ == rhs.nwords_ && "Bitset::operator&= width mismatch"); + return apply_words_(rhs, [](word_type x, word_type y) noexcept { return x & y; }); } - constexpr auto operator^=(const Bitset &rhs) noexcept -> Bitset & { - for (auto i = 0uz; i < kNumWords; ++i) - words_[i] ^= rhs.words_[i]; - return *this; + auto operator|=(const Bitset &rhs) noexcept -> Bitset & { + assert(nwords_ == rhs.nwords_ && "Bitset::operator|= width mismatch"); + return apply_words_(rhs, [](word_type x, word_type y) noexcept { return x | y; }); } - [[nodiscard]] constexpr auto operator~() const noexcept -> Bitset { + auto operator^=(const Bitset &rhs) noexcept -> Bitset & { + assert(nwords_ == rhs.nwords_ && "Bitset::operator^= width mismatch"); + return apply_words_(rhs, [](word_type x, word_type y) noexcept { return x ^ y; }); + } + + [[nodiscard]] auto operator~() const noexcept -> Bitset { Bitset r = *this; - for (auto i = 0uz; i < kNumWords; ++i) - r.words_[i] = ~r.words_[i]; + word_type *w = r.data(); + with_words_([w](auto n) { + for (size_t i = 0; i < n; ++i) + w[i] = ~w[i]; + }); r.sanitize_top(); return r; } - [[nodiscard]] friend constexpr auto operator&(const Bitset &lhs, const Bitset &rhs) noexcept -> Bitset { + [[nodiscard]] friend auto operator&(const Bitset &lhs, const Bitset &rhs) noexcept -> Bitset { Bitset r = lhs; r &= rhs; return r; } - [[nodiscard]] friend constexpr auto operator|(const Bitset &lhs, const Bitset &rhs) noexcept -> Bitset { + [[nodiscard]] friend auto operator|(const Bitset &lhs, const Bitset &rhs) noexcept -> Bitset { Bitset r = lhs; r |= rhs; return r; } - [[nodiscard]] friend constexpr auto operator^(const Bitset &lhs, const Bitset &rhs) noexcept -> Bitset { + [[nodiscard]] friend auto operator^(const Bitset &lhs, const Bitset &rhs) noexcept -> Bitset { Bitset r = lhs; r ^= rhs; return r; } - constexpr auto operator>>=(size_t pos) noexcept -> Bitset & { - if (pos >= NumBits) { - words_.fill(0); + auto operator>>=(size_t pos) noexcept -> Bitset & { + const size_t num_bits = size(); + word_type *w = data(); + if (pos >= num_bits) { + for (size_t i = 0; i < nwords_; ++i) + w[i] = 0; + return *this; + } + if (nwords_ <= 1) { + if (nwords_ == 1) { + w[0] >>= pos; + } return *this; } - if constexpr (kNumWords == 1) { - words_[0] >>= pos; + const size_t word_shift = pos / word_width; + const size_t limit = nwords_ - word_shift; + if (const size_t bit_shift = pos % word_width; bit_shift == 0) { + for (size_t i = 0; i < limit; ++i) + w[i] = w[i + word_shift]; } else { - const size_t word_shift = pos / word_width; - const size_t limit = kNumWords - word_shift; - if (const size_t bit_shift = pos % word_width; bit_shift == 0) { - for (size_t i = 0; i < limit; ++i) - words_[i] = words_[i + word_shift]; - } - else { - const size_t inv_shift = word_width - bit_shift; - for (size_t i = 0; i + 1 < limit; ++i) { - words_[i] = (words_[i + word_shift] >> bit_shift) | (words_[i + word_shift + 1] << inv_shift); - } - words_[limit - 1] = words_[kNumWords - 1] >> bit_shift; + const size_t inv_shift = word_width - bit_shift; + for (size_t i = 0; i + 1 < limit; ++i) { + w[i] = (w[i + word_shift] >> bit_shift) | (w[i + word_shift + 1] << inv_shift); } - for (size_t i = limit; i < kNumWords; ++i) - words_[i] = 0; + w[limit - 1] = w[nwords_ - 1] >> bit_shift; } + for (size_t i = limit; i < nwords_; ++i) + w[i] = 0; return *this; } - [[nodiscard]] constexpr auto operator>>(size_t pos) const noexcept -> Bitset { + [[nodiscard]] auto operator>>(size_t pos) const noexcept -> Bitset { Bitset r = *this; r >>= pos; return r; } - [[nodiscard]] constexpr auto operator==(const Bitset &o) const noexcept -> bool { - for (size_t i = 0; i < kNumWords; ++i) - if (words_[i] != o.words_[i]) - return false; - return true; + // Width first, or equality is asymmetric: the loops below run this->nwords_, so without it a + // default-constructed (width-0) bitset compares equal to everything while nothing compares equal + // to it. Unreachable while every bitset is built at a real width, but monomial keys live in a + // boost::unordered_flat_map, and an asymmetric operator== there is a silent corruption rather + // than a crash -- and the de-templated wire readers use exactly the + // default-construct-then-assign pattern that produces a width-0 operand. + [[nodiscard]] auto operator==(const Bitset &o) const noexcept -> bool { + if (nwords_ != o.nwords_) { + return false; + } + const word_type *a = data(); + const word_type *b = o.data(); + return with_words_([a, b](auto n) { + for (size_t i = 0; i < n; ++i) + if (a[i] != b[i]) + return false; + return true; + }); } - [[nodiscard]] static constexpr auto num_words() noexcept -> size_t { return kNumWords; } - [[nodiscard]] constexpr auto data() const noexcept -> const uint64_t * { return words_.data(); } - [[nodiscard]] constexpr auto data() noexcept -> uint64_t * { return words_.data(); } - [[nodiscard]] constexpr auto word(size_t i) const noexcept -> uint64_t { return words_[i]; } - - [[nodiscard]] constexpr auto find_first() const noexcept -> size_t { // NumBits if none - for (size_t i = 0; i < kNumWords; ++i) { - if (words_[i]) - return (i * word_width) + static_cast(std::countr_zero(words_[i])); - } - return NumBits; + [[nodiscard]] auto find_first() const noexcept -> size_t { // size() if none + const word_type *w = data(); + const size_t hit = with_words_([w](auto n) { + for (size_t i = 0; i < n; ++i) { + if (w[i]) + return (i * word_width) + static_cast(std::countr_zero(w[i])); + } + return static_cast(-1); + }); + return hit == static_cast(-1) ? size() : hit; } - [[nodiscard]] constexpr auto find_next(size_t pos) const noexcept -> size_t { // NumBits if none - if (++pos >= NumBits) - return NumBits; - if constexpr (kNumWords == 1) { - if (const uint64_t w = words_[0] >> pos; w) - return pos + static_cast(std::countr_zero(w)); - return NumBits; + [[nodiscard]] auto find_next(size_t pos) const noexcept -> size_t { // size() if none + const size_t num_bits = size(); + if (++pos >= num_bits) { + return num_bits; } - else { - size_t wi = pos / word_width; - if (const uint64_t w = words_[wi] >> (pos % word_width); w) - return pos + static_cast(std::countr_zero(w)); - for (++wi; wi < kNumWords; ++wi) { - if (words_[wi]) - return (wi * word_width) + static_cast(std::countr_zero(words_[wi])); - } - return NumBits; + const word_type *w = data(); + if (nwords_ <= 1) { + if (const uint64_t x = w[0] >> pos; x) + return pos + static_cast(std::countr_zero(x)); + return num_bits; + } + size_t wi = pos / word_width; + if (const uint64_t x = w[wi] >> (pos % word_width); x) + return pos + static_cast(std::countr_zero(x)); + for (++wi; wi < nwords_; ++wi) { + if (w[wi]) + return (wi * word_width) + static_cast(std::countr_zero(w[wi])); } + return num_bits; } // Stream output MSB→LSB (std::bitset convention). friend auto operator<<(std::ostream &os, const Bitset &bs) -> std::ostream & { - for (size_t i = NumBits; i-- > 0;) + for (size_t i = bs.size(); i-- > 0;) os << (bs.test(i) ? '1' : '0'); return os; } }; -} // namespace monoprop -template -struct SplitmixHash; +struct Bitset::FusedXor { + Bitset result; + size_t overlap; + size_t result_count; +}; -template -struct SplitmixHash> { +inline auto Bitset::fused_xor_into(const Bitset &gen, Bitset &dst) const noexcept -> FusedCounts { + assert(num_words() == gen.num_words() && "Bitset::fused_xor_into width mismatch"); + assert(num_words() == dst.num_words() && "Bitset::fused_xor_into destination width mismatch"); + const word_type *a = data(); + const word_type *b = gen.data(); + word_type *out = dst.data(); + return with_words_([a, b, out](auto n) { return detail::fused_xor_words(a, b, out, n); }); +} + +inline auto Bitset::fused_xor(const Bitset &gen) const noexcept -> FusedXor { + Bitset result(size()); + const auto counts = fused_xor_into(gen, result); + return {result, counts.overlap, counts.result_count}; +} + +// Bit-identical to the old per-width SplitmixHash>: same mix(), same per-word fold +// order, same "+i" per-word offset -- only the num_words()==1 dispatch moved from `if constexpr` to a +// runtime check. The values must not change: they drive MPI owner routing (see monomial_hash). +struct SplitmixHash { static constexpr auto mix(uint64_t x) noexcept -> uint64_t { x ^= x >> 30; x *= 0xbf58476d1ce4e5b9ULL; @@ -232,26 +615,86 @@ struct SplitmixHash> { return x; } - auto operator()(const monoprop::Bitset &bs) const noexcept -> size_t { - constexpr size_t W = monoprop::Bitset::num_words(); - if constexpr (W == 1) { + auto operator()(const Bitset &bs) const noexcept -> size_t { + const size_t w = bs.num_words(); + if (w == 1) { return static_cast(mix(bs.word(0))); } + uint64_t h = 0; + for (size_t i = 0; i < w; ++i) { + h ^= mix(bs.word(i) + static_cast(i)); + } + return static_cast(h); + } +}; + +namespace detail { + +// The per-term word ops with the word count supplied by the caller instead of read off the operand. +// +// The arithmetic is identical to Bitset's own methods; what differs is what the compiler knows. A +// Bitset method must load nwords_, compare it against the inline capacity and select a storage +// pointer on every call, and none of those three can be hoisted out of a loop the optimizer cannot +// see through -- which, on the per-term path, is every call. Handing a kernel a compile-time W and +// the word pointers the caller resolved once leaves a straight-line unrolled loop, which is what the +// per-width Bitset got for free. +// +// Preconditions, none of them checkable here: every pointer is a Bitset::data() of a bitset of +// exactly W words, and W <= kInlineWords so no operand is spilled. The only legal caller is one that +// bound W from a width it owns for the whole loop -- see DenseTermProductsW, which is the seam that +// binds it once per gate. +// +// Standing in for a Bitset method is also what decides membership: the scan's other bound-width word +// pass, fully_paired_words, answers a question about the algebra rather than the storage, so it lives +// beside its own oracle in AlgebraCommon.h instead. +template +struct WordKernel { + static_assert(W >= 1 && W <= Bitset::kInlineWords, + "the kernel covers the inline regime; above it the runtime loop already wins"); + + using word_type = Bitset::word_type; + + static auto clear(word_type *a) noexcept -> void { + for (size_t i = 0; i < W; ++i) { + a[i] = 0; + } + } + + // Bitset::fused_xor_into with W fixed -- the same pass, reached without the nwords_ load and + // storage-pointer select the method does per call. + static auto fused_xor_into(const word_type *a, const word_type *b, word_type *out) noexcept -> Bitset::FusedCounts { + return fused_xor_words(a, b, out, std::integral_constant{}); + } + + // Bitset::parity_and with W fixed, likewise. + [[nodiscard]] static auto parity_and(const word_type *a, const word_type *b) noexcept -> bool { + return (std::popcount(and_fold_words(a, b, std::integral_constant{})) & 1U) != 0; + } + + // SplitmixHash with W fixed. Must stay bit-identical to it: this value routes MPI ownership, so a + // divergence would move terms between ranks rather than merely run slower. Hence the W == 1 arm + // reproducing the same special case rather than folding into the loop. + [[nodiscard]] static auto splitmix(const word_type *a) noexcept -> size_t { + if constexpr (W == 1) { + return static_cast(SplitmixHash::mix(a[0])); + } else { uint64_t h = 0; for (size_t i = 0; i < W; ++i) { - h ^= mix(bs.word(i) + static_cast(i)); + h ^= SplitmixHash::mix(a[i] + static_cast(i)); } return static_cast(h); } } }; +} // namespace detail + +} // namespace monoprop + namespace std { -template -struct hash> { - auto operator()(const monoprop::Bitset &bs) const noexcept -> size_t { - return SplitmixHash>{}(bs); - } +template <> +struct hash { + auto operator()(const monoprop::Bitset &bs) const noexcept -> size_t { return monoprop::SplitmixHash{}(bs); } }; } // namespace std diff --git a/cpp/monoprop/CMakeLists.txt b/cpp/monoprop/CMakeLists.txt index 1b233211..c6ca6d19 100644 --- a/cpp/monoprop/CMakeLists.txt +++ b/cpp/monoprop/CMakeLists.txt @@ -22,6 +22,7 @@ target_compile_definitions( PUBLIC $<$:monoprop_ENABLE_MPI> $<$:monoprop_WIDE_TERM_INDEX> + monoprop_SPARSE_ROW_MIN_MODES=${monoprop_SPARSE_ROW_MIN_MODES} ) # flags to prepend @@ -100,6 +101,7 @@ target_compile_definitions( INTERFACE $<$:monoprop_ENABLE_MPI> $<$:monoprop_WIDE_TERM_INDEX> + monoprop_SPARSE_ROW_MIN_MODES=${monoprop_SPARSE_ROW_MIN_MODES} ) target_compile_features(monoprop INTERFACE cxx_std_23) diff --git a/cpp/monoprop/Utilities.h b/cpp/monoprop/Utilities.h index 992f12cf..1a142616 100644 --- a/cpp/monoprop/Utilities.h +++ b/cpp/monoprop/Utilities.h @@ -18,9 +18,12 @@ #include #include #include +#include #include #include "monoprop/Bitset.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/operator/RowAccess.h" #include "monoprop/monopropExport.h" namespace monoprop { @@ -30,51 +33,71 @@ struct MSb0 : BitOrdering {}; struct LSb0 : BitOrdering {}; namespace detail { -template -constexpr auto make_repeating_bitset(uint64_t pattern) -> Bitset { - constexpr size_t kNumWords = Bitset::num_words(); - Bitset bits; - for (size_t i = 0; i < kNumWords; ++i) +// n is an ordinary runtime argument: Bitset carries its width as data, so there is nothing for a +// template parameter of its own to supply. +inline auto make_repeating_bitset(size_t n, uint64_t pattern) -> Bitset { + Bitset bits(n); + const size_t num_words = bits.num_words(); + for (size_t i = 0; i < num_words; ++i) bits.data()[i] = pattern; - if constexpr (constexpr size_t kTopBits = N % 64; kTopBits != 0) { - constexpr uint64_t kTopMask = (uint64_t(1) << kTopBits) - 1; - bits.data()[kNumWords - 1] &= kTopMask; + if (const size_t top_bits = n % 64; top_bits != 0) { + const uint64_t top_mask = (uint64_t(1) << top_bits) - 1; + bits.data()[num_words - 1] &= top_mask; } return bits; } -template -constexpr auto even_bits() -> Bitset { - return make_repeating_bitset(0x5555555555555555ULL); +inline auto even_bits(size_t n) -> Bitset { + return make_repeating_bitset(n, 0x5555555555555555ULL); } - -template -constexpr auto odd_bits() -> Bitset { - return make_repeating_bitset(0xAAAAAAAAAAAAAAAAULL); +inline auto odd_bits(size_t n) -> Bitset { + return make_repeating_bitset(n, 0xAAAAAAAAAAAAAAAAULL); } } // namespace detail -// Under MSb0 the logical even positions are physically odd, so the pattern is swapped vs LSb0. -template -constexpr auto even_bits() -> Bitset { +// Under MSb0 the logical even positions are physically odd, so the pattern is swapped vs LSb0. The +// width is an ordinary argument: only the Ordering has to be a template parameter, since it selects +// which pattern at compile time and is never data. +template +auto even_bits(size_t n) -> Bitset { if constexpr (std::is_same_v) { - return detail::odd_bits(); + return detail::odd_bits(n); } else { - return detail::even_bits(); + return detail::even_bits(n); } }; // Same MSb0/LSb0 swap as even_bits(). -template -constexpr auto odd_bits() -> Bitset { +template +auto odd_bits(size_t n) -> Bitset { if constexpr (std::is_same_v) { - return detail::even_bits(); + return detail::even_bits(n); } else { - return detail::odd_bits(); + return detail::odd_bits(n); } }; +// Memoized even-bit mask for per-term code. A mask depends only on the storage width, which is fixed +// for a propagator's lifetime, so rebuilding one per term is pure waste -- and with Bitset +// runtime-width, building one is a full object construction, not a compile-time constant. +// +// thread_local rather than shared: the scan runs concurrently on the partitions' pinned masters, and +// a shared cache would need synchronisation on the hottest path in the library. The width only ever +// changes between propagators, so the miss branch is taken once per thread in practice. +// +// The reference is valid until the next call *on this thread* with a different width. Callers use it +// within a single expression or loop; do not store it across a width change. +template +[[nodiscard]] inline auto cached_even_bits(size_t n) -> const Bitset & { + thread_local Bitset cached; + if (thread_local auto cached_n = static_cast(-1); cached_n != n) [[unlikely]] { + cached = even_bits(n); + cached_n = n; + } + return cached; +} + inline auto n_choose_2(std::integral auto n) -> size_t { return static_cast(n * (n - 1) / 2); } @@ -91,4 +114,111 @@ auto join_with_separator(std::ranges::range auto const &values, std::string_view } return joined; } + +// A Majorana/Pauli index at or past the width of the system it is being applied to. +class AlgebraIndexOutOfRange : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// A coefficient with no real encoding under the algebra model: non-Hermitian for Majorana products, +// non-real for Pauli strings. +class NonEncodableCoefficient : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// Unchecked: `num_bits - 1 - bit_loc` underflows for an out-of-range index and Bitset::set is +// noexcept, so the result is an out-of-bounds write. Use indices_to_bitset_checked() for user input. +inline auto indices_to_bitset(const VecZ &arr, size_t num_bits) -> Bitset { + Bitset bs(num_bits); + for (const auto &bit_loc : arr) { + bs.set(num_bits - 1 - bit_loc); // MSb0 convention: index 0 maps to the top bit + } + return bs; +} + +// The two bounds are different quantities and neither implies the other, so they are separate +// arguments: max_index is the *logical* width (2 * logical_num_modes), which is what a caller's +// indices must fall inside, while num_bits is the *storage* width the result is built at. A +// propagator running fewer modes than its storage holds must still reject indices outside its own +// system, and storage rounds up. +inline auto indices_to_bitset_checked(const VecZ &arr, size_t max_index, size_t num_bits) -> Bitset { + for (const auto &bit_loc : arr) { + if (bit_loc >= max_index) { + throw AlgebraIndexOutOfRange( + std::format("Majorana/Pauli index {} is out of range; must be less than {}.", bit_loc, max_index)); + } + } + return indices_to_bitset(arr, num_bits); +} + +// O(popcount) via find_first/find_next rather than an O(num_bits) scan. +auto bitset_to_indices(const MonomialLike auto &bs) -> VecZ { + const auto pop = bs.count(); + VecZ indices(pop); + size_t idx = pop; + const size_t n = bs.size(); + for (size_t pos = bs.find_first(); pos < n; pos = bs.find_next(pos)) { + indices[--idx] = n - 1 - pos; + } + return indices; +} + +auto is_paired(const MonomialLike auto &mono, const auto &even_mask) -> bool { + // Paired = each mode's even bit and its odd partner agree (both set or both clear). Word loop + // rather than `(mono & m) ^ ((mono >> 1) & m)`, which built three runtime-width temporaries per + // call; (word >> 1) & m is within-word for the same reason as in cutoff_sums(). + const size_t nw = mono.num_words(); + for (size_t w = 0; w < nw; ++w) { + const uint64_t word = mono.word(w); + const uint64_t m = even_mask.word(w); + if (((word & m) ^ ((word >> 1) & m)) != 0) { + return false; + } + } + return true; +} + +auto is_paired(const MonomialLike auto &mono) -> bool { + return is_paired(mono, cached_even_bits(mono.size())); +} + +// `mono` is an index list, not a monomial, so there is no argument to deduce a width from -- hence the +// explicit num_bits, which sizes the bitset this builds. +inline auto is_paired(const VecZ &mono, size_t num_bits) -> bool { + return is_paired(indices_to_bitset(mono, num_bits)); +} + +// Rows carries no structural width of its own (unlike a MonomialLike argument), so the width is +// explicit here too. It must be the width of the rows themselves: the mask is compared against them +// pairwise, and a mismatch trips Bitset's width assertions. +template +auto is_fully_paired(const VecZ &inds, const Rows &op, size_t num_bits) -> VecZ { + VecZ result; + // Memoized rather than rebuilt: the reference stays valid across the loop because is_paired's + // two-argument form takes the mask and so never re-enters the cache with another width. + const auto &mask = cached_even_bits(num_bits); + for (const auto index : inds) { + const auto &op_row = materialize_row(op, index); + if (is_paired(op_row, mask)) { + result.push_back(index); + } + } + return result; +} + +// Occupation mask of the initial product state: the even index 2*i of each listed mode (Majorana) or +// qubit (Pauli) that starts in state 1. Both algebras read the same mask and differ only in the phase +// they score against it (majorana_state_phase / pauli_state_phase). +inline auto initial_state_mask(const VecZ &initial_state, size_t num_bits) -> Bitset { + // Set straight into the result rather than through indices_to_bitset: the index vector that would + // build costs an allocation and a second walk, for a mapping that is one multiply per mode. Same + // MSb0 convention indices_to_bitset applies. + Bitset mask(num_bits); + for (const auto &mode : initial_state) { + mask.set(num_bits - 1 - (2 * mode)); + } + return mask; +} } // namespace monoprop diff --git a/cpp/monoprop/Validation.cpp b/cpp/monoprop/Validation.cpp index 46e5252d..924c699e 100644 --- a/cpp/monoprop/Validation.cpp +++ b/cpp/monoprop/Validation.cpp @@ -116,7 +116,7 @@ auto validate_expected_initial_operator(size_t current_epoch, size_t expected_ep } } -auto validate_only_rotate_len_k_(std::optional only_rotate_len_k, size_t max_k) -> void { +auto validate_only_rotate_len_k(std::optional only_rotate_len_k, size_t max_k) -> void { if (!only_rotate_len_k.has_value()) { return; } diff --git a/cpp/monoprop/Validation.h b/cpp/monoprop/Validation.h index db93e622..56e187cc 100644 --- a/cpp/monoprop/Validation.h +++ b/cpp/monoprop/Validation.h @@ -45,7 +45,7 @@ monoprop_EXPORT auto validate_expected_graph_layers(size_t current_layers, size_ monoprop_EXPORT auto validate_expected_initial_operator(size_t current_epoch, size_t expected_epoch) -> void; // only_rotate_len_k is optional; when set it must satisfy 0 < k <= max_k. -monoprop_EXPORT auto validate_only_rotate_len_k_(std::optional only_rotate_len_k, size_t max_k) -> void; +monoprop_EXPORT auto validate_only_rotate_len_k(std::optional only_rotate_len_k, size_t max_k) -> void; monoprop_EXPORT auto expected_num_params(const VecZ ¶meter_mapping) -> size_t; diff --git a/cpp/monoprop/algebra/Algebra.h b/cpp/monoprop/algebra/Algebra.h index dec6350d..8948ce0a 100644 --- a/cpp/monoprop/algebra/Algebra.h +++ b/cpp/monoprop/algebra/Algebra.h @@ -24,6 +24,7 @@ #include #include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/algebra/CodesAlgebra.h" #include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/algebra/PauliAlgebra.h" #include "monoprop/core/Monomial.h" @@ -31,7 +32,6 @@ namespace monoprop { -template struct MajoranaAlgebra { static constexpr Basis basis = Basis::Majorana; static constexpr bool requires_support_cutoff = false; // length OR support cutoff both valid @@ -40,59 +40,72 @@ struct MajoranaAlgebra { // Built once per layer: the generator G and its fixed interleave mask W (see interleave_phase_mask). // G is stored by value so the context can outlive a caller's temporary; the cost is one bitset copy // per layer, cheaper than a lifetime contract on every call site. + // Both members are assigned by make_gen_context, so they carry the generator's width; a + // default-constructed GenContext would hold width-0 bitsets. struct GenContext { - Monomial gen; - Monomial interleave_mask; + Bitset gen; + Bitset interleave_mask; }; - static auto make_gen_context(const Monomial &gen) -> GenContext { - return GenContext{gen, interleave_phase_mask(gen)}; + static auto make_gen_context(const Bitset &gen) -> GenContext { + return GenContext{gen, interleave_phase_mask(gen)}; } - static auto generator(const GenContext &ctx) -> const Monomial & { return ctx.gen; } + static auto generator(const GenContext &ctx) -> const Bitset & { return ctx.gen; } // Ordering sign of mono·G via the per-layer mask (branch/scan-free). - static auto rotation_sign(const GenContext &ctx, - const Monomial &mono, - const Monomial & /*new_mono*/) -> int { + static auto rotation_sign(const GenContext &ctx, const Bitset &mono, const Bitset & /*new_mono*/) -> int { return mono.parity_and(ctx.interleave_mask) ? -1 : 1; } + // The same sign off word pointers the caller resolved once, with the word count bound by the + // caller rather than read off the operand. Same fold, same parity; see detail::WordKernel. + template + static auto rotation_sign_words(const GenContext &ctx, + const Bitset::word_type *mono, + const Bitset::word_type * /*new_mono*/) -> int { + return detail::WordKernel::parity_and(mono, ctx.interleave_mask.data()) ? -1 : 1; + } + // The same sign in support form. No GenContext: the interleave mask is dense by construction + // (roughly half the register), so the sparse form walks the two rows instead of carrying a mask, and + // the product row is not an argument either -- see codes_interleave_phase. + static auto codes_rotation_sign(const detail::SparseRow &mono, const detail::SparseRow &gen) -> int { + return detail::codes_interleave_phase(mono, gen); + } static auto emit_phase(int rotation_sign, size_t mono_pop, size_t gen_pop, size_t overlap) -> int { return rotation_sign * hermitian_phase(mono_pop, gen_pop, overlap); } // Anticommutation fold columns = G itself; odd |G| needs the per-row parity(|M|) correction. - static auto fold_generator(const Monomial &gen) -> Monomial { return gen; } - static auto fold_needs_odd_correction(const Monomial &gen) -> bool { return gen.count() % 2 != 0; } - - static auto encode_coeff(const std::complex &coeff, const Monomial &mono) -> double { - return monoprop::encode_coeff(coeff, mono); - } - static auto decode_coeff(const std::complex &coeff, const Monomial &mono) - -> std::complex { - return monoprop::decode_coeff(coeff, mono); - } - static auto state_phase(const Monomial &mono, const Monomial &state_mask) -> double { - return monoprop::majorana_state_phase(mono, state_mask); - } + static auto fold_generator(const Bitset &gen) -> Bitset { return gen; } + static auto fold_needs_odd_correction(const Bitset &gen) -> bool { return gen.count() % 2 != 0; } }; -template struct PauliAlgebra { static constexpr Basis basis = Basis::Pauli; static constexpr bool requires_support_cutoff = true; // the support cutoff measures Pauli weight static constexpr bool allows_basis_change = false; // the native encoding has no basis change struct GenContext { - PauliGenContext pauli_ctx; + PauliGenContext pauli_ctx; }; - static auto make_gen_context(const Monomial &gen) -> GenContext { - return GenContext{make_pauli_gen_context(gen)}; - } - static auto generator(const GenContext &ctx) -> const Monomial & { return ctx.pauli_ctx.gen; } + static auto make_gen_context(const Bitset &gen) -> GenContext { return GenContext{make_pauli_gen_context(gen)}; } + static auto generator(const GenContext &ctx) -> const Bitset & { return ctx.pauli_ctx.gen; } // Rotation-ready sign: already the negated raw product sign (see pauli_rotation_sign). - static auto rotation_sign(const GenContext &ctx, const Monomial &mono, const Monomial &new_mono) - -> int { - return pauli_rotation_sign(ctx.pauli_ctx, mono, new_mono); + static auto rotation_sign(const GenContext &ctx, const Bitset &mono, const Bitset &new_mono) -> int { + return pauli_rotation_sign(ctx.pauli_ctx, mono, new_mono); + } + // W is unused here and that is the point: this sign already loops over the generator's non-zero + // words only, so there is no trip count to bind -- what the word form removes is the storage-pointer + // select that mono.word(w) repeats on every access. + template + static auto rotation_sign_words(const GenContext &ctx, + const Bitset::word_type *mono, + const Bitset::word_type *new_mono) -> int { + return pauli_rotation_sign_words(ctx.pauli_ctx, mono, new_mono); + } + // Same exponent as above off the two rows; new_mono never has to exist, since a mode the generator + // misses contributes nothing (see codes_pauli_rotation_sign). + static auto codes_rotation_sign(const detail::SparseRow &mono, const detail::SparseRow &gen) -> int { + return detail::codes_pauli_rotation_sign(mono, gen); } // Pauli's rotation sign is already the emitted sine phase -- no Hermitian fold. static auto emit_phase(int rotation_sign, size_t /*mono_pop*/, size_t /*gen_pop*/, size_t /*overlap*/) -> int { @@ -100,19 +113,8 @@ struct PauliAlgebra { } // Anticommutation fold columns = J(G) = pair_swap(G); Pauli needs no odd-|G| row-parity correction. - static auto fold_generator(const Monomial &gen) -> Monomial { return pair_swap(gen); } - static auto fold_needs_odd_correction(const Monomial & /*gen*/) -> bool { return false; } - - static auto encode_coeff(const std::complex &coeff, const Monomial & /*mono*/) -> double { - return encode_pauli_coeff(coeff); - } - static auto decode_coeff(const std::complex &coeff, const Monomial & /*mono*/) - -> std::complex { - return decode_pauli_coeff(coeff.real()); - } - static auto state_phase(const Monomial &mono, const Monomial &state_mask) -> double { - return pauli_state_phase(mono, state_mask); - } + static auto fold_generator(const Bitset &gen) -> Bitset { return pair_swap(gen); } + static auto fold_needs_odd_correction(const Bitset & /*gen*/) -> bool { return false; } }; // Shape check only: the members the backbone actually calls are enforced by use, not by this concept. @@ -124,59 +126,67 @@ concept Algebra = requires { { A::allows_basis_change } -> std::convertible_to; }; -static_assert(Algebra>); -static_assert(Algebra>); +static_assert(Algebra); +static_assert(Algebra); // The single runtime->policy branch: the hot backbone passes a generic lambda and is then fully // specialized on the chosen algebra. Both arms must return the same type. -template +template auto with_algebra(Basis basis, F &&f) { if (basis == Basis::Pauli) { - return std::forward(f).template operator()>(); + return std::forward(f).template operator()(); } - return std::forward(f).template operator()>(); + return std::forward(f).template operator()(); } // Point-dispatch helpers for cold sites (per-layer / per-materialization) that carry a runtime Basis. -template -auto algebra_fold_generator(Basis basis, const Monomial &gen) -> Monomial { - return with_algebra(basis, [&]() { return A::fold_generator(gen); }); +template +auto algebra_fold_generator(Basis basis, const T &gen) -> T { + return with_algebra(basis, [&gen]() { return A::fold_generator(gen); }); } -template -auto algebra_fold_needs_odd_correction(Basis basis, const Monomial &gen) -> bool { - return with_algebra(basis, [&]() { return A::fold_needs_odd_correction(gen); }); +template +auto algebra_fold_needs_odd_correction(Basis basis, const T &gen) -> bool { + return with_algebra(basis, [&gen]() { return A::fold_needs_odd_correction(gen); }); } -template -auto algebra_encode_coeff(Basis basis, const std::complex &coeff, const Monomial &mono) -> double { - return with_algebra(basis, [&]() { return A::encode_coeff(coeff, mono); }); +// These three branch on Basis directly instead of going through with_algebra, and the policy classes +// carry no encode_coeff/decode_coeff/state_phase member as a result -- both would have been a +// width-agnostic passthrough to the free function called here, so an algebra *class* bought nothing and +// left the mapping written twice, with only one of the two reachable. The branch below is the whole +// mapping; see the note on state_phase_rows for why a per-scored-row branch is the right altitude here. +auto algebra_encode_coeff(Basis basis, const std::complex &coeff, const MonomialLike auto &mono) -> double { + return basis == Basis::Pauli ? encode_pauli_coeff(coeff) : monoprop::encode_coeff(coeff, mono); } -template -auto algebra_decode_coeff(Basis basis, const std::complex &coeff, const Monomial &mono) +auto algebra_decode_coeff(Basis basis, const std::complex &coeff, const MonomialLike auto &mono) -> std::complex { - return with_algebra(basis, [&]() { return A::decode_coeff(coeff, mono); }); + return basis == Basis::Pauli ? decode_pauli_coeff(coeff.real()) : monoprop::decode_coeff(coeff, mono); } -template -auto algebra_state_phase(Basis basis, const Monomial &mono, const Monomial &state_mask) -> double { - return with_algebra(basis, [&]() { return A::state_phase(mono, state_mask); }); +auto algebra_state_phase(Basis basis, const MonomialLike auto &mono, const auto &state_mask) -> double { + return basis == Basis::Pauli ? pauli_state_phase(mono, state_mask) : majorana_state_phase(mono, state_mask); } // Score each fully-paired term's diagonal element against the initial product state, emitting // sink(row, phase). A sink rather than a dense out[row] because the scored set is a vanishing // fraction of the rows. -template +// +// num_bits is the width of the rows in `store`, which the state mask must match. No width template +// parameter and no with_algebra: the only thing the algebra policy supplied here was A::state_phase, +// and algebra_state_phase above is the same branch without a compile-time width. The branch does move +// inside the loop, which is why this is spelled out rather than left implicit -- it is a per-*scored*-row +// branch on a value fixed for the propagator's lifetime, on a path that runs over the fully-paired +// terms only (~0.07% of rows) and not per term in the scan. +template auto algebra_score_state(Basis basis, const VecZ &paired_inds, const VecZ &initial_state, const Rows &store, + size_t num_bits, Sink &&sink) -> void { - with_algebra(basis, [&]() { - const auto state_mask = initial_state_mask(initial_state); - for (size_t i = 0; i < paired_inds.size(); ++i) { - const auto &row = materialize_row(store, paired_inds[i]); - sink(paired_inds[i], A::state_phase(row, state_mask)); - } - }); + const auto state_mask = initial_state_mask(initial_state, num_bits); + for (const auto &idx : paired_inds) { + const auto &row = materialize_row(store, idx); + sink(idx, algebra_state_phase(basis, row, state_mask)); + } } } // namespace monoprop diff --git a/cpp/monoprop/algebra/AlgebraCommon.h b/cpp/monoprop/algebra/AlgebraCommon.h index 80ef32b9..44d4d7f4 100644 --- a/cpp/monoprop/algebra/AlgebraCommon.h +++ b/cpp/monoprop/algebra/AlgebraCommon.h @@ -16,10 +16,10 @@ #include #include +#include #include #include #include -#include #include #include "monoprop/TypeAliases.h" @@ -27,101 +27,6 @@ #include "monoprop/detail/operator/RowAccess.h" namespace monoprop { - -// A Majorana/Pauli index at or past the width of the system it is being applied to. -class AlgebraIndexOutOfRange : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -// A coefficient with no real encoding under the algebra model: non-Hermitian for Majorana products, -// non-real for Pauli strings. -class NonEncodableCoefficient : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -// Unchecked: `2 * NumModes - 1 - bit_loc` underflows for an out-of-range index and Monomial::set is -// noexcept, so the result is an out-of-bounds write. Use indices_to_bitset_checked() for user input. -template -auto indices_to_bitset(const VecZ &arr) -> Monomial { - Monomial bs; - for (const auto &bit_loc : arr) { - bs.set(2 * NumModes - 1 - bit_loc); // MSb0 convention: index 0 maps to the top bit - } - return bs; -} - -// The bound is the logical width (2 * logical_num_modes), not the storage width 2 * NumModes: a -// propagator over fewer modes than its instantiation must still reject indices outside its own system. -template -auto indices_to_bitset_checked(const VecZ &arr, size_t max_index) -> Monomial { - for (const auto &bit_loc : arr) { - if (bit_loc >= max_index) { - throw AlgebraIndexOutOfRange( - std::format("Majorana/Pauli index {} is out of range; must be less than {}.", bit_loc, max_index)); - } - } - return indices_to_bitset(arr); -} - -// O(popcount) via find_first/find_next rather than an O(NumModes) scan. -template -auto bitset_to_indices(const Monomial &bs) -> VecZ { - const auto pop = bs.count(); - VecZ indices(pop); - size_t idx = pop; - for (size_t pos = bs.find_first(); pos < bs.size(); pos = bs.find_next(pos)) { - indices[--idx] = bs.size() - 1 - pos; - } - return indices; -} - -template -auto is_paired(const Monomial &mono, const Monomial &even_mask) -> bool { - // Paired = each mode's even bit and its odd partner agree (both set or both clear). - const auto even_bits_masked = mono & even_mask; - const auto odd_bits_masked = (mono >> 1) & even_mask; - return (even_bits_masked ^ odd_bits_masked).none(); -} - -template -auto is_paired(const Monomial &mono) -> bool { - const auto even_mask = even_bits<2 * NumModes, LSb0>(); - return is_paired(mono, even_mask); -} - -template -auto is_paired(const VecZ &mono) -> bool { - return is_paired(indices_to_bitset(mono)); -} - -template -auto is_fully_paired(const VecZ &inds, const Rows &op) -> VecZ { - VecZ result; - const auto mask = even_bits<2 * NumModes, LSb0>(); - for (const auto index : inds) { - const auto &op_row = materialize_row(op, index); - if (is_paired(op_row, mask)) { - result.push_back(index); - } - } - return result; -} - -// Occupation mask of the initial product state: the even index 2*i of each listed mode (Majorana) or -// qubit (Pauli) that starts in state 1. Both algebras read the same mask and differ only in the phase -// they score against it (majorana_state_phase / pauli_state_phase). -template -auto initial_state_mask(const VecZ &initial_state) -> Monomial { - VecZ bits; - bits.reserve(initial_state.size()); - for (const auto &mode : initial_state) { - bits.push_back(2 * mode); - } - return indices_to_bitset(bits); -} - // The per-mode sums the structural cutoffs measure, over the active modes only. struct CutoffSums { size_t xor_sum; // modes with exactly one of their two Majoranas set; 0 == fully paired @@ -129,93 +34,220 @@ struct CutoffSums { size_t or_sum; // modes with either Majorana present -- the support measure (JW Pauli weight) }; -template -[[gnu::always_inline]] inline auto cutoff_sums(const Monomial &mono, size_t logical_num_modes) -> CutoffSums { - const size_t active_bit_offset = 2 * (NumModes - logical_num_modes); - - if constexpr (Monomial::num_words() == 1) { - constexpr size_t num_bits = Monomial::size(); - constexpr uint64_t valid_mask = num_bits == 64 ? ~uint64_t{0} : ((uint64_t{1} << num_bits) - 1); - constexpr uint64_t even_mask = even_bits<2 * NumModes, LSb0>().word(0); - const uint64_t active_mask = - active_bit_offset == 0 ? valid_mask : (valid_mask & ~((uint64_t{1} << active_bit_offset) - 1)); - const uint64_t active_word = mono.word(0) & active_mask; - const uint64_t pair_mask = even_mask & active_mask; - const uint64_t first_pair = active_word & pair_mask; - const uint64_t second_pair = (active_word >> 1) & pair_mask; +// Everything cutoff_sums would otherwise derive for every term. It depends only on the storage width +// and the logical width, both fixed for a propagator's lifetime, so it is built once and carried by the +// cutoff functors below. +// +// This exists for a measured reason. With a compile-time width these were constant expressions; with a +// runtime width, rederiving them per term costs ~2.3 ns/term at one word -- the whole single-word +// kernel is only about that -- which measured as +16% on a 32-mode in-place propagation and +2-3% on +// the 120/127-qubit models. Note what did *not* work: routing the multi-word loop through +// detail::with_nwords, the way Bitset's own word ops do, is 50-66% *slower* than this in a plain -O3 +// build (fine under -march=native), so the per-word loop below stays an ordinary runtime loop. +struct CutoffMasks { + uint64_t active = 0; // single-word arm: valid bits AND the active window + uint64_t even_active = 0; // single-word arm: the even-bit pattern AND `active` + size_t active_bit_offset = 0; + size_t num_bits = 0; // the width these were built for; only checked in assertions + size_t first_active_word = 0; // multi-word arm: words below this one are entirely inactive + uint64_t first_word_mask = ~uint64_t{0}; // multi-word arm: the active bits within that word + + // Derived from num_bits/active_bit_offset rather than stored: each is a one-line fact about the + // other two fields, read at 3 call sites total. + [[nodiscard]] auto single_word() const -> bool { return num_bits <= 64; } + [[nodiscard]] auto whole_register() const -> bool { return active_bit_offset == 0; } + + [[nodiscard]] static auto make(size_t num_bits, size_t logical_num_modes) -> CutoffMasks { + CutoffMasks m; + const size_t num_modes = num_bits / 2; + m.num_bits = num_bits; + m.active_bit_offset = 2 * (num_modes - logical_num_modes); + if (m.single_word()) { + // The even-bit pattern spelled as a literal rather than via even_bits(): identical value, + // plain integer arithmetic, and no Bitset to construct. + const uint64_t valid = num_bits == 64 ? ~uint64_t{0} : ((uint64_t{1} << num_bits) - 1); + m.active = m.active_bit_offset == 0 ? valid : (valid & ~((uint64_t{1} << m.active_bit_offset) - 1)); + m.even_active = (0x5555555555555555ULL & valid) & m.active; + } + else { + // The same "mask off the inactive low bits" the single-word arm above does, split into the + // word to start at and the mask for that word. The active window is the *high* end of the + // register, so every word below first_active_word contributes nothing to any of the three + // sums and is skipped rather than masked. + m.first_active_word = m.active_bit_offset / Bitset::word_width; + const size_t within = m.active_bit_offset % Bitset::word_width; + m.first_word_mask = within == 0 ? ~uint64_t{0} : ~((uint64_t{1} << within) - 1); + } + return m; + } +}; + +[[gnu::always_inline]] inline auto cutoff_sums(const MonomialLike auto &mono, const CutoffMasks &masks) -> CutoffSums { + assert(mono.size() == masks.num_bits && "cutoff masks built for a different width"); + + // A runtime branch, not `if constexpr`: the width is data now. Still a branch rather than folded + // into the general loop because this arm skips the mask lookup and the loop entirely, and one word + // covers every model up to 32 modes. + if (masks.single_word()) { + const uint64_t active_word = mono.word(0) & masks.active; + const uint64_t first_pair = active_word & masks.even_active; + const uint64_t second_pair = (active_word >> 1) & masks.even_active; return {static_cast(std::popcount(first_pair ^ second_pair)), static_cast(std::popcount(active_word)), static_cast(std::popcount(first_pair | second_pair))}; } + // One pass over the words, with no Bitset temporaries. The `active & mask` / `(active >> 1) & mask` + // / `^` / `|` / `>>` chain this replaces built five of them per term, and since Stage 2b each is a + // full runtime-width object construction rather than a trivially copyable value. + // + // (word >> 1) & even_mask equals ((bits >> 1) & mask).word(w): a full-width shift carries the low + // bit of word w+1 into bit 63 of word w, which is an odd position and so masked off regardless. + // The same within-word-pairs argument pair_swap() and pauli_uv() already rely on. + // + // A narrower active window is masked, not shifted -- `mono >> active_bit_offset` would copy the + // whole monomial per term (and allocate, past kInlineWords), and it is the *common* case: the + // storage width rounds up to a whole 32-mode block, so the offset is non-zero for any logical + // width that is not a multiple of 32. Masking gives the same three sums because the offset is even, + // so a mode's two bits are dropped or kept together and every surviving mode keeps its parity -- + // exactly the argument the single-word arm above already rests on. word_mask applies to + // first_active_word only; the assignment in the loop is a register move rather than a branch. + const auto &mask = cached_even_bits(masks.num_bits); + size_t xor_sum = 0; + size_t popcount_sum = 0; + size_t or_sum = 0; + const size_t nw = mono.num_words(); + uint64_t word_mask = masks.first_word_mask; + for (size_t w = masks.first_active_word; w < nw; ++w) { + const uint64_t word = mono.word(w) & word_mask; + word_mask = ~uint64_t{0}; + const uint64_t m = mask.word(w); + const uint64_t first_pair = word & m; + const uint64_t second_pair = (word >> 1) & m; + xor_sum += static_cast(std::popcount(first_pair ^ second_pair)); + popcount_sum += static_cast(std::popcount(word)); + or_sum += static_cast(std::popcount(first_pair | second_pair)); + } + return {xor_sum, popcount_sum, or_sum}; +} - const auto active_mono = logical_num_modes == NumModes ? mono : (mono >> active_bit_offset); - const auto mask = even_bits<2 * NumModes, LSb0>(); - const auto first_pair = active_mono & mask; - const auto second_pair = (active_mono >> 1) & mask; - return {(first_pair ^ second_pair).count(), active_mono.count(), (first_pair | second_pair).count()}; +// Cold-path form: derives the masks per call. Every per-term caller goes through a cutoff functor, +// which holds them. +[[gnu::always_inline]] inline auto cutoff_sums(const MonomialLike auto &mono, size_t logical_num_modes) -> CutoffSums { + return cutoff_sums(mono, CutoffMasks::make(mono.size(), logical_num_modes)); } // Both cutoffs below keep a fully paired monomial (xor_sum == 0) unconditionally: those are the only // terms contributing to an expectation value against a product reference state, so bounding them by // length or support would discard signal. -template -auto length_cutoff(const Monomial &mono, unsigned int cutoff, size_t logical_num_modes) -> bool { - const auto sums = cutoff_sums(mono, logical_num_modes); +auto length_cutoff(const MonomialLike auto &mono, unsigned int cutoff, const CutoffMasks &masks) -> bool { + const auto sums = cutoff_sums(mono, masks); return sums.xor_sum == 0 || sums.popcount_sum <= cutoff; } -template -auto length_cutoff(const Monomial &mono, unsigned int cutoff) -> bool { - return length_cutoff(mono, cutoff, NumModes); +auto length_cutoff(const MonomialLike auto &mono, unsigned int cutoff, size_t logical_num_modes) -> bool { + return length_cutoff(mono, cutoff, CutoffMasks::make(mono.size(), logical_num_modes)); } -template -auto support_cutoff(const Monomial &mono, unsigned int cutoff, size_t logical_num_modes) -> bool { - const auto sums = cutoff_sums(mono, logical_num_modes); +// Whole-register overload: every mode is active. Reads the width off the instance -- a qualified +// decltype(mono)::size() would be ill-formed for a plain Bitset. +auto length_cutoff(const MonomialLike auto &mono, unsigned int cutoff) -> bool { + return length_cutoff(mono, cutoff, mono.size() / 2); +} + +auto support_cutoff(const MonomialLike auto &mono, unsigned int cutoff, const CutoffMasks &masks) -> bool { + const auto sums = cutoff_sums(mono, masks); return sums.xor_sum == 0 || sums.or_sum <= cutoff; } -template -auto support_cutoff(const Monomial &mono, unsigned int cutoff) -> bool { - return support_cutoff(mono, cutoff, NumModes); +auto support_cutoff(const MonomialLike auto &mono, unsigned int cutoff, size_t logical_num_modes) -> bool { + return support_cutoff(mono, cutoff, CutoffMasks::make(mono.size(), logical_num_modes)); +} + +auto support_cutoff(const MonomialLike auto &mono, unsigned int cutoff) -> bool { + return support_cutoff(mono, cutoff, mono.size() / 2); } namespace detail { -template +// The xor_sum == 0 clause above -- "every occupied mode has both its Majoranas" -- with the storage +// word count bound at compile time, for the per-gate scan kernel that has W and the words already (see +// DenseTermProductsW). It lives here and not beside the other bound-width word ops in Bitset.h because +// "paired" is a fact about the algebra and not about the storage, and because the mask literal it shares +// with CutoffMasks::make above is easier to keep honest in one file than in two. +// +// Folded with OR and tested against zero rather than summing popcounts: the caller only ever compares +// the sum to zero, and the two agree because each per-word term is non-negative. +// +// The even-bit mask is the literal rather than an argument, which is what makes this W loads instead of +// 2W: a storage width is a whole number of words, so even_bits is this pattern in every one of +// them. Its top-word trim at a non-word-multiple width is unobservable here -- bits above the logical +// width are never set, so they pair with themselves either way. (word >> 1) & mask cannot cross a word +// because a mode's two bits are 2m and 2m+1. +// +// Whole register only: a narrower active window would need the shift cutoff_sums applies, and getting +// it wrong would silently change which terms survive. +template +[[nodiscard]] [[gnu::always_inline]] inline auto fully_paired_words(const Bitset::word_type *a) noexcept -> bool { + constexpr Bitset::word_type kEven = 0x5555555555555555ULL; + Bitset::word_type unpaired = 0; + for (size_t i = 0; i < W; ++i) { + const Bitset::word_type word = a[i]; + unpaired |= (word & kEven) ^ ((word >> 1) & kEven); + } + return unpaired == 0; +} + +// Both hold their masks, so the per-term call does no width arithmetic. Real constructors rather than +// aggregate initialization, deliberately: the width used to arrive free from NumModes, and both +// remaining ways to get it wrong are silent. A logical width of 0 makes cutoff_sums' active window +// empty, so xor_sum is 0 and the cutoff keeps *everything*; masks that disagree with +// logical_num_modes do the same kind of damage. A constructor makes both unrepresentable -- with an +// aggregate, a designated initializer that omitted either field would just zero it. struct LengthCutoff { - unsigned int cutoff = 0; - size_t logical_num_modes = NumModes; + unsigned int cutoff; + size_t logical_num_modes; + CutoffMasks masks; - auto operator()(const Monomial &mono) const -> bool { - return length_cutoff(mono, cutoff, logical_num_modes); - } + LengthCutoff(unsigned int cutoff_, size_t logical_num_modes_, size_t num_bits) + : cutoff(cutoff_), + logical_num_modes(logical_num_modes_), + masks(CutoffMasks::make(num_bits, logical_num_modes_)) {} + + auto operator()(const Bitset &mono) const -> bool { return length_cutoff(mono, cutoff, masks); } }; -template struct SupportCutoff { - unsigned int cutoff = 0; - size_t logical_num_modes = NumModes; + unsigned int cutoff; + size_t logical_num_modes; + CutoffMasks masks; - auto operator()(const Monomial &mono) const -> bool { - return support_cutoff(mono, cutoff, logical_num_modes); - } + SupportCutoff(unsigned int cutoff_, size_t logical_num_modes_, size_t num_bits) + : cutoff(cutoff_), + logical_num_modes(logical_num_modes_), + masks(CutoffMasks::make(num_bits, logical_num_modes_)) {} + + auto operator()(const Bitset &mono) const -> bool { return support_cutoff(mono, cutoff, masks); } }; -template class CutoffEvaluator { public: - explicit CutoffEvaluator(const CutoffFn &cutoff_fn) + // The two target<>() probes recover the concrete functor behind the type-erased CutoffFn so the hot + // paths below can call it directly and read its cutoff. std::function::target() matches only on + // the *exact* stored type, so anything that wraps the functor -- a lambda, a different width, a + // structurally identical copy of the type -- yields nullptr and silently falls back to calling + // through the std::function. That is correct but slower, and invisible: no test and no bit-identity + // check can see it. cutoff_function() asserts the handshake at the point where the type is chosen. + explicit CutoffEvaluator(const CutoffFn &cutoff_fn) : cutoff_fn_(cutoff_fn), - length_cutoff_(cutoff_fn.template target>()), - support_cutoff_(cutoff_fn.template target>()) {} + length_cutoff_(cutoff_fn.target()), + support_cutoff_(cutoff_fn.target()) {} - auto length_cutoff() const -> const LengthCutoff * { return length_cutoff_; } + auto length_cutoff() const -> const LengthCutoff * { return length_cutoff_; } - auto support_cutoff() const -> const SupportCutoff * { return support_cutoff_; } + auto support_cutoff() const -> const SupportCutoff * { return support_cutoff_; } - auto operator()(const Monomial &mono) const -> bool { + auto operator()(const Bitset &mono) const -> bool { if (length_cutoff_ != nullptr) { return (*length_cutoff_)(mono); } @@ -227,7 +259,7 @@ class CutoffEvaluator { // Fast path when popcount(mono) is known: the predicate is `xor_sum==0 || (popcount/or_sum)<=cutoff`, // so popcount<=cutoff alone proves keep without reading the bitset (or_sum<=popcount makes support safe). - auto passes_with_popcount(const Monomial &mono, size_t popcount_sum) const -> bool { + auto passes_with_popcount(const Bitset &mono, size_t popcount_sum) const -> bool { if (length_cutoff_ != nullptr) { if (popcount_sum <= length_cutoff_->cutoff) { return true; @@ -256,10 +288,27 @@ class CutoffEvaluator { return std::nullopt; } + // The same bound counted in modes/qubits rather than slots, which is what a store keyed by mode + // (SparseRowStore) sizes its rows from. It is `cutoff` for both kinds and not max_slot_bound()/2: + // a support cutoff admits `cutoff` modes by definition, and a length cutoff of `cutoff` slots is + // worst-case `cutoff` singly-occupied modes -- halving would truncate that row. + // + // This bounds a *stored* row. A row being toggled in place transiently exceeds it, by as many + // modes as the generator touches, so a scratch row needs max_mode_bound() + generator locality. + auto max_mode_bound() const -> std::optional { + if (length_cutoff_ != nullptr) { + return length_cutoff_->cutoff; + } + if (support_cutoff_ != nullptr) { + return support_cutoff_->cutoff; + } + return std::nullopt; + } + private: - const CutoffFn &cutoff_fn_; - const LengthCutoff *length_cutoff_; - const SupportCutoff *support_cutoff_; + const CutoffFn &cutoff_fn_; + const LengthCutoff *length_cutoff_; + const SupportCutoff *support_cutoff_; }; } // namespace detail diff --git a/cpp/monoprop/algebra/CMakeLists.txt b/cpp/monoprop/algebra/CMakeLists.txt index ddc05bd5..be0c7083 100644 --- a/cpp/monoprop/algebra/CMakeLists.txt +++ b/cpp/monoprop/algebra/CMakeLists.txt @@ -6,6 +6,7 @@ target_sources( FILES "Algebra.h" "AlgebraCommon.h" + "CodesAlgebra.h" "MajoranaAlgebra.h" "PauliAlgebra.h" ) diff --git a/cpp/monoprop/algebra/CodesAlgebra.h b/cpp/monoprop/algebra/CodesAlgebra.h new file mode 100644 index 00000000..a2a06a67 --- /dev/null +++ b/cpp/monoprop/algebra/CodesAlgebra.h @@ -0,0 +1,317 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// The structural algebra on a sparse row's `codes` word, one function per dense counterpart in +// AlgebraCommon.h / PauliAlgebra.h / MajoranaAlgebra.h. Each is exact, not approximate: agreement with +// the dense version over the tests/data fixtures and randomized rows is asserted in +// cpp/tests/codes_algebra_tests.cpp, and that test is the gate on ever making these the default. +// +// Why any of this is possible in one word: a mode's two physical positions 2m, 2m+1 become the 2-bit +// field of slot j, so quantities the dense form derives from a per-word masked shift chain over the +// whole register become popcounts of two masks of a single word, independent of the storage width. +// With n = popcount(row_occupied_bits(codes)) and d = popcount(row_paired_bits(codes)): +// +// or_sum (support/Pauli weight) = n popcount_sum (length) = n + d xor_sum = n - d +// is_paired <=> d == n pair_swap = swap the two bits of every field +// Y letters = fields equal to 0b01 +// +// Nothing here reads a mode lane except codes_interleave_phase, which is inherently a two-row +// operation, and codes_cutoff_sums when a logical width narrower than the storage width makes some +// modes inactive. +// +// Names are prefixed rather than overloading the dense functions: while both representations are live +// a call site should say which one it means, and overload resolution between `MonomialLike auto` and +// SparseRow would decide that silently. + +#include +#include +#include +#include + +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/algebra/PauliAlgebra.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +namespace monoprop::detail { + +// Set bits of `codes` in slots strictly below `slot`. The dense counterpart is the prefix popcount an +// interleave scan maintains word by word. +[[nodiscard]] inline auto codes_popcount_below(RowCodes codes, size_t slot) noexcept -> size_t { + // A shift by 2*kRowMaxSlots would be undefined, and "below every slot" is the whole word anyway. + if (slot >= kRowMaxSlots) { + return static_cast(std::popcount(codes)); + } + return static_cast(std::popcount(codes & ((RowCodes{1} << (2 * slot)) - 1))); +} + +// or_sum / popcount_sum / xor_sum for a row whose every slot is inside the active window. Two popcounts +// and no reference to the storage width, where the dense cutoff_sums runs a masked shift chain per word +// and needs CutoffMasks to avoid rederiving the masks per term. +[[nodiscard]] inline auto codes_cutoff_sums(RowCodes codes) noexcept -> CutoffSums { + const auto n = static_cast(std::popcount(row_occupied_bits(codes))); + const auto d = static_cast(std::popcount(row_paired_bits(codes))); + return {n - d, n + d, n}; +} + +// The same, restricted to the active window. inactive_mode_prefix is the count of leading *physical* +// modes the logical width excludes -- storage_num_modes - logical_num_modes, half of +// CutoffMasks::active_bit_offset -- and the dense form applies it as `mono >> active_bit_offset`. +// +// The inactive modes are exactly the low ones, so they are a prefix of the ascending slots and drop out +// with one shift. A propagator's rows never carry them (a term is built from logical indices, which map +// into the window), so the common case is the zero-prefix early exit; the general path exists because +// the dense function it must agree with accepts such a monomial. +[[nodiscard]] inline auto codes_cutoff_sums(const SparseRow &row, size_t inactive_mode_prefix) noexcept -> CutoffSums { + if (inactive_mode_prefix == 0) { + return codes_cutoff_sums(row.codes); + } + const size_t n = row.num_slots(); + size_t inactive_slots = 0; + while (inactive_slots < n && row.mode(inactive_slots) < inactive_mode_prefix) { + ++inactive_slots; + } + if (inactive_slots >= kRowMaxSlots) { + return {0, 0, 0}; + } + return codes_cutoff_sums(row.codes >> (2 * inactive_slots)); +} + +// Both cutoffs keep a fully paired row unconditionally, exactly as the dense ones do: those are the +// only terms contributing to an expectation value against a product reference state. +[[nodiscard]] inline auto codes_length_cutoff(const SparseRow &row, + unsigned int cutoff, + size_t inactive_mode_prefix) noexcept -> bool { + const auto sums = codes_cutoff_sums(row, inactive_mode_prefix); + return sums.xor_sum == 0 || sums.popcount_sum <= cutoff; +} + +[[nodiscard]] inline auto codes_support_cutoff(const SparseRow &row, + unsigned int cutoff, + size_t inactive_mode_prefix) noexcept -> bool { + const auto sums = codes_cutoff_sums(row, inactive_mode_prefix); + return sums.xor_sum == 0 || sums.or_sum <= cutoff; +} + +// The counterparts of CutoffEvaluator::passes_with_popcount, one per concrete cutoff functor. Same +// shortcut and same reasoning: the predicate is `xor_sum == 0 || measure <= cutoff`, so a popcount +// already at or below the bound proves keep without reading the row at all (or_sum <= popcount_sum makes +// that sound for the support cutoff too). `popcount_sum` is the whole-register count, which can only +// exceed the active-window one, so the shortcut stays conservative when a logical width is narrower than +// the storage width. +// +// There is no evaluator argument: which cutoff a propagator has is fixed for its lifetime, so the caller +// resolves it once per gate rather than re-branching per term. +[[nodiscard]] inline auto codes_length_passes_with_popcount(const SparseRow &row, + unsigned int cutoff, + size_t popcount_sum, + size_t inactive_mode_prefix) noexcept -> bool { + return popcount_sum <= cutoff || codes_length_cutoff(row, cutoff, inactive_mode_prefix); +} + +[[nodiscard]] inline auto codes_support_passes_with_popcount(const SparseRow &row, + unsigned int cutoff, + size_t popcount_sum, + size_t inactive_mode_prefix) noexcept -> bool { + return popcount_sum <= cutoff || codes_support_cutoff(row, cutoff, inactive_mode_prefix); +} + +// Every occupied mode holds both of its positions, i.e. every field is 0b11. Unoccupied modes are not +// slots at all and are trivially paired, which is why this needs no window argument -- and matches the +// dense is_paired, which likewise checks the whole register. +[[nodiscard]] inline auto codes_is_paired(RowCodes codes) noexcept -> bool { + return row_paired_bits(codes) == row_occupied_bits(codes); +} + +// The pair-swap involution J: swap the two physical bits of every mode (u <-> v). Occupancy is +// preserved -- 0b01 <-> 0b10 and 0b11 is fixed -- so the mode lanes are untouched and the row's whole +// transform is this one word operation, against a per-word masked shift-and-or on the dense side. +[[nodiscard]] constexpr auto codes_pair_swap(RowCodes codes) noexcept -> RowCodes { + return ((codes & kRowLoBits) << 1) | ((codes >> 1) & kRowLoBits); +} + +// Y letters: v=1, u=0 under the JW image, so the field is exactly 0b01. +[[nodiscard]] inline auto codes_pauli_y_count(RowCodes codes) noexcept -> size_t { + const RowCodes v = codes & kRowLoBits; + const RowCodes u = (codes >> 1) & kRowLoBits; + return static_cast(std::popcount(v & ~u)); +} + +// Whether two Pauli strings anticommute: the symplectic inner product x_P.z_G + z_P.x_G mod 2, which +// dense-side is p.parity_and(pair_swap(g)). Sparse-side the AND is over the modes both rows occupy, so +// it is a merge of the two ascending lane arrays. +[[nodiscard]] inline auto codes_pauli_anticommutes(const SparseRow &p, const SparseRow &g) noexcept -> bool { + const size_t np = p.num_slots(); + const size_t ng = g.num_slots(); + unsigned int parity = 0; + size_t i = 0; + for (size_t k = 0; k < ng; ++k) { + const size_t g_mode = g.mode(k); + while (i < np && p.mode(i) < g_mode) { + ++i; + } + if (i < np && p.mode(i) == g_mode) { + // popcount of the pair-swapped generator field ANDed with p's, both 2 bits wide. + const unsigned int swapped = ((g.code(k) & 1U) << 1) | ((g.code(k) >> 1) & 1U); + parity ^= static_cast(std::popcount(p.code(i) & swapped)) & 1U; + } + } + return parity != 0; +} + +// Ordering sign (-1)^S of maj.gen, S = #{set bits of maj strictly below each set bit of gen} mod 2 over +// physical bit positions. Slots ascend in the mode and a mode's low position is 2*mode, so ascending +// slots are ascending positions and one merge walk over the two rows suffices -- O(slots), where the +// dense form is either a prefix-XOR scan over every word or a per-layer full-width mask W plus a +// parity_and per term. The mask has no sparse counterpart worth building: W is dense by construction +// (roughly half the register), so this replaces it with the direct walk instead. +[[nodiscard]] inline auto codes_interleave_phase(const SparseRow &maj, const SparseRow &gen) noexcept -> int { + const size_t nm = maj.num_slots(); + const size_t ng = gen.num_slots(); + unsigned int parity = 0; + size_t below_slots = 0; // maj slots at modes strictly below the current generator mode + for (size_t k = 0; k < ng; ++k) { + const size_t g_mode = gen.mode(k); + // Monotone across k, since generator modes ascend: the whole walk is one pass over each row. + while (below_slots < nm && maj.mode(below_slots) < g_mode) { + ++below_slots; + } + const size_t below = codes_popcount_below(maj.codes, below_slots); + const unsigned int g_code = gen.code(k); + // maj's bits at g_mode itself, if it occupies it: position 2*g_mode is below 2*g_mode+1 and so + // counts for the generator's high bit only. + const unsigned int m_code = (below_slots < nm && maj.mode(below_slots) == g_mode) ? maj.code(below_slots) : 0U; + if ((g_code & 1U) != 0U) { + parity ^= static_cast(below) & 1U; + } + if ((g_code & 2U) != 0U) { + parity ^= static_cast(below + (m_code & 1U)) & 1U; + } + } + return parity == 0 ? 1 : -1; +} + +// Pauli's per-term rotation sign, the counterpart of pauli_rotation_sign. Same exponent, one merge walk +// instead of a masked pass over the generator's nonzero words: +// e = g_y + sum(y_mono - y_new) + 2 * sum(v_mono & x_gen), sign = (e mod 4 == 1 ? -1 : +1) +// where per mode the code's low bit is v (physical position 2*mode) and its high bit is u, so a Y letter +// is the field 0b01 and x = u ^ v is one bit. +// +// The dense version restricts its sums to the words the generator occupies, on the grounds that +// elsewhere the two Y counts cancel and x_gen is zero. Per *mode* that argument is exact and tighter: a +// mode the generator misses has new_mono's field equal to mono's, so the Y terms cancel, and x_gen = 0 +// kills the cross term. So this walks the generator's slots and reads mono's field at each, which also +// means new_mono never has to exist -- the sign comes out of the same merge the toggle does. +[[nodiscard]] inline auto codes_pauli_rotation_sign(const SparseRow &mono, const SparseRow &gen) noexcept -> int { + auto delta = static_cast(codes_pauli_y_count(gen.codes)); + long cross = 0; + const size_t nm = mono.num_slots(); + const size_t ng = gen.num_slots(); + size_t i = 0; + for (size_t k = 0; k < ng; ++k) { + const size_t g_mode = gen.mode(k); + while (i < nm && mono.mode(i) < g_mode) { + ++i; + } + const unsigned int a = (i < nm && mono.mode(i) == g_mode) ? mono.code(i) : 0U; + const unsigned int b = gen.code(k); + delta += (a == 0b01U) ? 1 : 0; + delta -= ((a ^ b) == 0b01U) ? 1 : 0; + cross += ((a & 1U) != 0U && ((b ^ (b >> 1)) & 1U) != 0U) ? 1 : 0; + } + return mod4(delta + (2 * cross)) == 1 ? -1 : 1; +} + +// The product row of a term and a generator, and the overlap the emit phase needs. `codes` and +// `num_slots` describe the row written into `out_lanes`. +struct SparseProduct { + RowCodes codes = 0; + size_t num_slots = 0; + size_t overlap = 0; // popcount(mono & gen); 0 and meaningless when overflowed + bool overflowed = false; +}; + +// mono (+) gen: per mode the fields XOR, a mode whose field cancels to zero disappears, and the overlap +// is the popcount of the fields' AND. This is the dense fused_xor_into in support form, and it is the +// operation the whole representation exists for -- one merge over two ascending lane arrays, O(slots), +// against a pass over every storage word. +// +// out_lanes.size() lanes are available and must not exceed kRowMaxSlots (a codes word's worth). The +// product can occupy more modes than either input: up to mono's slots plus the generator's, so a +// scratch row needs CutoffEvaluator::max_mode_bound() + the generator's locality, not just the bound. +// When even that is not enough the result is reported as overflowed rather than truncated -- a truncated +// mode list keeps a plausible-looking codes word, which is exactly how the Stage 3 bench measured a +// capacity bug as if it were a speedup. On overflow the caller must fall back to the dense product; +// `overlap` is partial and is deliberately not returned. +[[nodiscard]] inline auto sparse_toggle(const SparseRow &mono, + const SparseRow &gen, + std::span out_lanes) noexcept -> SparseProduct { + assert(out_lanes.size() <= kRowMaxSlots && "sparse_toggle capacity exceeds one codes word"); + const size_t nm = mono.num_slots(); + const size_t ng = gen.num_slots(); + SparseProduct result; + size_t used = 0; + bool over = false; + + const auto emit = [&](size_t mode, unsigned int code) { + if (used == out_lanes.size()) { + over = true; + return; + } + out_lanes[used] = static_cast(mode); + result.codes |= static_cast(code) << (2 * used); + ++used; + }; + + size_t i = 0; + size_t j = 0; + while (!over && i < nm && j < ng) { + const size_t m_mode = mono.mode(i); + const size_t g_mode = gen.mode(j); + if (m_mode < g_mode) { + emit(m_mode, mono.code(i)); + ++i; + } + else if (g_mode < m_mode) { + emit(g_mode, gen.code(j)); + ++j; + } + else { + const unsigned int a = mono.code(i); + const unsigned int b = gen.code(j); + result.overlap += static_cast(std::popcount(a & b)); + if (const unsigned int c = a ^ b; c != 0U) { + emit(m_mode, c); + } + ++i; + ++j; + } + } + while (!over && i < nm) { + emit(mono.mode(i), mono.code(i)); + ++i; + } + while (!over && j < ng) { + emit(gen.mode(j), gen.code(j)); + ++j; + } + if (over) { + return SparseProduct{0, 0, 0, true}; + } + result.num_slots = used; + return result; +} + +} // namespace monoprop::detail diff --git a/cpp/monoprop/algebra/MajoranaAlgebra.h b/cpp/monoprop/algebra/MajoranaAlgebra.h index 33333b56..0c9819e9 100644 --- a/cpp/monoprop/algebra/MajoranaAlgebra.h +++ b/cpp/monoprop/algebra/MajoranaAlgebra.h @@ -36,8 +36,7 @@ inline constexpr auto POWERS_OF_MINUS_ONE = std::array{1, -1}; inline constexpr auto REAL_PARTS = std::array{1, 0, -1, 0}; // The i^C(|maj|,2) factor that makes a Majorana product Hermitian. -template -auto hermitian_coefficient(const Monomial &maj) -> std::complex { +auto hermitian_coefficient(const MonomialLike auto &maj) -> std::complex { const auto pop = maj.count(); return POWERS_OF_I[n_choose_2(pop) % 4]; } @@ -54,8 +53,7 @@ inline auto antihermitian_generator_correction(const VecZ &indices) -> std::comp // Diagonal element against the initial product state, whose occupation mask is state_mask // (initial_state_mask): (-1)^(|maj & state_mask| + |maj|/2) -- the pairing sign folds in on top of the // occupation parity. Only meaningful for fully-paired terms. -template -auto majorana_state_phase(const Monomial &maj, const Monomial &state_mask) -> double { +auto majorana_state_phase(const MonomialLike auto &maj, const auto &state_mask) -> double { const auto num_pairs = maj.count_and(state_mask); return POWERS_OF_MINUS_ONE[(num_pairs + maj.count() / 2) % 2]; } @@ -72,9 +70,8 @@ constexpr auto prefix_xor_64(uint64_t x) -> uint64_t { // Ordering sign (-1)^S of maj·gen, S = #{set bits of maj strictly below each set bit of gen} mod 2. // Reference spec: the hot path uses the equivalent per-layer mask form (see interleave_phase_mask). -template -auto interleave_phase(const Monomial &maj_bs, const Monomial &gen_bs) -> int { - constexpr size_t n_words = Monomial::num_words(); +auto interleave_phase(const MonomialLike auto &maj_bs, const auto &gen_bs) -> int { + const size_t n_words = maj_bs.num_words(); size_t parity = 0; uint64_t carry = 0; @@ -102,11 +99,15 @@ auto interleave_phase(const Monomial &maj_bs, const Monomial // Per-generator mask W collapsing the per-term interleave sign to one masked parity. // Identity: interleave_phase(M,G) = (−1)^{parity(M ∩ W)} with W = {c : #{g∈G : g>c} odd}, fixed for // the layer; the per-term sign is then one maj.parity_and(W) instead of the prefix-XOR scan. -template -auto interleave_phase_mask(const Monomial &gen) -> Monomial { - Monomial w; +template +auto interleave_phase_mask(const T &gen) -> T { + // Copy-then-reset for the same reason as change_basis: this needs a zero bitset at gen's width, and + // MonomialLike constrains operations, not constructors, so there is no width-argument construction + // to call on a deduced type. + T w = gen; + w.reset(); size_t above = 0; // #{g∈G : g>c}, maintained as c descends - for (size_t c = Monomial::size(); c-- > 0;) { + for (size_t c = gen.size(); c-- > 0;) { if ((above & 1U) != 0U) { w.set(c); } @@ -124,9 +125,10 @@ inline auto hermitian_phase(size_t maj_count, size_t gen_count, size_t overlap) // Selected slot i of the logical range owns the bit pair 2*(prefix+i), 2*(prefix+i)+1 — a paired // Majorana term sets both bits of every selected mode, so the two bits always travel together. -template -auto monomial_from_selector(const std::vector &selector, size_t inactive_mode_prefix) -> Monomial { - Monomial current; +// inline: a plain function in a header, where being a template used to supply the linkage. +inline auto monomial_from_selector(const std::vector &selector, size_t inactive_mode_prefix, size_t num_bits) + -> Bitset { + Bitset current(num_bits); for (size_t i = 0; i < selector.size(); ++i) { if (selector[i]) { const size_t bit_pair_offset = inactive_mode_prefix + i; @@ -137,32 +139,62 @@ auto monomial_from_selector(const std::vector &selector, size_t inactive_m return current; } -// All fully paired Majorana monomials with up to max_ones pairs, over the active logical modes only. -template -auto generate_paired_op(size_t max_ones, size_t logical_num_modes) -> MonomialList { - MonomialList combinations; +// How many monomials for_each_paired_op() yields: Sum_{k<=max_ones} C(logical_num_modes, k). Lets a +// caller size storage without generating anything first. Saturates rather than reports nonsense only +// at mode counts whose monomial list could not fit in memory anyway. +[[nodiscard]] inline auto count_paired_op(size_t max_ones, size_t logical_num_modes) -> size_t { + max_ones = std::min(max_ones, logical_num_modes); + size_t total = 0; + for (size_t k = 0, binomial = 1; k <= max_ones; ++k) { + total += binomial; + binomial = binomial * (logical_num_modes - k) / (k + 1); + } + return total; +} + +// Every fully paired Majorana monomial with up to max_ones pairs over the active logical modes, one +// at a time, in the same order generate_paired_op() lists them. +// +// A caller that keeps only a subset must use this and not generate_paired_op: the full list is +// count_paired_op() monomials, which in the Schrodinger picture is the entire term count, and a +// propagator with S partitions constructs S propagators that would each hold a complete copy at the +// same moment. Insertion order is load-bearing -- it fixes term indices and hence float accumulation +// order -- so this yields in exactly the list's order. +// num_bits is the storage width the monomials are built at; the logical modes occupy its *top* slots, +// so the inactive prefix is the difference between the two widths and not something logical_num_modes +// can supply on its own. +auto for_each_paired_op(size_t max_ones, size_t logical_num_modes, size_t num_bits, auto &&fn) -> void { // Clamp in pairs, not bits: max_ones counts pairs and bounds the fill over `selector`, one slot per mode. max_ones = std::min(max_ones, logical_num_modes); + auto selector = std::vector(logical_num_modes, false); - const size_t inactive_mode_prefix = NumModes - logical_num_modes; + const size_t inactive_mode_prefix = num_bits / 2 - logical_num_modes; for (size_t num_ones = 0; num_ones <= max_ones; ++num_ones) { std::fill(selector.begin(), selector.begin() + num_ones, true); do { - combinations.push_back(monomial_from_selector(selector, inactive_mode_prefix)); + fn(monomial_from_selector(selector, inactive_mode_prefix, num_bits)); } while (std::ranges::prev_permutation(selector).found); std::ranges::fill(selector, false); } +} +// All fully paired Majorana monomials with up to max_ones pairs, over the active logical modes only. +// Prefer for_each_paired_op() unless the whole list is genuinely needed at once. +inline auto generate_paired_op(size_t max_ones, size_t logical_num_modes, size_t num_bits) -> MonomialList { + MonomialList combinations; + combinations.reserve(count_paired_op(max_ones, logical_num_modes)); + for_each_paired_op(max_ones, logical_num_modes, num_bits, [&combinations](const auto &mono) { + combinations.push_back(mono); + }); return combinations; } -template -auto encode_coeff(const std::complex &coeff, const Monomial &maj) -> double { - const auto encoded = coeff / hermitian_coefficient(maj); +auto encode_coeff(const std::complex &coeff, const MonomialLike auto &maj) -> double { + const auto encoded = coeff / hermitian_coefficient(maj); if (std::abs(encoded.imag()) > 1e-10) { throw NonEncodableCoefficient("Non-Hermitian coeffs detected"); @@ -171,18 +203,27 @@ auto encode_coeff(const std::complex &coeff, const Monomial &m return encoded.real(); } -template -auto decode_coeff(const std::complex &coeff, const Monomial &maj) -> std::complex { - return coeff * hermitian_coefficient(maj); +auto decode_coeff(const std::complex &coeff, const MonomialLike auto &maj) -> std::complex { + return coeff * hermitian_coefficient(maj); } -template -auto change_basis(const Monomial &maj, const MonomialList &basis) -> Monomial { - Monomial new_maj; +// `basis` stays a plain (unconstrained) auto: it is a MonomialList, a container of monomials rather +// than a monomial, so it is not itself MonomialLike. Its elements' width always matches maj's at +// every call site -- required, since the XOR below asserts matching widths. +template +auto change_basis(const T &maj, const auto &basis) -> T { + const size_t width = maj.size(); + const size_t num_modes = width / 2; + // Copy-then-reset, rather than `Mono new_maj;` (width 0) or a width-argument constructor: + // MonomialLike constrains operations, not constructors, so a deduced Mono is not known to have + // one. The copied words are immediately overwritten; this path only runs when a basis change is + // configured. + T new_maj = maj; + new_maj.reset(); size_t pos = maj.find_first(); - while (pos < maj.size()) { - new_maj ^= materialize_row(basis, 2 * NumModes - pos - 1); + while (pos < width) { + new_maj ^= materialize_row(basis, 2 * num_modes - pos - 1); pos = maj.find_next(pos); } diff --git a/cpp/monoprop/algebra/PauliAlgebra.h b/cpp/monoprop/algebra/PauliAlgebra.h index 5cd5ef44..45f7d297 100644 --- a/cpp/monoprop/algebra/PauliAlgebra.h +++ b/cpp/monoprop/algebra/PauliAlgebra.h @@ -25,7 +25,9 @@ #include #include #include +#include #include +#include #include "monoprop/Bitset.h" #include "monoprop/TypeAliases.h" @@ -34,9 +36,8 @@ namespace monoprop { -template -[[nodiscard]] inline constexpr auto pauli_even_mask() -> Monomial { - return even_bits<2 * NumModes, LSb0>(); +[[nodiscard]] inline auto pauli_even_mask(size_t num_bits) -> Bitset { + return even_bits(num_bits); } namespace detail { @@ -51,11 +52,16 @@ struct PauliUv { // The pair-swap involution J: swap the two physical bits of every qubit pair (u <-> v). Stays inside // each word -- pairs are {2m, 2m+1}, so there is no cross-word carry. -template -[[nodiscard]] auto pair_swap(const Monomial &p) -> Monomial { - constexpr auto e_mask = pauli_even_mask(); - Monomial result; - for (size_t w = 0; w < Monomial::num_words(); ++w) { +// e_mask/result build from p.size()/p.num_words() (instance calls), not the qualified +// decltype(p)::size() other functions in this file use: p is only constrained MonomialLike, and a +// caller may hand this a plain Bitset (e.g. from a ^ b), whose width is data and so has no static +// size() to qualify-call. +template +[[nodiscard]] auto pair_swap(const T &p) -> T { + const auto &e_mask = cached_even_bits(p.size()); + Bitset result(p.size()); + const size_t nw = p.num_words(); + for (size_t w = 0; w < nw; ++w) { const uint64_t word = p.word(w); const uint64_t e = e_mask.word(w); result.data()[w] = ((word & e) << 1) | ((word >> 1) & e); @@ -64,11 +70,11 @@ template } // A Y letter has v=1, u=0. -template -[[nodiscard]] auto pauli_y_count(const Monomial &p) -> size_t { - constexpr auto e_mask = pauli_even_mask(); +[[nodiscard]] auto pauli_y_count(const MonomialLike auto &p) -> size_t { + const auto &e_mask = cached_even_bits(p.size()); size_t y = 0; - for (size_t w = 0; w < Monomial::num_words(); ++w) { + const size_t nw = p.num_words(); + for (size_t w = 0; w < nw; ++w) { const auto [v, u] = detail::pauli_uv(p.word(w), e_mask.word(w)); y += static_cast(std::popcount(v & ~u)); } @@ -77,38 +83,60 @@ template // Whether two Pauli strings anticommute (symplectic inner product is odd): // P.parity_and(pair_swap(G)) == (x_P . z_G + z_P . x_G) mod 2. -template -[[nodiscard]] auto pauli_anticommutes(const Monomial &p, const Monomial &g) -> bool { - return p.parity_and(pair_swap(g)); +[[nodiscard]] auto pauli_anticommutes(const MonomialLike auto &p, const auto &g) -> bool { + return p.parity_and(pair_swap(g)); } namespace detail { // Reduce a (possibly negative) i-power exponent to [0, 4). -[[nodiscard]] inline constexpr auto mod4(long e) -> int { +[[nodiscard]] constexpr auto mod4(long e) -> int { return static_cast(((e % 4) + 4) % 4); } } // namespace detail -// Per-generator context for the hot emit-sign kernel: nz_words lets pauli_rotation_sign() skip words -// outside G's support. -template +// One entry per word G occupies -- the only words the sign kernel below visits, since elsewhere the +// mono/new_mono Y counts cancel and x_gen is 0. All three fields are fixed for the layer, so they are +// derived once here rather than in the per-term loop, which used to rebuild G's x-plane from G's own +// word on every term and needed the even mask parked in the context to do it. +// +// Deriving them here is a simplification and not a speedup: measured pinned single-threaded, it moves +// the instruction count on either shipping model by under 0.05%, because the optimizer was already +// hoisting the derivation out of the inlined scan loop. +struct PauliGenWord { + size_t w; // storage word index, ascending + uint64_t e; // the even-bit mask for word w + uint64_t x_g; // G's x-plane in word w, aligned onto the even lane +}; + +// Per-generator context for the hot emit-sign kernel. Three members, not five: `words` carries its own +// length, and the even mask no longer has to be held here because nothing rebuilds it per term. +// +// `words` is a vector, not the std::array<..., num_words()> the word list was: with no compile-time +// width there is no bound to size an array by. It holds at most num_words() entries and is built once +// per layer, so the allocation is per layer while the reads are per term -- the same trade the retained +// LazyFold already makes for its columns. struct PauliGenContext final { - Monomial gen{}; + Bitset gen{}; size_t g_y = 0; - std::array::num_words()> nz_words{}; - size_t nz_count = 0; + std::vector words{}; }; // Call once per layer, not per term. -template -[[nodiscard]] auto make_pauli_gen_context(const Monomial &gen) -> PauliGenContext { - PauliGenContext ctx; +auto make_pauli_gen_context(const MonomialLike auto &gen) -> PauliGenContext { + PauliGenContext ctx; + const size_t nw = gen.num_words(); ctx.gen = gen; - ctx.g_y = pauli_y_count(gen); - for (size_t w = 0; w < Monomial::num_words(); ++w) { - if (gen.word(w) != 0) { - ctx.nz_words[ctx.nz_count++] = w; + ctx.g_y = pauli_y_count(gen); + const auto &e_mask = cached_even_bits(gen.size()); + ctx.words.reserve(nw); + for (size_t w = 0; w < nw; ++w) { + const uint64_t word = gen.word(w); + if (word == 0) { + continue; } + const uint64_t e = e_mask.word(w); + const auto [v_g, u_g] = detail::pauli_uv(word, e); + ctx.words.emplace_back(w, e, u_g ^ v_g); } return ctx; } @@ -118,31 +146,38 @@ template // raw product sign, so the emit site needs no extra negation (pinned by pauli_algebra_tests.cpp). // Loops only over gen's nonzero words (elsewhere mono/new_mono Y counts cancel and x_gen = 0). Exponent // e = g_y + Σ_w(yMono - yNew) + 2·Σ_w(v_mono & x_gen); raw sign = (e mod 4 == 1 ? +1 : -1), negated here. -template -[[gnu::always_inline]] inline auto pauli_rotation_sign(const PauliGenContext &ctx, - const Monomial &mono, - const Monomial &new_mono) -> int { - constexpr auto e_mask = pauli_even_mask(); - long delta = static_cast(ctx.g_y); +// +// Takes the two operands as word pointers, which is the form the per-gate kernel already has: it +// resolved them once, where mono.word(w) / new_mono.word(w) re-select a storage pointer on every one +// of the ctx.words accesses. Both must point at ctx.gen's width. +[[gnu::always_inline]] inline auto pauli_rotation_sign_words(const PauliGenContext &ctx, + const uint64_t *mono, + const uint64_t *new_mono) -> int { + auto delta = static_cast(ctx.g_y); long cross = 0; - for (size_t k = 0; k < ctx.nz_count; ++k) { - const size_t w = ctx.nz_words[k]; - const uint64_t e = e_mask.word(w); - const auto [v_m, u_m] = detail::pauli_uv(mono.word(w), e); - const auto [v_n, u_n] = detail::pauli_uv(new_mono.word(w), e); - const auto [v_g, u_g] = detail::pauli_uv(ctx.gen.word(w), e); + const size_t n = ctx.words.size(); + for (size_t k = 0; k < n; ++k) { + const auto [w, e, x_g] = ctx.words[k]; + const auto [v_m, u_m] = detail::pauli_uv(mono[w], e); + const auto [v_n, u_n] = detail::pauli_uv(new_mono[w], e); delta += std::popcount(v_m & ~u_m); delta -= std::popcount(v_n & ~u_n); - const uint64_t x_g = u_g ^ v_g; cross += std::popcount(v_m & x_g); } return detail::mod4(delta + 2 * cross) == 1 ? -1 : 1; } +// The monomial form of the above, for callers that hold bitsets rather than words. data() is where +// word(w) reads from, so this is the same computation and not a second one. +[[gnu::always_inline]] inline auto pauli_rotation_sign(const auto &ctx, + const MonomialLike auto &mono, + const auto &new_mono) -> int { + return pauli_rotation_sign_words(ctx, std::data(mono), std::data(new_mono)); +} + // Diagonal element = (-1)^{|Z ∩ occupied|} of a Z-only Pauli against the initial product // state. Only meaningful where is_paired holds; for a non-diagonal Pauli = 0. -template -[[nodiscard]] auto pauli_state_phase(const Monomial &mono, const Monomial &state_mask) -> double { +[[nodiscard]] auto pauli_state_phase(const MonomialLike auto &mono, const auto &state_mask) -> double { return (mono.count_and(state_mask) & 1) ? -1.0 : 1.0; } diff --git a/cpp/monoprop/core/CMakeLists.txt b/cpp/monoprop/core/CMakeLists.txt index d9faa730..d4bf6f18 100644 --- a/cpp/monoprop/core/CMakeLists.txt +++ b/cpp/monoprop/core/CMakeLists.txt @@ -1,3 +1,5 @@ +target_sources(monoprop-objs PRIVATE Monomial.cpp) + target_sources( monoprop PUBLIC diff --git a/cpp/monoprop/core/Monomial.cpp b/cpp/monoprop/core/Monomial.cpp new file mode 100644 index 00000000..3b40db62 --- /dev/null +++ b/cpp/monoprop/core/Monomial.cpp @@ -0,0 +1,69 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Monomial.h" + +#include +#include +#include + +namespace monoprop { +auto cutoff_type_str_2_enum(const std::string &cutoff_type) -> CutoffType { + if (cutoff_type == "length") { + return monoprop::CutoffType::Length; + } + + if (cutoff_type == "support") { + return monoprop::CutoffType::Support; + } + + throw std::invalid_argument( + std::format("Unknown CutoffType string: '{}'. Valid options are: 'length', 'support'.", cutoff_type)); +} + +auto cutoff_type_enum_2_str(CutoffType cutoff_type) -> std::string { + switch (cutoff_type) { + case monoprop::CutoffType::Length: + return "length"; + case monoprop::CutoffType::Support: + return "support"; + default: + throw std::invalid_argument("Unknown CutoffType enum value"); + } +} + +auto basis_str_2_enum(const std::string &basis) -> Basis { + if (basis == "majorana") { + return monoprop::Basis::Majorana; + } + + if (basis == "pauli") { + return monoprop::Basis::Pauli; + } + + throw std::invalid_argument( + std::format("Unknown Basis string: '{}'. Valid options are: 'majorana', 'pauli'.", basis)); +} + +auto basis_enum_2_str(Basis basis) -> std::string { + switch (basis) { + case monoprop::Basis::Majorana: + return "majorana"; + case monoprop::Basis::Pauli: + return "pauli"; + default: + throw std::invalid_argument("Unknown Basis enum value"); + } +} +} // namespace monoprop diff --git a/cpp/monoprop/core/Monomial.h b/cpp/monoprop/core/Monomial.h index 2e6f647f..6994a60a 100644 --- a/cpp/monoprop/core/Monomial.h +++ b/cpp/monoprop/core/Monomial.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include @@ -23,57 +24,74 @@ #include #include "monoprop/Bitset.h" +#include "monoprop/monopropExport.h" namespace monoprop { -template -using Monomial = Bitset<2 * NumModes>; +// A monomial is a Bitset of width 2 * num_modes, two bits per mode/qubit -- the width is data, so +// there is no monomial *type* to name and no header-level alias for one: spell Bitset, and carry the +// width with the value. Sites that need a NumModes deduce it from a monomial argument's own width +// (mono.size() / 2), never from a template parameter. + +// Structural stand-in for "a monomial, width unspecified", for the free functions in the algebra +// headers and elsewhere that read a monomial parameter generically. Instance (not qualified) calls, +// since a width only exists per value. +template +concept MonomialLike = requires(const T &t) { + { t.size() } -> std::convertible_to; + { t.num_words() } -> std::convertible_to; + { t.count() } -> std::convertible_to; + { t.find_first() } -> std::convertible_to; +}; -// Not the evolved operator's row storage -- that is detail::OperatorIndex (see detail/operator/OperatorIndex.h). -template -using MonomialList = std::vector>; +// Not the evolved operator's row storage -- that is detail::OperatorIndex (see +// detail/operator/RowAccess.h). +// +// Element-width caveat, from Bitset being runtime-width: a sized construction `MonomialList l(n)` +// fills with *width-0* bitsets, since nothing in the element type carries a width. Any site +// that sizes up front and assigns into slots afterwards must pass a fill value of the intended +// width, `MonomialList l(n, Bitset(num_bits))`; push_back-only sites need nothing. A width-0 element +// reaching a binary op trips the width assertions in Bitset.h. +using MonomialList = std::vector; -template struct MonomialHash final { using is_transparent = void; - auto operator()(const Monomial &arr) const noexcept -> size_t { - return SplitmixHash>{}(arr); - } + auto operator()(const Bitset &arr) const noexcept -> size_t { return SplitmixHash{}(arr); } }; -template struct MonomialEqual final { using is_transparent = void; - auto operator()(const Monomial &lhs, const Monomial &rhs) const noexcept -> bool { - return lhs == rhs; - } + auto operator()(const Bitset &lhs, const Bitset &rhs) const noexcept -> bool { return lhs == rhs; } }; -template -using MonomialMap = - boost::unordered_flat_map, double, MonomialHash, MonomialEqual>; - -template -inline auto monomial_hash(const Monomial &mono) noexcept -> size_t { - if constexpr (Monomial::num_words() == 1) { - return static_cast(SplitmixHash>::mix(mono.word(0))); - } - else { - return MonomialHash{}(mono); - } +using MonomialMap = boost::unordered_flat_map; + +// MPI owner routing (find_rank) hashes through here, so the value must not change: it decides which +// rank owns a term and, with it, probe order. It does not. The single-word fast path this used to +// select with `if constexpr` is the same one SplitmixHash::operator() now takes at runtime, and the +// wide arm was already a call to SplitmixHash via MonomialHash -- so both arms collapse into the one +// call below. Kept as a named function rather than inlined at the call sites: the name is what marks a +// hash as owner-routing (pinned) rather than an ordinary container hash. +inline auto monomial_hash(const Bitset &mono) noexcept -> size_t { + return SplitmixHash{}(mono); } // Structural keep/drop predicate applied to a monomial after each gate. -template -using CutoffFn = std::function &)>; +using CutoffFn = std::function; -enum class CutoffType { +enum class CutoffType : unsigned char { Length, // Keep if the monomial length (number of Majorana operators) <= cutoff (or fully paired) Support // Keep if the orbital support (number of distinct orbitals) <= cutoff (or fully paired) }; -enum class Basis : uint8_t { Majorana, Pauli }; +monoprop_EXPORT auto cutoff_type_str_2_enum(const std::string &cutoff_type) -> CutoffType; +monoprop_EXPORT auto cutoff_type_enum_2_str(CutoffType cutoff_type) -> std::string; + +enum class Basis : unsigned char { Majorana, Pauli }; + +monoprop_EXPORT auto basis_str_2_enum(const std::string &basis) -> Basis; +monoprop_EXPORT auto basis_enum_2_str(Basis basis) -> std::string; } // namespace monoprop diff --git a/cpp/monoprop/detail/EnvConfig.h b/cpp/monoprop/detail/EnvConfig.h index 30120417..ea96fa79 100644 --- a/cpp/monoprop/detail/EnvConfig.h +++ b/cpp/monoprop/detail/EnvConfig.h @@ -15,7 +15,9 @@ #pragma once #include +#include #include +#include #include // Single home for runtime environment configuration. Kept dependency-free by design, because it is @@ -23,9 +25,15 @@ // // monoprop_NUM_THREADS positive int (1..1e6), else ignored → num_threads // monoprop_PARTITIONS int N | "auto" | "off"; parsed where it is used (resolve_partition_count_) +// monoprop_ROW_STORE "auto" (default) | "dense" | "sparse"; unset == auto → row_store namespace monoprop::config { +// Which row backend a propagator builds on. Auto is the measured crossover +// (SparseRowStore::preferred_for_modes); the two explicit values force one backend for every +// propagator in the process, which is how the suite is run either way -- see row_store below. +enum class RowStore : std::uint8_t { Auto, Dense, Sparse }; + namespace detail { inline auto parse_positive_int(const char *text) -> std::optional { @@ -43,17 +51,40 @@ inline auto parse_positive_int(const char *text) -> std::optional { return static_cast(value); } +inline auto parse_row_store(const char *text) -> std::optional { + using enum RowStore; + if (text == nullptr || text[0] == '\0' || std::strcmp(text, "auto") == 0) { + return Auto; + } + if (std::strcmp(text, "dense") == 0) { + return Dense; + } + if (std::strcmp(text, "sparse") == 0) { + return Sparse; + } + return std::nullopt; +} + } // namespace detail struct Settings { std::optional num_threads; + // nullopt means monoprop_ROW_STORE held something unrecognized -- reported rather than ignored, + // unlike every other setting here: this one exists to prove the sparse backend was exercised, so a + // typo that silently fell back to auto would mean believing a configuration ran that never did. The + // throw is raised by the propagator, which has the exception types; this header stays + // dependency-free. + std::optional row_store = RowStore::Auto; }; -// Parse the environment once; the Settings are cached and shared across TUs. +// Parse the environment once; the Settings are cached and shared across TUs. Cached deliberately: a +// setting must not change between two propagators in one process, since the row backend is part of a +// monomial's hash and so of every cross-propagator comparison. inline auto get() -> const Settings & { static const Settings settings = [] { Settings s; s.num_threads = detail::parse_positive_int(std::getenv("monoprop_NUM_THREADS")); + s.row_store = detail::parse_row_store(std::getenv("monoprop_ROW_STORE")); return s; }(); return settings; diff --git a/cpp/monoprop/detail/evolution/CosineRecompute.h b/cpp/monoprop/detail/evolution/CosineRecompute.h index 653f7da4..f7c8a322 100644 --- a/cpp/monoprop/detail/evolution/CosineRecompute.h +++ b/cpp/monoprop/detail/evolution/CosineRecompute.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -39,10 +40,16 @@ namespace monoprop::detail { -// Reconstruct a layer's generator Monomial from the raw words stored on its LayerCore. -template -inline auto generator_from_words(const std::vector &gw) -> Monomial { - Monomial gen{}; +// Reconstruct a layer's generator from the raw words stored on its LayerCore. +// +// num_bits is passed rather than recovered as gw.size() * 64: the stored words are the generator's word +// count, which does not pin down its bit width -- a width that is not a whole multiple of 64 rounds up +// to the same word count, and the round trip would silently widen it (Bitset carries the used bits of +// the last word, so `size()` and the top-bit mask would both come back wrong). The caller has the real +// width in hand; the operator the generator is applied against is the one that defines it. +inline auto generator_from_words(const std::vector &gw, size_t num_bits) -> Bitset { + Bitset gen(num_bits); + assert(gw.size() == gen.num_words() && "generator words must match the operator's storage width"); std::memcpy(gen.data(), gw.data(), gw.size() * sizeof(uint64_t)); return gen; } @@ -61,15 +68,14 @@ struct FoldMask { uint64_t last_mask = ~uint64_t{0}; }; -template -inline auto make_fold_mask(const InvertedIndex &sc, - const Monomial &gen, +inline auto make_fold_mask(const auto &sc, + const MonomialLike auto &gen, uint64_t scaled_count, Basis basis = Basis::Majorana) -> FoldMask { FoldMask s; // Pauli folds J(G) and never needs the odd-|G| parity correction (see Scan.h); Majorana applies it // when |G| is odd. Truncation bounds are basis-independent. - s.g_odd = algebra_fold_needs_odd_correction(basis, gen); + s.g_odd = algebra_fold_needs_odd_correction(basis, gen); const size_t full = sc.words(); s.mask_words = std::min(full, static_cast((scaled_count + 63) / 64)); s.last_word = (s.mask_words == 0) ? 0 : s.mask_words - 1; @@ -79,7 +85,8 @@ inline auto make_fold_mask(const InvertedIndex &sc, // A layer's cosine fold materialised into one buffer. Backs the pare materializer and the // recompute-equivalence test oracle. -template +// No width parameter: every member is a byte/word count or a heap buffer, and none was ever sized by +// one. It was templated only because its producer was. struct FoldCache { std::vector combined; // the generator's columns XOR-combined over [0, fold.mask_words) FoldMask fold; @@ -89,31 +96,26 @@ struct FoldCache { }; // The odd-|G| row-parity words for a fold, or nullptr when the correction does not apply. -template -inline auto fold_row_parity(const InvertedIndex &sc, const FoldMask &f) -> const uint64_t * { +inline auto fold_row_parity(const auto &sc, const FoldMask &f) -> const uint64_t * { return f.g_odd ? sc.row_parity_words() : nullptr; } -template -auto make_fold_cache(const InvertedIndex &sc, - const Monomial &gen, - uint64_t scaled_count, - Basis basis) -> FoldCache { - FoldCache p; - p.fold = make_fold_mask(sc, gen, scaled_count, basis); - p.row_parity = fold_row_parity(sc, p.fold); +auto make_fold_cache(const auto &sc, const MonomialLike auto &gen, uint64_t scaled_count, Basis basis) -> FoldCache { + FoldCache p; + p.fold = make_fold_mask(sc, gen, scaled_count, basis); + p.row_parity = fold_row_parity(sc, p.fold); // generator_words stores the real G; re-derive the fold generator (J(G) for Pauli) as the scan did. - const auto fold_gen = algebra_fold_generator(basis, gen); - const auto gen_columns = build_even_parity_generator_columns(fold_gen); + const auto fold_gen = algebra_fold_generator(basis, gen); + const auto gen_columns = build_even_parity_generator_columns(fold_gen); // One combine over [0, mask_words): words >= mask_words are never read, so dropping them is exact. p.combined.resize(p.fold.mask_words); // combine_columns_block zero-fills if (p.fold.mask_words != 0) { - combine_columns_block(sc, - {gen_columns.indices.data(), gen_columns.count}, - p.combined.data(), - 0, - p.fold.mask_words); + combine_columns_block(sc, + {gen_columns.indices.data(), gen_columns.count}, + p.combined.data(), + 0, + p.fold.mask_words); } return p; } @@ -134,8 +136,7 @@ auto make_fold_cache(const InvertedIndex &sc, return bits; } -template -[[gnu::always_inline]] inline auto fold_word(const FoldCache &p, size_t wi) -> uint64_t { +[[gnu::always_inline]] inline auto fold_word(const FoldCache &p, size_t wi) -> uint64_t { return apply_fold_mask(p.combined[wi], wi, p.fold, p.row_parity); } @@ -150,30 +151,25 @@ template // Metadata to recompute a layer's cosine fold on the fly, with no per-layer cos buffer. // -// `columns` is heap-sized to |G| (typically 2-4) rather than reusing EvenParityGeneratorColumns' fixed -// std::array: a LazyFold is retained per graph layer, 4 KB each at NumModes=256. -template +// `columns` is heap-sized to |G| (typically 2-4): a LazyFold is retained per graph layer, and sizing it +// by the register width instead would have cost 4 KB each at 256 modes. Carries no width parameter, for +// the same reason as FoldCache. struct LazyFold { std::vector columns; FoldMask fold; }; -template -auto make_lazy_fold(const InvertedIndex &sc, - const Monomial &gen, - uint64_t scaled_count, - Basis basis) -> LazyFold { - LazyFold r; - r.fold = make_fold_mask(sc, gen, scaled_count, basis); - const auto fold_gen = algebra_fold_generator(basis, gen); - const auto columns = build_even_parity_generator_columns(fold_gen); +auto make_lazy_fold(const auto &sc, const MonomialLike auto &gen, uint64_t scaled_count, Basis basis) -> LazyFold { + LazyFold r; + r.fold = make_fold_mask(sc, gen, scaled_count, basis); + const auto fold_gen = algebra_fold_generator(basis, gen); + const auto columns = build_even_parity_generator_columns(fold_gen); r.columns.assign(columns.indices.begin(), columns.indices.begin() + columns.count); return r; } // The recompute analogue of fold_word, over a freshly-built block word (bb = the block's first fold word). -template -[[gnu::always_inline]] inline auto recipe_fold_word(const LazyFold &r, +[[gnu::always_inline]] inline auto recipe_fold_word(const LazyFold &r, const uint64_t *blk, size_t bb, size_t wi, @@ -183,19 +179,17 @@ template // Append a layer's cosine-set indices to `out`, walking the same blocks as scale_cos_* rather than sharing // a visitor with them, so the scaling kernels stay verbatim. -template -auto cos_indices_lazy(const InvertedIndex &sc, const LazyFold &r, std::vector &out) - -> void { +auto cos_indices_lazy(const auto &sc, const LazyFold &r, std::vector &out) -> void { const size_t mask_words = r.fold.mask_words; - const uint64_t *row_parity = fold_row_parity(sc, r.fold); + const uint64_t *row_parity = fold_row_parity(sc, r.fold); std::vector &blk = column_block_scratch(); for (size_t bb = 0; bb < mask_words; bb += kColumnBlockWords) { const size_t be = std::min(bb + kColumnBlockWords, mask_words); - combine_columns_block(sc, {r.columns.data(), r.columns.size()}, blk.data(), bb, be); + combine_columns_block(sc, {r.columns.data(), r.columns.size()}, blk.data(), bb, be); for (size_t wi = bb; wi < be; ++wi) { - for_each_cos_index(wi * 64, - recipe_fold_word(r, blk.data(), bb, wi, row_parity), - [&out](size_t i) { out.push_back(static_cast(i)); }); + for_each_cos_index(wi * 64, recipe_fold_word(r, blk.data(), bb, wi, row_parity), [&out](size_t i) { + out.push_back(static_cast(i)); + }); } } } @@ -206,43 +200,38 @@ inline auto cos_indices_mask(const CosMask &cos, std::vector &out) -> } } -template -auto scale_cos_lazy(const InvertedIndex &sc, const LazyFold &r, double *coeff, double cos_val) - -> void { +auto scale_cos_lazy(const auto &sc, const LazyFold &r, double *coeff, double cos_val) -> void { const size_t mask_words = r.fold.mask_words; - const uint64_t *row_parity = fold_row_parity(sc, r.fold); + const uint64_t *row_parity = fold_row_parity(sc, r.fold); std::vector &blk = column_block_scratch(); for (size_t bb = 0; bb < mask_words; bb += kColumnBlockWords) { const size_t be = std::min(bb + kColumnBlockWords, mask_words); - combine_columns_block(sc, {r.columns.data(), r.columns.size()}, blk.data(), bb, be); + combine_columns_block(sc, {r.columns.data(), r.columns.size()}, blk.data(), bb, be); for (size_t wi = bb; wi < be; ++wi) { - for_each_cos_index(wi * 64, recipe_fold_word(r, blk.data(), bb, wi, row_parity), [&](size_t i) { - coeff[i] *= cos_val; - }); + for_each_cos_index(wi * 64, + recipe_fold_word(r, blk.data(), bb, wi, row_parity), + [&coeff, &cos_val](size_t i) { coeff[i] *= cos_val; }); } } } -template -auto accumulate_cos_lazy(const InvertedIndex &sc, - const LazyFold &r, - double *state, - double *ham, - double cos_val, - double sec_val) -> double { +auto accumulate_cos_lazy(const auto &sc, const LazyFold &r, double *state, double *ham, double cos_val, double sec_val) + -> double { const size_t mask_words = r.fold.mask_words; - const uint64_t *row_parity = fold_row_parity(sc, r.fold); + const uint64_t *row_parity = fold_row_parity(sc, r.fold); double loc = 0.0; std::vector &blk = column_block_scratch(); for (size_t bb = 0; bb < mask_words; bb += kColumnBlockWords) { const size_t be = std::min(bb + kColumnBlockWords, mask_words); - combine_columns_block(sc, {r.columns.data(), r.columns.size()}, blk.data(), bb, be); + combine_columns_block(sc, {r.columns.data(), r.columns.size()}, blk.data(), bb, be); for (size_t wi = bb; wi < be; ++wi) { - for_each_cos_index(wi * 64, recipe_fold_word(r, blk.data(), bb, wi, row_parity), [&](size_t i) { - loc += state[i] * ham[i]; - ham[i] *= sec_val; - state[i] *= cos_val; - }); + for_each_cos_index(wi * 64, + recipe_fold_word(r, blk.data(), bb, wi, row_parity), + [&loc, &state, &ham, &sec_val, &cos_val](size_t i) { + loc += state[i] * ham[i]; + ham[i] *= sec_val; + state[i] *= cos_val; + }); } } return loc; @@ -252,7 +241,7 @@ inline auto scale_cos_mask(double *coeff, const CosMask &cos, double cos_val) -> const size_t n = cos.blocks.size(); for (size_t k = 0; k < n; ++k) { const auto [base, bits] = cos.blocks[k]; - for_each_cos_index(base, bits, [&](size_t i) { coeff[i] *= cos_val; }); + for_each_cos_index(base, bits, [&coeff, &cos_val](size_t i) { coeff[i] *= cos_val; }); } } inline auto accumulate_cos_mask(double *state, double *ham, const CosMask &cos, double cos_val, double sec_val) @@ -261,7 +250,7 @@ inline auto accumulate_cos_mask(double *state, double *ham, const CosMask &cos, double loc = 0.0; for (size_t k = 0; k < n; ++k) { const auto [base, bits] = cos.blocks[k]; - for_each_cos_index(base, bits, [&](size_t i) { + for_each_cos_index(base, bits, [&loc, &state, &ham, &sec_val, &cos_val](size_t i) { loc += state[i] * ham[i]; ham[i] *= sec_val; state[i] *= cos_val; @@ -270,11 +259,10 @@ inline auto accumulate_cos_mask(double *state, double *ham, const CosMask &cos, return loc; } -template -inline auto fold_to_cos_mask(const FoldCache &p) -> CosMask { +inline auto fold_to_cos_mask(const FoldCache &p) -> CosMask { CosMask c; for (size_t wi = 0; wi < p.fold.mask_words; ++wi) { - const uint64_t b = fold_word(p, wi); + const uint64_t b = fold_word(p, wi); if (b) { c.blocks.emplace_back(wi * 64, b); c.total_count += static_cast(std::popcount(b)); @@ -283,20 +271,18 @@ inline auto fold_to_cos_mask(const FoldCache &p) -> CosMask { return c; } // Cos-index count without materialising the blocks; for diagnostics (graph_size). -template -inline auto fold_popcount(const FoldCache &p) -> size_t { +inline auto fold_popcount(const FoldCache &p) -> size_t { size_t total = 0; for (size_t wi = 0; wi < p.fold.mask_words; ++wi) { - total += static_cast(std::popcount(fold_word(p, wi))); + total += static_cast(std::popcount(fold_word(p, wi))); } return total; } -template -inline auto fold_to_indices(const FoldCache &p) -> VecZ { +inline auto fold_to_indices(const FoldCache &p) -> VecZ { VecZ inds; for (size_t wi = 0; wi < p.fold.mask_words; ++wi) { - for_each_cos_index(wi * 64, fold_word(p, wi), [&](size_t i) { inds.push_back(i); }); + for_each_cos_index(wi * 64, fold_word(p, wi), [&inds](size_t i) { inds.push_back(i); }); } return inds; } diff --git a/cpp/monoprop/detail/evolution/LayerBuilder.h b/cpp/monoprop/detail/evolution/LayerBuilder.h index 597e6782..fa172ff3 100644 --- a/cpp/monoprop/detail/evolution/LayerBuilder.h +++ b/cpp/monoprop/detail/evolution/LayerBuilder.h @@ -14,7 +14,7 @@ #pragma once -// Umbrella header for build_layer(); the implementation lives in the sibling layer_build/ headers. +// Umbrella header for build_layer(); the implementation lives in the sibling layer_build/ headers. // Pivot split: M and its partner M⊕G differ in every column of G including the pivot (G's lowest set // column), so exactly one of the pair carries it — leader (pivot clear) vs follower (pivot set). Visiting // leaders then the still-unmatched followers touches each pair once, no sort and no dedup. diff --git a/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt b/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt index eb4c9853..33842166 100644 --- a/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt +++ b/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt @@ -9,4 +9,5 @@ target_sources( "FusedApply.h" "Resolve.h" "Scan.h" + "TermProduct.h" ) diff --git a/cpp/monoprop/detail/evolution/layer_build/Common.h b/cpp/monoprop/detail/evolution/layer_build/Common.h index e3c203d3..38f4c23f 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Common.h +++ b/cpp/monoprop/detail/evolution/layer_build/Common.h @@ -16,8 +16,11 @@ #include #include +#include #include #include +#include +#include #include #include #include @@ -25,6 +28,8 @@ #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" #include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" namespace monoprop::detail { @@ -44,7 +49,7 @@ struct MatchedEpochSet { // Wraps once per 65535 gates; without the fill a stale stamp on a row reused after a truncation aliases. auto begin_gate(size_t n) -> void { if (cur_ == std::numeric_limits::max()) { - std::fill(epoch_.begin(), epoch_.end(), Stamp{0}); + std::ranges::fill(epoch_, Stamp{0}); cur_ = 0; } ++cur_; @@ -104,15 +109,47 @@ struct FusedContract { std::vector cross_half; // R>1: one half per cross-rank query (resolver +φ, querier −φ) }; -// Queries ride flat VecZ buffers: kQueryWords elements per query (W monomial words + one ±1 phase word). +// Queries ride flat VecZ buffers: one header word holding the record count, then query_words(nw) elements +// per query (nw payload words + one ±1 phase word), then -- support form only -- a tail of dense escape +// monomials for the queries no fixed-stride sparse record can hold. // The source index is not in the payload — the resolver answers by position; the querier holds src_idx_r[r][q]. -template -inline constexpr size_t kQueryWords = mpi_detail::kWords + 1; +// +// Functions of the word count rather than width-derived constants: every caller either has a monomial +// to ask (`mono.num_words()`) or the operator (`op.num_bits()`). +constexpr auto query_words(size_t num_words) -> size_t { + return num_words + 1; +} + +// The record count leads the buffer rather than being divided out of its size, for two reasons: a +// support-form buffer carries a tail after its records, so size/stride is not the count; and the resolver +// has nothing but the received buffer to derive it from, where the querier still has its parallel source +// array. One word per (rank, pass) buffer. +// +// A stream nothing was pushed to may be an empty buffer rather than a zero header -- the scan allocates +// per rank and only some ranks are queried -- so both must read as zero records. +inline constexpr size_t kQueryHeaderWords = 1; + +[[nodiscard]] inline auto query_buffer() -> VecZ { + return VecZ(kQueryHeaderWords, 0); +} +[[nodiscard]] inline auto query_record_count(const VecZ &buf) -> size_t { + return buf.size() < kQueryHeaderWords ? 0 : buf[0]; +} +[[nodiscard]] constexpr auto query_record_offset(size_t q, size_t stride) -> size_t { + return kQueryHeaderWords + (q * stride); +} +// Where the escape tail starts: right after the last record. Both sides derive it from the header and the +// stride, so an escape's own index into the tail is independent of either -- which is what lets the fused +// re-layout below move the records without touching the tail. +[[nodiscard]] inline auto query_tail_offset(const VecZ &buf, size_t stride) -> size_t { + return query_record_offset(query_record_count(buf), stride); +} // Fused query+value record width (R>1): the plain query record plus one trailing word holding the source's // pre-cos coeff (v_src, bit-cast from double), so query + value ride a single alltoallv instead of two. -template -inline constexpr size_t kQueryWordsFused = kQueryWords + 1; +constexpr auto query_words_fused(size_t num_words) -> size_t { + return query_words(num_words) + 1; +} // The unsigned-int intermediate normalizes the ±1 sign bit into a fixed 32-bit pattern so the round-trip // is exact for any VecZ element width. Edit encode/decode as a pair. @@ -132,45 +169,401 @@ inline auto decode_value(size_t word) -> double { return std::bit_cast(word); } -template -inline auto query_push(VecZ &buf, const Monomial &mono, int phase) -> void { - mpi_detail::append_monomial_words(mono, buf); +// The two buffers a scan pushes a query into. A record cannot be appended once the tail has started, so +// the tail accumulates separately and is concatenated when the scan is done -- an escape's index names a +// position within the tail, so the concatenation moves nothing it refers to. +struct QueryOut { + VecZ &records; + VecZ &escapes; +}; + +// The scan's single exit: fold each stream's escapes in behind its records. +inline auto append_escape_tail(VecZ &records, VecZ &escapes) -> void { + if (escapes.empty()) { + return; + } + records.insert(records.end(), escapes.begin(), escapes.end()); + escapes.clear(); +} + +// Appends one record and bumps the header. Every push must precede the escape tail, which the scan +// guarantees by collecting escapes in a buffer of their own and concatenating once the scan is done. +inline auto query_push(VecZ &buf, const Bitset &mono, int phase) -> void { + assert(buf.size() >= kQueryHeaderWords && "a query buffer must be created with query_buffer()"); + mpi_detail::append_monomial_words(mono, buf); buf.push_back(encode_phase(phase)); + ++buf[0]; } // The mono + phase words occupy the same leading offsets in the plain and fused record, so readers differ -// only in the per-record stride QW (defaulted to the plain width). -template > -inline auto query_read(const VecZ &buf, size_t q, Monomial &mono_out, int &phase_out) -> void { - const size_t base = q * QW; - mono_out = mpi_detail::read_monomial_from_words(buf, base); - phase_out = decode_phase(buf[base + mpi_detail::kWords]); +// only in the per-record stride: query_words for a plain record, query_words_fused for a fused one. +// +// The word count comes from mono_out, which as the destination already carries the record's width. +inline auto query_read(const VecZ &buf, size_t q, size_t stride, Bitset &mono_out, int &phase_out) -> void { + const size_t base = query_record_offset(q, stride); + mpi_detail::read_monomial_from_words(buf, base, mono_out); + phase_out = decode_phase(buf[base + mono_out.num_words()]); +} + +// A query record in support form. Deliberately the same *shape* as the dense one -- a fixed-stride +// payload followed by the phase word -- so every stride computation, alltoallv count and reader offset +// above holds with `num_words` reinterpreted as the payload word count. Only the payload differs: +// `sparse_lane_words` words of four uint16 mode lanes each plus one codes word, against the monomial's +// full word count. That is what the format is for: a 1024-mode monomial is 32 words, where a 12-slot row +// is 3 lane words plus 1. +// +// The stride needs a capacity, and it must be one every rank derives identically without communication -- +// SparseRowStore::scratch_slots_for(cutoff mode bound, widest generator), the same value the scan's +// scratch row uses, since a query carries exactly such a product. Ranks already owe each other this kind +// of agreement for the hash width (see find_rank). +// +// A row past that capacity has no sparse record: the caller must fall back to the dense one, and the +// push below asserts rather than truncating. +constexpr auto sparse_lane_words(size_t capacity) -> size_t { + return (capacity + 3) / 4; +} +constexpr auto sparse_payload_words(size_t capacity) -> size_t { + return sparse_lane_words(capacity) + 1; +} + +inline auto sparse_query_push(VecZ &buf, const SparseRow &row, size_t capacity, int phase) -> void { + assert(buf.size() >= kQueryHeaderWords && "a query buffer must be created with query_buffer()"); + const size_t lane_words = sparse_lane_words(capacity); + const size_t n = row.num_slots(); + assert(n <= capacity && "sparse_query_push row exceeds the record capacity"); + const size_t base = buf.size(); + // Zero-filled, so the lanes past the row's own are deterministic; the reader ignores them, taking the + // slot count off the codes word. + buf.resize(base + lane_words + 2, 0); + for (size_t j = 0; j < n; ++j) { + buf[base + (j / 4)] |= static_cast(row.modes[j]) << (16 * (j % 4)); + } + buf[base + lane_words] = static_cast(row.codes); + buf[base + lane_words + 1] = encode_phase(phase); + ++buf[0]; +} + +// The record left behind by a query no sparse record can hold, and it is not a corner case to be sized +// away: a query is M ⊕ G and a fully paired product escapes the cutoff, so nothing bounds its support. +// +// The record keeps its place and its stride -- which is what leaves every offset, alltoallv count and +// compaction in the engine as plain arithmetic -- and says where to find the monomial instead: lane 0 +// carries the store's own overflow marker and the codes slot carries the escape's index into the buffer's +// tail. Both conventions are SparseRowStore's for a spilled row, deliberately: one representation of "too +// wide for a codes word", not two. +// +// `escapes` accumulates the tail separately during the scan, because a record cannot be appended after +// the tail has started; the scan concatenates the two once it is done. An escape's index is its position +// in that tail, so it survives the concatenation and the fused re-layout alike. +inline auto sparse_query_push_escape(VecZ &buf, VecZ &escapes, const Bitset &mono, size_t capacity, int phase) -> void { + assert(buf.size() >= kQueryHeaderWords && "a query buffer must be created with query_buffer()"); + const size_t lane_words = sparse_lane_words(capacity); + const size_t base = buf.size(); + buf.resize(base + lane_words + 2, 0); + buf[base] = static_cast(SparseRowStore::kOverflowLane); + buf[base + lane_words] = escapes.size() / mono.num_words(); + buf[base + lane_words + 1] = encode_phase(phase); + ++buf[0]; + mpi_detail::append_monomial_words(mono, escapes); +} + +// Reading a record's shape needs the lane word count, which the reader has from the capacity; taking the +// record base as an argument keeps this usable from both the plain and the fused stride. +[[nodiscard]] inline auto sparse_record_is_escape(const VecZ &buf, size_t base) -> bool { + return buf[base] == static_cast(SparseRowStore::kOverflowLane); +} + +// lanes_out must hold `capacity` lanes. Writes only the row's own, for the same reason +// read_monomial_from_words overwrites whole words: what a previous record left beyond them cannot be read. +inline auto sparse_query_read(const VecZ &buf, + size_t q, + size_t stride, + size_t capacity, + RowMode *lanes_out, + RowCodes &codes_out, + int &phase_out) -> void { + const size_t base = query_record_offset(q, stride); + const size_t lane_words = sparse_lane_words(capacity); + assert(!sparse_record_is_escape(buf, base) && "an escaped record has no row to read"); + codes_out = static_cast(buf[base + lane_words]); + const size_t n = row_slot_count(codes_out); + for (size_t j = 0; j < n; ++j) { + lanes_out[j] = static_cast((buf[base + (j / 4)] >> (16 * (j % 4))) & 0xFFFFU); + } + phase_out = decode_phase(buf[base + lane_words + 1]); +} + +// Owned storage for a batch of query keys in whichever form a store keys its rows by, plus the record +// reader that fills it. Both resolve paths -- the self-resolve batch in Engine.h and the incoming probe in +// Resolve.h -- want exactly this, and both used to hand-roll it: allocate once, keep the storage across +// layers, overwrite every element before reading it, and hand the whole run to find_batch contiguously. +// +// Grow-only and never cleared by ensure()/read within one object's lifetime: an element is overwritten +// whole before any read, so a per-batch rebuild bought nothing and cost a construction per query. Whether +// that lifetime spans layers depends on the call site, not the class. The thread_local batch in Resolve.h +// keeps its storage across layers on purpose, so its resting footprint is the largest layer's worth until +// the thread exits (peak RSS is unchanged -- the peak was always reached *during* a layer). Engine.h's +// keys_ does not: it is a plain member of a LayerBuildEngine built fresh per build_layer call, so it +// starts default-constructed and pays ensure()'s construction cost every layer -- required, not a missed +// optimization, because retain()'s handles index into storage that must not survive past the layer that +// produced them. +// +// configure() must be called before any use and re-called if the extent changes -- a thread servicing two +// propagators of different widths must not reuse elements sized for the other. +class DenseQueryKeys { +public: + using key_type = Bitset; + + // num_bits is the monomial storage width: query_read memcpys the destination's full word count, so the + // destination is what fixes the record width. capacity is the support form's row capacity, which a + // dense record has no use for -- both batches take both so the call sites need no branch. + auto configure(size_t num_bits, size_t /*capacity*/) -> void { + if (extent_ != num_bits) { + keys_.clear(); + retained_words_.clear(); + retained_count_ = 0; + extent_ = num_bits; + view_ = Bitset(extent_); + } + } + auto ensure(size_t n) -> void { + if (keys_.size() < n) { + // resize only constructs the new tail; elements an earlier layer built keep their storage. + keys_.resize(n, Bitset(extent_)); + } + } + // Slots are refilled for every batch, so anything a caller must still read afterwards has to be + // retained first (see retain below). Nothing per-batch to reset on this side. + auto begin_batch() -> void {} + [[nodiscard]] auto read_record(const VecZ &buf, size_t q, size_t stride, size_t slot) -> int { + int phase = 0; + query_read(buf, q, stride, keys_[slot], phase); + return phase; + } + [[nodiscard]] auto data() const -> const key_type * { return keys_.data(); } + [[nodiscard]] auto operator[](size_t slot) const -> const key_type & { return keys_[slot]; } + + // Copies slot's key into storage that lives as long as this batch, and returns its handle. The + // deferred self-miss list is what needs it: it is read after the batch has been refilled several times + // over, once both resolve passes are done. + // + // The retained keys are a flat word arena at the batch's own width rather than a second MonomialList: + // a Bitset is sized for the widest inline width whatever its own is, so a vector of them carried 72 + // bytes per key where a 128-bit monomial needs 16, and one is retained per term the layer inserts. + // The support-form batch below already keeps its retained rows in an arena, for the same reason. + [[nodiscard]] auto retain(size_t slot) -> size_t { + const size_t words = Bitset::words_for(extent_); + const auto *src = keys_[slot].data(); + retained_words_.insert(retained_words_.end(), src, src + words); + return retained_count_++; + } + // Returns a reference to a scratch monomial refilled per call, so at most one retained key may be + // read at a time. Both readers satisfy that: insert_absent_terms writes slot k's row and is done with + // the key before asking for k+1, and its bulk_insert hashes one key per slot. The support form's + // retained() has the same one-at-a-time contract -- its view points into an arena that must not have + // grown since. + [[nodiscard]] auto retained(size_t handle) const -> const key_type & { + const size_t words = Bitset::words_for(extent_); + std::memcpy(view_.data(), retained_words_.data() + (handle * words), words * sizeof(Bitset::word_type)); + return view_; + } + +private: + MonomialList keys_ = {}; + // Handle h occupies words [h * words_for(extent_), (h+1) * words_for(extent_)). + DefaultInitVector retained_words_ = {}; + size_t retained_count_ = 0; + mutable Bitset view_{0}; + size_t extent_ = 0; +}; + +// The support-form counterpart: one lane array for the whole batch plus a parallel array of keys viewing +// into it, since find_batch wants the keys contiguous and a SparseRow is only a pointer and a word. +// +// The escaped records are why the key is a SparseRowKey rather than a SparseRow: one of those queries has +// no row, so its key points at a monomial materialized out of the buffer's tail. That storage is a deque +// on purpose -- push_back must not invalidate a key handed out for an earlier slot, and a vector's would. +class SparseQueryKeys { +public: + using key_type = SparseRowKey; + + // capacity is the row capacity in slots -- the same value that fixes the record stride, since a record + // holds exactly that many lanes. num_bits sizes the escape monomials. + auto configure(size_t num_bits, size_t capacity) -> void { + if (capacity_ != capacity || num_bits_ != num_bits) { + lanes_.clear(); + keys_.clear(); + escapes_.clear(); + retained_lanes_.clear(); + retained_bases_.clear(); + retained_escapes_.clear(); + retained_.clear(); + capacity_ = capacity; + num_bits_ = num_bits; + } + } + auto ensure(size_t n) -> void { + if (keys_.size() >= n) { + return; + } + lanes_.resize(n * capacity_); + keys_.resize(n); + // Every view is rebuilt, not just the new tail: the resize above may have moved lanes_, which + // would leave the existing views pointing into freed storage. + for (size_t i = 0; i < n; ++i) { + keys_[i] = SparseRowKey{.row = SparseRow{&lanes_[i * capacity_], 0}}; + } + } + // Hygiene rather than correctness: a key always points at the entry read for it, so stale entries are + // unreachable -- they would just accumulate for the whole resolve. Dropped per batch because that is + // the granularity at which slots are refilled anyway. + auto begin_batch() -> void { escapes_.clear(); } + [[nodiscard]] auto read_record(const VecZ &buf, size_t q, size_t stride, size_t slot) -> int { + const size_t base = query_record_offset(q, stride); + const size_t lane_words = sparse_lane_words(capacity_); + if (sparse_record_is_escape(buf, base)) { + // The tail entry this record named. Its offset needs the record count and the stride, both of + // which the buffer and the caller already carry. + const size_t tail = query_tail_offset(buf, stride); + const size_t words = Bitset::words_for(num_bits_); + escapes_.emplace_back(num_bits_); + mpi_detail::read_monomial_from_words(buf, tail + (buf[base + lane_words] * words), escapes_.back()); + keys_[slot].spilled = &escapes_.back(); + return decode_phase(buf[base + lane_words + 1]); + } + int phase = 0; + keys_[slot].spilled = nullptr; + // The lane pointer stays the one ensure() set: a record's lanes are read into this slot's own run. + sparse_query_read(buf, q, stride, capacity_, &lanes_[slot * capacity_], keys_[slot].row.codes, phase); + return phase; + } + [[nodiscard]] auto data() const -> const key_type * { return keys_.data(); } + [[nodiscard]] auto operator[](size_t slot) const -> const key_type & { return keys_[slot]; } + + // See DenseQueryKeys::retain. A retained key owns its lanes here too, in a second arena, and an escaped + // one owns its monomial -- the batch's own escape storage is dropped every batch. + [[nodiscard]] auto retain(size_t slot) -> size_t { + const size_t handle = retained_.size(); + // A base is recorded for every handle, escaped or not, so retained_bases_ stays indexable by + // handle; an escaped key's is simply never read. + retained_bases_.push_back(retained_lanes_.size()); + if (keys_[slot].is_spilled()) { + retained_escapes_.push_back(*keys_[slot].spilled); + // .row left empty: is_spilled() sends every read to .spilled instead. + retained_.push_back(SparseRowKey{.row = {}, .spilled = &retained_escapes_.back()}); + return handle; + } + retained_lanes_.resize(retained_bases_.back() + capacity_); + const size_t n = keys_[slot].row.num_slots(); + std::copy_n(keys_[slot].row.modes, + n, + retained_lanes_.begin() + static_cast(retained_bases_.back())); + // The lane array grows, so a key cannot hold a pointer into it; retained() rebuilds the view. + retained_.push_back(SparseRowKey{.row = SparseRow{nullptr, keys_[slot].row.codes}}); + return handle; + } + [[nodiscard]] auto retained(size_t handle) const -> key_type { + const key_type &key = retained_[handle]; + if (key.is_spilled()) { + return key; + } + return SparseRowKey{.row = SparseRow{&retained_lanes_[retained_bases_[handle]], key.row.codes}}; + } + +private: + DefaultInitVector lanes_ = {}; + std::vector keys_ = {}; + std::deque escapes_ = {}; + // Retained keys, whose storage must outlive the batch's own (see retain). retained_bases_ is parallel + // to retained_ but indexed only for the non-escaped ones -- an escaped key's entry is unread. + DefaultInitVector retained_lanes_ = {}; + std::vector retained_bases_ = {}; + std::deque retained_escapes_ = {}; + std::vector retained_ = {}; + size_t capacity_ = 0; + size_t num_bits_ = 0; +}; + +// A query key as a dense monomial, for the handful of places that need one -- the Schrodinger fresh-insert +// scoring, which has no codes form. Returns a reference when the key already is one and a value otherwise, +// so callers bind with `const auto &` to extend the temporary, exactly as materialize_row documents. +[[nodiscard]] inline auto key_monomial(const Bitset &key, size_t /*num_bits*/) -> const Bitset & { + return key; +} +[[nodiscard]] inline auto key_monomial(const SparseRowKey &key, size_t num_bits) -> Bitset { + if (key.is_spilled()) { + return *key.spilled; + } + return sparse_row_to_bitset(key.row, num_bits); } +// The payload width of one query record for a store, in VecZ words -- the quantity every stride, alltoallv +// count and record offset in Engine.h derives from. An overload per store rather than one accessor on +// MPOperator, because it is a property of the wire format the store is queried through, not of the store. +// +// Dense rows put the monomial's own words on the wire. The support form will put lane words plus the codes +// word: 5 words for a 12-slot row against 33 for a 1024-mode monomial, but slightly *wider* just above the +// store's own crossover -- 5 against 4 at 96 modes, break-even near 128 modes. That band is why the record +// form is tied to the store rather than chosen per layer: a runtime record form would double the engine's +// template instantiations again, to save a word in a narrow range. +[[nodiscard]] inline auto query_payload_words_for(const OperatorIndex &store, size_t /*capacity*/) -> size_t { + return Bitset::words_for(store.num_bits()); +} +[[nodiscard]] inline auto query_payload_words_for(const SparseRowStore & /*store*/, size_t capacity) -> size_t { + return sparse_payload_words(capacity); +} + +// Which key batch a store's query records arrive in. Explicit specializations rather than a member +// typedef on the stores: the record codec lives here, and a store must not depend on the wire format it +// is queried through. +// +// Each store is queried in the form it keys its rows by, so a resolve never converts: the dense store +// receives monomials, the support form receives rows (and, for the queries no row can hold, the escape +// monomials its tail carries). +template +struct QueryKeysFor; +template <> +struct QueryKeysFor { + using type = DenseQueryKeys; +}; +template <> +struct QueryKeysFor { + using type = SparseQueryKeys; +}; + // No monomial reconstruction: process_responses needs only the phase. -template > -inline auto query_phase(const VecZ &buf, size_t q) -> int { - return decode_phase(buf[q * QW + mpi_detail::kWords]); +inline auto query_phase(const VecZ &buf, size_t q, size_t num_words) -> int { + return decode_phase(buf[query_record_offset(q, query_words(num_words)) + num_words]); } -template -inline auto query_value(const VecZ &buf, size_t q) -> double { - return decode_value(buf[q * kQueryWordsFused + mpi_detail::kWords + 1]); +inline auto query_value(const VecZ &buf, size_t q, size_t num_words) -> double { + return decode_value(buf[query_record_offset(q, query_words_fused(num_words)) + num_words + 1]); } -// Requires v.size() == q.size()/kQueryWords: exactly one value per query record. -template -inline auto build_fused_query_value(const VecZ &q, const std::vector &v, VecZ &out) -> void { - constexpr size_t W = kQueryWords; - const size_t nq = q.empty() ? 0 : q.size() / W; +// Requires v.size() == query_record_count(q): exactly one value per query record. +// +// The escape tail rides along unchanged. It can, because an escape's index names its position *within the +// tail* rather than an offset into the buffer -- so widening every record by a value word moves the tail +// without renumbering anything in it. +inline auto build_fused_query_value(const VecZ &q, const std::vector &v, VecZ &out, size_t num_words) -> void { out.clear(); - out.reserve(nq * kQueryWordsFused); + if (q.size() < kQueryHeaderWords) { + // No stream at all rather than an empty one: the self entry is cleared once resolved inline, and it + // must stay empty so the alltoallv sends nothing to self. + return; + } + const size_t W = query_words(num_words); + const size_t nq = query_record_count(q); + const size_t tail = query_tail_offset(q, W); + out.reserve(kQueryHeaderWords + (nq * query_words_fused(num_words)) + (q.size() - tail)); + out.push_back(nq); for (size_t i = 0; i < nq; ++i) { out.insert(out.end(), - q.begin() + static_cast(i * W), - q.begin() + static_cast((i + 1) * W)); + q.begin() + static_cast(query_record_offset(i, W)), + q.begin() + static_cast(query_record_offset(i + 1, W))); out.push_back(encode_value(v[i])); } + out.insert(out.end(), q.begin() + static_cast(tail), q.end()); } } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index d073f00c..36b38b76 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -42,10 +42,9 @@ namespace monoprop::detail { // scaling; only freshly inserted half-terms can be absent (see tests/test_infinite_cutoff.py), and those // sit in [combined_size, op.size()). Scan cos bits and inserted endpoint bits are disjoint, so only the // seam word can carry both — bitwise-or that one, append the rest, keeping blocks ascending/disjoint. -template -inline auto append_inserted_endpoints(CosMask &cos_all, size_t combined_size, const MPOperator &op) -> void { +inline auto append_inserted_endpoints(CosMask &cos_all, size_t combined_size, const auto &op) -> void { const size_t cos_lo = combined_size; - const size_t cos_hi = op.store->size(); + const size_t cos_hi = std::size(op); CosineWordBuilder end_b; for (size_t idx = cos_lo; idx < cos_hi; ++idx) { end_b.push_index(idx); @@ -67,11 +66,17 @@ inline auto append_inserted_endpoints(CosMask &cos_all, size_t combined_size, co // Graph-build sink: accumulates the per-rank PartnerAcc endpoints and assembles a LayerCore at finalize. // wants_values=false — the scan captures no coeffs and every rotation records only (index, phase). -template struct GraphSink { static constexpr bool wants_values = false; - static constexpr size_t kStride = kQueryWords; using Response = TermIndex; + // The record stride is not a constant: it is derived from the + // monomial word count now, which the sink is handed at construction. + size_t num_words = 0; + // The support form's row capacity, which fixes what a record's lanes hold. Zero and unread for a dense + // record; carried by both sinks so the resolve path needs no branch to configure its key batch. + size_t record_capacity = 0; + [[nodiscard]] auto stride() const -> size_t { return query_words(num_words); } + [[nodiscard]] auto capacity() const -> size_t { return record_capacity; } static auto init_response() -> Response { return std::numeric_limits::max(); } size_t R; @@ -81,7 +86,12 @@ struct GraphSink { size_t def_out_base_ = 0; std::vector in_base_; // cross-rank per-rank base into acc[s].in_entries (set in prepare) - GraphSink(size_t R_, size_t my_rank_) : R(R_), my_rank(my_rank_), acc(R_) {} + GraphSink(size_t num_words_, size_t record_capacity_, size_t R_, size_t my_rank_) + : num_words(num_words_), + record_capacity(record_capacity_), + R(R_), + my_rank(my_rank_), + acc(R_) {} auto self_hit(size_t src, size_t found, int phase, double /*v_src*/) -> void { acc[my_rank].in_entries.push_back({found, phase}); @@ -105,9 +115,9 @@ struct GraphSink { std::vector & /*scratch*/) -> std::vector & { return queries; } - auto prepare(const IncomingProbe & /*pr*/, + auto prepare(const auto & /*pr*/, size_t rank_count, - MPOperator & /*op*/, + MPOperator & /*op*/, const std::vector> &responses) -> void { in_base_.assign(rank_count, 0); for (size_t s = 0; s < rank_count; ++s) { @@ -115,18 +125,14 @@ struct GraphSink { acc[s].in_entries.resize(in_base_[s] + responses[s].size()); } } - auto on_resolved(size_t g, - size_t s, - size_t q, - size_t ip, - const IncomingProbe &pr, - const std::vector & /*incoming*/) -> Response { + auto on_resolved(size_t g, size_t s, size_t q, size_t ip, const auto &pr, const std::vector & /*incoming*/) + -> Response { acc[s].in_entries[in_base_[s] + q] = {ip, pr.phase_of[g]}; return static_cast(ip); } auto process_reserve(const std::vector> & /*inc_r*/, size_t /*rank_count*/, - size_t /*my_rank*/) -> void {} + size_t /*my_rank*/) const -> void {} auto on_response_block(size_t r, const std::vector &resp, const std::vector &srcs, @@ -137,14 +143,14 @@ struct GraphSink { out.resize(base + nq); for (size_t q = 0; q < nq; ++q) { assert(resp[q] != std::numeric_limits::max() && "resolver must insert absent cross-rank terms"); - out[base + q] = {srcs[q], query_phase(qbuf, q)}; + out[base + q] = {srcs[q], query_phase(qbuf, q, num_words)}; } } // Drains the per-rank accumulators into the LayerCore's sin_send/sin_recv lists (layout derivation: // see cross_rank_sin_recv_index). cos covers all anticommuting indices, endpoints included, since the // sin_recv apply only adds the sine term. - auto finalize(CosMask &&cos_all, CosMask *out_cos, size_t combined_size, MPOperator &op) + auto finalize(CosMask &&cos_all, CosMask *out_cos, size_t combined_size, const MPOperator &op) -> std::shared_ptr { std::vector partners(R); for (size_t r = 0; r < R; ++r) { @@ -173,7 +179,7 @@ struct GraphSink { } } if (out_cos != nullptr) { - append_inserted_endpoints(cos_all, combined_size, op); + append_inserted_endpoints(cos_all, combined_size, op); *out_cos = std::move(cos_all); } return build_layer_storage_unified(partners, my_rank); @@ -183,11 +189,13 @@ struct GraphSink { // Fused ContractImmediately sink: applies each resolved rotation directly to op_coeffs via the // FusedContract record streams (no LayerCore — finalize returns nullptr). wants_values=true: the scan // captures the signed pre-cos v_src, and resolve reads v_tgt from op_coeffs (·inv_cos under the cos sweep). -template struct ContractSink { static constexpr bool wants_values = true; - static constexpr size_t kStride = kQueryWordsFused; using Response = double; + size_t num_words = 0; // see GraphSink::num_words + size_t record_capacity = 0; // see GraphSink::record_capacity + [[nodiscard]] auto stride() const -> size_t { return query_words_fused(num_words); } + [[nodiscard]] auto capacity() const -> size_t { return record_capacity; } static auto init_response() -> Response { return 0.0; } size_t R; @@ -196,11 +204,12 @@ struct ContractSink { const VecD &op_coeffs; // the very array the scan read, not a copy bool fused_scale; // fused cos sweep active: hit v_tgt recovered as stored·inv_cos double inv_cos; - bool schrodinger; // fresh cross-rank miss coeff: 0 (Heisenberg) vs state-scored (Schrödinger) - Basis basis; // Pauli vs Majorana state scoring of fresh cross-rank Schrödinger misses - size_t def_base_ = 0; // deferred self-insert base into fc.inserts - size_t cross_base_ = 0; // cross-rank resolver-half base into fc.cross_half - Monomial state_mask_{}; // Schrödinger fresh-insert scoring mask (empty in Heisenberg) + bool schrodinger; // fresh cross-rank miss coeff: 0 (Heisenberg) vs state-scored (Schrödinger) + Basis basis; // Pauli vs Majorana state scoring of fresh cross-rank Schrödinger misses + size_t def_base_ = 0; // deferred self-insert base into fc.inserts + size_t cross_base_ = 0; // cross-rank resolver-half base into fc.cross_half + Bitset state_mask_{}; // Schrödinger fresh-insert scoring mask (empty in Heisenberg) + size_t num_bits_ = 0; // operator storage width, for materializing a key that has no dense form // No constructor on purpose: as an aggregate the call site names each field, so the two adjacent // bools cannot be swapped silently. GraphSink keeps its ctor because it sizes `acc` from R. @@ -209,7 +218,7 @@ struct ContractSink { // R=1 hot loop, where a real call is a measurable regression on the Pauli benches. [[gnu::always_inline]] auto self_hit(size_t src, size_t found, int phase, double v_src) -> void { const double v_tgt = fused_scale ? op_coeffs[found] * inv_cos : op_coeffs[found]; - fc.hits.push_back(RotationRec{src, found, v_src, v_tgt, static_cast(phase)}); + fc.hits.emplace_back(src, found, v_src, v_tgt, static_cast(phase)); } // Deferred self-miss insert: v_tgt filled later (after op_coeffs is extended by the apply). auto prepare_deferred(size_t n_miss) -> void { @@ -226,37 +235,36 @@ struct ContractSink { -> std::vector & { scratch.resize(queries.size()); for (size_t r = 0; r < queries.size(); ++r) { - build_fused_query_value(queries[r], vals[r], scratch[r]); + build_fused_query_value(queries[r], vals[r], scratch[r], num_words); } return scratch; } - auto prepare(const IncomingProbe &pr, + auto prepare(const auto &pr, size_t /*rank_count*/, - MPOperator &op, + const MPOperator &op, const std::vector> & /*responses*/) -> void { - state_mask_ = schrodinger ? initial_state_mask(op.initial_state) : Monomial{}; + num_bits_ = op.num_bits(); + state_mask_ = schrodinger ? initial_state_mask(op.initial_state, op.num_bits()) : Bitset(op.num_bits()); cross_base_ = fc.cross_half.size(); fc.cross_half.resize(cross_base_ + pr.nq_total); } - auto on_resolved(size_t g, - size_t s, - size_t q, - size_t ip, - const IncomingProbe &pr, - const std::vector &incoming) -> Response { + auto on_resolved(size_t g, size_t s, size_t q, size_t ip, const auto &pr, const std::vector &incoming) + -> Response { double v_tgt; if (ip < pr.base) { v_tgt = fused_scale ? op_coeffs[ip] * inv_cos : op_coeffs[ip]; } else if (schrodinger) { - v_tgt = - is_paired(pr.mono[g]) ? algebra_state_phase(basis, pr.mono[g], state_mask_) : 0.0; + // The one arm that needs a dense monomial: this scoring has no codes form, so a support-form + // key materializes here. It is a fresh cross-rank Schrodinger miss, so per gate it is rare. + const auto &mono = key_monomial(pr.mono[g], num_bits_); + v_tgt = is_paired(mono) ? algebra_state_phase(basis, mono, state_mask_) : 0.0; } else { v_tgt = 0.0; // Heisenberg fresh insert } fc.cross_half[cross_base_ + g] = HalfRotationRec{ip, - query_value(incoming[s], q), + query_value(incoming[s], q, num_words), static_cast(pr.phase_of[g]), /*is_insert=*/ip >= pr.base}; return v_tgt; @@ -277,33 +285,38 @@ struct ContractSink { const VecZ &qbuf) -> void { const size_t nq = rval.size(); for (size_t q = 0; q < nq; ++q) { - const auto nphase = static_cast(-query_phase(qbuf, q)); - fc.cross_half.push_back(HalfRotationRec{srcs[q], rval[q], nphase, /*is_insert=*/false}); + const auto nphase = static_cast(-query_phase(qbuf, q, num_words)); + fc.cross_half.emplace_back(srcs[q], rval[q], nphase, /*is_insert=*/false); } } // No LayerCore in the fused path → nullptr. Two-pass fused (k>0 / cos==0 fallback) appends inserted // endpoints so the immediate cos scale covers them; the fused cos sweep covers them in-place instead. - auto finalize(CosMask &&cos_all, CosMask *out_cos, size_t combined_size, MPOperator &op) + auto finalize(CosMask &&cos_all, CosMask *out_cos, size_t combined_size, const MPOperator &op) const -> std::shared_ptr { if (out_cos != nullptr && !fused_scale) { - append_inserted_endpoints(cos_all, combined_size, op); + append_inserted_endpoints(cos_all, combined_size, op); *out_cos = std::move(cos_all); } return nullptr; } }; -// Owns build_layer's machinery over a compile-time Sink policy. combined_size = the pre-layer operator size. -template +// Owns build_layer's machinery over a compile-time Sink policy and a concrete row backend. +// combined_size = the pre-layer operator size. Store is a template parameter, not reached through +// local_op, because it fixes the key batch type and the row writes: build_layer binds it once. +template struct LayerBuildEngine { struct DeferredSelfMiss { - Monomial mono; + // A handle into the key batch's retained storage, not a copy: the key is in whatever form the store + // is keyed by, and the batch is the only thing that knows how to own one (see Keys::retain). + size_t key; size_t src; int phase; double v_src = 0.0; // ContractSink only: op_pre[src] captured at scan emit; 0 for GraphSink }; - MPOperator &local_op; // scanned, looked up, and grown by the inserts + MPOperator &local_op; // scanned, looked up, and grown by the inserts + Store &store; // local_op's live backend, bound by build_layer mpi::Comm comm; size_t R; size_t my_rank; @@ -319,15 +332,29 @@ struct LayerBuildEngine { // Fused query+value send scratch (ContractSink, R>1): shared by a gate's two exchange passes. std::vector combined_qv_; Sink sink; - - LayerBuildEngine(MPOperator &local_op_, + // The *plain* query record's payload width. Self-resolve records are plain even under the fused sink, + // whose wider stride applies only to the cross-rank wire, so this is not sink.stride(). Kept as its + // own field rather than read off sink: Sink is a minimal concept (see RecordingSink in + // evolution_detail_tests.cpp), and only the two production sinks happen to also carry num_words. + size_t words_ = 0; + // The support form's row capacity (see GraphSink::record_capacity), for the key batch below. + size_t record_capacity_ = 0; + // Self-resolve key batch, configured once off the operator. Separate from the incoming probe's batch: + // the two are live at the same time on the resolve path. + typename QueryKeysFor::type keys_; + + LayerBuildEngine(MPOperator &local_op_, + Store &store_, mpi::Comm comm_, size_t R_, size_t my_rank_, MatchedEpochSet &matched_scratch, size_t combined_size_, + size_t payload_words, + size_t record_capacity, Sink &&sink_) : local_op(local_op_), + store(store_), comm(comm_), R(R_), my_rank(my_rank_), @@ -335,7 +362,10 @@ struct LayerBuildEngine { combined_size(combined_size_), queries_r(R_), src_idx_r(R_), - sink(std::move(sink_)) { + sink(std::move(sink_)), + words_(payload_words), + record_capacity_(record_capacity) { + keys_.configure(local_op_.num_bits(), record_capacity_); matched.begin_gate(combined_size); } @@ -347,7 +377,7 @@ struct LayerBuildEngine { if constexpr (Sink::wants_values) { lv = &src_val_r[my_rank]; } - const size_t nq = lq.empty() ? 0 : lq.size() / kQueryWords; + const size_t nq = query_record_count(lq); resolve_range_(lq, ls, lv, 0, nq, is_leader_pass); lq.clear(); ls.clear(); @@ -377,19 +407,19 @@ struct LayerBuildEngine { if (R <= 1) { return; } - std::vector &send = sink.send_buffer(queries_r, src_val_r, combined_qv_); + const std::vector &send = sink.send_buffer(queries_r, src_val_r, combined_qv_); std::vector> inc_q; mpi::begin_alltoallv(send, comm).wait_into(inc_q); - auto resp = resolve_incoming(inc_q, local_op, R, is_leader_pass, matched, combined_size, sink); + auto resp = resolve_incoming(inc_q, local_op, store, R, is_leader_pass, matched, combined_size, sink); std::vector resp_recv = response_recv_counts(); std::vector> inc_r; mpi::begin_alltoallv(resp, comm, /*skip_self=*/false, &resp_recv).wait_into(inc_r); - process_responses(inc_r, src_idx_r, queries_r, R, my_rank, sink); + process_responses(inc_r, src_idx_r, queries_r, R, my_rank, sink); } // Followers a leader already matched must not be re-resolved over the wire, so compact them out. auto drop_matched_cross_rank_followers() -> void { - constexpr size_t W = kQueryWords; + const size_t W = query_words(words_); for (size_t r = 0; r < R; ++r) { if (r == my_rank) { continue; @@ -402,15 +432,16 @@ struct LayerBuildEngine { v = &src_val_r[r]; } const size_t nq = s.size(); + assert(nq == query_record_count(q) && "the source array must be parallel to the records"); size_t kept = 0; for (size_t k = 0; k < nq; ++k) { if (matched.is_marked(s[k])) { continue; } if (kept != k) { - std::copy(q.begin() + static_cast(k * W), - q.begin() + static_cast((k + 1) * W), - q.begin() + static_cast(kept * W)); + std::copy(q.begin() + static_cast(query_record_offset(k, W)), + q.begin() + static_cast(query_record_offset(k + 1, W)), + q.begin() + static_cast(query_record_offset(kept, W))); } s[kept] = s[k]; if (v != nullptr) { @@ -418,7 +449,18 @@ struct LayerBuildEngine { } ++kept; } - q.resize(kept * W); + // The escape tail follows the records down, contents untouched. Its entries keep their + // positions relative to each other, so the indices the surviving records carry stay right -- + // including the entries a dropped record orphaned, which ride along as unread padding rather + // than being renumbered. + const size_t tail = query_record_offset(nq, W); + const size_t tail_words = q.size() - tail; + const size_t kept_end = query_record_offset(kept, W); + std::copy(q.begin() + static_cast(tail), + q.end(), + q.begin() + static_cast(kept_end)); + q.resize(kept_end + tail_words); + q[0] = kept; s.resize(kept); if (v != nullptr) { v->resize(kept); @@ -435,11 +477,13 @@ struct LayerBuildEngine { if (n_miss == 0) { return; } - auto key_at = [&](size_t k) -> const Monomial & { return deferred_self_misses[k].mono; }; + // decltype(auto): the dense batch hands back a reference to its own storage, the support form a + // small value view over its lane arena. + auto key_at = [this](size_t k) -> decltype(auto) { return keys_.retained(deferred_self_misses[k].key); }; sink.prepare_deferred(n_miss); - insert_absent_terms(local_op, n_miss, key_at, [&](size_t k, size_t base) { + insert_absent_terms(local_op, store, n_miss, key_at, [this](size_t k, size_t base) { const auto &m = deferred_self_misses[k]; - assign_row(*local_op.store, base + k, m.mono); + assign_row(store, base + k, keys_.retained(m.key)); sink.emit_deferred(k, base + k, m.src, m.phase, m.v_src); }); } @@ -455,7 +499,7 @@ struct LayerBuildEngine { auto response_recv_counts() const -> std::vector { std::vector counts(R); for (size_t r = 0; r < R; ++r) { - counts[r] = static_cast(queries_r[r].size() / kQueryWords); + counts[r] = static_cast(query_record_count(queries_r[r])); } return counts; } @@ -469,21 +513,26 @@ struct LayerBuildEngine { size_t lo, size_t hi, bool is_leader_pass) -> void { - const size_t op_size = local_op.store->size(); - std::array, kResolveBatch> keys; + const size_t op_size = store.size(); + auto &keys = keys_; + // Here rather than in the constructor: the engine is built per gate, including for gates that + // resolve nothing, and ensure() is a no-op once the batch is sized. + keys.ensure(kResolveBatch); std::array phases; std::array srcs; std::array vals; std::array found; + const size_t stride = query_words(words_); // loop-invariant: words_ is fixed for the whole call size_t q = lo; while (q < hi) { size_t m = 0; + keys.begin_batch(); for (; q < hi && m < kResolveBatch; ++q) { const size_t src = ls[q]; if (!is_leader_pass && matched.is_marked(src)) { continue; // follower already matched by a leader → not an independent rotation } - query_read(lq, q, keys[m], phases[m]); + phases[m] = keys.read_record(lq, q, stride, m); srcs[m] = src; if constexpr (Sink::wants_values) { vals[m] = (*lv)[q]; @@ -493,7 +542,7 @@ struct LayerBuildEngine { if (m == 0) { break; } - local_op.store->find_batch(keys.data(), m, found.data()); + store.find_batch(keys.data(), m, found.data()); for (size_t j = 0; j < m; ++j) { double v_src = 0.0; if constexpr (Sink::wants_values) { @@ -508,7 +557,7 @@ struct LayerBuildEngine { sink.self_hit(srcs[j], found[j], phases[j], v_src); } else { - deferred_self_misses.push_back({keys[j], srcs[j], phases[j], v_src}); + deferred_self_misses.push_back({keys.retain(j), srcs[j], phases[j], v_src}); } } } @@ -521,10 +570,12 @@ static inline auto empty_coeffs() -> const VecD & { } // Primary-path layer builder: one fused scan, then two resolve passes into the chosen sink. See LayerBuilder.h. -template -auto build_layer(MPOperator &local_op, - const Monomial &gen, - const CutoffFn &cutoff_fn, +// local_op and gen are deduced from their argument types; cutoff_fn is left a plain auto (a +// std::function, so neither MonomialLike nor otherwise width-bearing). Nothing below names a width: the +// sinks and the engine take theirs from the operator, and every monomial built here takes it from gen. +auto build_layer(auto &local_op, + const MonomialLike auto &gen, + const auto &cutoff_fn, const std::optional &atol, std::optional> local_coeffs, const std::optional &upper_atol, @@ -532,20 +583,21 @@ auto build_layer(MPOperator &local_op, std::optional only_rotate_len_k, MatchedEpochSet &matched_scratch, mpi::Comm comm, + size_t logical_num_modes, CosMask *out_cos = nullptr, FusedContract *fused_contract = nullptr, bool schrodinger = false, VecD *fused_scale_coeffs = nullptr, bool *fused_scale_out = nullptr, Basis basis = Basis::Majorana) -> std::shared_ptr { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); - const size_t my_rank = static_cast(mpi::rank(comm)); - const size_t R = static_cast(mpi::size(comm)); + validate_only_rotate_len_k(only_rotate_len_k, 2 * logical_num_modes); + const auto my_rank = static_cast(mpi::rank(comm)); + const auto R = static_cast(mpi::size(comm)); // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). const bool use_fused = (fused_contract != nullptr); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); const auto &coeffs = local_coeffs.value_or(empty_coeffs()).get(); - const CutoffEvaluator cut_eval{cutoff_fn}; + const CutoffEvaluator cut_eval{cutoff_fn}; // Fused cos sweep: fold the per-gate cosine scale into the scan's own coefficient pass. No length cap only (a // popcount>k hit is outside the per-index cos set, so 1/cos recovery would be wrong) and cos!=0 (else @@ -560,79 +612,146 @@ auto build_layer(MPOperator &local_op, } assert(fused_scale_coeffs == nullptr || (local_coeffs && &local_coeffs->get() == fused_scale_coeffs)); - FusedScanResult fused = [&] { - double *const sweep_ptr = fused_scale ? fused_scale_coeffs->data() : nullptr; - return with_algebra(basis, [&]() { - return fused_find_and_collect(local_op, - gen, - cut_eval, - cut_st, - coeffs, - only_rotate_len_k, - R, - my_rank, - /*capture_values=*/use_fused, - sweep_ptr, - cos_build); + // The single place the row backend is bound: everything from the scan to the inserts is templated on + // it (the per-term kernel, the key batch, the row writes), and binding it here means the choice costs + // one branch per layer instead of one per term. Both arms are instantiated, so this doubles the + // scan/engine template instantiations -- the same trade with_algebra already makes for Basis. + std::shared_ptr storage = + local_op.with_store([&gen, + &cut_eval, + &fused_scale, + &fused_scale_coeffs, + &basis, + &local_op, + &cut_st, + &coeffs, + &only_rotate_len_k, + &R, + &my_rank, + &logical_num_modes, + &use_fused, + &cos_build, + &comm, + &matched_scratch, + &out_cos, + &fused_contract, + &schrodinger](S &store) -> std::shared_ptr { + // The record shape, derived once per layer and passed to everything that reads a record: the scan's + // kernel, the engine's key batch and the sink's strides. Both numbers are functions of the store, the + // generator and the cutoff, so every rank computes the same pair without communication. + const size_t record_capacity = sparse_record_capacity(gen, cut_eval); + const size_t record_payload_words = query_payload_words_for(store, record_capacity); + + FusedScanResult fused = [&] { + double *const sweep_ptr = fused_scale ? fused_scale_coeffs->data() : nullptr; + return with_algebra(basis, + [&local_op, + &store, + &gen, + &cut_eval, + &cut_st, + &coeffs, + &only_rotate_len_k, + &R, + &my_rank, + &logical_num_modes, + &use_fused, + &sweep_ptr, + &cos_build]() { + // Third and last thing bound once per layer, beside the algebra and the + // backend: the storage word count, so the scan's per-term word loops have a + // compile-time trip count. + return with_kernel_width( + gen.num_words(), + [&local_op, + &store, + &gen, + &cut_eval, + &cut_st, + &coeffs, + &only_rotate_len_k, + &R, + &my_rank, + &logical_num_modes, + &use_fused, + &sweep_ptr, + &cos_build](std::integral_constant) { + return fused_find_and_collect(local_op, + store, + gen, + cut_eval, + cut_st, + coeffs, + only_rotate_len_k, + R, + my_rank, + logical_num_modes, + /*capture_values=*/use_fused, + sweep_ptr, + cos_build); + }); + }); + }(); + + CosMask cos_all; + if (fused.cos_blocks.size() == 1) { + // The serial scan produces a single cosine block set — take it wholesale. + cos_all = std::move(fused.cos_blocks[0]); + } + else { + // Cosine block sets are disjoint and ascending; concatenate in order. + for (const auto &block : fused.cos_blocks) { + cos_all.total_count += block.total_count; + cos_all.blocks.insert(cos_all.blocks.end(), block.blocks.begin(), block.blocks.end()); + } + } + fused.cos_blocks = std::vector{}; + + auto run = [&](Sink sink) -> std::shared_ptr { + LayerBuildEngine eng(local_op, + store, + comm, + R, + my_rank, + matched_scratch, + /*combined_size=*/store.size(), + record_payload_words, + record_capacity, + std::move(sink)); + eng.run_exchange(/*is_leader_pass=*/true, + std::move(fused.leader_queries), + std::move(fused.leader_src), + std::move(fused.leader_val)); + eng.run_exchange(/*is_leader_pass=*/false, + std::move(fused.follower_queries), + std::move(fused.follower_src), + std::move(fused.follower_val)); + + return eng.finish(std::move(cos_all), out_cos); + }; + + if (use_fused) { + const double inv_cos = fused_scale ? 1.0 / cos_build : 1.0; // pre-cos recovery factor for hit v_tgt + return run(ContractSink{.num_words = record_payload_words, + .record_capacity = record_capacity, + .R = R, + .my_rank = my_rank, + .fc = *fused_contract, + .op_coeffs = coeffs, + .fused_scale = fused_scale, + .inv_cos = inv_cos, + .schrodinger = schrodinger, + .basis = basis}); + } + return run(GraphSink{record_payload_words, record_capacity, R, my_rank}); }); - }(); - - CosMask cos_all; - if (fused.cos_blocks.size() == 1) { - // The serial scan produces a single cosine block set — take it wholesale. - cos_all = std::move(fused.cos_blocks[0]); - } - else { - // Cosine block sets are disjoint and ascending; concatenate in order. - for (const auto &block : fused.cos_blocks) { - cos_all.total_count += block.total_count; - cos_all.blocks.insert(cos_all.blocks.end(), block.blocks.begin(), block.blocks.end()); - } - } - fused.cos_blocks = std::vector{}; - - auto run = [&](Sink sink) -> std::shared_ptr { - LayerBuildEngine eng(local_op, - comm, - R, - my_rank, - matched_scratch, - /*combined_size=*/local_op.store->size(), - std::move(sink)); - eng.run_exchange(/*is_leader_pass=*/true, - std::move(fused.leader_queries), - std::move(fused.leader_src), - std::move(fused.leader_val)); - eng.run_exchange(/*is_leader_pass=*/false, - std::move(fused.follower_queries), - std::move(fused.follower_src), - std::move(fused.follower_val)); - - return eng.finish(std::move(cos_all), out_cos); - }; - - std::shared_ptr storage; - if (use_fused) { - const double inv_cos = fused_scale ? 1.0 / cos_build : 1.0; // pre-cos recovery factor for hit v_tgt - storage = run(ContractSink{.R = R, - .my_rank = my_rank, - .fc = *fused_contract, - .op_coeffs = coeffs, - .fused_scale = fused_scale, - .inv_cos = inv_cos, - .schrodinger = schrodinger, - .basis = basis}); - } - else { - storage = run(GraphSink{R, my_rank}); - } // Recompute metadata rides with the layer so it survives every graph transform. scaled_count is the // post-insert operator size: the fold truncated to it reproduces the "all anticommuting" cos // bit-for-bit with no stored bitmap. Fused mode has no LayerCore to stamp. if (storage != nullptr) { - storage->generator_words.assign(gen.data(), gen.data() + mpi_detail::kWords); - storage->scaled_count = static_cast(local_op.store->size()); + storage->generator_words.assign(gen.data(), gen.data() + gen.num_words()); + storage->scaled_count = static_cast(local_op.size()); } return storage; diff --git a/cpp/monoprop/detail/evolution/layer_build/Resolve.h b/cpp/monoprop/detail/evolution/layer_build/Resolve.h index ef90d91f..c0dff5dd 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Resolve.h +++ b/cpp/monoprop/detail/evolution/layer_build/Resolve.h @@ -16,6 +16,7 @@ #include #include +#include #include #include "monoprop/TypeAliases.h" @@ -31,31 +32,50 @@ namespace monoprop::detail { // the next index base+j in (sender,record) order, so the assignment (and multi-rank bit-exactness) cannot // drift between resolvers. Queries are source⊕G over globally-distinct sources, ⊕G injective ⇒ queries // pairwise distinct ⇒ misses distinct and absent. -template -struct IncomingProbe { - std::vector goff; // rank_count+1 flat offsets: g = goff[s] + q - DefaultInitVector sender_of; // g → sender rank - DefaultInitVector> mono; // g → deserialized query monomial - DefaultInitVector phase_of; // g → query phase - DefaultInitVector idx_of; // g → resolved index (hit: < base; miss: base+j) - std::vector miss_g; // j → the g that became miss j (Phase 4 reads mono[miss_g[j]]) - size_t base = 0; // op size before the miss inserts (the miss-index base) +// Key is whichever form the store being probed is keyed by (QueryKeysFor): a monomial for the dense store, +// a row-or-escape key for the support form. Named `Key` rather than fixed because the whole point of the +// two record forms is that a resolve never converts one into the other. +template +struct IncomingProbeT { + std::vector goff; // rank_count+1 flat offsets: g = goff[s] + q + DefaultInitVector sender_of; // g → sender rank + // Not owned: a view over the thread_local key batch probe_incoming_queries fills, which is why only + // one IncomingProbe may be live per thread at a time (see the note there). Every element is + // full-width and fully overwritten before anything reads it. + std::span mono; // g → deserialized query key + DefaultInitVector phase_of; // g → query phase + DefaultInitVector idx_of; // g → resolved index (hit: < base; miss: base+j) + std::vector miss_g; // j → the g that became miss j (Phase 4 reads mono[miss_g[j]]) + size_t base = 0; // op size before the miss inserts (the miss-index base) size_t nq_total = 0; }; -// Phases 1-2, read-only w.r.t. operator contents. QW = per-record stride: the plain query width, or -// kQueryWordsFused for the fused resolver. The caller runs Phase 3, then insert_incoming_misses. -template > -auto probe_incoming_queries(const std::vector &incoming, // serialized, one VecZ per sender - MPOperator &op, - size_t rank_count) -> IncomingProbe { - constexpr size_t W = QW; - IncomingProbe pr; +// The probe over a store, spelled once so callers need not name the key form. +template +using IncomingProbeFor = IncomingProbeT::type::key_type>; + +// Phases 1-2, read-only w.r.t. operator contents. query_stride is the per-record width: the plain query +// width, or the fused one for the fused resolver. The caller runs Phase 3, then insert_incoming_misses. +// No width parameter: the monomials are built at `op.num_bits()`, which is the width of the rows they are +// probed against, so the probe monomials and those rows provably share a width. The stride stays an +// ordinary argument. +// inline is load-bearing: this is a plain function defined in a header, so without it every TU that +// includes this emits its own definition and the link fails. +template +inline auto probe_incoming_queries(const std::vector &incoming, // serialized, one VecZ per sender + const MPOperator &op, + Store &store, + size_t rank_count, + size_t query_stride, + size_t record_capacity) -> IncomingProbeFor { + using Keys = typename QueryKeysFor::type; + IncomingProbeFor pr; pr.goff.assign(rank_count + 1, 0); for (size_t s = 0; s < rank_count; ++s) { - const size_t nq = incoming[s].empty() ? 0 : incoming[s].size() / W; - pr.goff[s + 1] = pr.goff[s] + nq; + // Off the buffer's own header, not its size: a support-form buffer carries an escape tail after its + // records, so size/stride is not the record count. + pr.goff[s + 1] = pr.goff[s] + query_record_count(incoming[s]); } pr.nq_total = pr.goff[rank_count]; if (pr.nq_total == 0) { @@ -70,21 +90,42 @@ auto probe_incoming_queries(const std::vector &incoming, // serialized, on } // Phase 1 (read-only): deserialize, then probe with the group-prefetch batch find. - pr.mono.resize(pr.nq_total); + // + // The keys come from a batch that outlives the call, grow-only, rather than being constructed per + // resolve. Constructing them was measurable: a dense element is 72 bytes whose constructor zeroes all + // eight inline words whatever the operator's real width, and above that width it allocates -- so a + // layer paid nq_total of those, every layer, only to overwrite every word immediately (the record + // reader overwrites whole, so no stale bit from a previous layer can survive). Reusing the buffer is + // worth -64% to -72% of this phase at 2-8 words and -41% at 16. + // + // thread_local, matching the scan's `nz`: each partition master gets its own and reuses its + // capacity across layers. Two consequences, both load-bearing: + // - At most one IncomingProbe may be live per thread. Only resolve_incoming builds one, holds it + // to the end of the call, and calls nothing that builds another; the self-resolve path has its + // own separate batch (Engine.h's keys_). + // - The extent is part of the batch's state, which is why configure() is called every time rather + // than once: a thread servicing two propagators of different storage widths must not reuse + // elements sized for the other, or the reader would write a wide record into a narrow element. + // It holds the largest layer's worth of keys until the thread exits, where before it was + // released after every layer. Peak RSS is unchanged -- the peak was always reached *during* a + // layer -- but the resting footprint after one is not; the same trade `nz` already makes. Engine.h's + // own `keys_` does not make this trade: it is a plain member of a LayerBuildEngine built fresh per + // build_layer call, so it starts default-constructed every layer (see DenseQueryKeys's comment). + thread_local Keys scratch; + scratch.configure(op.num_bits(), record_capacity); + scratch.ensure(pr.nq_total); + scratch.begin_batch(); + pr.mono = std::span(scratch.data(), pr.nq_total); pr.phase_of.resize(pr.nq_total); pr.idx_of.resize(pr.nq_total); for (size_t g = 0; g < pr.nq_total; ++g) { const size_t s = pr.sender_of[g]; const size_t q = g - pr.goff[s]; - Monomial m; - int ph = 0; - query_read(incoming[s], q, m, ph); - pr.mono[g] = m; - pr.phase_of[g] = ph; + pr.phase_of[g] = scratch.read_record(incoming[s], q, query_stride, g); } { - const size_t op_size = op.store->size(); - op.store->find_batch(pr.mono.data(), pr.nq_total, pr.idx_of.data()); + const size_t op_size = store.size(); + store.find_batch(pr.mono.data(), pr.nq_total, pr.idx_of.data()); for (size_t g = 0; g < pr.nq_total; ++g) { if (pr.idx_of[g] >= op_size) { // kNotFound is size_t max → also lands here pr.idx_of[g] = kMissingIndex; @@ -93,7 +134,7 @@ auto probe_incoming_queries(const std::vector &incoming, // serialized, on } // Phase 2 ((sender,query) prefix order): each miss takes the next index base+j. - pr.base = op.store->size(); + pr.base = store.size(); for (size_t g = 0; g < pr.nq_total; ++g) { if (pr.idx_of[g] == kMissingIndex) { pr.idx_of[g] = pr.base + pr.miss_g.size(); @@ -105,17 +146,18 @@ auto probe_incoming_queries(const std::vector &incoming, // serialized, on // Phase 4 (bulk insert of the distinct absent terms) into op slots [base, base+n_miss). Call after the // caller's Phase-3 scatter, which reads pre-insert op_coeffs for hits and needs base == op.size(). -template -auto insert_incoming_misses(MPOperator &op, const IncomingProbe &pr) -> void { +// op/pr are deduced from their argument types (MPOperator/IncomingProbe). +auto insert_incoming_misses(auto &op, auto &store, const auto &pr) -> void { const size_t n_miss = pr.miss_g.size(); if (n_miss == 0) { return; } - insert_absent_terms( + insert_absent_terms( op, + store, n_miss, - [&](size_t j) -> const Monomial & { return pr.mono[pr.miss_g[j]]; }, - [&](size_t j, size_t base) { assign_row(*op.store, base + j, pr.mono[pr.miss_g[j]]); }); + [&pr](size_t j) -> decltype(auto) { return pr.mono[pr.miss_g[j]]; }, + [&store, &pr](size_t j, size_t base) { assign_row(store, base + j, pr.mono[pr.miss_g[j]]); }); } // resolve_incoming / process_responses are the picture-independent cross-rank exchange skeletons; what @@ -125,16 +167,20 @@ auto insert_incoming_misses(MPOperator &op, const IncomingProbe +// op is deduced (MPOperator, an argument type); Sink stays named -- it is referenced by name below +// (typename Sink::Response). No width is named anywhere here any more: the record stride is a member of +// the sink and the monomial width comes off the operator. +template auto resolve_incoming(const std::vector &incoming, // serialized, one VecZ per sender - MPOperator &op, + auto &op, + auto &store, size_t rank_count, bool is_leader_pass, MatchedEpochSet &matched, size_t combined_size, // pre-layer op size: bounds the matched set Sink &sink) -> std::vector> { using Resp = typename Sink::Response; - const IncomingProbe pr = probe_incoming_queries(incoming, op, rank_count); + const auto pr = probe_incoming_queries(incoming, op, store, rank_count, sink.stride(), sink.capacity()); std::vector> responses(rank_count); for (size_t s = 0; s < rank_count; ++s) { responses[s].assign(pr.goff[s + 1] - pr.goff[s], Sink::init_response()); @@ -156,13 +202,15 @@ auto resolve_incoming(const std::vector &incoming, // serialized, one VecZ } } - insert_incoming_misses(op, pr); + insert_incoming_misses(op, store, pr); return responses; } // Querier rank (any cross-rank sink): fold each resolver response into a querier-side record. The self/ // local rank was already resolved inline, so it is skipped here. inc_r[r][q] answers query q from rank r. -template +// Unlike resolve_incoming, nothing here touches the operator, so no argument needs deducing at all -- +// Sink stays named for the same reason as above. +template auto process_responses(const std::vector> &inc_r, const std::vector> &src_idx, const std::vector &queries, // serialized query buffers (for phase recovery) diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index d5b8a77e..1f3195fd 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "monoprop/TypeAliases.h" @@ -29,6 +30,7 @@ #include "monoprop/core/Monomial.h" #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/TermProduct.h" #include "monoprop/detail/graph_encoding/MPGraphEncodingTypes.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/operator/InvertedIndex.h" @@ -52,16 +54,19 @@ inline auto build_majorana_evolution_cutoff_state(const std::optional &a .use_coeff_checks = check_atol || check_upper_atol}; } -template +// indices is a vector, not the std::array it was: with no compile-time width there is +// no bound to size an array by, and the array was sized for the whole register while only |G| entries +// (typically 2-4) are ever used. `count` stays alongside it so the existing +// {indices.data(), count} spans keep working unchanged. struct EvenParityGeneratorColumns { - std::array::size()> indices{}; + std::vector indices{}; size_t count = 0; }; -// Set columns in ascending bit order. -template -auto build_even_parity_generator_columns(const Monomial &gen_mono) -> EvenParityGeneratorColumns { - EvenParityGeneratorColumns columns; +// Set columns in ascending bit order. Called once per layer, not per term. +auto build_even_parity_generator_columns(const MonomialLike auto &gen_mono) -> EvenParityGeneratorColumns { + EvenParityGeneratorColumns columns; + columns.indices.resize(gen_mono.count()); for (size_t bit_idx = gen_mono.find_first(); bit_idx < gen_mono.size(); bit_idx = gen_mono.find_next(bit_idx)) { columns.indices[columns.count++] = bit_idx; } @@ -80,8 +85,9 @@ struct EvenParityNzWord { // `pivot_col` is read separately from `gen_cols` so a caller can fold a transformed generator while // splitting on the untransformed one. `g_odd` XORs the per-row parity(|M|) correction (row_parity_ptr) // in before followers are derived. -template -inline auto even_parity_scan_pass1(const InvertedIndex &sc, +// `sc` stays a deduced `auto`: InvertedIndex is no longer a template, but the fold-cache tests also +// bind this to a stand-in exposing the same column accessors. +inline auto even_parity_scan_pass1(const auto &sc, std::span gen_cols, size_t pivot_col, size_t wlo, @@ -103,7 +109,7 @@ inline auto even_parity_scan_pass1(const InvertedIndex &sc, // nonzero overlap, so no-anticommuter blocks skip it) via a deferred follower fix-up — bit-identical // to eager expansion. auto fold_range = [&](size_t bb, size_t be) { - combine_columns_block(sc, gen_cols, blk.data(), bb, be); + combine_columns_block(sc, gen_cols, blk.data(), bb, be); const size_t nz_block_start = nz.size(); for (size_t wi = bb; wi < be; ++wi) { uint64_t overlap = blk[wi - bb]; @@ -122,13 +128,13 @@ inline auto even_parity_scan_pass1(const InvertedIndex &sc, foll = overlap & pivot_dense_ptr[wi]; n_foll += static_cast(std::popcount(foll)); } - nz.push_back(EvenParityNzWord{wi * 64, overlap, foll}); + nz.emplace_back(wi * 64, overlap, foll); } if (pivot_dense || nz.size() == nz_block_start) { return; // dense pivot already folded in, or no anticommuting term — nothing to expand } std::vector &pblk = pivot_column_block_scratch(); - combine_columns_block(sc, std::span(&pivot_col, 1), pblk.data(), bb, be); + combine_columns_block(sc, std::span(&pivot_col, 1), pblk.data(), bb, be); const uint64_t *pw = pblk.data(); for (size_t k = nz_block_start; k < nz.size(); ++k) { EvenParityNzWord &e = nz[k]; @@ -157,25 +163,11 @@ inline auto rotation_dynamic_gate(std::optional only_rotate_len_k, return true; } -// phase_factor is the basis-specific sign only: Majorana interleave_phase, still to be folded with -// hermitian_phase at emit; Pauli pauli_rotation_sign, already rotation-ready. -template -[[gnu::always_inline]] inline auto emit_term_products(const OperatorIndex &ham, - size_t i, - const typename A::GenContext &ctx, - Monomial &new_mono, - size_t &overlap, - int &phase_factor) -> void { - Monomial mono; - ham.for_each_position(i, [&](size_t pos) { mono.set(pos); }); - const Monomial &gen = A::generator(ctx); - new_mono = mono ^ gen; - overlap = mono.count_and(gen); - phase_factor = A::rotation_sign(ctx, mono, new_mono); -} - struct FusedScanResult { - std::vector cos_blocks; // ascending, disjoint, chunk order + std::vector cos_blocks; // ascending, disjoint, chunk order + // No escape tails here: they are folded into the query buffers before this is returned, so a + // consumer only ever sees the finished streams. They live as locals of the scan for exactly as long + // as they are pushed to. std::vector leader_queries; // size R: serialized leader queries per owner rank std::vector> leader_src; // size R: parallel to leader_queries (source op idx) std::vector follower_queries; // size R: serialized follower queries per owner rank @@ -191,27 +183,46 @@ struct FusedScanResult { // deterministic. `fused_scale_coeffs` (no length cap only; must alias coeffs.data()) scales every anticommuting // coeff in place by `fused_scale_cos`=cos(2·build_angle), so no cosine set is built and a hit's stored // value is post-cos (resolve recovers it via 1/cos). -template -auto fused_find_and_collect(const MPOperator &op, - const Monomial &gen, - const CutoffEvaluator &cutoff_eval, +// op, store, gen and cutoff_eval are deduced from their argument types; no width is named anywhere below +// any more -- the monomials this builds take theirs from `gen`, which is the operator's storage width. +// +// `store` is a separate argument rather than reached through `op`, and its concrete type is what selects +// the per-term kernel: build_layer has already bound the backend, and re-entering that dispatch here +// would put a branch on the per-term path, which is the one place it cannot go. +// +// W is the storage word count, bound by build_layer through with_kernel_width for the same reason and at +// the same seam as the backend and the algebra; 0 means "not specialized" (see TermProductsFor). It is a +// template parameter of the scan rather than something the kernel is handed, so that the per-term code +// below stays an ordinary function body: wrapping it in a generic lambda instead measured 2-3% slower +// even on the unspecialized arm, which does no different work. +template +auto fused_find_and_collect(const auto &op, + const Store &store, + const MonomialLike auto &gen, + const auto &cutoff_eval, const CutoffContext &cut_st, const VecD &coeffs, std::optional only_rotate_len_k, size_t rank_count, size_t my_rank, + size_t logical_num_modes, bool capture_values = false, double *fused_scale_coeffs = nullptr, double fused_scale_cos = 1.0) -> FusedScanResult { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); + validate_only_rotate_len_k(only_rotate_len_k, 2 * logical_num_modes); const size_t gen_pop = gen.count(); - const auto ectx = A::make_gen_context(gen); FusedScanResult res; - res.leader_queries.assign(rank_count, VecZ{}); + // Header-initialized, not empty: a record push bumps the count in place, so the header has to be there + // before the first one -- including on the per-rank streams nothing is ever pushed to. + res.leader_queries.assign(rank_count, query_buffer()); res.leader_src.assign(rank_count, std::vector{}); - res.follower_queries.assign(rank_count, VecZ{}); + res.follower_queries.assign(rank_count, query_buffer()); res.follower_src.assign(rank_count, std::vector{}); + // Drained into the query buffers by append_escape_tail before this function returns, so they are + // locals; nothing outside this scan ever sees an unfinished stream. + std::vector leader_escapes(rank_count); + std::vector follower_escapes(rank_count); // Sized to R even on the early-return paths below so the fused engine's per-rank src_val_r access // is always in bounds (parallel to leader_src / follower_src). if (capture_values) { @@ -224,9 +235,9 @@ auto fused_find_and_collect(const MPOperator &op, // pair_swap(G), so pauli_anticommutes = parity(|M ∩ J(G)|), and Pauli never needs the odd-|G| // correction since parity(|G ∩ J(G)|)=0). The pivot splitting each pair is a set bit of the real G // (gen.find_first()), not J(G) — A and A⊕G differ exactly on G's bits. - const Monomial fold_gen = A::fold_generator(gen); + const auto fold_gen = A::fold_generator(gen); const bool g_odd = A::fold_needs_odd_correction(gen); - const auto gen_columns = build_even_parity_generator_columns(fold_gen); + const auto gen_columns = build_even_parity_generator_columns(fold_gen); if (gen_columns.count == 0) { return res; } @@ -236,7 +247,7 @@ auto fused_find_and_collect(const MPOperator &op, return res; } const uint64_t *const row_parity_ptr = g_odd ? inverted_index.row_parity_words() : nullptr; - const size_t n = op.store->size(); + const size_t n = store.size(); // The fused sweep writes fused_scale_coeffs[i] for every anticommuting i < n, so it must be the // very array the reads come from and cover the full operator — a violation corrupts 1/cos recovery. assert(fused_scale_coeffs == nullptr || (fused_scale_coeffs == coeffs.data() && coeffs.size() >= n)); @@ -267,36 +278,42 @@ auto fused_find_and_collect(const MPOperator &op, auto &fq = res.follower_queries; auto &fs = res.follower_src; auto &fv = res.follower_val; + auto &lesc = leader_escapes; + auto &fesc = follower_escapes; + + // Per-gate, not per-term: the kernel's product scratch is overwritten whole per term, so + // constructing it per term would buy nothing and cost a width derivation, an inline-capacity test + // and -- above that capacity -- a heap allocation, every term. It takes the generator's width, + // which is the operator's storage width, so every word op inside stays on Bitset's matched-width + // path. Which kernel this is follows from the store (TermProductsFor); the scan below names no + // representation. + typename TermProductsFor::type products(gen, cutoff_eval); - // The dynamic gate runs before emit_term_products, so a gate-rejected term computes no products. + // The dynamic gate runs before the product, so a gate-rejected term computes none. // abs_c/v_src come from the caller's coeff read, not re-read. auto emit = [&](size_t mono_pop, size_t i, double abs_c, double v_src, bool is_follower) { if (!rotation_dynamic_gate(only_rotate_len_k, mono_pop, cut_st, abs_c)) { return; } - Monomial new_mono; - size_t overlap = 0; - int phase_factor = 0; - emit_term_products(*op.store, i, ectx, new_mono, overlap, phase_factor); + const auto [overlap, phase_factor] = products.product(store, i); // Structural cutoff on the partner M⊕G, unless upper_atol rescues it (CutoffContext::is_above_upper). const size_t new_pop = mono_pop + gen_pop - 2 * overlap; - const bool struct_pass = cutoff_eval.passes_with_popcount(new_mono, new_pop); - if (!struct_pass && !cut_st.is_above_upper(abs_c)) { + if (const bool struct_pass = products.passes(new_pop); !struct_pass && !cut_st.is_above_upper(abs_c)) { return; } const int phase = A::emit_phase(phase_factor, mono_pop, gen_pop, overlap); // Single rank: every partner is self-owned, skip the O(W) hash; multi-rank routes by owner. - const size_t r_prime = (rank_count == 1) ? my_rank : (monomial_hash(new_mono) % rank_count); + const size_t r_prime = (rank_count == 1) ? my_rank : products.owner(rank_count); const size_t source = i; if (is_follower) { - query_push(fq[r_prime], new_mono, phase); + products.push(QueryOut{fq[r_prime], fesc[r_prime]}, phase); fs[r_prime].push_back(source); if (capture_values) { fv[r_prime].push_back(v_src); } } else { - query_push(lq[r_prime], new_mono, phase); + products.push(QueryOut{lq[r_prime], lesc[r_prime]}, phase); ls[r_prime].push_back(source); if (capture_values) { lv[r_prime].push_back(v_src); @@ -313,23 +330,26 @@ auto fused_find_and_collect(const MPOperator &op, nz.clear(); // pass 1 clears it on entry; the skip must too (thread_local reuse) } else { - even_parity_scan_pass1(inverted_index, - gen_cols, - gen.find_first(), - /*wlo=*/0, - /*whi=*/word_count, - last_word, - last_word_mask, - g_odd, - row_parity_ptr, - nz, - n_anti, - n_foll); + even_parity_scan_pass1(inverted_index, + gen_cols, + gen.find_first(), + /*wlo=*/0, + /*whi=*/word_count, + last_word, + last_word_mask, + g_odd, + row_parity_ptr, + nz, + n_anti, + n_foll); } if (rank_count == 1) { - lq[my_rank].reserve((n_anti - n_foll) * kQueryWords); + // The record width comes off the kernel, not off the generator: it is a property of the form + // a query is pushed in, which is the kernel's business and not the monomial's. + const size_t record_words = products.record_words(); + lq[my_rank].reserve(kQueryHeaderWords + ((n_anti - n_foll) * record_words)); ls[my_rank].reserve(n_anti - n_foll); - fq[my_rank].reserve(n_foll * kQueryWords); + fq[my_rank].reserve(kQueryHeaderWords + (n_foll * record_words)); fs[my_rank].reserve(n_foll); } auto derive_coeff = [&](size_t i) -> std::pair { @@ -345,15 +365,15 @@ auto fused_find_and_collect(const MPOperator &op, if (word_aligned_cos && fused_scale_coeffs != nullptr) { // Fused cos sweep: scaling in place here is what replaces building a cosine set. for (uint64_t m = w.overlap; m; m &= m - 1) { - const size_t tz = static_cast(std::countr_zero(m)); - const size_t i = w.base + tz; + const auto tz = static_cast(std::countr_zero(m)); + const auto i = w.base + tz; const double v_src = fused_scale_coeffs[i]; fused_scale_coeffs[i] = v_src * fused_scale_cos; const double abs_c = std::abs(v_src); if (cut_st.is_below_sin(abs_c)) { continue; } - const size_t mono_pop = op.store->popcount(i); + const size_t mono_pop = row_popcount(store, i); const bool is_follower = (w.foll >> tz) & 1u; emit(mono_pop, i, abs_c, v_src, is_follower); } @@ -363,13 +383,13 @@ auto fused_find_and_collect(const MPOperator &op, // gate before the popcount row read — deferring popcount saves random packed-row loads. cos_b.push_word(w.base, w.overlap); for (uint64_t m = w.overlap; m; m &= m - 1) { - const size_t tz = static_cast(std::countr_zero(m)); - const size_t i = w.base + tz; + const auto tz = static_cast(std::countr_zero(m)); + const auto i = w.base + tz; const auto [v_src, abs_c] = derive_coeff(i); if (cut_st.is_below_sin(abs_c)) { continue; } - const size_t mono_pop = op.store->popcount(i); + const size_t mono_pop = row_popcount(store, i); const bool is_follower = (w.foll >> tz) & 1u; emit(mono_pop, i, abs_c, v_src, is_follower); } @@ -378,9 +398,9 @@ auto fused_find_and_collect(const MPOperator &op, // Orbital gate active: it needs mono_pop, and the per-index cosine push covers only // orbital-passing terms, so the popcount row read must precede both. for (uint64_t m = w.overlap; m; m &= m - 1) { - const size_t tz = static_cast(std::countr_zero(m)); - const size_t i = w.base + tz; - const size_t mono_pop = op.store->popcount(i); + const auto tz = static_cast(std::countr_zero(m)); + const auto i = w.base + tz; + const size_t mono_pop = row_popcount(store, i); if (mono_pop > static_cast(*only_rotate_len_k)) { continue; } @@ -393,6 +413,12 @@ auto fused_find_and_collect(const MPOperator &op, } res.cos_blocks.push_back(cos_b.finish()); } + // The one place a stream is finished. The early returns above are all before the first push, so their + // escape buffers are empty and skipping this is a no-op for them. + for (size_t r = 0; r < rank_count; ++r) { + append_escape_tail(res.leader_queries[r], leader_escapes[r]); + append_escape_tail(res.follower_queries[r], follower_escapes[r]); + } return res; } diff --git a/cpp/monoprop/detail/evolution/layer_build/TermProduct.h b/cpp/monoprop/detail/evolution/layer_build/TermProduct.h new file mode 100644 index 00000000..cc071005 --- /dev/null +++ b/cpp/monoprop/detail/evolution/layer_build/TermProduct.h @@ -0,0 +1,443 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// The scan's per-term kernel. For one anticommuting term it answers five questions -- the product M(+)G, +// the overlap the emitted phase needs, the basis rotation sign, whether the product survives the +// structural cutoff, and (for a survivor) its owner rank and query record -- and those five are exactly +// what changes when a row stops being a bitset. So they are gathered behind one per-gate object, chosen +// off the store type by TermProductsFor, rather than spread over the scan's emit lambda. +// +// The call sequence per term is product() -> passes() -> owner() -> push(), each reading the product the +// previous left. That is stateful on purpose: the product is per-gate scratch, since a term must not pay +// to construct the storage its product goes into (see the note on the scratch monomials below). + +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/algebra/CodesAlgebra.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +namespace monoprop::detail { + +// phase_factor is the basis-specific sign only: Majorana interleave_phase, still to be folded with +// hermitian_phase at emit; Pauli pauli_rotation_sign, already rotation-ready. +// ham and the two monomials are deduced from their argument types; A stays the sole explicit template +// argument at call sites, same as before. +// +// `mono` and `new_mono` are scratch owned by the caller for the whole gate, not locals: both are +// overwritten whole here, and a term must not pay for constructing them. Constructing a Bitset means +// deriving the word count from a runtime width, testing it against the inline capacity and -- above +// that capacity -- allocating; with a compile-time width all of that used to fold away to nothing, so +// the scratch is what keeps it off the per-term path. +template +[[gnu::always_inline]] inline auto emit_term_products(const auto &ham, + size_t i, + const typename A::GenContext &ctx, + MonomialLike auto &mono, + MonomialLike auto &new_mono, + size_t &overlap, + int &phase_factor) -> void { + const auto &gen = A::generator(ctx); + // for_each_position only sets bits, so the scratch has to start clear; reset() keeps the width, + // unlike assigning a default-constructed Bitset. + mono.reset(); + for_each_row_position(ham, i, [&mono](size_t pos) { mono.set(pos); }); + // One pass instead of two: mono ^ gen and popcount(mono & gen) are both always needed here, so + // fused_xor_into() computes them together, straight into new_mono (see Bitset::fused_xor_into). + // Its result_count (popcount of the XOR) goes unused -- the caller already has new_pop for free + // via mono_pop + gen_pop - 2*overlap -- so it is not threaded through here. + overlap = mono.fused_xor_into(gen, new_mono).overlap; + phase_factor = A::rotation_sign(ctx, mono, new_mono); +} + +// What one term's product yields. The product's popcount is not here: the caller gets it for free as +// mono_pop + gen_pop - 2*overlap, where the product itself would have to count it. +struct TermProduct { + size_t overlap = 0; + int phase_factor = 0; +}; + +// Dense rows -- the representation the engine has always used, and the reference the sparse one below +// must match term for term. +template +class DenseTermProducts { +public: + // gen fixes the width of both scratch monomials, and it is the operator's storage width, so every + // word op below stays on Bitset's matched-width path. cutoff_eval is held by reference: it borrows + // the caller's CutoffFn already, so it outlives no less than this does. + DenseTermProducts(const Bitset &gen, const CutoffEvaluator &cutoff_eval) + : ctx_(A::make_gen_context(gen)), + cutoff_(&cutoff_eval), + mono_(gen.size()), + new_mono_(gen.size()) {} + + template + [[gnu::always_inline]] auto product(const Store &store, size_t i) -> TermProduct { + TermProduct out; + emit_term_products(store, i, ctx_, mono_, new_mono_, out.overlap, out.phase_factor); + return out; + } + + [[nodiscard]] auto passes(size_t new_pop) const -> bool { + return cutoff_->passes_with_popcount(new_mono_, new_pop); + } + // find_rank's expression, minus its n_ranks == 0 guard, which the scan's rank_count > 1 + // short-circuit already covers. Owner routing is monomial_hash everywhere, including that initial + // distribution, so this must not become anything else. + [[nodiscard]] auto owner(size_t rank_count) const -> size_t { return monomial_hash(new_mono_) % rank_count; } + auto push(QueryOut out, int phase) const -> void { query_push(out.records, new_mono_, phase); } + + // Record width for the per-rank query reserves, which run before the first product. + [[nodiscard]] auto record_words() const -> size_t { return query_words(new_mono_.num_words()); } + + // The product monomial, for the sparse emitter's fallback and for the differential tests. + [[nodiscard]] auto product_row() const -> const Bitset & { return new_mono_; } + +private: + typename A::GenContext ctx_; + const CutoffEvaluator *cutoff_; + Bitset mono_; + Bitset new_mono_; +}; + +// Thrown by DenseTermProductsW's constructor: with_kernel_width picks W from the store's row word +// count and the generator's own width is independent of that (a stale generator bank, a basis +// change that resized the store but not the gates, ...), so unlike the per-term word ops in +// Bitset.h -- deliberately assert-only, since Release must keep their loops bare -- this binding +// happens once per gate. A real branch there costs nothing next to the per-term work it guards, so +// it stays a check even in Release rather than silently reading past W words of gen/mono/new_mono. +class KernelWidthMismatch : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// The dense kernel with the storage word count bound at compile time, chosen once per gate by the +// scan (see with_kernel_width). Answers exactly what DenseTermProducts answers, in the same order +// and to the same values -- what differs is that every word loop inside has a known trip count and +// every operand's storage pointer is resolved once here instead of on each access. +// +// Why this is a separate class rather than a W parameter on DenseTermProducts: only the hot answers +// are worth specializing, and a fallback is still needed for W outside the inline regime, where the +// kernel's inline-operand precondition does not hold. +// +// The cutoff is specialized only for a length cutoff over the whole register. Both other cases -- a +// support cutoff, or an active window narrower than the storage width -- keep going through the +// evaluator. Not for lack of trying: a support arm folding or_sum the same way was measured and cost +// about 1% everywhere, gaining nothing even on the Pauli models that use it, because their per-term +// time is not in the cutoff. A narrow window would need a third kernel, and getting its shift wrong +// would silently change which terms survive. +template +class DenseTermProductsW { +public: + DenseTermProductsW(const Bitset &gen, const CutoffEvaluator &cutoff_eval) + : ctx_(A::make_gen_context(gen)), + cutoff_(&cutoff_eval), + mono_(gen.size()), + new_mono_(gen.size()), + gen_words_(A::generator(ctx_).data()), + mono_words_(mono_.data()), + new_words_(new_mono_.data()) { + if (gen.num_words() != W) { + throw KernelWidthMismatch( + std::format("DenseTermProductsW bound against a generator of {} words; the kernel's W must " + "be the generator's word count.", + W, + gen.num_words())); + } + if (const auto *length = cutoff_eval.length_cutoff(); length != nullptr && length->masks.whole_register()) { + length_cutoff_ = length->cutoff; + } + } + + // The word pointers below point into this object's own bitsets, so a copy would leave the copy + // reading and writing the original's storage. Nothing copies this -- it is the scan's per-gate + // local -- so the case is made unrepresentable rather than documented. + DenseTermProductsW(const DenseTermProductsW &) = delete; + auto operator=(const DenseTermProductsW &) -> DenseTermProductsW & = delete; + DenseTermProductsW(DenseTermProductsW &&) = delete; + auto operator=(DenseTermProductsW &&) -> DenseTermProductsW & = delete; + + template + [[gnu::always_inline]] auto product(const Store &store, size_t i) -> TermProduct { + WordKernel::clear(mono_words_); + // Straight to the words: Bitset::set would re-select the storage pointer for every set bit, + // and a row carries one per surviving slot. + for_each_row_position(store, i, [this](size_t pos) { + mono_words_[pos / Bitset::word_width] |= uint64_t{1} << (pos % Bitset::word_width); + }); + const auto counts = WordKernel::fused_xor_into(mono_words_, gen_words_, new_words_); + return {counts.overlap, A::template rotation_sign_words(ctx_, mono_words_, new_words_)}; + } + + [[nodiscard]] auto passes(size_t new_pop) const -> bool { + if (length_cutoff_.has_value()) { + // Same two clauses as CutoffEvaluator::passes_with_popcount for a length cutoff, in the + // same order: the popcount test proves keep without reading the monomial, and the paired + // test is the xor_sum == 0 clause that rescues a fully paired term of any length. + return new_pop <= *length_cutoff_ || fully_paired_words(new_words_); + } + return cutoff_->passes_with_popcount(new_mono_, new_pop); + } + [[nodiscard]] auto owner(size_t rank_count) const -> size_t { + return WordKernel::splitmix(new_words_) % rank_count; + } + auto push(QueryOut out, int phase) const -> void { query_push(out.records, new_mono_, phase); } + + [[nodiscard]] auto record_words() const -> size_t { return query_words(W); } + [[nodiscard]] auto product_row() const -> const Bitset & { return new_mono_; } + + // Whether passes() answers off the words or through the evaluator. Exists for the differential + // tests, which otherwise cannot tell a run that exercised the word cutoff from one that compared + // the evaluator against itself -- the same reason SparseTermProducts::fell_back() is observable. + [[nodiscard]] auto uses_word_cutoff() const -> bool { return length_cutoff_.has_value(); } + +private: + typename A::GenContext ctx_; + const CutoffEvaluator *cutoff_; + Bitset mono_; + Bitset new_mono_; + // Resolved once, in the constructor's order: each is the data() of a member above, which does not + // move for this object's lifetime because no member below is ever resized or reassigned. + const uint64_t *gen_words_; + uint64_t *mono_words_; + uint64_t *new_words_; + std::optional length_cutoff_ = std::nullopt; +}; + +// The slot capacity a support-form query record is cut to, which is also the scan's scratch product +// capacity -- a query carries exactly such a product, so one number has to serve both or a product that +// fits the scratch would not fit the record. A product occupies at most the cutoff's mode bound plus the +// generator's own modes; an absent bound means the cutoff has no codes form and no term will reach the +// toggle, but the row is sized anyway so the capacity is never zero. +// +// Every rank derives this from the same circuit and cutoff, which is what lets it fix a wire stride with no +// communication -- the same agreement find_rank already needs for the hash width. +[[nodiscard]] inline auto sparse_record_capacity(const Bitset &gen, const CutoffEvaluator &cutoff_eval) -> size_t { + return SparseRowStore::scratch_slots_for(cutoff_eval.max_mode_bound().value_or(SparseRowStore::kMaxSlots), + occupied_mode_count(gen)); +} + +// Support-form rows. The five answers split three ways: product, overlap and rotation sign come off the +// codes word and the two mode lists (sparse_toggle plus A::codes_rotation_sign, O(slots) where the dense +// form is O(storage words)); the cutoff is a popcount test that reads the row only when the bound is +// exceeded; and push() writes the row itself, so a term that survives touches a storage word only when it +// escaped the sparse form or when owner() needs the dense hash at R>1. A term the cutoff rejects touches +// none at all, which is the whole point. +// +// Three cases fall back to the dense kernel for that term, none of them rare enough to assert away: a +// spilled store row (no view exists), a product past the scratch capacity (sparse_toggle reports it +// rather than truncating), and a generator too wide for one codes word. A cutoff that is neither of the +// two concrete functors has no codes form either, and falls back for every term of the gate. +template +class SparseTermProducts { +public: + SparseTermProducts(const Bitset &gen, const CutoffEvaluator &cutoff_eval) + : fallback_(gen, cutoff_eval), + dense_(gen.size()) { + gen_lanes_.reserve(SparseRowStore::kMaxSlots); + bool gen_fits = true; + for_each_mode_slot(gen, [this, &gen_fits](size_t mode, unsigned int code) { + if (gen_lanes_.size() == SparseRowStore::kMaxSlots) { + gen_fits = false; + return; + } + gen_codes_ |= static_cast(code) << (2 * gen_lanes_.size()); + gen_lanes_.push_back(static_cast(mode)); + }); + + // Which cutoff, its bound, and the inactive-mode prefix, all fixed for the propagator's + // lifetime. active_bit_offset counts physical bits and a mode spans two, hence the halving. + if (const auto *length = cutoff_eval.length_cutoff(); length != nullptr) { + kind_ = Kind::Length; + cutoff_value_ = length->cutoff; + inactive_mode_prefix_ = length->masks.active_bit_offset / 2; + } + else if (const auto *support = cutoff_eval.support_cutoff(); support != nullptr) { + kind_ = Kind::Support; + cutoff_value_ = support->cutoff; + inactive_mode_prefix_ = support->masks.active_bit_offset / 2; + } + sparse_usable_ = gen_fits && kind_ != Kind::None; + // Shared with the record stride rather than derived here: a product that fits the scratch has to fit + // the record it is pushed into. + capacity_ = sparse_record_capacity(gen, cutoff_eval); + out_lanes_.resize(capacity_); + } + + template + [[gnu::always_inline]] auto product(const Store &store, size_t i) -> TermProduct { + dense_valid_ = false; + if (sparse_usable_ && !store.spilled(i)) { + const SparseRow mono = store.view(i); + const auto toggled = sparse_toggle(mono, generator(), std::span(out_lanes_.data(), capacity_)); + if (!toggled.overflowed) { + fallback_used_ = false; + product_ = toggled; + return {toggled.overlap, A::codes_rotation_sign(mono, generator())}; + } + } + fallback_used_ = true; + return fallback_.product(store, i); + } + + [[nodiscard]] auto passes(size_t new_pop) const -> bool { + if (fallback_used_) { + return fallback_.passes(new_pop); + } + if (kind_ == Kind::Length) { + return codes_length_passes_with_popcount(product_row(), cutoff_value_, new_pop, inactive_mode_prefix_); + } + return codes_support_passes_with_popcount(product_row(), cutoff_value_, new_pop, inactive_mode_prefix_); + } + // Still the dense hash, and it has to be: owner routing is monomial_hash everywhere, including + // find_rank's initial distribution, and the store's own probe hash is a different function for a + // different purpose. So a *multi-rank* run still materializes once per surviving term here -- moving + // that would mean changing find_rank too. A serial run never calls this (the scan short-circuits at + // rank_count == 1), so it materializes only for the terms that escape. + auto owner(size_t rank_count) -> size_t { return monomial_hash(dense_row()) % rank_count; } + + // The row when there is one, and the dense escape when there is not. Which of the two is not a tuning + // choice: a fully paired product escapes the cutoff, so nothing bounds a query's support. + auto push(QueryOut out, int phase) -> void { + if (fallback_used_) { + sparse_query_push_escape(out.records, out.escapes, dense_row(), capacity_, phase); + return; + } + sparse_query_push(out.records, product_row(), capacity_, phase); + } + [[nodiscard]] auto record_words() const -> size_t { return query_words(sparse_payload_words(capacity_)); } + + // The product in support form. Meaningless when the term fell back to the dense kernel. + [[nodiscard]] auto product_row() const -> SparseRow { return SparseRow{out_lanes_.data(), product_.codes}; } + [[nodiscard]] auto fell_back() const -> bool { return fallback_used_; } + // The record's lane capacity, which is also this kernel's scratch capacity -- see sparse_record_capacity. + [[nodiscard]] auto record_capacity() const -> size_t { return capacity_; } + +private: + enum class Kind : uint8_t { None, Length, Support }; + + [[nodiscard]] auto generator() const -> SparseRow { return SparseRow{gen_lanes_.data(), gen_codes_}; } + + // Memoized because owner() and push() both want it and only push() runs unconditionally. + auto dense_row() -> const Bitset & { + if (fallback_used_) { + return fallback_.product_row(); + } + if (!dense_valid_) { + dense_.reset(); // as in the dense kernel: the slot walk only sets bits + fill_from_sparse_row(product_row(), dense_); + dense_valid_ = true; + } + return dense_; + } + + DenseTermProducts fallback_; + std::vector gen_lanes_ = {}; + RowCodes gen_codes_ = 0; + DefaultInitVector out_lanes_ = {}; + SparseProduct product_ = {}; + Bitset dense_; + size_t capacity_ = 0; + size_t inactive_mode_prefix_ = 0; + unsigned int cutoff_value_ = 0; + Kind kind_ = Kind::None; + bool sparse_usable_ = false; + bool fallback_used_ = true; + bool dense_valid_ = false; +}; + +// Which kernel a store wants at a given bound width. Explicit specializations for the same reason +// QueryKeysFor has them: a store must not know what the scan does with its rows, and an unhandled +// store must fail to compile rather than pick a default. W == 0 is the unspecialized arm, and it is +// the only one the sparse store has -- a sparse row's per-term work is O(slots), not O(storage words), +// so is deliberately left incomplete rather than silently dense. +template +struct TermProductsFor; +template +struct TermProductsFor { + using type = DenseTermProducts; +}; +template +struct TermProductsFor { + using type = DenseTermProductsW; +}; +template +struct TermProductsFor { + using type = SparseTermProducts; +}; + +// Bind the storage word count once per gate and let the scan build a kernel that knows it. +// +// Like with_algebra and with_store, this turns one runtime gate-wide property into a compile-time one +// at a single seam. Everything after that seam is templated on it. Doing this once per gate keeps +// compile time under control; encoding Bitset width in the type was too expensive. +// +// We use tag dispatch instead of passing in a kernel object. The body needs to declare the kernel as a +// local, and passing by reference was ~3% slower on the unspecialized arm because locals optimize better. +// +// Only W in [1, kNarrowKernelWords] is specialized. Above that, the runtime loop is already as fast or +// faster, and fewer code paths are better for instruction cache. +// +// This cap is fixed here, not a build option. Unlike monoprop_SPARSE_ROW_MIN_MODES (ISA-dependent), +// this is a storage-word count and maps to the same width regime on every machine. +// +// kNarrowKernelWords = 4 means 128 storage modes, which covers all current shipped models. Measured +// benefit is about 10% at 2-4 words, fading by 7 words, for about 11% extra `.text` from the four +// specializations. Raising the cap to Bitset::kInlineWords is correct but not faster; setting it to 0 +// restores the pre-seam code path for re-measurement. +inline constexpr size_t kNarrowKernelWords = 4; +static_assert(kNarrowKernelWords <= Bitset::kInlineWords, + "a specialized kernel assumes its operands are inline; above kInlineWords they spill"); + +// Whether a store's per-term work has a storage-word trip count for W to bind at all. A trait rather +// than an is_same_v in the dispatch, so a new backend states its own answer next to the kernel it +// asks for above, instead of silently inheriting "no". +template +inline constexpr bool kBindsKernelWidth = false; +template <> +inline constexpr bool kBindsKernelWidth = true; + +// W as the dispatch will actually pass it: itself while the build specializes that width, 0 once it +// does not. Mapping the arm rather than shortening the dispatch is what makes raising the cap a +// one-line change with no new arm. +template +inline constexpr size_t kCappedKernelWidth = W <= kNarrowKernelWords ? W : 0; + +template +[[gnu::always_inline]] inline auto with_kernel_width(size_t num_words, F &&f) -> decltype(auto) { + // The width regime, not the arm count, is what with_nwords' contract asks a caller to gate on: + // above kInlineWords the words are on the heap, so there is no width to bind even in principle -- + // the kernel's precondition is that every operand is inline. + if constexpr (kBindsKernelWidth) { + if (num_words >= 1 && num_words <= Bitset::kInlineWords) { + return with_nwords(num_words, [&f](std::integral_constant) -> decltype(auto) { + return f(std::integral_constant>{}); + }); + } + } + return f(std::integral_constant{}); +} + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt b/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt index 6afe6999..2dfda6be 100644 --- a/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt +++ b/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt @@ -1,3 +1,5 @@ +target_sources(monoprop-objs PRIVATE MonomialPropagator.cpp) + target_sources( monoprop PUBLIC @@ -5,5 +7,4 @@ target_sources( TYPE HEADERS FILES "MonomialPropagatorCommon.h" - "MonomialPropagator.inl" ) diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.cpp similarity index 59% rename from cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl rename to cpp/monoprop/detail/monomial_propagator/MonomialPropagator.cpp index 34814906..e7d0b013 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once +#include "monoprop/MonomialPropagator.h" #include #include @@ -41,63 +41,34 @@ namespace monoprop { -// The ranks disagree on S. The count comes from partitions= or the environment on -// every rank independently, so the fix is to the launch, and it may belong to a different rank. -class PartitionCountMismatch : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -// The requested operation does not agree with the graph this propagator currently holds -- either it -// requires no stored graph, or its parameter_mapping matches neither the stored layer nor gate count. -// The caller recovers by contracting or rebuilding the graph, not by fixing an isolated argument. -class GraphStateConflict : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -// The (basis, cutoff_type, basis_change) triple is inconsistent: a Pauli basis with a Length cutoff or -// a basis change, or a basis-change table that is not 2*logical_num_modes rows. -class CutoffConfigError : public std::invalid_argument { -public: - using std::invalid_argument::invalid_argument; -}; - -// A coefficient-informed build_graph() was given fewer parameter values than replaying the stored graph -// as a seed needs. -class SeedParametersTooShort : public std::invalid_argument { -public: - using std::invalid_argument::invalid_argument; -}; - -template -MonomialPropagator::MonomialPropagator(const OperatorDict &initial_operator, - unsigned int cutoff, - const VecZ &initial_state, - std::optional schrodinger_cutoff, - mpi::Comm comm, - std::optional lower_atol, - std::optional upper_atol, - CutoffType cutoff_type, - std::optional> basis_change, - size_t logical_num_modes, - Basis basis, - size_t partitions, - PartitionChildFactory child_factory) +MonomialPropagator::MonomialPropagator(const OperatorDict &initial_operator, + unsigned int cutoff, + const VecZ &initial_state, + size_t num_modes, + std::optional schrodinger_cutoff, + mpi::Comm comm, + std::optional lower_atol, + std::optional upper_atol, + CutoffType cutoff_type, + std::optional> basis_change, + Basis basis, + size_t partitions, + PartitionChildFactory child_factory) : schrodinger_{schrodinger_cutoff.has_value()}, comm_{comm}, - mp_op_{}, + mp_op_(2 * detail::storage_modes_for(num_modes)), graph_(schrodinger_cutoff.has_value()), + cutoff_type_{cutoff_type}, + basis_{basis}, cutoff_{cutoff}, + num_modes_{num_modes}, lower_atol_{lower_atol}, upper_atol_{upper_atol}, - logical_num_modes_{logical_num_modes}, - cutoff_type_{cutoff_type}, - basis_change_{basis_change}, - basis_{basis} { - if (logical_num_modes_ == 0 || logical_num_modes_ > NumModes) { - throw PropagatorConfigError( - std::format("logical_num_modes ({}) must be in the range [1, {}].", logical_num_modes_, NumModes)); + basis_change_{basis_change} { + // The storage width is derived from this one, so it can no longer be too narrow for the system; + // zero modes is the only width left to reject. + if (num_modes_ == 0) { + throw PropagatorConfigError("num_modes must be at least 1."); } validate_cutoff_config_(cutoff_type_, basis_change_); @@ -120,66 +91,101 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope "partitions= / monoprop_PARTITIONS / monoprop_NUM_THREADS so R*S is a consistent world."); } if (n_partitions > 1) { - PartitionChildFactory factory = - child_factory ? std::move(child_factory) : PartitionChildFactory{[=](mpi::Comm partition_comm) { - return std::make_unique>(initial_operator, - cutoff, - initial_state, - schrodinger_cutoff, - partition_comm, - lower_atol, - upper_atol, - cutoff_type, - basis_change, - logical_num_modes, - basis, - /*partitions=*/1); - }}; - partition_group_ = std::make_unique>(static_cast(n_partitions), - factory, - comm); + // The partitions hash-partition one operator between them, so they must store at the same width + // as the facade -- which they do by construction, the width being a pure function of num_modes. + PartitionChildFactory factory = child_factory + ? std::move(child_factory) + : PartitionChildFactory{[initial_operator, + cutoff, + initial_state, + num_modes, + schrodinger_cutoff, + lower_atol, + upper_atol, + cutoff_type, + basis_change, + basis](mpi::Comm partition_comm) { + return std::make_unique(initial_operator, + cutoff, + initial_state, + num_modes, + schrodinger_cutoff, + partition_comm, + lower_atol, + upper_atol, + cutoff_type, + basis_change, + basis, + /*partitions=*/1); + }}; + partition_group_ = + std::make_unique(static_cast(n_partitions), factory, comm); return; } - const size_t num_ranks = static_cast(mpi::size(comm_)); - const size_t my_rank = static_cast(mpi::rank(comm_)); - MonomialList local_heisenberg_terms; + const auto num_ranks = static_cast(mpi::size(comm_)); + const auto my_rank = static_cast(mpi::rank(comm_)); + MonomialList local_heisenberg_terms; double core_term = 0.0; + const size_t storage_bits = mp_op_.num_bits(); // invariant for this loop: no store swap happens until below for (const auto &[indices, coefficient] : initial_operator) { - const auto majorana_bitset = indices_to_bitset_checked(indices, 2 * logical_num_modes_); - const auto encoded_coeff = algebra_encode_coeff(basis_, coefficient, majorana_bitset); + const auto majorana_bitset = indices_to_bitset_checked(indices, 2 * num_modes_, storage_bits); + const auto encoded_coeff = algebra_encode_coeff(basis_, coefficient, majorana_bitset); // Store the core term separately as it is orders of magnitude larger than the other terms if (indices.empty()) { core_term = encoded_coeff; continue; } - if (my_rank == find_rank(majorana_bitset, num_ranks)) { + if (my_rank == find_rank(majorana_bitset, num_ranks)) { mp_op_.init_op_map[majorana_bitset] = encoded_coeff; local_heisenberg_terms.push_back(majorana_bitset); } } auto sc = schrodinger_cutoff.value_or(cutoff + 2); - sc = std::min(sc, static_cast(2 * logical_num_modes_)); - auto op = schrodinger_ ? generate_paired_op(sc / 2 + sc % 2, logical_num_modes_) : local_heisenberg_terms; + sc = std::min(sc, static_cast(2 * num_modes_)); - const size_t expected_local_terms = std::max(1, op.size() / std::max(1, num_ranks)); - // Must run before the store: packed_inline_width_() derives the packed-row width from cutoff_fn_. + // Schrodinger's initial basis is streamed, not listed: it is the whole term count (~11.0M at 128 + // modes / cutoff 6), only the ~1/num_ranks share this rank owns is kept, and with S partitions + // every one of the S propagators would hold its own complete copy at the same moment. Heisenberg's + // list is one entry per owned initial-operator term, so it is already small and stays a list. + const size_t max_pairs = sc / 2 + sc % 2; + const size_t total_terms = schrodinger_ ? count_paired_op(max_pairs, num_modes_) : local_heisenberg_terms.size(); + + const size_t expected_local_terms = std::max(1, total_terms / std::max(1, num_ranks)); + // Must run before the store: target_row_width_() derives the row width from cutoff_fn_. regenerate_cutoff_fn_(); - mp_op_.store = std::make_unique>(packed_inline_width_()); - mp_op_.store->reserve(expected_local_terms); - // Store replaced: drop the stale lazy inverted index so it rebuilds against the new store. - mp_op_.inverted_index_.reset(); + // Replaces the store MPOperator's constructor made: same width, but now with the cutoff-derived row + // width, which is only knowable after regenerate_cutoff_fn_() above. set_store() drops the stale lazy + // inverted index with it. A plain set_store() rather than resize_row_store_if_needed_(), because + // nothing has been inserted yet -- there are no rows to migrate. + // + // Which backend: the crossover is on the *storage* width, not the logical one, because what the dense + // representation costs is one pass per storage word. + if (const bool sparse = use_sparse_rows_(); sparse) { + mp_op_.set_store(std::make_unique(mp_op_.num_bits(), target_row_width_(sparse))); + } + else { + mp_op_.set_store(std::make_unique(mp_op_.num_bits(), target_row_width_(sparse))); + } + mp_op_.reserve_terms(expected_local_terms); size_t i = 0; // The initial monomials are distinct, so emplace (insert-if-absent) is an assigning insert here. - for (size_t r = 0; r < op.size(); ++r) { - const auto &mono = materialize_row(op, r); - if (my_rank == find_rank(mono, num_ranks)) { + const auto insert_if_owned = [this, &my_rank, &num_ranks, &i](const auto &mono) { + if (my_rank == find_rank(mono, num_ranks)) { mp_op_.append_term(mono); - mp_op_.store->emplace(mono, i++); + mp_op_.index_term(mono, i++); + } + }; + if (schrodinger_) { + for_each_paired_op(max_pairs, num_modes_, mp_op_.num_bits(), insert_if_owned); + } + else { + for (size_t r = 0; r < local_heisenberg_terms.size(); ++r) { + insert_if_owned(materialize_row(local_heisenberg_terms, r)); } } @@ -189,38 +195,35 @@ MonomialPropagator::MonomialPropagator(const OperatorDict &initial_ope initialize_operator_caches_(); } -template -MonomialPropagator::~MonomialPropagator() = default; +MonomialPropagator::~MonomialPropagator() = default; -template -MonomialPropagator::MonomialPropagator(const MonomialPropagator &other) +MonomialPropagator::MonomialPropagator(const MonomialPropagator &other) : schrodinger_(other.schrodinger_), comm_(other.comm_), cutoff_fn_(other.cutoff_fn_), mp_op_(other.mp_op_), graph_(other.graph_), matched_scratch_(other.matched_scratch_), + cutoff_type_(other.cutoff_type_), + basis_(other.basis_), cutoff_(other.cutoff_), - lower_atol_(other.lower_atol_), - upper_atol_(other.upper_atol_), core_term_(other.core_term_), initial_operator_epoch_(other.initial_operator_epoch_), - logical_num_modes_(other.logical_num_modes_), - cutoff_type_(other.cutoff_type_), - basis_change_(other.basis_change_), - basis_(other.basis_), + num_modes_(other.num_modes_), partition_group_(other.partition_group_ - ? std::make_unique>(*other.partition_group_) - : nullptr) {} + ? std::make_unique(*other.partition_group_) + : nullptr), + lower_atol_(other.lower_atol_), + upper_atol_(other.upper_atol_), + basis_change_(other.basis_change_) {} -template -auto MonomialPropagator::resolve_partition_count_(size_t requested, mpi::Comm comm) -> size_t { +auto MonomialPropagator::resolve_partition_count_(size_t requested, mpi::Comm comm) -> size_t { if (requested >= 1) { return requested; } // One serial partition per physical core, capped by monoprop_NUM_THREADS. On a multi-rank comm this // engages only when threads were explicitly requested, so a pure-MPI run is not oversubscribed. - const auto compute_auto = [&]() -> size_t { + const auto compute_auto = [&]() { const int ranks = mpi::size(comm); size_t cores = detail::partition::enumerate_physical_cores().size(); if (cores == 0) { @@ -251,27 +254,20 @@ auto MonomialPropagator::resolve_partition_count_(size_t requested, mp // Partition fan-out vocabulary; the declarations record which helper is legal where. -template -auto MonomialPropagator::for_each_partition_(const std::function &fn) -> void { - partition_group_->run_on_all([&](int r) { fn(partition_group_->partition(r)); }); +auto MonomialPropagator::for_each_partition_(const std::function &fn) -> void { + for_each_partition_indexed_([&fn](int, MonomialPropagator &p) { fn(p); }); } -template -template -auto MonomialPropagator::map_partitions_(Fn fn) -> std::vector { - return detail::partition::map_partitions(*partition_group_, fn); +auto MonomialPropagator::for_each_partition_indexed_(const std::function &fn) -> void { + partition_group_->run_on_all([this, &fn](int r) { fn(r, partition_group_->partition(r)); }); } -template -template -auto MonomialPropagator::map_partitions_indexed_(Fn fn) -> std::vector { - return detail::partition::collect_on_all(*partition_group_, - [&](int r) -> R { return fn(r, partition_group_->partition(r)); }); +auto MonomialPropagator::partition_count_() const -> size_t { + return static_cast(partition_group_->partition_count()); } -template template -auto MonomialPropagator::concat_partitions_(Fn fn) -> R { +auto MonomialPropagator::concat_partitions_(Fn fn) -> R { const auto per_partition = map_partitions_(fn); size_t total = 0; for (const auto &v : per_partition) { @@ -285,9 +281,8 @@ auto MonomialPropagator::concat_partitions_(Fn fn) -> R { return merged; } -template template -auto MonomialPropagator::fold_partitions_(Proj proj, Accumulate accumulate) const -> R { +auto MonomialPropagator::fold_partitions_(Proj proj, Accumulate accumulate) const -> R { R total{}; for (int r = 0; r < partition_group_->partition_count(); ++r) { accumulate(total, proj(partition_group_->partition(r))); @@ -295,24 +290,20 @@ auto MonomialPropagator::fold_partitions_(Proj proj, Accumulate accumu return total; } -template template -auto MonomialPropagator::sum_partitions_(Proj proj) const -> R { +auto MonomialPropagator::sum_partitions_(Proj proj) const -> R { return fold_partitions_(proj, [](R &total, const R &value) { total += value; }); } -template -auto MonomialPropagator::first_partition_() const -> const MonomialPropagator & { +auto MonomialPropagator::first_partition_() const -> const MonomialPropagator & { return partition_group_->partition(0); } -template -auto MonomialPropagator::partitioned_size_() const -> size_t { +auto MonomialPropagator::partitioned_size_() const -> size_t { return sum_partitions_([](const MonomialPropagator &s) { return s.size(); }); } -template -auto MonomialPropagator::partitioned_graph_size_() const -> std::pair { +auto MonomialPropagator::partitioned_graph_size_() const -> std::pair { // One pass: graph_size() recomputes the cosine-only count, so it must not be projected twice. return fold_partitions_([](const MonomialPropagator &s) { return s.graph_size(); }, [](std::pair &total, const std::pair &value) { @@ -321,63 +312,97 @@ auto MonomialPropagator::partitioned_graph_size_() const -> std::pair< }); } -template -auto MonomialPropagator::partitioned_graph_layers_() const -> size_t { +auto MonomialPropagator::partitioned_graph_layers_() const -> size_t { return first_partition_().graph_layers(); } -template -auto MonomialPropagator::partitioned_core_term_() const -> double { +auto MonomialPropagator::partitioned_core_term_() const -> double { return first_partition_().core_term(); } -template -auto MonomialPropagator::partitioned_operator_memory_usage_() const - -> detail::MPOperatorMemoryBreakdown { +auto MonomialPropagator::partitioned_operator_memory_usage_() const -> detail::MPOperatorMemoryBreakdown { return sum_partitions_([](const MonomialPropagator &s) { return s.operator_memory_usage(); }); } -template -auto MonomialPropagator::partitioned_graph_memory_usage_() const -> GraphMemoryBreakdown { +auto MonomialPropagator::partitioned_graph_memory_usage_() const -> GraphMemoryBreakdown { return sum_partitions_([](const MonomialPropagator &s) { return s.graph_memory_usage(); }); } -template -auto MonomialPropagator::packed_inline_width_() const -> size_t { - constexpr size_t kMax = detail::OperatorIndex::kMaxInlinePositions; - constexpr size_t kDefault = detail::OperatorIndex::kDefaultInlinePositions; +auto MonomialPropagator::row_width_bound_() const -> size_t { + constexpr size_t kDefault = detail::OperatorIndex::kDefaultInlinePositions; + // Schrödinger's initial term set is every fully paired monomial with up to ceil(schrodinger_cutoff/2) + // occupied modes -- schrodinger_cutoff is an independent user knob, not cutoff_, so a bound derived + // from cutoff_fn_ below can be far narrower than what the initial fill actually needs. Both row-store + // backends would otherwise spill most of the initial rows into their overflow map. if (schrodinger_) { return kDefault; } // The bound is already in physical slots (CutoffEvaluator::max_slot_bound), so nothing to scale. - const auto bound = detail::CutoffEvaluator(cutoff_fn_).max_slot_bound(); - if (!bound) { - return kDefault; + const auto bound = detail::CutoffEvaluator(cutoff_fn_).max_slot_bound(); + return bound.value_or(kDefault); +} + +auto MonomialPropagator::use_sparse_rows_() const -> bool { + const auto &settings = config::get(); + if (!settings.row_store) { + throw PropagatorConfigError( + R"(monoprop_ROW_STORE must be "auto", "dense" or "sparse". Unset it to pick by system size.)"); + } + using enum monoprop::config::RowStore; + switch (*settings.row_store) { + case Dense: + return false; + case Sparse: + return true; + case Auto: + break; } - return std::min(*bound, kMax); + return detail::SparseRowStore::preferred_for_modes(storage_num_modes()); } -template -auto MonomialPropagator::apply_initial_operator_(const OperatorDict &op_dict) - -> std::pair, VecD> { +auto MonomialPropagator::target_row_width_(bool sparse) const -> size_t { + // Each backend applies its own cap, so both answers are directly comparable against the width the + // built store reports -- the comparison resize_row_store_if_needed_() makes. + if (sparse) { + return detail::SparseRowStore::slots_for_bound(row_width_bound_()); + } + return detail::OperatorIndex::inline_width_for_bound(row_width_bound_()); +} + +// A no-op resize (the common case: update_lower_atol/update_upper_atol never call this at all, and most +// update_cutoff_type/update_basis_change changes leave the structural bound where it was) costs one +// comparison. An actual resize is an O(current term count) migration -- see MPOperator::resize_store() +// -- paid once per call that moves the bound, not per term for the rest of the propagator's life. +auto MonomialPropagator::resize_row_store_if_needed_() -> void { + // The installed store, not use_sparse_rows_(): the backend is chosen once at construction, and + // re-deriving it from the environment here would be the same decision made twice, one of them from + // configuration rather than from what is actually installed. + const size_t target = target_row_width_(mp_op_.rows_are_sparse()); + if (mp_op_.row_width() != target) { + mp_op_.resize_store(target); + } +} + +auto MonomialPropagator::apply_initial_operator_(const OperatorDict &op_dict) -> std::pair { ++initial_operator_epoch_; if (partition_group_) { // The facade holds no local terms of its own, so the return is empty. - for_each_partition_([&](MonomialPropagator &s) { s.update_initial_operator(op_dict); }); + for_each_partition_([&op_dict](MonomialPropagator &s) { s.update_initial_operator(op_dict); }); return {}; } - const size_t num_ranks = static_cast(mpi::size(comm_)); - const size_t my_rank = static_cast(mpi::rank(comm_)); + const auto num_ranks = static_cast(mpi::size(comm_)); + const auto my_rank = static_cast(mpi::rank(comm_)); OperatorDict new_op; + const size_t storage_bits = mp_op_.num_bits(); // invariant across this loop for (const auto &[ind, coeff] : op_dict) { - const auto mono = indices_to_bitset_checked(ind, 2 * logical_num_modes_); + const auto mono = indices_to_bitset_checked(ind, 2 * num_modes_, storage_bits); if (ind.empty()) { // Core term, store in all - core_term_ = algebra_encode_coeff(basis_, coeff, mono); + core_term_ = algebra_encode_coeff(basis_, coeff, mono); continue; } - if (my_rank == find_rank(mono, num_ranks)) { - const auto mono_indices = bitset_to_indices(mono); + if (my_rank == find_rank(mono, num_ranks)) { + const auto mono_indices = bitset_to_indices(mono); new_op[mono_indices] = coeff; } } @@ -385,12 +410,12 @@ auto MonomialPropagator::apply_initial_operator_(const OperatorDict &o return mp_op_.update_initial_operator(new_op, schrodinger_); } -template -auto MonomialPropagator::graph_data() const -> std::vector { +auto MonomialPropagator::graph_data() const -> std::vector { require_single_partition_("graph_data()"); std::vector layers; const auto num_layers = graph_.layers(); layers.reserve(num_layers); + const size_t storage_bits = mp_op_.num_bits(); // invariant across this loop for (size_t i = 0; i < num_layers; ++i) { const auto traversal = graph_.get_layer_traversal(i); const size_t rank_count = traversal.cross_rank_rank_count(); @@ -400,8 +425,9 @@ auto MonomialPropagator::graph_data() const -> std::vector // The exported shape stays dense (callers index by rank), but it is filled by scattering the // occupied slots rather than by asking every possible slot how much it holds. - std::vector b_data(rank_count), d_data(rank_count); - traversal.for_each_occupied_slot([&](size_t rank, const detail::CrossRankSlotView &slot) { + std::vector b_data(rank_count); + std::vector d_data(rank_count); + traversal.for_each_occupied_slot([&b_data, &d_data](size_t rank, const detail::CrossRankSlotView &slot) { const size_t count = slot.sin_send_count; VecZ sin_send_indices(count); VecI b_phases(count, 0); @@ -421,25 +447,25 @@ auto MonomialPropagator::graph_data() const -> std::vector if (const CosMask *stored = traversal.stored_cos(); stored != nullptr) { cos_inds.reserve(stored->total_count); for (const auto &[base, bits] : stored->blocks) { - detail::for_each_cos_index(base, bits, [&](size_t idx) { cos_inds.push_back(idx); }); + detail::for_each_cos_index(base, bits, [&cos_inds](size_t idx) { cos_inds.push_back(idx); }); } } else if (const auto &gw = traversal.generator_words(); !gw.empty()) { - const auto gen = detail::generator_from_words(gw); - auto p = detail::make_fold_cache(mp_op_.inverted_index(), gen, traversal.scaled_count(), basis_); - cos_inds = detail::fold_to_indices(p); + const auto gen = detail::generator_from_words(gw, storage_bits); + auto p = detail::make_fold_cache(mp_op_.inverted_index(), gen, traversal.scaled_count(), basis_); + cos_inds = detail::fold_to_indices(p); } layers.emplace_back(std::move(cos_inds), std::move(local_cyc_data), std::move(b_data), std::move(d_data)); } return layers; } -template -auto MonomialPropagator::cos_index_count_() const -> size_t { +auto MonomialPropagator::cos_index_count_() const -> size_t { // Only a pared layer stores a cosine set; otherwise recompute the fold here. Cosine-only = cos-scaled // minus the rotation endpoints, saturating at 0. size_t total = 0; const auto num_layers = graph_.layers(); + const size_t storage_bits = mp_op_.num_bits(); // invariant across this loop for (size_t i = 0; i < num_layers; ++i) { const auto traversal = graph_.get_layer_traversal(i); size_t cos_total = 0; @@ -447,10 +473,9 @@ auto MonomialPropagator::cos_index_count_() const -> size_t { cos_total = traversal.num_cos_inds(); } else if (const auto &gw = traversal.generator_words(); !gw.empty()) { - const auto gen = detail::generator_from_words(gw); - const auto fold = - detail::make_fold_cache(mp_op_.inverted_index(), gen, traversal.scaled_count(), basis_); - cos_total = detail::fold_popcount(fold); + const auto gen = detail::generator_from_words(gw, storage_bits); + const auto fold = detail::make_fold_cache(mp_op_.inverted_index(), gen, traversal.scaled_count(), basis_); + cos_total = detail::fold_popcount(fold); } const size_t endpoints = traversal.total_rotation_endpoints(); total += (cos_total > endpoints) ? (cos_total - endpoints) : 0; @@ -458,11 +483,9 @@ auto MonomialPropagator::cos_index_count_() const -> size_t { return total; } -template -auto MonomialPropagator::validate_cutoff_config_(CutoffType cutoff_type, - const std::optional> &basis_change) const - -> void { - with_algebra(basis_, [&]() { +auto MonomialPropagator::validate_cutoff_config_(CutoffType cutoff_type, + const std::optional> &basis_change) const -> void { + with_algebra(basis_, [&cutoff_type, &basis_change]() { if (A::requires_support_cutoff && cutoff_type != CutoffType::Support) { throw CutoffConfigError("Pauli basis requires cutoff_type == Support " "(Length has no Pauli-weight meaning under the Pauli encoding)."); @@ -472,32 +495,31 @@ auto MonomialPropagator::validate_cutoff_config_(CutoffType cutoff_typ "(the encoding is already the Jordan-Wigner image)."); } }); - // regenerate_cutoff_fn_ indexes rows [0, 2*logical_num_modes) unconditionally, so a short + // regenerate_cutoff_fn_ indexes rows [0, 2*num_modes) unconditionally, so a short // basis_change is an out-of-bounds read. - if (basis_change.has_value() && basis_change->size() != 2 * logical_num_modes_) { - throw CutoffConfigError(std::format("basis_change must have exactly 2*logical_num_modes ({}) rows; got {}.", - 2 * logical_num_modes_, + if (basis_change.has_value() && basis_change->size() != 2 * num_modes_) { + throw CutoffConfigError(std::format("basis_change must have exactly 2*num_modes ({}) rows; got {}.", + 2 * num_modes_, basis_change->size())); } } -template -auto MonomialPropagator::regenerate_cutoff_fn_() -> void { +auto MonomialPropagator::regenerate_cutoff_fn_() -> void { + const size_t storage_bits = mp_op_.num_bits(); if (basis_change_.has_value()) { - MonomialList basis; - basis.reserve(2 * logical_num_modes_); - for (size_t i = 0; i < 2 * logical_num_modes_; ++i) { - basis.push_back(indices_to_bitset_checked(basis_change_.value()[i], 2 * logical_num_modes_)); + MonomialList basis; + basis.reserve(2 * num_modes_); + for (size_t i = 0; i < 2 * num_modes_; ++i) { + basis.push_back(indices_to_bitset_checked(basis_change_.value()[i], 2 * num_modes_, storage_bits)); } - cutoff_fn_ = detail::cutoff_function_basis_change(cutoff_type_, cutoff_, basis, logical_num_modes_); + cutoff_fn_ = detail::cutoff_function_basis_change(cutoff_type_, cutoff_, std::move(basis), num_modes_); } else { - cutoff_fn_ = detail::cutoff_function(cutoff_type_, cutoff_, logical_num_modes_); + cutoff_fn_ = detail::cutoff_function(cutoff_type_, cutoff_, num_modes_, storage_bits); } } -template -auto MonomialPropagator::initialize_operator_caches_() -> void { +auto MonomialPropagator::initialize_operator_caches_() -> void { (void)mp_op_.get_operator(); // Heisenberg warms the sparse state only; densifying here would defeat it. Schrödinger's dense vector // IS the live evolved vector. @@ -512,13 +534,11 @@ auto MonomialPropagator::initialize_operator_caches_() -> void { mp_op_.shrink_state_to_fit(); } -template -auto MonomialPropagator::current_picture_coeffs_() -> const VecD & { +auto MonomialPropagator::current_picture_coeffs_() -> const VecD & { return schrodinger_ ? mp_op_.dense_state() : mp_op_.get_operator(); } -template -auto MonomialPropagator::extend_coeffs_from_current_picture_if_needed_(VecD &coeffs) -> void { +auto MonomialPropagator::extend_coeffs_from_current_picture_if_needed_(VecD &coeffs) -> void { if (coeffs.size() >= mp_op_.size()) { return; } @@ -534,12 +554,11 @@ auto MonomialPropagator::extend_coeffs_from_current_picture_if_needed_ coeffs.resize(mp_op_.size(), 0.0); } -template -auto MonomialPropagator::evolve_mode_build_graph_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecZ &gate_indices, - std::optional only_rotate_len_k) -> void { +auto MonomialPropagator::evolve_mode_build_graph_(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + const VecZ &gate_indices, + std::optional only_rotate_len_k) -> void { const auto majoranas_size = majoranas.size(); run_gate_loop_(majoranas, only_rotate_len_k, @@ -557,14 +576,13 @@ auto MonomialPropagator::evolve_mode_build_graph_(const std::vector -auto MonomialPropagator::evolve_mode_graph_with_coeffs_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecZ &gate_indices, - const VecD ¶meters, - const VecD &operator_coeffs, - std::optional only_rotate_len_k) -> void { +auto MonomialPropagator::evolve_mode_graph_with_coeffs_(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + const VecZ &gate_indices, + const VecD ¶meters, + const VecD &operator_coeffs, + std::optional only_rotate_len_k) -> void { auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, 1.0); auto coeffs = operator_coeffs; const auto majoranas_size = majoranas.size(); @@ -592,12 +610,11 @@ auto MonomialPropagator::evolve_mode_graph_with_coeffs_(const std::vec }); } -template -auto MonomialPropagator::evolve_mode_contract_immediately_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecD ¶meters, - std::optional only_rotate_len_k) -> void { +auto MonomialPropagator::evolve_mode_contract_immediately_(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + const VecD ¶meters, + std::optional only_rotate_len_k) -> void { auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, 1.0); // Called for the side effect alone: it returns a reference to the very vector selected below. (void)current_picture_coeffs_(); @@ -618,18 +635,19 @@ auto MonomialPropagator::evolve_mode_contract_immediately_(const std:: }); } -template -auto MonomialPropagator::build_graph(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - std::optional gate_indices, - std::optional parameters, - std::optional only_rotate_len_k) -> void { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * logical_num_modes_); +auto MonomialPropagator::build_graph(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + std::optional gate_indices, + std::optional parameters, + std::optional only_rotate_len_k) -> void { + validate_only_rotate_len_k(only_rotate_len_k, 2 * num_modes_); if (partition_group_) { - for_each_partition_([&](MonomialPropagator &s) { - s.build_graph(majoranas, parameter_mapping, gen_coeffs, gate_indices, parameters, only_rotate_len_k); - }); + for_each_partition_( + [&majoranas, ¶meter_mapping, &gen_coeffs, &gate_indices, ¶meters, &only_rotate_len_k]( + MonomialPropagator &s) { + s.build_graph(majoranas, parameter_mapping, gen_coeffs, gate_indices, parameters, only_rotate_len_k); + }); return; } if (majoranas.empty()) { @@ -661,8 +679,8 @@ auto MonomialPropagator::build_graph(const std::vector &majorana // realistic coefficients. That graph covers the parameter prefix [0, m). VecD seed; if (graph_layers() > 0) { - const auto existing = graph_gate_arrays_(); - const size_t m = expected_num_params(existing.first); + const auto [existing_mapping, existing_gen_coeffs] = graph_gate_arrays_(); + const size_t m = expected_num_params(existing_mapping); // The per-mapping check above only covers this call's indices, which may all sit above the // prefix the stored graph needs. Truncating instead would replay the existing graph at a silently // different point on the axis, and map_params would fail one layer down on the sliced vector. @@ -690,17 +708,17 @@ auto MonomialPropagator::build_graph(const std::vector &majorana } } -template -auto MonomialPropagator::propagate(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecD ¶meters, - std::optional only_rotate_len_k) -> void { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * logical_num_modes_); +auto MonomialPropagator::propagate(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + const VecD ¶meters, + std::optional only_rotate_len_k) -> void { + validate_only_rotate_len_k(only_rotate_len_k, 2 * num_modes_); if (partition_group_) { - for_each_partition_([&](MonomialPropagator &s) { - s.propagate(majoranas, parameter_mapping, gen_coeffs, parameters, only_rotate_len_k); - }); + for_each_partition_( + [&majoranas, ¶meter_mapping, &gen_coeffs, ¶meters, &only_rotate_len_k](MonomialPropagator &s) { + s.propagate(majoranas, parameter_mapping, gen_coeffs, parameters, only_rotate_len_k); + }); return; } if (majoranas.empty()) { @@ -718,11 +736,10 @@ auto MonomialPropagator::propagate(const std::vector &majoranas, evolve_mode_contract_immediately_(majoranas, parameter_mapping, gen_coeffs, parameters, only_rotate_len_k); } -template template -auto MonomialPropagator::run_gate_loop_(const std::vector &majoranas, - std::optional only_rotate_len_k, - EvolutionFunc evolution_func) -> void { +auto MonomialPropagator::run_gate_loop_(const std::vector &majoranas, + std::optional only_rotate_len_k, + EvolutionFunc evolution_func) -> void { // Serial per partition; parallelism comes from partitioning the operator across cores. for (size_t i = 0; i < majoranas.size(); ++i) { const auto idx = !schrodinger_ ? majoranas.size() - 1 - i : i; @@ -733,66 +750,69 @@ auto MonomialPropagator::run_gate_loop_(const std::vector &major initialize_operator_caches_(); } -template -auto MonomialPropagator::build_evolve_result_(const VecZ &gen_vec, - std::optional only_rotate_len_k, - std::optional> coeffs, - std::optional param, - CosMask *out_cos, - detail::FusedContract *fused_contract, - VecD *fused_scale_coeffs, - bool *fused_scale) -> std::shared_ptr { +auto MonomialPropagator::build_evolve_result_(const VecZ &gen_vec, + std::optional only_rotate_len_k, + std::optional> coeffs, + std::optional param, + CosMask *out_cos, + detail::FusedContract *fused_contract, + VecD *fused_scale_coeffs, + bool *fused_scale) -> std::shared_ptr { // The only place a gate generator's indices are bounds-checked: nothing between the public entry // points and here constrains them. - const auto gen_mono = indices_to_bitset_checked(gen_vec, 2 * logical_num_modes_); + const auto gen_mono = indices_to_bitset_checked(gen_vec, 2 * num_modes_, mp_op_.num_bits()); // The cos-recompute metadata is written onto the returned LayerCore. - return detail::build_layer(mp_op_, - gen_mono, - cutoff_fn_, - lower_atol_, - coeffs, - upper_atol_, - param, - only_rotate_len_k, - matched_scratch_, - comm_, - out_cos, - fused_contract, - schrodinger_, - fused_scale_coeffs, - fused_scale, - basis_); + return detail::build_layer(mp_op_, + gen_mono, + cutoff_fn_, + lower_atol_, + coeffs, + upper_atol_, + param, + only_rotate_len_k, + matched_scratch_, + comm_, + num_modes_, + out_cos, + fused_contract, + schrodinger_, + fused_scale_coeffs, + fused_scale, + basis_); } -template -auto MonomialPropagator::propagate_one_(const VecZ &gen_vec, - std::optional only_rotate_len_k, - std::optional> coeffs, - std::optional param, - size_t param_index, - double gen_coeff, - size_t gate_index) -> void { +auto MonomialPropagator::propagate_one_(const VecZ &gen_vec, + std::optional only_rotate_len_k, + std::optional> coeffs, + std::optional param, + size_t param_index, + double gen_coeff, + size_t gate_index) -> void { graph_.append(build_evolve_result_(gen_vec, only_rotate_len_k, coeffs, param), param_index, gen_coeff, gate_index); } -template -auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, +namespace { +// Forward-declared because evolve_operator_with_recompute_ below is defined before it. Internal +// linkage: nothing outside this file builds cosine callbacks. +// num_bits is the operator's monomial storage width; the layers' stored generator words are replayed +// back into monomials at it (see generator_from_words). +auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, const MPGraphView &graph, + size_t num_bits, Basis basis = Basis::Majorana) -> detail::CosCallbacks; +} // namespace -template -auto MonomialPropagator::evolve_operator_with_recompute_(VecD &&coeffs, - const MPGraphView &graph, - const VecD ¶ms) -> VecD { +auto MonomialPropagator::evolve_operator_with_recompute_(VecD &&coeffs, + const MPGraphView &graph, + const VecD ¶ms) const -> VecD { const auto &inverted_index = mp_op_.inverted_index(); // Only the scale side is consumed; build both through the shared builder for consistency. - auto cos_scale = build_cos_callbacks(inverted_index, graph, basis_).scale; + auto cos_scale = build_cos_callbacks(inverted_index, graph, mp_op_.num_bits(), basis_).scale; return evolve_operator(std::move(coeffs), graph, params, comm_, cos_scale); } -template -auto MonomialPropagator::n_gates() const -> size_t { +auto MonomialPropagator::n_gates() const -> size_t { if (partition_group_) { return first_partition_().n_gates(); } @@ -807,10 +827,10 @@ auto MonomialPropagator::n_gates() const -> size_t { return any ? max_gate + 1 : 0; } -template -auto MonomialPropagator::set_parameter_mapping(const VecZ ¶meter_mapping) -> void { +auto MonomialPropagator::set_parameter_mapping(const VecZ ¶meter_mapping) -> void { if (partition_group_) { - for_each_partition_([&](MonomialPropagator &s) { s.set_parameter_mapping(parameter_mapping); }); + for_each_partition_( + [¶meter_mapping](MonomialPropagator &s) { s.set_parameter_mapping(parameter_mapping); }); return; } const size_t count = graph_.layers(); @@ -850,8 +870,7 @@ auto MonomialPropagator::set_parameter_mapping(const VecZ ¶meter_m } } -template -auto MonomialPropagator::graph_gate_arrays_() const -> std::pair { +auto MonomialPropagator::graph_gate_arrays_() const -> std::pair { if (partition_group_) { return first_partition_().graph_gate_arrays_(); } @@ -868,13 +887,15 @@ auto MonomialPropagator::graph_gate_arrays_() const -> std::pair -auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, const MPGraphView &graph, Basis basis) - -> detail::CosCallbacks { +namespace { +auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, + const MPGraphView &graph, + size_t num_bits, + Basis basis) -> detail::CosCallbacks { struct LayerCos { bool recomputes_cos = false; - detail::LazyFold recipe{}; // used iff recomputes_cos - const CosMask *filtered = nullptr; // points into a pruned layer's stored cos + detail::LazyFold recipe{}; // used iff recomputes_cos + const CosMask *filtered = nullptr; // points into a pruned layer's stored cos }; auto cache = std::make_shared>(); cache->reserve(graph.layers()); @@ -888,8 +909,8 @@ auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, else { entry.recomputes_cos = true; const auto t = layer.traversal(); - const auto gen = detail::generator_from_words(t.generator_words()); - entry.recipe = detail::make_lazy_fold(inverted_index, gen, t.scaled_count(), basis); + const auto gen = detail::generator_from_words(t.generator_words(), num_bits); + entry.recipe = detail::make_lazy_fold(inverted_index, gen, t.scaled_count(), basis); } cache->push_back(std::move(entry)); } @@ -901,7 +922,7 @@ auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, detail::scale_cos_mask(c, *e.filtered, v); } else { - detail::scale_cos_lazy(*sc, e.recipe, c, v); + detail::scale_cos_lazy(*sc, e.recipe, c, v); } }; detail::LayerCosAccumulate cos_acc = [cache, sc](size_t i, double *s, double *h, double v, double sec) { @@ -909,7 +930,7 @@ auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, if (!e.recomputes_cos) { return detail::accumulate_cos_mask(s, h, *e.filtered, v, sec); } - return detail::accumulate_cos_lazy(*sc, e.recipe, s, h, v, sec); + return detail::accumulate_cos_lazy(*sc, e.recipe, s, h, v, sec); }; detail::LayerCosIndices cos_inds = [cache, sc](size_t i, std::vector &out) { const auto &e = (*cache)[i]; @@ -917,18 +938,16 @@ auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, detail::cos_indices_mask(*e.filtered, out); return; } - detail::cos_indices_lazy(*sc, e.recipe, out); + detail::cos_indices_lazy(*sc, e.recipe, out); }; return {.scale = std::move(cos_scale), .accumulate = std::move(cos_acc), .indices = std::move(cos_inds)}; } +} // namespace -template template -auto MonomialPropagator::make_functional_(Fn &&func, std::optional pare_threshold) +auto MonomialPropagator::make_functional_(Fn &&func, std::optional pare_threshold) -> std::function { - auto gate_arrays = graph_gate_arrays_(); - auto parameter_mapping = std::move(gate_arrays.first); - auto gen_coeffs = std::move(gate_arrays.second); + auto [parameter_mapping, gen_coeffs] = graph_gate_arrays_(); const auto num_params = expected_num_params(parameter_mapping); // Nothing here needs a dense state: energy only dots it against the evolved operator, and the gradient @@ -958,11 +977,12 @@ auto MonomialPropagator::make_functional_(Fn &&func, std::optional graph; if (pare_threshold.has_value()) { - auto full_cos_of_layer = [this, &inverted_index](size_t i) -> CosMask { + const size_t storage_bits = mp_op_.num_bits(); // invariant across every layer this lambda is called for + auto full_cos_of_layer = [this, &inverted_index, storage_bits](size_t i) { const auto layer = graph_.get_layer_traversal(i); - const auto gen = detail::generator_from_words(layer.generator_words()); - const auto combined = detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), basis_); - return detail::fold_to_cos_mask(combined); + const auto gen = detail::generator_from_words(layer.generator_words(), storage_bits); + const auto combined = detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), basis_); + return detail::fold_to_cos_mask(combined); }; // Threshold the picture's driving vector: the Hamiltonian in Schrödinger, the state otherwise. const auto keep = schrodinger_ ? indices_above(op, *pare_threshold) : state.indices_above(*pare_threshold); @@ -976,9 +996,9 @@ auto MonomialPropagator::make_functional_(Fn &&func, std::optional(inverted_index, graph->replay_view(), basis_); + auto cos = build_cos_callbacks(inverted_index, graph->replay_view(), mp_op_.num_bits(), basis_); - return [func = std::move(func), + return [func = std::forward(func), core_term, state = std::move(state), op = std::move(op), @@ -1006,64 +1026,63 @@ auto MonomialPropagator::make_functional_(Fn &&func, std::optional -auto MonomialPropagator::expectation_value_functional(std::optional pare_threshold) +auto MonomialPropagator::expectation_value_functional(std::optional pare_threshold) -> std::function { if (partition_group_) { // Each partition allreduces internally, so partition 0 is the global value. The group is captured by // raw pointer, so the returned callable must not outlive this propagator. - auto fns = std::make_shared>>( - map_partitions_([&](MonomialPropagator &s) { return s.expectation_value_functional(pare_threshold); })); + auto fns = std::make_shared>>(map_partitions_( + [&pare_threshold](MonomialPropagator &s) { return s.expectation_value_functional(pare_threshold); })); auto *grp = partition_group_.get(); - return [grp, fns](const VecD ¶ms) -> double { - return detail::partition::collect_on_all(*grp, - [&](int r) { return (*fns)[static_cast(r)](params); })[0]; + return [grp, fns](const VecD ¶ms) { + return detail::partition::collect_on_all(*grp, [&fns, ¶ms](int r) { + return (*fns)[static_cast(r)](params); + })[0]; }; } return make_functional_(ev_fn, pare_threshold); } -template -auto MonomialPropagator::expectation_value_and_gradient_functional(std::optional pare_threshold) +auto MonomialPropagator::expectation_value_and_gradient_functional(std::optional pare_threshold) -> std::function(const VecD &)> { if (partition_group_) { - auto fns = std::make_shared(const VecD &)>>>(map_partitions_( - [&](MonomialPropagator &s) { return s.expectation_value_and_gradient_functional(pare_threshold); })); + auto fns = std::make_shared(const VecD &)>>>( + map_partitions_([&pare_threshold](MonomialPropagator &s) { + return s.expectation_value_and_gradient_functional(pare_threshold); + })); auto *grp = partition_group_.get(); - return [grp, fns](const VecD ¶ms) -> std::pair { - return detail::partition::collect_on_all(*grp, - [&](int r) { return (*fns)[static_cast(r)](params); })[0]; + return [grp, fns](const VecD ¶ms) { + return detail::partition::collect_on_all(*grp, [&fns, ¶ms](int r) { + return (*fns)[static_cast(r)](params); + })[0]; }; } return make_functional_(ev_and_grad_fn, pare_threshold); } -template -auto MonomialPropagator::expectation_value(const VecD ¶meters) -> double { +auto MonomialPropagator::expectation_value(const VecD ¶meters) -> double { if (partition_group_) { // Each partition allreduces internally, so every partition returns the global value; take partition 0. - return map_partitions_([&](MonomialPropagator &s) { return s.expectation_value(parameters); })[0]; + return map_partitions_([¶meters](MonomialPropagator &s) { return s.expectation_value(parameters); })[0]; } return expectation_value_functional(std::nullopt)(parameters); } -template -auto MonomialPropagator::expectation_value_and_gradient(const VecD ¶meters) -> std::pair { +auto MonomialPropagator::expectation_value_and_gradient(const VecD ¶meters) -> std::pair { if (partition_group_) { // As in expectation_value(): the gradient is allreduced inside each partition. - return map_partitions_([&](MonomialPropagator &s) { return s.expectation_value_and_gradient(parameters); })[0]; + return map_partitions_( + [¶meters](MonomialPropagator &s) { return s.expectation_value_and_gradient(parameters); })[0]; } return expectation_value_and_gradient_functional(std::nullopt)(parameters); } -template -auto MonomialPropagator::contract_partially(const VecD ¶meters, bool inplace) -> VecD { +auto MonomialPropagator::contract_partially(const VecD ¶meters, bool inplace) -> VecD { if (partition_group_) { - return concat_partitions_([&](MonomialPropagator &s) { return s.contract_partially(parameters, inplace); }); + return concat_partitions_( + [¶meters, &inplace](MonomialPropagator &s) { return s.contract_partially(parameters, inplace); }); } - const auto gate_arrays = graph_gate_arrays_(); - const auto ¶meter_mapping = gate_arrays.first; - const auto &gen_coeffs = gate_arrays.second; + const auto [parameter_mapping, gen_coeffs] = graph_gate_arrays_(); validate_parameters_length(parameters, parameter_mapping); if (parameters.empty()) { @@ -1103,15 +1122,14 @@ auto MonomialPropagator::contract_partially(const VecD ¶meters, bo return evolved_op; } -template -auto MonomialPropagator::evolved_operator_terms(const VecD ¶meters, double atol) +auto MonomialPropagator::evolved_operator_terms(const VecD ¶meters, double atol) -> std::vector>> { using Term = std::pair>; - // `p` is always unpartitioned here (a partition, or *this), so indexing() is available. - const auto collect = [&](MonomialPropagator &p) -> std::vector { + // `p` is always unpartitioned here (a partition, or *this), so for_each_term() is available. + const auto collect = [this, ¶meters, &atol](MonomialPropagator &p) { std::vector terms; const VecD evolved = p.contract_partially(parameters, false); - p.indexing().for_each([&](const auto &mono, size_t idx) { + p.for_each_term([this, &evolved, &atol, &terms](const auto &mono, size_t idx) { if (idx >= evolved.size()) { return; } @@ -1120,10 +1138,10 @@ auto MonomialPropagator::evolved_operator_terms(const VecD ¶meters return; } // Round to drop anti-hermitian numerical noise (Majorana un-applies the Hermitian phase). - const auto decoded = algebra_decode_coeff(basis_, coeff, mono); + const auto decoded = algebra_decode_coeff(basis_, coeff, mono); const std::complex rounded(std::round(decoded.real() * 1e12) / 1e12, std::round(decoded.imag() * 1e12) / 1e12); - terms.emplace_back(bitset_to_indices(mono), rounded); + terms.emplace_back(bitset_to_indices(mono), rounded); }); return terms; }; diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h b/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h index de221f30..4d784481 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h @@ -14,7 +14,9 @@ #pragma once +#include #include +#include #include "monoprop/TypeAliases.h" #include "monoprop/algebra/AlgebraCommon.h" @@ -27,34 +29,54 @@ class UnknownCutoffTypeError : public std::runtime_error { UnknownCutoffTypeError() : std::runtime_error("Unknown cutoff type") {} }; -template -auto cutoff_function(CutoffType cutoff_type, unsigned int cutoff, size_t logical_num_modes = NumModes) - -> CutoffFn { +// logical_num_modes and num_bits are both required: they used to arrive free from NumModes. num_bits is +// the storage width the functor's masks are built for, and it has to match the monomials the functor +// will be handed -- asserted in cutoff_sums. +// +// The stored type is load-bearing, not just an implementation choice. CutoffEvaluator recovers the +// concrete functor with std::function::target(), which matches on the *exact* type -- so wrapping +// either functor in a lambda, or storing a structurally identical but distinct type, would disengage +// the scan's fast paths while still computing the right answer. Nothing downstream can detect that: +// the results are unchanged, so the tests pass and the bit-identity check stays green, and only a +// benchmark would notice. The asserts below pin the handshake here, where the type is chosen. +inline auto cutoff_function(CutoffType cutoff_type, unsigned int cutoff, size_t logical_num_modes, size_t num_bits) + -> CutoffFn { switch (cutoff_type) { - case CutoffType::Length: - return detail::LengthCutoff{cutoff, logical_num_modes}; - case CutoffType::Support: - return detail::SupportCutoff{cutoff, logical_num_modes}; + case CutoffType::Length: { + CutoffFn fn = LengthCutoff{cutoff, logical_num_modes, num_bits}; + assert(fn.target() != nullptr && "CutoffEvaluator's length fast path would not engage"); + return fn; + } + case CutoffType::Support: { + CutoffFn fn = SupportCutoff{cutoff, logical_num_modes, num_bits}; + assert(fn.target() != nullptr && "CutoffEvaluator's support fast path would not engage"); + return fn; + } default: throw UnknownCutoffTypeError(); } } -template -auto cutoff_function_basis_change(CutoffType cutoff_type, - unsigned int cutoff, - const MonomialList &basis, - size_t logical_num_modes = NumModes) -> CutoffFn { +// Lambdas by design, so CutoffEvaluator's target<>() probes find nothing and it calls through the +// std::function. That is the intended behaviour here, not the silent miss described above: the +// predicate is length/support applied to the *mapped* monomial, so the fast paths -- which read a raw +// popcount and a bare cutoff -- do not apply. +// `basis` is taken by value and moved into the closure: it is one Bitset per Majorana, and every caller +// builds it as a local it discards straight after. +inline auto cutoff_function_basis_change(CutoffType cutoff_type, + unsigned int cutoff, + MonomialList basis, + size_t logical_num_modes) -> CutoffFn { switch (cutoff_type) { case CutoffType::Length: - return [cutoff, logical_num_modes, basis_copy = basis](const Monomial &mono) { - const auto mapped_mono = change_basis(mono, basis_copy); - return length_cutoff(mapped_mono, cutoff, logical_num_modes); + return [cutoff, logical_num_modes, basis_copy = std::move(basis)](const Bitset &mono) { + const auto mapped_mono = change_basis(mono, basis_copy); + return length_cutoff(mapped_mono, cutoff, logical_num_modes); }; case CutoffType::Support: - return [cutoff, logical_num_modes, basis_copy = basis](const Monomial &mono) { - const auto mapped_mono = change_basis(mono, basis_copy); - return support_cutoff(mapped_mono, cutoff, logical_num_modes); + return [cutoff, logical_num_modes, basis_copy = std::move(basis)](const Bitset &mono) { + const auto mapped_mono = change_basis(mono, basis_copy); + return support_cutoff(mapped_mono, cutoff, logical_num_modes); }; default: throw UnknownCutoffTypeError(); diff --git a/cpp/monoprop/detail/mpi/CheckedCount.h b/cpp/monoprop/detail/mpi/CheckedCount.h index 703e06f5..d8a8dd61 100644 --- a/cpp/monoprop/detail/mpi/CheckedCount.h +++ b/cpp/monoprop/detail/mpi/CheckedCount.h @@ -44,8 +44,8 @@ inline auto checked_mpi_count(long long value, const char *what = "Aggregate MPI return static_cast(value); } -// A buffer size is not int-bounded to begin with: a per-peer payload is kWords + 1 elements -// per term, so a single wide-mode query round reaches INT_MAX well inside the term space +// A buffer size is not int-bounded to begin with: a per-peer payload is one monomial's words plus a +// phase word per term, so a single wide-mode query round reaches INT_MAX well inside the term space // monoprop_WIDE_TERM_INDEX advertises. Kept separate from the `long long` overload so a size_t above // LLONG_MAX cannot sign-flip on the way into the check. inline auto checked_mpi_count(size_t value, const char *what = "MPI count") -> int { diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index 3a2d20ab..d6713924 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -18,6 +18,8 @@ #include #include +#include + #include "monoprop/MPGraph.h" #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" @@ -27,20 +29,25 @@ namespace monoprop::mpi_detail { static_assert(sizeof(size_t) == sizeof(uint64_t), "MPI serialization assumes 64-bit size_t"); -template -inline constexpr size_t kWords = Monomial::num_words(); - -template -inline auto append_monomial_words(const Monomial &mono, VecZ &buffer) -> void { +// The per-monomial wire width is the monomial's own word count, so a caller with no monomial in hand +// carries the count as a value. +inline auto append_monomial_words(const Bitset &mono, VecZ &buffer) -> void { const auto *src = mono.data(); - for (size_t i = 0; i < kWords; ++i) + const size_t nw = mono.num_words(); + for (size_t i = 0; i < nw; ++i) buffer.push_back(src[i]); } -template -inline auto read_monomial_from_words(const VecZ &buffer, size_t start) -> Monomial { - Monomial mono; - std::memcpy(mono.data(), &buffer[start], kWords * sizeof(uint64_t)); +// mono_out supplies the width: it is the destination, so it already knows how wide the record is, and +// reading into it avoids constructing a bitset per query on the resolve path. +inline auto read_monomial_from_words(const VecZ &buffer, size_t start, Bitset &mono_out) -> void { + assert(mono_out.num_words() != 0 && "read_monomial_from_words needs a pre-sized destination"); + std::memcpy(mono_out.data(), &buffer[start], mono_out.num_words() * sizeof(uint64_t)); +} + +inline auto read_monomial_from_words(const VecZ &buffer, size_t start, size_t num_bits) -> Bitset { + Bitset mono(num_bits); + read_monomial_from_words(buffer, start, mono); return mono; } @@ -49,12 +56,14 @@ inline auto read_monomial_from_words(const VecZ &buffer, size_t start) -> Monomi namespace monoprop { // Stateless and identical on every rank, so all ranks agree on a term's owner without communication. -template -auto find_rank(const Monomial &mono, const size_t n_ranks) -> size_t { +// The ranks must also agree on the *width* they hash at: a monomial's storage width is part of its +// hash (SplitmixHash folds every word), so two ranks disagreeing about it would disagree about owners. +// Every rank derives it from the same propagator settings, so they do. +inline auto find_rank(const Bitset &mono, const size_t n_ranks) -> size_t { if (n_ranks == 0) { return 0; } - return monomial_hash(mono) % n_ranks; + return monomial_hash(mono) % n_ranks; } } // namespace monoprop diff --git a/cpp/monoprop/detail/operator/CMakeLists.txt b/cpp/monoprop/detail/operator/CMakeLists.txt index ea0266a1..66003ed3 100644 --- a/cpp/monoprop/detail/operator/CMakeLists.txt +++ b/cpp/monoprop/detail/operator/CMakeLists.txt @@ -8,4 +8,6 @@ target_sources( "MPOperator.h" "OperatorIndex.h" "RowAccess.h" + "RowHashTable.h" + "SparseRowStore.h" ) diff --git a/cpp/monoprop/detail/operator/InvertedIndex.h b/cpp/monoprop/detail/operator/InvertedIndex.h index f5a77116..f2a2df6f 100644 --- a/cpp/monoprop/detail/operator/InvertedIndex.h +++ b/cpp/monoprop/detail/operator/InvertedIndex.h @@ -35,9 +35,7 @@ namespace monoprop::detail { // served. Columns are stored in two tiers, bit-identical to all-dense: dense (density ≥ // 1/kPromoteDensityInv) full-height uint64 vectors; sparse an ascending set-row list scatter-expanded at // scan time. Promotion is one-way (the operator is append-only). -template struct InvertedIndex { - static constexpr size_t kNumColumns = Monomial::size(); static constexpr size_t kPromoteDensityInv = 64; struct Column { @@ -48,9 +46,17 @@ struct InvertedIndex { bool is_dense = false; }; - std::array cols{}; + // One column per bit position of a monomial, so the count is the storage bit width. Runtime-sized + // -- the column count is the operator's bit width, which is data. + std::vector cols; size_t row_count = 0; + // num_columns must be the *storage* bit width (2 * storage modes), matching the monomials whose + // positions fill_rows() scatters -- for_each_row_position indexes cols[bit] unchecked. + explicit InvertedIndex(size_t num_columns) : cols(num_columns) {} + + [[nodiscard]] auto num_columns() const -> size_t { return cols.size(); } + // Parity of |M| per row, packed 1 bit/row: bit r = popcount(row r) & 1. Built on first use and only // by odd-|G| generators, so even-parity workloads never allocate it. mutable std::vector row_parity_; // empty == not built @@ -115,7 +121,7 @@ struct InvertedIndex { for (size_t row_idx = base; row_idx < new_total_rows; ++row_idx) { const size_t w = row_idx >> 6U; const uint64_t row_bit = uint64_t{1} << (row_idx & 63U); - for_each_row_position(op, row_idx, [this, w, row_bit, row_idx](size_t bit) { + for_each_row_position(op, row_idx, [this, w, row_bit, row_idx](size_t bit) { Column &col = cols[bit]; if (col.is_dense) { col.words[w] |= row_bit; @@ -125,7 +131,7 @@ struct InvertedIndex { } }); } - for (size_t c = 0; c < kNumColumns; ++c) { + for (size_t c = 0; c < cols.size(); ++c) { Column &col = cols[c]; if (!col.is_dense && col.set_rows.size() * kPromoteDensityInv >= row_count) { promote_to_dense(c); @@ -153,12 +159,11 @@ struct InvertedIndex { // Count per-column set bits first and decide tiers from the final density, so the fill never has // to promote. - using Counts = std::array; - Counts counts{}; + auto counts = std::vector(cols.size(), 0); for (size_t row_idx = 0; row_idx < size; ++row_idx) { - for_each_row_position(op, row_idx, [&counts](size_t bit) { ++counts[bit]; }); + for_each_row_position(op, row_idx, [&counts](size_t bit) { ++counts[bit]; }); } - for (size_t c = 0; c < kNumColumns; ++c) { + for (size_t c = 0; c < cols.size(); ++c) { const size_t count = counts[c]; Column &col = cols[c]; if (count * kPromoteDensityInv >= size) { @@ -186,15 +191,20 @@ struct InvertedIndex { row_parity_.resize((row_count + 63) / 64, 0); for (size_t j = 0; j < n; ++j) { const size_t r = base + j; - if (row_popcount(op, r) & 1U) { + if (row_popcount(op, r) & 1U) { row_parity_[r >> 6] |= (uint64_t{1} << (r & 63)); } } } } + // The column vector itself is counted, not just the payload it points at: it is one Column per bit + // position, so it is the only term here that grows with the mode count rather than with the operator, + // and an accounting that omitted it could not answer whether the index is what caps a wide run. + auto columns_bytes() const -> size_t { return cols.capacity() * sizeof(Column); } + auto memory_bytes() const -> size_t { - size_t total = 0; + size_t total = columns_bytes(); for (const auto &col : cols) { total += col.words.capacity() * sizeof(uint64_t); total += col.set_rows.capacity() * sizeof(TermIndex); @@ -203,7 +213,8 @@ struct InvertedIndex { return total; } - // Diagnostic tier split of memory_bytes(): {dense_bytes, sparse_bytes, dense_columns}. + // Diagnostic tier split of memory_bytes(): {dense_bytes, sparse_bytes, dense_columns}. The + // width-driven remainder is columns_bytes() plus the row-parity bitmap. auto tier_memory_bytes() const -> std::array { std::array out{0, 0, 0}; for (const auto &col : cols) { @@ -237,8 +248,10 @@ inline auto pivot_column_block_scratch() -> std::vector & { // XOR a generator's inverted-index columns for fold words [bb, be) into blk[0 .. be-bb): dense columns // XOR their words directly, sparse columns lower_bound to the block's row range. XOR associativity means // any block decomposition reproduces the full-width fold bit-for-bit. -template -[[gnu::always_inline]] inline auto combine_columns_block(const InvertedIndex &sc, +// `sc` stays a deduced `auto`: InvertedIndex is no longer a template, but the parameter is also bound +// by the fold-cache tests to a stand-in with the same column accessors, so naming the type here would +// narrow it for no gain. +[[gnu::always_inline]] inline auto combine_columns_block(const auto &sc, std::span cols, uint64_t *blk, size_t bb, diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index 774baf6c..f9b84439 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -26,7 +26,6 @@ #include #include -#include "monoprop/TypeAliases.h" #include "monoprop/Utilities.h" #include "monoprop/core/Monomial.h" #include "monoprop/detail/operator/InvertedIndex.h" @@ -34,22 +33,16 @@ // Forward-declared to break an include cycle with algebra/Algebra.h. namespace monoprop { -template -auto is_fully_paired(const VecZ &inds, const Rows &op) -> VecZ; - -template -auto indices_to_bitset(const VecZ &arr) -> Monomial; - -// Each binds the runtime Basis to its algebra model internally, so no basis branch is needed here. -template +// Branches on the runtime Basis internally, so no basis branch is needed here. +template auto algebra_score_state(Basis basis, const VecZ &paired_inds, const VecZ &initial_state, const Rows &store, + size_t num_bits, Sink &&sink) -> void; -template -auto algebra_encode_coeff(Basis basis, const std::complex &coeff, const Monomial &mono) -> double; +auto algebra_encode_coeff(Basis basis, const std::complex &coeff, const MonomialLike auto &mono) -> double; } // namespace monoprop namespace monoprop::detail { @@ -59,11 +52,22 @@ class OperatorTermNotFound : public std::runtime_error { using std::runtime_error::runtime_error; }; -template struct MPOperator { - // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr (keeping MPOperator - // itself cheaply movable). Always non-null. - std::unique_ptr> store{std::make_unique>()}; + // The row store is one of two backends, chosen per propagator from its storage mode count + // (SparseRowStore::preferred_for_modes) and then fixed for the propagator's lifetime. Exactly one + // of these is non-null. + // + // Two pointers rather than the compile-time alias this was, because the choice is data. And rather + // than a virtual interface, because the scan asks the store for a row per anticommuting term: a + // branch or an indirect call on that path is not affordable. with_store() binds the concrete type + // once per layer instead -- the same shape as with_algebra() for a runtime Basis, and the reason + // build_layer is a template. Everything off that path goes through the forwarding accessors below, + // which pay one well-predicted branch. + // + // Heap-owned because neither store is copyable or movable (single-writer, and their views borrow + // their arrays), which keeps MPOperator itself cheaply movable. + std::unique_ptr dense_rows = nullptr; + std::unique_ptr sparse_rows = nullptr; VecD op_coeffs; // Only fully-paired terms score nonzero (see score_new_state_rows_), which on production models is // ~0.07% of the rows -- a dense vector here is 99.9% zeros. state_rows_ is strictly ascending: rows are @@ -74,18 +78,24 @@ struct MPOperator { // The dense state: empty in Heisenberg unless a caller asks dense_state() to cache one; in Schrödinger // it is the live coefficient vector evolution mutates in place. VecD state_coeffs; - MonomialMap init_op_map{}; + MonomialMap init_op_map{}; VecZ initial_state; // Set once at propagator construction. Basis basis{Basis::Majorana}; - mutable std::optional> inverted_index_{std::nullopt}; + mutable std::optional inverted_index_{std::nullopt}; + + // num_bits is the storage bit width of every monomial this operator holds. No default constructor: + // the width used to come free from NumModes, and a default-constructed store would be a width-0 + // one that silently mis-sizes every monomial built from it. The backend starts dense and the + // propagator replaces it via set_store() once the cutoff -- and with it the row width -- is known. + explicit MPOperator(size_t num_bits) : dense_rows(std::make_unique(num_bits)) {} - MPOperator() noexcept = default; MPOperator(MPOperator &&) noexcept = default; MPOperator &operator=(MPOperator &&) noexcept = default; MPOperator(const MPOperator &other) - : store(other.store->clone()), + : dense_rows(other.dense_rows ? other.dense_rows->clone() : nullptr), + sparse_rows(other.sparse_rows ? other.sparse_rows->clone() : nullptr), op_coeffs(other.op_coeffs), state_rows_(other.state_rows_), state_vals_(other.state_vals_), @@ -96,23 +106,111 @@ struct MPOperator { basis(other.basis), inverted_index_(other.inverted_index_) {} - auto size() const -> size_t { return store->size(); } + // Binds the live store to a concrete type for the duration of the call. Both arms are instantiated, + // so `f` must be a generic lambda and must return the same type from each. + template + [[gnu::always_inline]] auto with_store(F &&f) -> decltype(auto) { + if (sparse_rows) { + return f(*sparse_rows); + } + return f(*dense_rows); + } + template + [[gnu::always_inline]] auto with_store(F &&f) const -> decltype(auto) { + if (sparse_rows) { + return f(*sparse_rows); + } + return f(*dense_rows); + } + + // Installs a backend, dropping the lazy inverted index with it: the index addresses the old rows, + // and leaving it would let a stale one answer for the new store until its row count happened to + // disagree. One overload per backend rather than a tag, so a call site names the choice. + auto set_store(std::unique_ptr rows) -> void { + dense_rows = std::move(rows); + sparse_rows.reset(); + inverted_index_.reset(); + } + auto set_store(std::unique_ptr rows) -> void { + sparse_rows = std::move(rows); + dense_rows.reset(); + inverted_index_.reset(); + } + [[nodiscard]] auto rows_are_sparse() const -> bool { return sparse_rows != nullptr; } + + // The live backend's own row-width parameter, off the store rather than a member of its own for the + // same reason as num_bits(): a caller deciding whether resize_store() has anything to do must read + // the width that would actually be resized. Both stores spell it row_width(), so this is a plain + // forward like every other accessor here -- a backend detected by which accessor it happens to have + // would put the one decision a third backend has to get right outside the seam. + [[nodiscard]] auto row_width() const -> size_t { + return with_store([](const auto &rows) { return rows.row_width(); }); + } + + // Rebuilds the live backend at a new row width, migrating every existing row rather than dropping + // them the way set_store() would: OperatorIndex::resized() / SparseRowStore::resized() re-flow each + // row at the new width while preserving its index, so a caller with terms already inserted (e.g. a + // cutoff change after the initial fill) keeps them at the same row number. The lazy inverted index is + // dropped with it, exactly as set_store() would. + auto resize_store(size_t new_width) -> void { + if (sparse_rows) { + sparse_rows = sparse_rows->resized(new_width); + } + else { + dense_rows = dense_rows->resized(new_width); + } + inverted_index_.reset(); + } + + auto size() const -> size_t { + return with_store([](const auto &rows) { return rows.size(); }); + } + + // Off the store, not a member of its own, so the width driving row reconstruction and the width + // driving the monomials handed to it cannot drift apart. The copy constructor needs no extra + // work for the same reason: clone() carries the width across. + [[nodiscard]] auto num_bits() const -> size_t { + return with_store([](const auto &rows) { return rows.num_bits(); }); + } + // The per-word loops are sized in words, not bits. + [[nodiscard]] auto num_words() const -> size_t { return Bitset::words_for(num_bits()); } // Does not keep the lazy inverted index in sync: appends happen during setup, before the index is // first materialized, so a later append just makes inverted_index() rebuild via its staleness guard. - auto append_term(const Monomial &mono) -> void { store->push_back(mono); } + auto append_term(const Bitset &mono) -> void { + with_store([&mono](auto &rows) { rows.push_back(mono); }); + } + + // Both are setup-path forwards, kept here rather than exposing a store, so nothing outside has to + // know which backend is live. + auto reserve_terms(size_t n) -> void { + with_store([&n](auto &rows) { rows.reserve(n); }); + } + auto index_term(const Bitset &mono, size_t row) -> void { + with_store([&mono, &row](auto &rows) { rows.emplace(mono, row); }); + } + [[nodiscard]] auto find(const Bitset &mono) const -> std::optional { + return with_store([&mono](const auto &rows) { return rows.find(mono); }); + } + // This rank's terms as fn(monomial, row), in the index's slot order. Materializes each row. + template + auto for_each_term(Fn &&fn) const -> void { + with_store([&fn](const auto &rows) { rows.for_each(std::forward(fn)); }); + } - // Resync the inverted index after a bulk growth of `store`, preserving has_value() ⟹ rows()==store.size(). + // Resync the inverted index after a bulk growth of the store, preserving has_value() ⟹ rows()==size(). auto reindex_after_growth(size_t base, size_t n) -> void { if (inverted_index_.has_value()) { - inverted_index_->append_rows(*store, base, n); + with_store([this, &base, &n](const auto &rows) { inverted_index_->append_rows(rows, base, n); }); } } - auto inverted_index() const -> const InvertedIndex & { - if (!inverted_index_.has_value() || inverted_index_->rows() != store->size()) { - inverted_index_.emplace(); - inverted_index_->rebuild(*store); + auto inverted_index() const -> const InvertedIndex & { + if (!inverted_index_.has_value() || inverted_index_->rows() != size()) { + // Column count is the storage bit width, taken off the store so it cannot drift from the + // monomials whose positions rebuild() scatters. + inverted_index_.emplace(num_bits()); + with_store([this](const auto &rows) { inverted_index_->rebuild(rows); }); } return *inverted_index_; } @@ -131,12 +229,19 @@ struct MPOperator { const auto before = init_op_map.size(); erase_if(init_op_map, [this](const auto &kv) { - const auto found = store->find(kv.first); + const auto found = find(kv.first); if (found) { op_coeffs[*found] = kv.second; } return found.has_value(); }); + // Erasing does not give the slot array back, and a drained map is the normal end state: every + // initial-operator term is materialized as a row by the time the caches are warmed, so without + // this the propagator carries an empty map sized for the whole initial operator for its whole + // life -- 2.5 MB behind zero entries for a 20k-term observable. + // + // rehash(0) rather than clear(): it shrinks to what the entries left behind need, which is the + // whole array once the map has drained and a smaller one while terms are still pending. if (init_op_map.size() != before) { init_op_map.rehash(0); } @@ -188,24 +293,23 @@ struct MPOperator { // Heisenberg rejects a term absent from both (new monomials may have no graph paths); Schrödinger // admits them freely (the state was already evolved). Returns the supplied terms with their encoded // coefficients, in order. - auto update_initial_operator(const OperatorDict &op_dict, bool schrodinger) - -> std::pair, VecD> { - MonomialMap new_op_map; - std::pair, VecD> new_grad_op; + auto update_initial_operator(const OperatorDict &op_dict, bool schrodinger) -> std::pair { + MonomialMap new_op_map; + auto [new_grad_terms, new_grad_coeffs] = std::pair{}; VecD new_op_coeffs(size(), 0.0); for (const auto &[k, v] : op_dict) { - // Unchecked by design: the only caller bounds-checks against its logical_num_modes_. - const auto mono = indices_to_bitset(k); - const auto rank_evolved_op = store->find(mono); + // Unchecked by design: the only caller bounds-checks against its num_modes_. + const auto mono = indices_to_bitset(k, num_bits()); + const auto rank_evolved_op = find(mono); const auto rank_init_op = init_op_map.find(mono); - const auto coeff = algebra_encode_coeff(basis, v, mono); + const auto coeff = algebra_encode_coeff(basis, v, mono); if (!schrodinger) { if (rank_init_op != init_op_map.end()) { new_op_map[mono] = coeff; } - else if (rank_evolved_op) { + else if (rank_evolved_op.has_value()) { new_op_coeffs[*rank_evolved_op] = coeff; } else { @@ -214,20 +318,20 @@ struct MPOperator { } } else { - if (rank_evolved_op) { + if (rank_evolved_op.has_value()) { new_op_coeffs[*rank_evolved_op] = coeff; } else { new_op_map[mono] = coeff; } } - new_grad_op.first.push_back(mono); - new_grad_op.second.push_back(coeff); + new_grad_terms.push_back(mono); + new_grad_coeffs.push_back(coeff); } init_op_map = std::move(new_op_map); op_coeffs = std::move(new_op_coeffs); - return new_grad_op; + return {std::move(new_grad_terms), std::move(new_grad_coeffs)}; } auto score_new_state_rows_() -> void { @@ -237,14 +341,16 @@ struct MPOperator { VecZ new_inds(size() - state_scored_rows_); std::iota(new_inds.begin(), new_inds.end(), state_scored_rows_); // NOLINT(modernize-use-ranges) - const auto paired_inds = is_fully_paired(new_inds, *store); - state_rows_.reserve(state_rows_.size() + paired_inds.size()); - state_vals_.reserve(state_vals_.size() + paired_inds.size()); - - // The algebra picks the diagonal ⟨b|·|b⟩ phase of each fully-paired term. - algebra_score_state(basis, paired_inds, initial_state, *store, [this](size_t row, double phase) { - state_rows_.push_back(static_cast(row)); - state_vals_.push_back(phase); + with_store([this, &new_inds](const auto &rows) { + const auto paired_inds = is_fully_paired(new_inds, rows, num_bits()); + state_rows_.reserve(state_rows_.size() + paired_inds.size()); + state_vals_.reserve(state_vals_.size() + paired_inds.size()); + + // The algebra picks the diagonal ⟨b|·|b⟩ phase of each fully-paired term. + algebra_score_state(basis, paired_inds, initial_state, rows, num_bits(), [this](size_t row, double phase) { + state_rows_.push_back(static_cast(row)); + state_vals_.push_back(phase); + }); }); state_scored_rows_ = size(); @@ -263,13 +369,17 @@ struct MPOperator { // Callers must pass pairwise-distinct, currently-absent keys: bulk_insert then skips duplicate probes and // slot k deterministically lands at base+k. Call after any pass that reads pre-insert op state // (op.size() must equal the returned base). -template -inline auto insert_absent_terms(MPOperator &op, size_t n, KeyAt &&key_at, PerSlot &&per_slot) -> size_t { - const size_t base = op.store->grow_rows_geometric(n); +// +// `store` is passed alongside `op` rather than taken off it: every caller is inside build_layer, which +// has already bound the concrete backend, and re-entering with_store() here would bind it a second time +// per insert batch for nothing. +template +inline auto insert_absent_terms(auto &op, auto &store, size_t n, KeyAt &&key_at, auto &&per_slot) -> size_t { + const size_t base = store.grow_rows_geometric(n); for (size_t k = 0; k < n; ++k) { per_slot(k, base); } - op.store->bulk_insert(n, base, std::forward(key_at)); + store.bulk_insert(n, base, std::forward(key_at)); op.reindex_after_growth(base, n); return base; } @@ -279,7 +389,19 @@ inline auto unordered_flat_map_storage_bytes(const FlatMap &map) -> size_t { return sizeof(FlatMap) + map.bucket_count() * (sizeof(typename FlatMap::value_type) + sizeof(unsigned char)); } -template +// The slot array plus what the keys own outside it. A monomial key wider than Bitset's inline capacity +// points at its own allocation, so slots alone under-report a wide operator's map by more than the slots +// themselves: 20k keys at 1024 modes hold 5.1 MB of words behind 2.5 MB of slots. +inline auto monomial_map_bytes(const MonomialMap &map) -> size_t { + size_t total = unordered_flat_map_storage_bytes(map); + for (const auto &kv : map) { + total += kv.first.heap_bytes(); + } + return total; +} + +// No width parameter: nothing in here is width-dependent, and it never was -- every field is a byte +// count. struct MPOperatorMemoryBreakdown final { size_t operator_terms_bytes{0uz}; size_t op_coeffs_bytes{0uz}; @@ -295,6 +417,9 @@ struct MPOperatorMemoryBreakdown final { // never double-count. size_t inverted_index_dense_bytes{0uz}; // of inverted_index_bytes: full-height bitmap columns size_t inverted_index_sparse_bytes{0uz}; // of inverted_index_bytes: ascending set-row lists + // of inverted_index_bytes: the Column vector itself, one entry per bit position. The only term that + // scales with the mode count instead of the operator, so it is what a width sweep has to watch. + size_t inverted_index_columns_bytes{0uz}; size_t inverted_index_dense_columns{0uz}; size_t operator_terms_slack_bytes{0uz}; // of operator_terms_bytes: unused geometric-growth capacity // of state_coeffs_bytes: entries of the state that are not exactly 0.0 @@ -318,6 +443,7 @@ struct MPOperatorMemoryBreakdown final { matched_scratch_bytes += o.matched_scratch_bytes; inverted_index_dense_bytes += o.inverted_index_dense_bytes; inverted_index_sparse_bytes += o.inverted_index_sparse_bytes; + inverted_index_columns_bytes += o.inverted_index_columns_bytes; inverted_index_dense_columns += o.inverted_index_dense_columns; operator_terms_slack_bytes += o.operator_terms_slack_bytes; state_coeffs_nonzero += o.state_coeffs_nonzero; @@ -326,17 +452,19 @@ struct MPOperatorMemoryBreakdown final { } }; -template -inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorMemoryBreakdown { - MPOperatorMemoryBreakdown breakdown; - breakdown.operator_terms_bytes = op.store->memory_bytes(); +inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorMemoryBreakdown { + MPOperatorMemoryBreakdown breakdown; + op.with_store([&breakdown](const auto &rows) { + breakdown.operator_terms_bytes = rows.memory_bytes(); + breakdown.indexing_bytes = rows.index_estimated_memory_bytes(); + breakdown.operator_terms_slack_bytes = rows.slack_bytes(); + }); breakdown.op_coeffs_bytes = op.op_coeffs.capacity() * sizeof(double); // Every representation of the state at once: the sparse scored set plus the dense vector. breakdown.state_coeffs_bytes = op.state_coeffs.capacity() * sizeof(double) + op.state_rows_.capacity() * sizeof(TermIndex) + op.state_vals_.capacity() * sizeof(double); - breakdown.indexing_bytes = op.store->index_estimated_memory_bytes(); - breakdown.init_operator_bytes = unordered_flat_map_storage_bytes(op.init_op_map); + breakdown.init_operator_bytes = monomial_map_bytes(op.init_op_map); breakdown.init_operator_entries = op.init_op_map.size(); breakdown.initial_state_bytes = op.initial_state.capacity() * sizeof(size_t); if (op.inverted_index_.has_value()) { @@ -344,9 +472,9 @@ inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorM const auto tiers = op.inverted_index_->tier_memory_bytes(); breakdown.inverted_index_dense_bytes = tiers[0]; breakdown.inverted_index_sparse_bytes = tiers[1]; + breakdown.inverted_index_columns_bytes = op.inverted_index_->columns_bytes(); breakdown.inverted_index_dense_columns = tiers[2]; } - breakdown.operator_terms_slack_bytes = op.store->slack_bytes(); // State phases are unit-magnitude, so at rest the scored count IS the nonzero count; a live vector needs a scan. breakdown.state_coeffs_nonzero = op.state_coeffs.empty() diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 8a3c29b2..a6ef076e 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -19,78 +19,191 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/operator/RowHashTable.h" namespace monoprop::detail { -class TermIndexCeilingReached : public std::runtime_error { +// The requested monomial width needs bit positions wider than PosT can hold. Thrown rather than +// asserted: with the compile-time mode ceiling gone, the width is user data. +class OperatorIndexWidthUnsupported : public std::runtime_error { public: using std::runtime_error::runtime_error; }; // Operator-term store: entropy-packed position-list rows plus a keyless open-addressing hash index over -// those rows. Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), slots 1..c = -// ascending set-bit positions; stride_ is fixed for the container's life so row offsets stay stable. +// those rows. Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), +// slots 1..c = ascending set-bit positions; stride_ is fixed for the container's life so row offsets stay +// stable, and so is the slot's width (see kNarrowPositions). // inline_width_ is a free parameter -- any width is correct, over-long rows spill losslessly to overflow. // Single-writer: one partition, one thread; parallelism is cross-partition. -template class OperatorIndex { public: - using value_type = Monomial; - using key_type = Monomial; + // The store carries its width as data (num_bits_), so a row is a plain Bitset and the width is not + // recoverable from the type. + using value_type = Bitset; + using key_type = Bitset; using mapped_type = size_t; - using PosT = std:: - conditional_t<(2 * NumModes <= 256), uint8_t, std::conditional_t<(2 * NumModes <= 65536), uint16_t, uint32_t>>; - static constexpr size_t kDefaultInlinePositions = 11; // A weight-w Pauli needs 2w positions; 32 covers the common case inline at the supported Pauli // cutoffs (2*cutoff <= 32 for cutoff <= 16). static constexpr size_t kMaxInlinePositions = 32; - static constexpr PosT kOverflowMarker = std::numeric_limits::max(); - - static_assert((2 * NumModes) - 1 <= std::numeric_limits::max(), - "OperatorIndex PosT too narrow for 2*NumModes positions"); - static_assert(kMaxInlinePositions < std::numeric_limits::max(), - "kOverflowMarker sentinel must not collide with a valid popcount"); - - // Valid term indices are < kIndexCeiling (check_index_fits throws at the ceiling), so the - // all-ones TermIndex is free to mark an empty slot. - static constexpr size_t kIndexCeiling = static_cast(std::numeric_limits::max()); - static constexpr TermIndex kEmptySlot = std::numeric_limits::max(); - // find_batch's "absent" result; same value as detail::kMissingIndex (not included here — the - // operator store must not depend on evolution headers). - static constexpr size_t kNotFound = std::numeric_limits::max(); - - explicit OperatorIndex(size_t inline_width = kDefaultInlinePositions) - : inline_width_(std::clamp(inline_width, 1, kMaxInlinePositions)), - stride_(1 + inline_width_) {} + + // A slot holds one bit position, so the narrowest integer that indexes num_bits_ is what a row costs + // per slot: uint8_t at or below kNarrowPositions, uint16_t above it. The width is data, so it is a + // member (narrow_) rather than the conditional_t on 2*NumModes it used to be, and the payload type is + // bound per call by with_rows(). One fixed uint16_t is the obvious simplification and was rejected on + // measurement: it doubles the entire operator's row footprint for every system at or below + // kNarrowPositions/2 modes, which is where both shipping models sit. + // + // Row payload only -- never a hash input, never serialized, never an owner-routing input -- so the + // width cannot change results, only footprint. That also means a term-and-energy baseline diff cannot + // see a regression here; the operator_terms_bytes checks in the unit tests are the gate. + static constexpr size_t kNarrowPositions = static_cast(std::numeric_limits::max()) + 1; + static constexpr size_t kMaxPositions = static_cast(std::numeric_limits::max()) + 1; + + // The all-ones slot value, per row width. It only ever occupies slot 0, whose value is a popcount + // bounded by inline_width_ <= kMaxInlinePositions, so it collides with no popcount at either width + // -- asserted below on the narrower type, which covers both. A *position* slot may legitimately + // hold the marker value (position 255 under uint8_t) and is never compared against it. + template + static constexpr P kOverflowMarker = std::numeric_limits

::max(); + + static_assert(kMaxInlinePositions < kOverflowMarker, + "the overflow marker must not collide with a valid popcount at either row width"); + + static constexpr size_t kIndexCeiling = RowHashTable::kIndexCeiling; + static constexpr size_t kNotFound = RowHashTable::kNotFound; + + // num_bits is the storage bit width of every monomial this store will hold; row() reconstructs at + // exactly that width, and a wrong one would change num_words() and therefore the hash, the probe + // order and MPI owner routing. It also fixes the row payload width for the store's life, so a store + // holds exactly one of the two row arrays and narrow_ is never reassigned. The check replaces the + // static_assert that used to guard set()'s narrowing cast, which can no longer be a compile-time + // assertion. + explicit OperatorIndex(size_t num_bits, size_t inline_width = kDefaultInlinePositions) + : num_bits_(num_bits), + inline_width_(std::clamp(inline_width, 1, kMaxInlinePositions)), + stride_(1 + inline_width_), + narrow_(num_bits <= kNarrowPositions) { + if (num_bits > kMaxPositions) { + throw OperatorIndexWidthUnsupported( + std::format("OperatorIndex supports at most {} bit positions ({} modes); got {} bits ({} modes).", + kMaxPositions, + kMaxPositions / 2, + num_bits, + num_bits / 2)); + } + } + + [[nodiscard]] auto num_bits() const noexcept -> size_t { return num_bits_; } + [[nodiscard]] auto inline_width() const noexcept -> size_t { return inline_width_; } + // The backend-neutral spelling of the line above, so a caller holding either store asks the same + // question of both (see MPOperator::row_width). + [[nodiscard]] auto row_width() const noexcept -> size_t { return inline_width_; } + + // Inline position count for a row-width bound, clamped exactly as the constructor clamps its + // argument. Mirrors SparseRowStore::slots_for_bound, and exists for the same reason: a caller + // comparing a target width against a built store's row_width() must compare like with like, or the + // comparison never converges at a bound of 0 and re-migrates every row on every settings change. + [[nodiscard]] static auto inline_width_for_bound(size_t width_bound) noexcept -> size_t { + return std::clamp(width_bound, 1, kMaxInlinePositions); + } OperatorIndex(const OperatorIndex &) = delete; OperatorIndex &operator=(const OperatorIndex &) = delete; OperatorIndex(OperatorIndex &&) = delete; OperatorIndex &operator=(OperatorIndex &&) = delete; +private: + // The dispatcher sits here, mid-class, rather than with the other private members at the bottom: a + // deduced (decltype(auto)) return type is not available to a caller that appears earlier in the class + // body -- the body is a complete-class context for name lookup, but not for return-type deduction. + // + // Binds the row payload type for one call. narrow_ is fixed at construction, so the branch is a + // load-and-test on a member that never changes: predicted, and amortized over the row loop inside f. + // The dispatch is per call rather than hoisted into the store type on purpose -- the row width is not + // part of the seam the scan is templated on (see the with_store note in MPOperator), and making it so + // would double every downstream instantiation to save a predicted branch per row. + template + [[gnu::always_inline]] auto with_rows(F &&f) const -> decltype(auto) { + return narrow_ ? f(rows8_) : f(rows16_); + } + template + [[gnu::always_inline]] auto with_rows(F &&f) -> decltype(auto) { + return narrow_ ? f(rows8_) : f(rows16_); + } + + // Row i's address, for the prefetch hint, which discards the element type anyway. + [[nodiscard]] auto row_addr(size_t i) const noexcept -> const void * { + return with_rows([this, &i](const auto &rows) -> const void * { return rows.data() + (i * stride_); }); + } + + // The two row arrays differ only in element size, so everything that counts bytes rather than + // reading a slot is plain arithmetic off this and needs no type bound. + [[nodiscard]] auto slot_bytes() const noexcept -> size_t { return narrow_ ? sizeof(uint8_t) : sizeof(uint16_t); } + [[nodiscard]] auto row_slots_capacity() const -> size_t { + return with_rows([](const auto &rows) { return rows.capacity(); }); + } + [[nodiscard]] auto row_bytes_capacity() const -> size_t { return row_slots_capacity() * slot_bytes(); } + +public: // Called only on an idle store, so it needs no synchronization. [[nodiscard]] auto clone() const -> std::unique_ptr { - auto out = std::make_unique(inline_width_); - out->rows_ = rows_; + auto out = std::make_unique(num_bits_, inline_width_); + // Both, not with_rows(): the dead one is empty, and copying it is cheaper than a dispatch. + out->rows8_ = rows8_; + out->rows16_ = rows16_; out->size_ = size_; out->overflow_ = overflow_; - out->reserve_index(table_.count); - for (const Slot &e : table_.slots) { - if (e.idx != kEmptySlot) { - out->insert_slot_(e.idx, e.h); + out->table_ = table_; // RowHashTable is rule-of-zero copyable; a plain copy preserves slot order exactly. + return out; + } + + // Same term set at a different inline_width_, e.g. after a cutoff change moves the bound rows are + // sized from. Every row's monomial is re-flowed through set() at the new stride, which decides + // inline-vs-overflow the same way a fresh insert would; the hash index is copied as-is, since + // fold_hash depends only on the monomial, never on inline_width_, so no rehash is needed. Row index i + // is preserved for every row -- load-bearing, since callers key op_coeffs, state_rows_/state_vals_ + // and the evolution graph by this same index. + [[nodiscard]] auto resized(size_t new_inline_width) const -> std::unique_ptr { + auto out = std::make_unique(num_bits_, new_inline_width); + out->size_ = size_; + // The position slots are copied across rather than reflowed through row(i)/set(i, mono): row() + // would materialize a fresh Bitset from the slots (and allocate, past kInlineWords) and set() + // would immediately re-walk it to rebuild them, a double pass per row over the whole operator. + // Only a row that changes regime -- spilled, or too long for the narrower width -- needs the + // dense form. Same argument SparseRowStore::resized already makes for its own reflow. + // + // out shares num_bits_, so it shares narrow_ and therefore the payload type P. + with_rows([this, &out](const DefaultInitVector

&src) { + auto &dst = out->rows_ref

(); + dst.resize(size_ * out->stride_); + for (size_t i = 0; i < size_; ++i) { + const P *s = &src[i * stride_]; + const P c = s[0]; + if (c == kOverflowMarker

|| static_cast(c) > out->inline_width_) { + // Widening can bring a spilled row back inline and narrowing can push an inline row + // out, so either way the regime is re-decided by set(). + out->set(i, row(i)); + continue; + } + std::memcpy(&dst[i * out->stride_], s, (1 + static_cast(c)) * sizeof(P)); } - } + }); + out->table_ = table_; // RowHashTable is rule-of-zero copyable; a plain copy preserves slot order exactly. return out; } @@ -105,12 +218,11 @@ class OperatorIndex { auto grow_rows_geometric(size_t n) -> size_t { const size_t base = size_; if (capacity() < base + n) { - const size_t cap = capacity(); - reserve_rows(std::max(base + n, cap + (cap / 2) + 1)); + reserve_rows(geometric_row_capacity(base, n, capacity())); } // Default-init grow, not a zeroing resize: every freshly grown row is overwritten by set() // before any read, so a tail zero-fill would be wasted bandwidth. - rows_.resize((base + n) * stride_); + with_rows([this, &base, &n](auto &rows) { rows.resize((base + n) * stride_); }); size_ = base + n; return base; } @@ -120,286 +232,166 @@ class OperatorIndex { // Row i may be grown-but-uninitialized or hold a prior value, so the row header is never pre-read // (freshly grown headers are indeterminate); a stale overflow entry at i, if any, is dropped. auto set(size_t i, const value_type &mono) -> void { - const size_t c = mono.count(); - PosT *row = &rows_[i * stride_]; - if (c > inline_width_) { - row[0] = kOverflowMarker; - overflow_[i] = mono; - return; - } - if (!overflow_.empty()) { - overflow_.erase(i); - } - row[0] = static_cast(c); - PosT *out = row + 1; - for (size_t b = mono.find_first(); b < mono.size(); b = mono.find_next(b)) { - *out++ = static_cast(b); - } + with_rows([this, &i, &mono](DefaultInitVector

&rows) { + const size_t c = mono.count(); + P *row = &rows[i * stride_]; + if (c > inline_width_) { + row[0] = kOverflowMarker

; + overflow_[i] = mono; + return; + } + if (!overflow_.empty()) { + overflow_.erase(i); + } + row[0] = static_cast

(c); + P *out = row + 1; + for (size_t b = mono.find_first(); b < mono.size(); b = mono.find_next(b)) { + *out++ = static_cast

(b); + } + }); } [[nodiscard]] auto row(size_t i) const -> value_type { - const PosT c = rows_[i * stride_]; - if (c == kOverflowMarker) { - return overflow_.at(i); - } - value_type mono; - const PosT *pos = &rows_[(i * stride_) + 1]; - for (size_t j = 0; j < c; ++j) { - mono.set(pos[j]); - } - return mono; + return with_rows([this, &i](const DefaultInitVector

&rows) { + const P c = rows[i * stride_]; + if (c == kOverflowMarker

) { + return overflow_.at(i); + } + value_type mono(num_bits_); + const P *pos = &rows[(i * stride_) + 1]; + for (size_t j = 0; j < c; ++j) { + mono.set(pos[j]); + } + return mono; + }); } template auto for_each_position(size_t i, Fn &&fn) const -> void { - const PosT c = rows_[i * stride_]; - if (c == kOverflowMarker) { - const auto &m = overflow_.at(i); - for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { - fn(b); + with_rows([this, &i, &fn](const DefaultInitVector

&rows) { + const P c = rows[i * stride_]; + if (c == kOverflowMarker

) { + const auto &m = overflow_.at(i); + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + fn(b); + } + return; } - return; - } - const PosT *pos = &rows_[(i * stride_) + 1]; - for (size_t j = 0; j < c; ++j) { - fn(static_cast(pos[j])); - } + const P *pos = &rows[(i * stride_) + 1]; + for (size_t j = 0; j < c; ++j) { + fn(static_cast(pos[j])); + } + }); } [[nodiscard]] auto popcount(size_t i) const -> size_t { - if (const PosT c = rows_[i * stride_]; c != kOverflowMarker) { - return c; - } - return overflow_.at(i).count(); - } - [[nodiscard]] auto memory_bytes() const -> size_t { - size_t total = rows_.capacity() * sizeof(PosT); - total += overflow_.size() * (sizeof(value_type) + sizeof(size_t) + 24); - return total; + return with_rows([this, &i](const DefaultInitVector

&rows) { + if (const P c = rows[i * stride_]; c != kOverflowMarker

) { + return static_cast(c); + } + return overflow_.at(i).count(); + }); } + [[nodiscard]] auto memory_bytes() const -> size_t { return row_bytes_capacity() + spilled_rows_bytes(overflow_); } auto find(const key_type &key) const -> std::optional { - const uint32_t h = fold_hash(key); - if (table_.count == 0) { - return std::nullopt; - } - size_t s = spread(h) & table_.mask; - for (;; s = (s + 1) & table_.mask) { - const Slot &e = table_.slots[s]; - if (e.idx == kEmptySlot) { - return std::nullopt; - } - if (e.h == h && row_eq_key(static_cast(e.idx), key)) { - return static_cast(e.idx); - } - } + return table_.find(fold_hash(key), [this, &key](size_t i) { return row_eq_key(i, key); }); } // Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. Same result as n // find() calls, but overlaps dram misses via a per-group hash/probe/confirm pipeline. An h // collision falls back to an exact find. must not run concurrently with inserts. - auto find_batch(const key_type *keys, size_t n, size_t *out) const -> void { - static constexpr size_t G = 16; // keys prefetched together per pipeline pass - std::array hh; - std::array sp; - std::array cand; - for (size_t base = 0; base < n; base += G) { - const size_t g = std::min(G, n - base); - for (size_t j = 0; j < g; ++j) { - hh[j] = fold_hash(keys[base + j]); - sp[j] = spread(hh[j]); - __builtin_prefetch(&table_.slots[sp[j] & table_.mask], 0, 0); - } - for (size_t j = 0; j < g; ++j) { - cand[j] = kEmptySlot; - if (table_.count == 0) { - continue; - } - cand[j] = probe_hash_match_(hh[j], sp[j] & table_.mask); - if (cand[j] != kEmptySlot) { - __builtin_prefetch(&rows_[static_cast(cand[j]) * stride_], 0, 0); - } - } - for (size_t j = 0; j < g; ++j) { - if (cand[j] != kEmptySlot && row_eq_key(static_cast(cand[j]), keys[base + j])) { - out[base + j] = static_cast(cand[j]); - } - else if (cand[j] != kEmptySlot) { - const auto v = find(keys[base + j]); - out[base + j] = v ? *v : kNotFound; - } - else { - out[base + j] = kNotFound; - } - } - } + // Templated on the key type rather than taking `const key_type *`: any Key that binds to + // const key_type& works, since that is all fold_hash/row_eq_key need. SparseRowStore's twin takes + // its own key type through the same signature, which is what lets Resolve.h call one spelling. + template + auto find_batch(const Key *keys, size_t n, size_t *out) const -> void { + table_.find_batch( + keys, + n, + out, + [](const Key &key) { return fold_hash(key); }, + [this](size_t i) { __builtin_prefetch(row_addr(i), 0, 0); }, + [this](size_t i, const Key &key) { return row_eq_key(i, key); }); } // Insert-or-no-op. Row at `value` must already be written (the confirm reads dense rows). auto emplace(const key_type &key, mapped_type value) -> void { - check_index_fits(value); - const uint32_t h = fold_hash(key); - table_.rehash_if_needed(); - size_t s = spread(h) & table_.mask; - while (table_.slots[s].idx != kEmptySlot) { - if (table_.slots[s].h == h && row_eq_key(static_cast(table_.slots[s].idx), key)) { - return; - } - s = (s + 1) & table_.mask; - } - table_.slots[s] = Slot{static_cast(value), h}; - ++table_.count; + table_.emplace(fold_hash(key), value, [this, &key](size_t i) { return row_eq_key(i, key); }); } // Insert n distinct rows with consecutive indices [base, base+n). Rows must already be written. template auto bulk_insert(size_t n, mapped_type base, KeyFn &&key_at) -> void { - if (n == 0) { - return; - } - check_index_fits(base + n - 1); - for (size_t k = 0; k < n; ++k) { - insert_slot_(static_cast(base + k), fold_hash(key_at(k))); - } + table_.insert_distinct_range(base, n, [&key_at](size_t k) { return fold_hash(key_at(k)); }); } template auto for_each(Func &&fn) const -> void { - for (const Slot &e : table_.slots) { - if (e.idx != kEmptySlot) { - fn(row(static_cast(e.idx)), static_cast(e.idx)); - } - } + table_.for_each_slot( + [this, &fn](TermIndex idx, uint32_t) { fn(row(static_cast(idx)), static_cast(idx)); }); } // Diagnostic: the part of memory_bytes() that is unused geometric-growth capacity. [[nodiscard]] auto slack_bytes() const -> size_t { - return (rows_.capacity() * sizeof(PosT)) - (std::min(rows_.capacity(), size_ * stride_) * sizeof(PosT)); + const size_t cap = row_slots_capacity(); + return (cap - std::min(cap, size_ * stride_)) * slot_bytes(); } - auto index_estimated_memory_bytes() const -> size_t { - return sizeof(OperatorIndex) + (table_.slots.capacity() * sizeof(Slot)); - } + auto index_estimated_memory_bytes() const -> size_t { return sizeof(OperatorIndex) + table_.slot_bytes(); } private: - struct Slot { - TermIndex idx = kEmptySlot; - uint32_t h = 0; - }; - - // First slot on h's probe chain whose stored hash matches, or kEmptySlot if the chain ends first. - // Matches on h alone and leaves the dense-row comparison to the caller — that deferral is what lets - // find_batch prefetch the row between probe and confirm, so do not fold row_eq_key in here (find() - // deliberately keeps its own confirming variant). `start` must already be masked; the table must not - // be mutated concurrently. - [[gnu::always_inline]] auto probe_hash_match_(uint32_t h, size_t start) const -> TermIndex { - for (size_t s = start;; s = (s + 1) & table_.mask) { - const Slot &e = table_.slots[s]; - if (e.idx == kEmptySlot) { - return kEmptySlot; - } - if (e.h == h) { - return e.idx; - } + // SplitmixHash directly, which is exactly what MonomialHash forwarded to -- the hash is + // unchanged, and must stay so: it drives probe order and monomial_hash % rank_count owner routing + // (plan invariant 2). + static uint32_t fold_hash(const key_type &q) noexcept { return RowHashTable::fold(SplitmixHash{}(q)); } + + // The row array of a given payload type, for the one caller that has P bound but needs the array on + // *another* store of the same width (resized()); with_rows cannot express that. + template + [[nodiscard]] auto rows_ref() noexcept -> DefaultInitVector

& { + if constexpr (std::is_same_v) { + return rows8_; + } + else { + return rows16_; } } - static uint32_t fold_hash(const key_type &q) noexcept { - const size_t full = MonomialHash{}(q); - return static_cast(full ^ (static_cast(full) >> 32)); - } - // Avalanche the cached 32-bit fold into a full-width hash (splitmix64 finalizer): the stored h - // is only an equality pre-filter, so it must be re-mixed before its low bits drive table bucketing. - static size_t spread(uint32_t h) noexcept { - uint64_t x = static_cast(h) * 0x9E3779B97F4A7C15ULL; - x ^= x >> 30; - x *= 0xBF58476D1CE4E5B9ULL; - x ^= x >> 27; - x *= 0x94D049BB133111EBULL; - x ^= x >> 31; - return static_cast(x); - } - - // One open-addressing table: power-of-2 slot count, linear probing, max load factor 0.7 - // (the group-prefetch win erodes at high load — longer probe chains add un-prefetched reads). - struct Table { - std::vector slots = std::vector(kMinSlots, Slot{}); - size_t mask = kMinSlots - 1; - size_t count = 0; - - auto rehash_if_needed() -> void { - if ((count + 1) * 10 >= slots.size() * 7) { - rehash_to(slots.size() * 2); - } - } - auto rehash_to(size_t new_cap) -> void { - new_cap = std::bit_ceil(std::max(new_cap, kMinSlots)); - if (new_cap <= slots.size()) { - return; - } - std::vector old = std::move(slots); - slots.assign(new_cap, Slot{}); - mask = new_cap - 1; - for (const Slot &e : old) { - if (e.idx == kEmptySlot) { - continue; - } - size_t s = spread(e.h) & mask; - while (slots[s].idx != kEmptySlot) { - s = (s + 1) & mask; - } - slots[s] = e; - } - } - }; - static constexpr size_t kMinSlots = 16; - // Slot count for `n` entries at ≤0.7 load. - static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } - - [[nodiscard]] auto capacity() const -> size_t { return rows_.capacity() / stride_; } - auto reserve_rows(size_t n) -> void { rows_.reserve(n * stride_); } - auto reserve_index(size_t n) -> void { table_.rehash_to(slots_for_(n + 1)); } - - // Insert (idx, h) into the table with no duplicate probe — callers on this path insert provably distinct - // keys (⊕G-injective miss batches, clone re-insertion). - auto insert_slot_(TermIndex idx, uint32_t h) -> void { - table_.rehash_if_needed(); - size_t s = spread(h) & table_.mask; - while (table_.slots[s].idx != kEmptySlot) { - s = (s + 1) & table_.mask; - } - table_.slots[s] = Slot{idx, h}; - ++table_.count; + [[nodiscard]] auto capacity() const -> size_t { return row_slots_capacity() / stride_; } + auto reserve_rows(size_t n) -> void { + with_rows([this, &n](auto &rows) { rows.reserve(n * stride_); }); } + auto reserve_index(size_t n) -> void { table_.reserve(n); } // Compare row i against key q without materializing the row (the find confirm). Reads the // popcount byte first, so a false h prefilter match usually costs one byte compare. [[nodiscard]] auto row_eq_key(size_t i, const key_type &q) const -> bool { - const PosT c = rows_[i * stride_]; - if (c == kOverflowMarker) { - return overflow_.at(i) == q; - } - if (q.count() != static_cast(c)) { - return false; - } - const PosT *pos = &rows_[(i * stride_) + 1]; - for (size_t j = 0; j < c; ++j) { - if (!q.test(pos[j])) { + return with_rows([this, &i, &q](const DefaultInitVector

&rows) { + const P c = rows[i * stride_]; + if (c == kOverflowMarker

) { + return overflow_.at(i) == q; + } + if (q.count() != static_cast(c)) { return false; } - } - return true; - } - - static auto check_index_fits(size_t value) -> void { - if (value >= kIndexCeiling) { - throw TermIndexCeilingReached("OperatorIndex: operator index reached the TermIndex ceiling; rebuild with " - "-Dmonoprop_WIDE_TERM_INDEX (this partition's term count exceeded ~2^32)."); - } + const P *pos = &rows[(i * stride_) + 1]; + for (size_t j = 0; j < c; ++j) { + if (!q.test(pos[j])) { + return false; + } + } + return true; + }); } - DefaultInitVector rows_ = {}; + // Exactly one is ever non-empty, selected by narrow_ -- the same one-live-backend shape MPOperator + // uses for its two stores. + DefaultInitVector rows8_ = {}; + DefaultInitVector rows16_ = {}; + size_t num_bits_ = 0; size_t size_ = 0; size_t inline_width_ = kMaxInlinePositions; size_t stride_ = 1 + kMaxInlinePositions; + bool narrow_ = false; // Lossless side-map for rows whose popcount exceeds inline_width_. std::unordered_map overflow_ = {}; - Table table_ = {}; + RowHashTable table_ = {}; }; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/operator/RowAccess.h b/cpp/monoprop/detail/operator/RowAccess.h index 3e7ed0cf..a5d90e7e 100644 --- a/cpp/monoprop/detail/operator/RowAccess.h +++ b/cpp/monoprop/detail/operator/RowAccess.h @@ -14,10 +14,11 @@ #pragma once -// One row-reader/writer vocabulary over both operator backends: the dense MonomialList and the packed -// detail::OperatorIndex. Templates parameterized on the row store (`Rows`) call these unqualified, so -// every such template must include this header — ADL cannot reach monoprop:: from an argument in -// monoprop::detail, and a later declaration is not found for an already-parsed template definition. +// One row-reader/writer vocabulary over all three operator backends: the dense MonomialList, the packed +// detail::OperatorIndex and the fixed-width-lane detail::SparseRowStore. Templates parameterized on the +// row store (`Rows`) call these unqualified, so every such template must include this header — ADL +// cannot reach monoprop:: from an argument in monoprop::detail, and a later declaration is not found for +// an already-parsed template definition. #include #include @@ -25,49 +26,71 @@ #include "monoprop/core/Monomial.h" #include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" namespace monoprop { -// materialize_row() returns a const ref (dense backend, zero-copy) or a fresh value (packed backend), -// so callers must bind with `const auto&` to extend the temporary's lifetime. -template -[[nodiscard]] inline auto materialize_row(const std::vector> &op, size_t i) - -> const Monomial & { +// materialize_row() returns a const ref (dense backend, zero-copy) or a fresh value (packed/sparse +// backend), so callers must bind with `const auto&` to extend the temporary's lifetime. +template +[[nodiscard]] inline auto materialize_row(const std::vector &op, size_t i) -> decltype(auto) { return op[i]; } -template -inline auto assign_row(std::vector> &op, size_t i, const Monomial &mono) -> void { +template +inline auto assign_row(std::vector &op, size_t i, const M &mono) -> void { op[i] = mono; } -template -[[nodiscard]] inline auto row_popcount(const std::vector> &op, size_t i) -> size_t { +template +[[nodiscard]] inline auto row_popcount(const std::vector &op, size_t i) -> size_t { return op[i].count(); } // Visits row i's set-bit positions ascending, without materializing a dense bitset when the backend can // avoid it. Hot: the even-parity inverted index is the heaviest per-row op reader. -template -inline auto for_each_row_position(const std::vector> &op, size_t i, Fn &&fn) -> void { +template +inline auto for_each_row_position(const std::vector &op, size_t i, Fn &&fn) -> void { const auto &m = op[i]; - for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + const size_t n = m.size(); + for (size_t b = m.find_first(); b < n; b = m.find_next(b)) { fn(b); } } -template -[[nodiscard]] inline auto materialize_row(const detail::OperatorIndex &op, size_t i) -> Monomial { +// Structural stand-in for "a row store shaped like detail::OperatorIndex/detail::SparseRowStore": exposes +// value_type and a row(i) accessor returning it, so the overloads below take a store without naming its +// row type -- the same idea as the MonomialLike overloads above, one level up (op itself is not +// MonomialLike; its rows are). A std::vector has no row(), so this never collides with the overloads above. +template +concept RowStoreLike = requires(const T &t, size_t i) { + typename T::value_type; + { t.row(i) } -> std::same_as; +}; + +template +[[nodiscard]] inline auto materialize_row(const Op &op, size_t i) -> typename Op::value_type { return op.row(i); } -template -inline auto assign_row(detail::OperatorIndex &op, size_t i, const Monomial &mono) -> void { +template +inline auto assign_row(Op &op, size_t i, const typename Op::value_type &mono) -> void { op.set(i, mono); } -template -[[nodiscard]] inline auto row_popcount(const detail::OperatorIndex &op, size_t i) -> size_t { +// A row written from a key that is already in the store's own form -- what the insert of an absent term +// does once the query record it came from was read in that form. Only the support-form store has a form +// of its own, so this is the one accessor with a backend-specific overload rather than a generic one: +// there is no such thing as an OperatorIndex-shaped key that is not simply a monomial. +inline auto assign_row(detail::SparseRowStore &op, size_t i, const detail::SparseRow &row) -> void { + op.set(i, row); +} +inline auto assign_row(detail::SparseRowStore &op, size_t i, const detail::SparseRowKey &key) -> void { + op.set(i, key); +} + +template +[[nodiscard]] inline auto row_popcount(const Op &op, size_t i) -> size_t { return op.popcount(i); } -template -inline auto for_each_row_position(const detail::OperatorIndex &op, size_t i, Fn &&fn) -> void { +template +inline auto for_each_row_position(const Op &op, size_t i, Fn &&fn) -> void { op.for_each_position(i, std::forward(fn)); } diff --git a/cpp/monoprop/detail/operator/RowHashTable.h b/cpp/monoprop/detail/operator/RowHashTable.h new file mode 100644 index 00000000..686e8128 --- /dev/null +++ b/cpp/monoprop/detail/operator/RowHashTable.h @@ -0,0 +1,290 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" + +namespace monoprop::detail { + +// The next row-array capacity for a geometric (1.5x) grow by `n` rows from `base` (the pre-growth size), +// given the current capacity. Never exact-fit: an exact fit would realloc the whole operator every layer. +// Shared by OperatorIndex and SparseRowStore, whose grow_rows_geometric() differ only in which arrays +// that capacity gets applied to. +[[nodiscard]] inline auto geometric_row_capacity(size_t base, size_t n, size_t capacity) noexcept -> size_t { + return std::max(base + n, capacity + (capacity / 2) + 1); +} + +// What a row store's spilled rows cost outside its own arrays: the map node per entry (key, mapped +// value and ~24 bytes of std::unordered_map node and bucket overhead) plus whatever each spilled +// monomial owns past its inline words. Shared for the same reason as the capacity rule above -- the +// node-overhead estimate is a single number that must not be corrected in one store and not the other, +// which would skew operator_memory_breakdown() for one backend only. +template +[[nodiscard]] inline auto spilled_rows_bytes(const OverflowMap &overflow) -> size_t { + size_t total = overflow.size() * (sizeof(typename OverflowMap::mapped_type) + sizeof(size_t) + 24); + for (const auto &[key, value] : overflow) { + total += value.heap_bytes(); + } + return total; +} + +class TermIndexCeilingReached : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// The keyless open-addressing index a row store puts over its rows: power-of-2 slot count, linear +// probing, max load factor 0.7 (the group-prefetch win erodes at higher load -- longer probe chains add +// un-prefetched reads). A slot holds a row index plus a 32-bit hash used only as an equality +// pre-filter, so the table never stores or compares a key itself. +// +// Keyless is why the hash and the equality test arrive as callables rather than as members: the whole +// point is that the caller owns the row representation. `eq(row_index)` confirms a pre-filter hit +// against the caller's rows, and no operation here reads a row. +// +// Shared by every row store, and that is load-bearing rather than tidiness: the layout this produces +// fixes the iteration order of for_each_slot(), which sets the order of a propagator's user-visible +// evolved-term list and therefore its floating-point accumulation order. Two stores that keyed rows +// through separate copies of this logic could diverge on that while both looking correct. +// +// Single-writer, matching its owners: one partition, one thread; parallelism is cross-partition. +class RowHashTable { +public: + // Valid row indices are < kIndexCeiling (check_index_fits throws at the ceiling), so the all-ones + // TermIndex is free to mark an empty slot. + static constexpr size_t kIndexCeiling = static_cast(std::numeric_limits::max()); + static constexpr TermIndex kEmptySlot = std::numeric_limits::max(); + // find_batch's "absent" result; same value as detail::kMissingIndex (not included here -- the + // operator store must not depend on evolution headers). + static constexpr size_t kNotFound = std::numeric_limits::max(); + + [[nodiscard]] auto count() const noexcept -> size_t { return count_; } + [[nodiscard]] auto slot_bytes() const -> size_t { return slots_.capacity() * sizeof(Slot); } + + // Slot capacity for `n` rows at <= 0.7 load. The +1 keeps a table reserved for exactly n rows off + // the rehash threshold on the n-th insert. + auto reserve(size_t n) -> void { rehash_to(slots_for_(n + 1)); } + + // The 32-bit form of a full-width row hash, which is what a slot stores as its equality + // pre-filter. Each store supplies its own full-width hash and folds it here, so the two agree on + // the pre-filter's format by construction. + [[nodiscard]] static constexpr auto fold(size_t full) noexcept -> uint32_t { + return static_cast(full ^ (static_cast(full) >> 32)); + } + + static auto check_index_fits(size_t value) -> void { + if (value >= kIndexCeiling) { + throw TermIndexCeilingReached("operator index reached the TermIndex ceiling; rebuild with " + "-Dmonoprop_WIDE_TERM_INDEX (this partition's term count exceeded ~2^32)."); + } + } + + // eq(row_index) -> bool confirms a hash pre-filter hit. + template + auto find(uint32_t h, Eq &&eq) const -> std::optional { + if (count_ == 0) { + return std::nullopt; + } + size_t s = spread(h) & mask_; + for (;; s = (s + 1) & mask_) { + const Slot &e = slots_[s]; + if (e.idx == kEmptySlot) { + return std::nullopt; + } + if (e.h == h && eq(static_cast(e.idx))) { + return static_cast(e.idx); + } + } + } + + // Insert-or-no-op. The row at `value` must already be written -- eq reads it. + template + auto emplace(uint32_t h, size_t value, Eq &&eq) -> void { + check_index_fits(value); + rehash_if_needed(); + size_t s = spread(h) & mask_; + while (slots_[s].idx != kEmptySlot) { + if (slots_[s].h == h && eq(static_cast(slots_[s].idx))) { + return; + } + s = (s + 1) & mask_; + } + slots_[s] = Slot{static_cast(value), h}; + ++count_; + } + + // Insert with no duplicate probe -- callers on this path insert provably distinct keys + // (+G-injective miss batches, clone re-insertion). + // insert_distinct over consecutive row indices [base, base + n), hashing each through hash_at(k). + // The stores' bulk_insert is this and nothing else, so it lives here rather than once per backend. + template + auto insert_distinct_range(size_t base, size_t n, HashFn &&hash_at) -> void { + if (n == 0) { + return; + } + check_index_fits(base + n - 1); + for (size_t k = 0; k < n; ++k) { + insert_distinct(static_cast(base + k), hash_at(k)); + } + } + + auto insert_distinct(TermIndex idx, uint32_t h) -> void { + rehash_if_needed(); + size_t s = spread(h) & mask_; + while (slots_[s].idx != kEmptySlot) { + s = (s + 1) & mask_; + } + slots_[s] = Slot{idx, h}; + ++count_; + } + + // Group-prefetch batch find: out[i] = row index of keys[i], or kNotFound. Same result as n find() + // calls, but overlaps dram misses via a per-group hash/probe/confirm pipeline. An h collision falls + // back to an exact find. Must not run concurrently with inserts. + // + // The three callables are what make the pipeline possible without the table knowing a row: + // hash(key) -> uint32_t, prefetch_row(row_index) issued between probe and confirm (which is the + // whole reason confirmation is deferred rather than folded into the probe), and + // eq(row_index, key) -> bool. + template + auto find_batch(const Key *keys, size_t n, size_t *out, Hash &&hash, PrefetchRow &&prefetch_row, Eq &&eq) const + -> void { + static constexpr size_t G = 16; // keys prefetched together per pipeline pass + std::array hh; + std::array sp; + std::array cand; + for (size_t base = 0; base < n; base += G) { + const size_t g = std::min(G, n - base); + for (size_t j = 0; j < g; ++j) { + hh[j] = hash(keys[base + j]); + sp[j] = spread(hh[j]); + __builtin_prefetch(&slots_[sp[j] & mask_], 0, 0); + } + for (size_t j = 0; j < g; ++j) { + cand[j] = kEmptySlot; + if (count_ == 0) { + continue; + } + cand[j] = probe_hash_match_(hh[j], sp[j] & mask_); + if (cand[j] != kEmptySlot) { + prefetch_row(static_cast(cand[j])); + } + } + for (size_t j = 0; j < g; ++j) { + if (cand[j] != kEmptySlot && eq(static_cast(cand[j]), keys[base + j])) { + out[base + j] = static_cast(cand[j]); + } + else if (cand[j] != kEmptySlot) { + const auto v = find(hh[j], [&eq, &keys, &base, &j](size_t i) { return eq(i, keys[base + j]); }); + out[base + j] = v ? *v : kNotFound; + } + else { + out[base + j] = kNotFound; + } + } + } + } + + // Occupied slots in table order, as fn(row_index, stored_hash). That order is the store's iteration + // order; see the class comment on why it must not drift between stores. + template + auto for_each_slot(Fn &&fn) const -> void { + for (const Slot &e : slots_) { + if (e.idx != kEmptySlot) { + fn(e.idx, e.h); + } + } + } + +private: + struct Slot { + TermIndex idx = kEmptySlot; + uint32_t h = 0; + }; + + static constexpr size_t kMinSlots = 16; + + static auto slots_for_(size_t n) -> size_t { return std::bit_ceil(std::max(kMinSlots, (n * 10 / 7) + 1)); } + + // Avalanche the cached 32-bit fold into a full-width hash (splitmix64 finalizer): the stored h is + // only an equality pre-filter, so it must be re-mixed before its low bits drive table bucketing. + static auto spread(uint32_t h) noexcept -> size_t { + uint64_t x = static_cast(h) * 0x9E3779B97F4A7C15ULL; + x ^= x >> 30; + x *= 0xBF58476D1CE4E5B9ULL; + x ^= x >> 27; + x *= 0x94D049BB133111EBULL; + x ^= x >> 31; + return static_cast(x); + } + + // First slot on h's probe chain whose stored hash matches, or kEmptySlot if the chain ends first. + // Matches on h alone and leaves confirmation to the caller -- that deferral is what lets find_batch + // prefetch the row between probe and confirm, so do not fold eq in here (find() deliberately keeps + // its own confirming variant). `start` must already be masked; the table must not be mutated + // concurrently. + [[gnu::always_inline]] auto probe_hash_match_(uint32_t h, size_t start) const -> TermIndex { + for (size_t s = start;; s = (s + 1) & mask_) { + const Slot &e = slots_[s]; + if (e.idx == kEmptySlot) { + return kEmptySlot; + } + if (e.h == h) { + return e.idx; + } + } + } + + auto rehash_if_needed() -> void { + if ((count_ + 1) * 10 >= slots_.size() * 7) { + rehash_to(slots_.size() * 2); + } + } + + auto rehash_to(size_t new_cap) -> void { + new_cap = std::bit_ceil(std::max(new_cap, kMinSlots)); + if (new_cap <= slots_.size()) { + return; + } + std::vector old = std::move(slots_); + slots_.assign(new_cap, Slot{}); + mask_ = new_cap - 1; + for (const Slot &e : old) { + if (e.idx == kEmptySlot) { + continue; + } + size_t s = spread(e.h) & mask_; + while (slots_[s].idx != kEmptySlot) { + s = (s + 1) & mask_; + } + slots_[s] = e; + } + } + + std::vector slots_ = std::vector(kMinSlots, Slot{}); + size_t mask_ = kMinSlots - 1; + size_t count_ = 0; +}; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/operator/SparseRowStore.h b/cpp/monoprop/detail/operator/SparseRowStore.h new file mode 100644 index 00000000..ca89e8b1 --- /dev/null +++ b/cpp/monoprop/detail/operator/SparseRowStore.h @@ -0,0 +1,775 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/operator/RowHashTable.h" + +// Logical mode count at or above which the sparse rows are the cheaper backend. Build-time and not a +// runtime knob because what moves the crossover is the target ISA, which is fixed when the translation +// unit is compiled: dense costs one pass per storage word and sparse is flat in the width, so without a +// vector popcount the dense pass degrades an order of magnitude sooner. Set from CMake off the arch +// flags actually emitted (see the top-level CMakeLists for the measured values). +// +// Hard error rather than a fallback default: the value is a usage requirement of monoprop-objs, so a +// translation unit reaching here without it did not inherit that target's requirements, and any +// fallback would differ from the value the rest of the library was compiled with. kMinModes reaches +// inline definitions (MPOperator, with_store), so under LTO that disagreement is an ODR violation +// resolving to one arbitrary answer -- a silently wrong backend choice rather than a build failure. +#ifndef monoprop_SPARSE_ROW_MIN_MODES +#error \ + "monoprop_SPARSE_ROW_MIN_MODES is undefined: link against the monoprop-objs target rather than adding its include paths by hand." +#endif + +namespace monoprop::detail { + +// The requested width needs mode indices wider than ModeT can hold, or more slots than a codes word +// has room for. Thrown rather than asserted, for the same reason OperatorIndexWidthUnsupported is: +// with the compile-time mode ceiling gone, the width is user data. +class SparseRowStoreUnsupported : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// A sparse row's two storage types, at namespace scope rather than inside the store: the algebra that +// reads rows (CodesAlgebra.h) must not depend on the container that owns them. +using RowMode = uint16_t; +using RowCodes = uint64_t; + +// Two bits per slot in one RowCodes. A wider row is representable in the mode lanes but would put the +// algebra back on a multi-word loop, which is the whole cost the support form removes. +inline constexpr size_t kRowMaxSlots = 32; +inline constexpr RowCodes kRowLoBits = 0x5555555555555555ULL; // bit 2j of every slot + +// Bit 2j of each slot: set iff slot j is occupied at all. popcount is n, the support measure. +[[nodiscard]] constexpr auto row_occupied_bits(RowCodes codes) noexcept -> RowCodes { + return (codes | (codes >> 1)) & kRowLoBits; +} +// Bit 2j of each slot: set iff slot j holds both of its positions. popcount is d. +[[nodiscard]] constexpr auto row_paired_bits(RowCodes codes) noexcept -> RowCodes { + return codes & (codes >> 1) & kRowLoBits; +} +[[nodiscard]] constexpr auto row_slot_count(RowCodes codes) noexcept -> size_t { + return static_cast(std::popcount(row_occupied_bits(codes))); +} + +// Non-owning view of one row: ascending mode lanes plus the codes word. The lane array is only read +// below num_slots(), which the codes word determines -- so a view stays valid over a padded row and +// carries no length of its own. It borrows the store's arrays, so it must not outlive them, and a row +// mutation invalidates it the way an iterator would. +struct SparseRow { + const RowMode *modes = nullptr; + RowCodes codes = 0; + + [[nodiscard]] auto num_slots() const noexcept -> size_t { return row_slot_count(codes); } + [[nodiscard]] auto mode(size_t j) const noexcept -> size_t { return static_cast(modes[j]); } + // The 2-bit field of slot j: 0b01 is physical position 2*mode alone, 0b10 is 2*mode+1 alone, 0b11 + // the paired mode. + [[nodiscard]] auto code(size_t j) const noexcept -> unsigned int { + return static_cast((codes >> (2 * j)) & 0b11U); + } +}; + +// A row *key* that may be too wide for a codes word: `spilled` non-null means the key is that dense +// monomial and `row` is unread. These are the two shapes a stored row already has, and a query needs +// both for the same reason a stored row does -- a query is M ⊕ G, and a fully paired product escapes +// the cutoff, so nothing bounds its support. +// +// Deliberately not folded into SparseRow, which is what the per-term algebra reads: that one is a view +// of something that fits, and a branch on every read of it is the cost the support form exists to +// avoid. The branch belongs here, on the probe path, where it runs once per query. +struct SparseRowKey { + SparseRow row; + const Bitset *spilled = nullptr; + + [[nodiscard]] auto is_spilled() const noexcept -> bool { return spilled != nullptr; } +}; + +// Visits a *dense* monomial as (mode, code) slots, ascending: the same sequence a SparseRow over the +// same monomial yields. Positions arrive ascending, so a mode's two positions are adjacent and one pass +// closes each slot before opening the next. +template +inline auto for_each_mode_slot(const Bitset &mono, Fn &&fn) -> void { + size_t pos = mono.find_first(); + while (pos < mono.size()) { + const size_t mode = pos >> 1; + unsigned int code = 1U << (pos & 1U); + pos = mono.find_next(pos); + if (pos < mono.size() && (pos >> 1) == mode) { + code |= 1U << (pos & 1U); + pos = mono.find_next(pos); + } + fn(mode, code); + } +} + +// Occupied modes in a dense monomial, via the same slot walk as sparse_row_hash/dense_row_equals below -- +// what a spilled row's occupied_modes() reports, and what a per-gate generator's mode count also needs +// (see sparse_record_capacity in layer_build/TermProduct.h). +[[nodiscard]] inline auto occupied_mode_count(const Bitset &mono) -> size_t { + size_t n = 0; + for_each_mode_slot(mono, [&n](size_t, unsigned int) { ++n; }); + return n; +} + +// The row hash, as an accumulator over (mode, code) slots. One definition with two walkers -- a sparse +// row and a dense monomial -- because a keyed store must hold both and hash them identically: a fully +// paired term escapes the cutoff, so a row can occupy more modes than any codes word holds and has to +// spill to the dense side map, where it still needs to be findable. +// +// That requirement is why the mix is *sequential* rather than an XOR-fold of slot-indexed terms, which +// is what the plan's "fixed-width mix over the padded row" would have been. Sequential mixing is +// positional without packing a slot index into the mixed word, so it does not care how many slots there +// are -- and it depends on neither the row capacity nor the padding, so two stores tuned to different +// capacities agree, which matters because the hash decides probe order. +class SparseRowHasher { +public: + auto add(size_t mode, unsigned int code) noexcept -> void { + h_ = SplitmixHash::mix(h_ ^ ((static_cast(mode) << 2) | code)); + } + [[nodiscard]] auto value() const noexcept -> size_t { return static_cast(h_); } + +private: + // Nonzero, so an empty row does not hash to zero and every slot count starts from a mixed state. + uint64_t h_ = 0x9E3779B97F4A7C15ULL; +}; + +[[nodiscard]] inline auto sparse_row_hash(const SparseRow &row) noexcept -> size_t { + SparseRowHasher hasher; + const size_t n = row.num_slots(); + for (size_t j = 0; j < n; ++j) { + hasher.add(row.mode(j), row.code(j)); + } + return hasher.value(); +} + +[[nodiscard]] inline auto sparse_row_hash(const Bitset &mono) noexcept -> size_t { + SparseRowHasher hasher; + for_each_mode_slot(mono, [&hasher](size_t mode, unsigned int code) { hasher.add(mode, code); }); + return hasher.value(); +} + +// Dispatches to whichever shape the key holds, so a batch of keys hashes identically whether or not any +// of them spilled. The two arms must agree with the store's own row hash, which is what makes a spilled +// row findable by either form. +[[nodiscard]] inline auto sparse_row_hash(const SparseRowKey &key) noexcept -> size_t { + return key.is_spilled() ? sparse_row_hash(*key.spilled) : sparse_row_hash(key.row); +} + +// Whether a dense monomial and a sparse row hold the same slots, without materializing either. Used +// where one side is a spilled row (no codes word) and the other is a query. +[[nodiscard]] inline auto dense_row_equals(const Bitset &mono, const SparseRow &row) -> bool { + const size_t n = row.num_slots(); + size_t j = 0; + bool equal = true; + for_each_mode_slot(mono, [&equal, &j, &n, &row](size_t mode, unsigned int code) { + if (!equal) { + return; + } + if (j >= n || row.mode(j) != mode || row.code(j) != code) { + equal = false; + return; + } + ++j; + }); + return equal && j == n; +} + +// Writes a sparse row's occupied slots into `mono`, which must already be at the row's width and +// cleared -- a fresh Bitset(num_bits), or one a caller reset itself before refilling it. The shared body +// behind every dense materialization of a SparseRow below and in layer_build/Common.h and TermProduct.h. +inline auto fill_from_sparse_row(const SparseRow &row, Bitset &mono) -> void { + const size_t n = row.num_slots(); + for (size_t j = 0; j < n; ++j) { + const unsigned int code = row.code(j); + if ((code & 1U) != 0U) { + mono.set(2 * row.mode(j)); + } + if ((code & 2U) != 0U) { + mono.set((2 * row.mode(j)) + 1); + } + } +} + +// Materializes a sparse row as a fresh dense monomial at the given width. +[[nodiscard]] inline auto sparse_row_to_bitset(const SparseRow &row, size_t num_bits) -> Bitset { + Bitset mono(num_bits); + fill_from_sparse_row(row, mono); + return mono; +} + +// Operator-term store in support form: each row is a fixed-width list of the *modes* it occupies plus +// one word holding two bits per occupied mode. It is the third backend behind the four TypeAliases.h +// row accessors, alongside std::vector and OperatorIndex, and agrees with both through them +// (cpp/tests/row_accessor_tests.cpp). +// +// Layout, structure-of-arrays: modes_ is `slots_per_row_` ModeT lanes per row, ascending, padded with +// kPadLane; the codes array is one word per row, at the narrowest of three widths that holds +// 2 * slots_per_row_ bits (see CodesWidth). The two live in separate arrays on purpose -- the cutoff and +// pairing algebra reads only the codes, so evaluating it over a run of rows is a sequential walk that +// never touches a mode list, and narrowing that array puts proportionally more rows on each line of it. +// +// A codes word packs slot j (the j-th occupied mode, ascending) into bits 2j and 2j+1: bit 2j marks physical +// position 2*mode, bit 2j+1 marks 2*mode+1. The whole cutoff algebra follows from that one word -- +// with occupied = (codes | codes>>1) & 0x5555..., paired = codes & (codes>>1) & 0x5555..., +// n = popcount(occupied) and d = popcount(paired) give or_sum = n, popcount_sum = n + d and +// xor_sum = n - d, independent of the storage width. popcount() below is the first consumer; the rest +// arrives with the algebra port. +// +// Rows wider than slots_per_row_ spill losslessly to a side map. They are not a corner case to be +// ruled out by sizing: a fully-paired term escapes the cutoff (xor_sum == 0 is kept unconditionally), +// so support is genuinely unbounded no matter what the cutoff is. They are rare -- ~0.07% of rows on +// production models, per MPOperator.h -- and an all-0b11 row needs only its mode list, so a second +// cheap row kind is available if that ever stops being true. +// +// The keyless index over the rows is the shared RowHashTable, the same one OperatorIndex uses, so both +// stores produce the same slot layout for the same insertion sequence. What differs is only what a key +// is: rows hash through sparse_row_hash and confirm through a codes compare plus a lane memcmp, where +// OperatorIndex hashes a whole Bitset. That hash is *not* the dense one, so a store swap changes probe +// order, MPI owner routing and therefore floating-point accumulation order -- the deliberate +// re-baseline, not a regression. +// +// static_assert cannot express it, so: SparseRowStore is interchangeable with OperatorIndex through the +// TypeAliases.h accessors and through find/emplace/bulk_insert/find_batch, and cpp/tests are what hold +// that. It is not a subclass of anything and nothing dispatches on it. +// +// Single-writer, like OperatorIndex: one partition, one thread; parallelism is cross-partition. +class SparseRowStore { +public: + using value_type = Bitset; + using key_type = Bitset; + using mapped_type = size_t; + using ModeT = RowMode; + using CodesT = RowCodes; + + static constexpr size_t kMaxSlots = kRowMaxSlots; + static constexpr size_t kDefaultSlots = 8; + + // The top two ModeT values are markers, so a valid mode index is at most kPadLane - 2. kPadLane + // fills the unused lanes of a short row (fixed, so two equal rows have equal lanes); kOverflowLane + // sits in lane 0 of a spilled row, where it cannot be confused with the empty row's kPadLane. + static constexpr ModeT kPadLane = std::numeric_limits::max(); + static constexpr ModeT kOverflowLane = static_cast(kPadLane - 1); + static constexpr size_t kMaxModes = static_cast(kOverflowLane); // exclusive bound + + static constexpr size_t kIndexCeiling = RowHashTable::kIndexCeiling; + static constexpr size_t kNotFound = RowHashTable::kNotFound; + + // The Stage 3 crossover, as a predicate rather than a bare number so the rule has one home. + static constexpr size_t kMinModes = monoprop_SPARSE_ROW_MIN_MODES; + [[nodiscard]] static constexpr auto preferred_for_modes(size_t num_modes) noexcept -> bool { + return num_modes >= kMinModes; + } + + // num_bits is the storage bit width of every monomial this store will hold, exactly as for + // OperatorIndex: row() reconstructs at that width, and a wrong one changes num_words() and with it + // the hash, the probe order and MPI owner routing. slots_per_row is the per-row mode capacity -- + // any value is correct, since over-long rows spill; size it from CutoffEvaluator::max_mode_bound(). + explicit SparseRowStore(size_t num_bits, size_t slots_per_row = kDefaultSlots) + : num_bits_(num_bits), + slots_per_row_(std::clamp(slots_per_row, 1, kMaxSlots)), + codes_width_(codes_width_for(slots_per_row_)) { + if (((num_bits + 1) / 2) > kMaxModes) { + throw SparseRowStoreUnsupported( + std::format("SparseRowStore supports at most {} modes ({} bits); got {} bits ({} modes).", + kMaxModes, + 2 * kMaxModes, + num_bits, + (num_bits + 1) / 2)); + } + } + + SparseRowStore(const SparseRowStore &) = delete; + SparseRowStore &operator=(const SparseRowStore &) = delete; + SparseRowStore(SparseRowStore &&) = delete; + SparseRowStore &operator=(SparseRowStore &&) = delete; + +private: + // Storage width of the codes array. A codes word carries two bits per slot, so a store sized from a + // cutoff bound only ever sets 2 * slots_per_row_ of the 64 bits a CodesT has -- 12 at cutoff 6 and 16 + // at cutoff 8, the two shipping models. Narrowing the storage recovers the rest, 6 bytes per row at + // both, which was this backend's whole per-row gap to OperatorIndex's (1 + inline_width) payloads. + // + // Only the array narrows. Every reader still sees a CodesT, zero-extended on load, so CodesAlgebra.h, + // sparse_row_hash, SparseRow and the cutoff algebra are untouched -- and with them the term set, the + // values and the probe order. Rows are payload, never a hash input and never serialized, so nothing + // here is visible to a baseline diff; the footprint gate is the memory_bytes() case in + // cpp/tests/sparse_row_store_tests.cpp. + enum class CodesWidth : uint8_t { Narrow, Medium, Wide }; + + [[nodiscard]] static constexpr auto codes_width_for(size_t slots) noexcept -> CodesWidth { + if ((2 * slots) <= 16) { + return CodesWidth::Narrow; + } + if ((2 * slots) <= 32) { + return CodesWidth::Medium; + } + return CodesWidth::Wide; + } + + // Sits mid-class for the reason OperatorIndex::with_rows does: a deduced return type is not available + // to a caller that appears earlier in the class body. + // + // Binds the codes storage type for one call. codes_width_ is fixed at construction, so the branch is + // a load-and-test on a member that never changes -- predicted, and one per row read rather than per + // slot. Not hoisted into the store type, for the same reason the row payload is not: the codes width + // is not part of the seam the scan is templated on (see with_store in MPOperator), and making it so + // would triple every downstream instantiation to save a predicted branch. + template + [[gnu::always_inline]] auto with_codes(this Self&& self, F&& f) + -> decltype(auto) { + switch (self.codes_width_) { + case CodesWidth::Narrow: + return f(self.codes16_); + case CodesWidth::Medium: + return f(self.codes32_); + default: + return f(self.codes64_); + } + } + + [[nodiscard]] auto load_codes(size_t i) const -> CodesT { + return with_codes([i](const auto &codes) -> CodesT { return static_cast(codes[i]); }); + } + + // The narrowing cast is exact rather than checked: a row reaches here only after it is known to fit + // slots_per_row_ slots, and slot j occupies bits 2j and 2j+1, so no bit at or above 2 * slots_per_row_ + // is ever set. The assert is what holds that when a caller hands over a SparseRow it built itself. + auto store_codes(size_t i, CodesT codes) -> void { + with_codes([i, codes](auto &store) { + using ElemT = typename std::remove_cvref_t::value_type; + assert(codes == static_cast(static_cast(codes)) && "codes word wider than its storage"); + store[i] = static_cast(codes); + }); + } + + auto resize_codes(size_t n) -> void { + with_codes([n](auto &codes) { codes.resize(n); }); + } + auto reserve_codes(size_t n) -> void { + with_codes([n](auto &codes) { codes.reserve(n); }); + } + [[nodiscard]] auto codes_capacity() const -> size_t { + return with_codes([](const auto &codes) { return codes.capacity(); }); + } + // The three arrays differ only in element size, so everything that counts bytes rather than reading a + // word is plain arithmetic off this and needs no type bound. + [[nodiscard]] auto codes_bytes() const noexcept -> size_t { + switch (codes_width_) { + case CodesWidth::Narrow: + return sizeof(uint16_t); + case CodesWidth::Medium: + return sizeof(uint32_t); + default: + return sizeof(CodesT); + } + } + +public: + [[nodiscard]] auto num_bits() const noexcept -> size_t { return num_bits_; } + [[nodiscard]] auto slots_per_row() const noexcept -> size_t { return slots_per_row_; } + // The backend-neutral spelling of the line above, so a caller holding either store asks the same + // question of both (see MPOperator::row_width). + [[nodiscard]] auto row_width() const noexcept -> size_t { return slots_per_row_; } + [[nodiscard]] auto size() const noexcept -> size_t { return size_; } + + // Called only on an idle store, so it needs no synchronization. + [[nodiscard]] auto clone() const -> std::unique_ptr { + auto out = std::make_unique(num_bits_, slots_per_row_); + out->modes_ = modes_; + // Exactly one of the three is non-empty, and out shares slots_per_row_ so it shares codes_width_. + out->codes16_ = codes16_; + out->codes32_ = codes32_; + out->codes64_ = codes64_; + out->size_ = size_; + out->overflow_ = overflow_; + out->table_ = table_; // RowHashTable is rule-of-zero copyable; a plain copy preserves slot order exactly. + return out; + } + + // Same term set at a different slots_per_row_, e.g. after a cutoff change moves the bound rows are + // sized from. Every row's monomial is re-flowed through set() at the new stride, which decides + // inline-vs-overflow the same way a fresh insert would; the hash index is copied as-is, since + // fold_hash (via sparse_row_hash) depends only on the monomial, never on slots_per_row_, so no rehash + // is needed. Row index i is preserved for every row -- load-bearing, since callers key op_coeffs, + // state_rows_/state_vals_ and the evolution graph by this same index. + [[nodiscard]] auto resized(size_t new_slots_per_row) const -> std::unique_ptr { + auto out = std::make_unique(num_bits_, new_slots_per_row); + out->modes_.resize(size_ * out->slots_per_row_); + out->resize_codes(size_); + out->size_ = size_; + // Reflow via view()/overflow_ directly rather than row(i): row() would materialize a fresh + // Bitset from the slots and set() would immediately re-walk it to rebuild them, a double pass + // this store's own non-allocating set(SparseRow) / set(value_type) overloads make unnecessary. + for (size_t i = 0; i < size_; ++i) { + if (spilled(i)) { + out->set(i, overflow_.at(i)); + } + else { + out->set(i, view(i)); + } + } + out->table_ = table_; // RowHashTable is rule-of-zero copyable; a plain copy preserves slot order exactly. + return out; + } + + auto reserve(size_t n) -> void { + reserve_rows_(n); + table_.reserve(n); + } + + // Returns the pre-growth size (the caller's insert base). Growth is geometric (1.5x), never + // exact-fit: an exact fit would realloc the whole operator every layer. Rows only -- the table grows + // on its own load factor, and pre-sizing it per layer would rehash for nothing. + auto grow_rows_geometric(size_t n) -> size_t { + const size_t base = size_; + if (capacity() < base + n) { + reserve_rows_(geometric_row_capacity(base, n, capacity())); + } + // Default-init grow, not a zeroing resize: every freshly grown row is overwritten by set() + // before any read, so a tail zero-fill would be wasted bandwidth. + modes_.resize((base + n) * slots_per_row_); + resize_codes(base + n); + size_ = base + n; + return base; + } + + auto push_back(const value_type &mono) -> void { set(grow_rows_geometric(1), mono); } + + // Row i may be grown-but-uninitialized or hold a prior value, so nothing in the row is pre-read; a + // stale overflow entry at i, if any, is dropped. + auto set(size_t i, const value_type &mono) -> void { + ModeT *lanes = &modes_[i * slots_per_row_]; + CodesT codes = 0; + size_t used = 0; + bool overflows = false; + // The slot walk is shared with the hash, so the two cannot disagree about what a row's slots are. + // Lanes come out ascending because the walk is. + for_each_mode_slot(mono, [&overflows, &used, &lanes, &codes, this](size_t mode, unsigned int code) { + if (overflows) { + return; + } + if (used == slots_per_row_) { + overflows = true; + return; + } + lanes[used] = static_cast(mode); + codes |= static_cast(code) << (2 * used); + ++used; + }); + if (overflows) { + lanes[0] = kOverflowLane; + store_codes(i, 0); + overflow_[i] = mono; + return; + } + if (!overflow_.empty()) { + overflow_.erase(i); + } + for (size_t j = used; j < slots_per_row_; ++j) { + lanes[j] = kPadLane; + } + store_codes(i, codes); + } + + // The row form of set(), and the write the support form exists for: the lanes are already ascending + // and the codes word already says what each holds, so this copies `n` lanes and one word where the + // dense overload walks the monomial's storage words. + // + // A row wider than this store's capacity still has to spill, and a spilled row is held densely, so + // that arm materializes. It cannot be asserted away: the capacity is sized from the cutoff and a + // fully paired term escapes the cutoff. + auto set(size_t i, const SparseRow &row) -> void { + const size_t n = row.num_slots(); + // Contiguity from slot 0 is the representation's invariant -- num_slots() counts occupied slots + // and the lanes are read from 0 -- so a row with a hole would silently lose its high slots here. + assert((n >= kRowMaxSlots || (row.codes >> (2 * n)) == 0) && "SparseRow slots must be contiguous from slot 0"); + ModeT *lanes = &modes_[i * slots_per_row_]; + if (n > slots_per_row_) { + lanes[0] = kOverflowLane; + store_codes(i, 0); + overflow_[i] = to_monomial_(row); + return; + } + // Hygiene, not correctness: spilled() reads lane 0, so a stale entry here is already unreachable + // -- it would just keep a monomial alive for the store's lifetime. + if (!overflow_.empty()) { + overflow_.erase(i); + } + if (n != 0) { + std::memcpy(lanes, row.modes, n * sizeof(ModeT)); + } + // Padding is load-bearing for the empty row and only for it: with n == 0 nothing above writes a + // lane, so lane 0 would keep a previous occupant's kOverflowLane and the row would read as spilled. + for (size_t j = n; j < slots_per_row_; ++j) { + lanes[j] = kPadLane; + } + store_codes(i, row.codes); + } + + // Whichever shape the key holds. The spilled arm is the dense set(), so a key that arrived too wide + // for a codes word lands in the side map exactly as the dense path would have put it. + auto set(size_t i, const SparseRowKey &key) -> void { + if (key.is_spilled()) { + set(i, *key.spilled); + return; + } + set(i, key.row); + } + + [[nodiscard]] auto row(size_t i) const -> value_type { + if (spilled(i)) { + return overflow_.at(i); + } + return sparse_row_to_bitset(view(i), num_bits_); + } + + // Ascending, matching the dense backends: slots are stored ascending in the mode, and within a mode + // position 2*mode precedes 2*mode+1. + template + auto for_each_position(size_t i, Fn &&fn) const -> void { + if (spilled(i)) { + const auto &m = overflow_.at(i); + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + fn(b); + } + return; + } + for_each_slot(i, [&fn](size_t mode, unsigned int code) { + if ((code & 1U) != 0U) { + fn(2 * mode); + } + if ((code & 2U) != 0U) { + fn((2 * mode) + 1); + } + }); + } + + // Visits (mode, code) per occupied slot, ascending in the mode; code is the 2-bit field, so 0b01 is + // position 2*mode alone, 0b10 is 2*mode+1 alone and 0b11 is the paired mode. Row i must not be + // spilled -- a spilled row has no slots, and its lane 0 marker would read as a mode. + template + auto for_each_slot(size_t i, Fn &&fn) const -> void { + assert(!spilled(i) && "SparseRowStore::for_each_slot on a spilled row"); + const ModeT *lanes = &modes_[i * slots_per_row_]; + const CodesT codes = load_codes(i); + for (size_t j = 0; j < slots_per_row_ && lanes[j] != kPadLane; ++j) { + fn(static_cast(lanes[j]), static_cast((codes >> (2 * j)) & 0b11U)); + } + } + + // The row's codes word. Meaningless for a spilled row -- ask spilled(i) first; the algebra port + // will need the same guard, which is why the spill is kept rare rather than made general. + [[nodiscard]] auto codes(size_t i) const -> CodesT { return load_codes(i); } + + // What the codes algebra reads. Borrows this store's arrays, so it is invalidated by anything that + // reallocates them (grow_rows_geometric, reserve) or rewrites row i; row i must not be spilled. + [[nodiscard]] auto view(size_t i) const -> SparseRow { + assert(!spilled(i) && "SparseRowStore::view on a spilled row"); + return SparseRow{&modes_[i * slots_per_row_], load_codes(i)}; + } + + [[nodiscard]] auto spilled(size_t i) const -> bool { return modes_[i * slots_per_row_] == kOverflowLane; } + + // Occupied modes -- the support measure, or_sum. + [[nodiscard]] auto slot_count(size_t i) const -> size_t { + if (spilled(i)) { + return occupied_modes(overflow_.at(i)); + } + return row_slot_count(load_codes(i)); + } + + // Set bits -- the length measure, popcount_sum = n + d straight off the codes word. + [[nodiscard]] auto popcount(size_t i) const -> size_t { + if (spilled(i)) { + return overflow_.at(i).count(); + } + const CodesT codes = load_codes(i); + return row_slot_count(codes) + static_cast(std::popcount(row_paired_bits(codes))); + } + + // --- the keyless index over those rows --------------------------------------------------------- + // + // A key is a SparseRow or a Bitset, and both hash through sparse_row_hash, so the two are + // interchangeable at a call site. Prefer the row: it is what the scan holds, and it is the only form + // that needs no slot walk to hash. The Bitset form is what a caller still holding a monomial uses. + + auto find(const SparseRow &key) const -> std::optional { return find_hashed_(key); } + auto find(const key_type &key) const -> std::optional { return find_hashed_(key); } + auto find(const SparseRowKey &key) const -> std::optional { return find_hashed_(key); } + + // Insert-or-no-op. The row at `value` must already be written -- the confirm reads it. + template + auto emplace(const Key &key, mapped_type value) -> void { + table_.emplace(fold_hash(key), value, [&key, this](size_t i) { return row_eq_key(i, key); }); + } + + // Insert n distinct rows with consecutive indices [base, base+n). Rows must already be written. + template + auto bulk_insert(size_t n, mapped_type base, KeyFn &&key_at) -> void { + table_.insert_distinct_range(base, n, [&key_at](size_t k) { return fold_hash(key_at(k)); }); + } + + // out[i] = row index of keys[i], or kNotFound. Same result as n find() calls; see + // RowHashTable::find_batch for why the row prefetch sits between probe and confirm. Both arrays are + // prefetched: the confirm reads the codes word first and the lanes only if it matches, but they are + // separate allocations and so separate cache misses. + template + auto find_batch(const Key *keys, size_t n, size_t *out) const -> void { + table_.find_batch( + keys, + n, + out, + [](const Key &key) { return fold_hash(key); }, + [this](size_t i) { + with_codes([i](const auto &codes) { __builtin_prefetch(&codes[i], 0, 0); }); + __builtin_prefetch(&modes_[i * slots_per_row_], 0, 0); + }, + [this](size_t i, const Key &key) { return row_eq_key(i, key); }); + } + + // Rows in table order (for_each_slot walks the slot array, i.e. hash/probe order, not ascending row + // index -- see the class comment above), as fn(row_index). Not the row itself: a spilled row has no + // view, so what a caller wants off the index is the index. + template + auto for_each_index(Fn &&fn) const -> void { + table_.for_each_slot([&fn](TermIndex idx, uint32_t) { fn(static_cast(idx)); }); + } + + // OperatorIndex's signature, fn(monomial, row_index), so the two stores are interchangeable at the + // one call site that wants both. Materializes each row, which for_each_index does not -- prefer that + // where the index alone will do. + template + auto for_each(Fn &&fn) const -> void { + for_each_index([&fn, this](size_t i) { fn(row(i), i); }); + } + + [[nodiscard]] auto indexed_count() const noexcept -> size_t { return table_.count(); } + + [[nodiscard]] auto index_estimated_memory_bytes() const -> size_t { + return sizeof(SparseRowStore) + table_.slot_bytes(); + } + + [[nodiscard]] auto memory_bytes() const -> size_t { + return (modes_.capacity() * sizeof(ModeT)) + (codes_capacity() * codes_bytes()) + spilled_rows_bytes(overflow_); + } + + // Diagnostic: the part of memory_bytes() that is unused geometric-growth capacity. + [[nodiscard]] auto slack_bytes() const -> size_t { + const size_t lanes = modes_.capacity() - std::min(modes_.capacity(), size_ * slots_per_row_); + const size_t words = codes_capacity() - std::min(codes_capacity(), size_); + return (lanes * sizeof(ModeT)) + (words * codes_bytes()); + } + + // Slot count for a cutoff bound in modes (CutoffEvaluator::max_mode_bound()), clamped to what one + // codes word holds. A bound above kMaxSlots is not an error: the rows that exceed it spill. + [[nodiscard]] static auto slots_for_bound(size_t mode_bound) noexcept -> size_t { + return std::clamp(mode_bound, 1, kMaxSlots); + } + + // Slot count for a row in flight rather than a row at rest: a product occupies up to the term's modes + // plus the generator's, so a scan scratch row and a wire record both need the cutoff's mode bound plus + // the widest generator's locality. Every rank derives this from the same circuit and cutoff, so they + // agree on it without communication -- which is what lets it fix a wire stride. + [[nodiscard]] static auto scratch_slots_for(size_t mode_bound, size_t max_generator_modes) noexcept -> size_t { + return std::clamp(mode_bound + max_generator_modes, 1, kMaxSlots); + } + +private: + // Spilled rows have no codes word, so their support is counted the dense way. + [[nodiscard]] static auto occupied_modes(const value_type &mono) -> size_t { return occupied_mode_count(mono); } + + // The 32-bit fold the table stores as its equality pre-filter, over the full-width row hash. + template + static auto fold_hash(const Key &key) noexcept -> uint32_t { + return RowHashTable::fold(sparse_row_hash(key)); + } + + template + auto find_hashed_(const Key &key) const -> std::optional { + return table_.find(fold_hash(key), [&key, this](size_t i) { return row_eq_key(i, key); }); + } + + // The find confirm, against a row query. Codes first: one word compare rejects nearly every + // pre-filter false positive, and it is what fixes the lane compare's length -- equal codes means + // equal slot counts, so only the occupied lanes can differ. Comparing the padded width instead (as + // the plan sketched) would be the same cost but would silently mismatch any query a caller left + // unpadded, and the padding is capacity-dependent where a key must not be. + [[nodiscard]] auto row_eq_key(size_t i, const SparseRow &key) const -> bool { + if (spilled(i)) { + return dense_row_equals(overflow_.at(i), key); + } + if (load_codes(i) != key.codes) { + return false; + } + const size_t k = row_slot_count(key.codes); + return k == 0 || std::memcmp(&modes_[i * slots_per_row_], key.modes, k * sizeof(ModeT)) == 0; + } + + // The same against a monomial query, which is the only form that can match a spilled row exactly. + [[nodiscard]] auto row_eq_key(size_t i, const key_type &key) const -> bool { + if (spilled(i)) { + return overflow_.at(i) == key; + } + return dense_row_equals(key, view(i)); + } + + [[nodiscard]] auto row_eq_key(size_t i, const SparseRowKey &key) const -> bool { + return key.is_spilled() ? row_eq_key(i, *key.spilled) : row_eq_key(i, key.row); + } + + // A row at this store's width. Only the spill arms need it: everything else reads slots in place. + [[nodiscard]] auto to_monomial_(const SparseRow &row) const -> value_type { + return sparse_row_to_bitset(row, num_bits_); + } + + auto reserve_rows_(size_t n) -> void { + modes_.reserve(n * slots_per_row_); + reserve_codes(n); + } + + [[nodiscard]] auto capacity() const -> size_t { return codes_capacity(); } + + DefaultInitVector modes_ = {}; + // Exactly one is ever non-empty, selected by codes_width_ -- the same one-live-arm shape + // OperatorIndex uses for its row payload. + DefaultInitVector codes16_ = {}; + DefaultInitVector codes32_ = {}; + DefaultInitVector codes64_ = {}; + size_t num_bits_ = 0; + size_t size_ = 0; + size_t slots_per_row_ = kDefaultSlots; + // Declared after slots_per_row_: the constructor derives it from the clamped value. + CodesWidth codes_width_ = codes_width_for(kDefaultSlots); + // Lossless side-map for rows occupying more than slots_per_row_ modes. + std::unordered_map overflow_ = {}; + RowHashTable table_ = {}; +}; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/partition/CMakeLists.txt b/cpp/monoprop/detail/partition/CMakeLists.txt index 19d4b69b..ba7365dd 100644 --- a/cpp/monoprop/detail/partition/CMakeLists.txt +++ b/cpp/monoprop/detail/partition/CMakeLists.txt @@ -6,6 +6,7 @@ target_sources( FILES "CpuTopology.h" "PartitionGroup.h" + "StagedCollect.h" ) target_sources(monoprop-objs PRIVATE CpuTopology.cpp) diff --git a/cpp/monoprop/detail/partition/PartitionGroup.h b/cpp/monoprop/detail/partition/PartitionGroup.h index 85000680..7b3252c9 100644 --- a/cpp/monoprop/detail/partition/PartitionGroup.h +++ b/cpp/monoprop/detail/partition/PartitionGroup.h @@ -42,17 +42,20 @@ namespace monoprop { -template -class MonomialPropagator; // completed before any PartitionGroup member body is instantiated (Impl.h) +// Only detail/monomial_propagator/MonomialPropagator.cpp includes this header, and it does so *after* +// MonomialPropagator's definition -- which this file now requires rather than merely prefers. The +// member bodies below are ordinary functions, not templates, so they are parsed where they are written +// instead of at instantiation, and make_unique needs the complete type right there. +// A future include from anywhere earlier fails loudly on the incomplete type; it cannot go wrong quietly. +class MonomialPropagator; namespace detail::partition { -template class PartitionGroup { public: // Builds each partition's propagator via `factory(partition_comm)` ON its master thread, so heap allocations // are first-touched on the owning core. `factory` must build a partitions=1 propagator. - using Factory = std::function>(mpi::Comm)>; + using Factory = std::function(mpi::Comm)>; // `parent` is the enclosing communicator (size R): R == 1 ⇒ an in-process ShmComm; R > 1 ⇒ a // HybridComm folding R ranks x S partitions into one flat P=R*S world. @@ -68,7 +71,7 @@ class PartitionGroup { // The masters are already running, so a ctor throw must not escape: ~PartitionGroup would never run, // and destroying joinable threads during unwinding calls std::terminate. try { - run_on_all([&](int r) { partitions_[static_cast(r)] = factory(comm_for_(r)); }); + run_on_all([this, &factory](int r) { partitions_[static_cast(r)] = factory(comm_for_(r)); }); } catch (...) { stop_and_join_(); @@ -90,7 +93,7 @@ class PartitionGroup { cpusets_ = topo_partition_cpusets(n_, node_rank_, node_size_, node_mask_); start_masters_(); try { // see the primary ctor: a throw past live masters would std::terminate - run_on_all([&](int r) { + run_on_all([this, &src](int r) { auto p = src.partitions_[static_cast(r)]->clone_(); // virtual: keeps the derived type p->comm_ = comm_for_(r); // PartitionGroup is a friend of MonomialPropagator partitions_[static_cast(r)] = std::move(p); @@ -106,8 +109,8 @@ class PartitionGroup { ~PartitionGroup() { stop_and_join_(); } auto partition_count() const -> int { return n_; } - auto partition(int s) -> MonomialPropagator & { return *partitions_[static_cast(s)]; } - auto partition(int s) const -> const MonomialPropagator & { return *partitions_[static_cast(s)]; } + auto partition(int s) -> MonomialPropagator & { return *partitions_[static_cast(s)]; } + auto partition(int s) const -> const MonomialPropagator & { return *partitions_[static_cast(s)]; } // Run `body(partition_rank)` on all masters, block until every one finishes, then rethrow the first // exception raised (peers were released via poison, so a throw on one master never hangs the rest). @@ -125,9 +128,9 @@ class PartitionGroup { cv_start_.notify_all(); { std::unique_lock lk(m_); - cv_done_.wait(lk, [&] { return done_count_ == n_; }); + cv_done_.wait(lk, [this] { return done_count_ == n_; }); } - for (auto &e : errs_) { + for (const auto &e : errs_) { if (e) { std::rethrow_exception(e); } @@ -164,7 +167,7 @@ class PartitionGroup { auto discover_node_peers_() -> void { #ifdef monoprop_ENABLE_MPI if (parent_.kind == mpi::Comm::Kind::Mpi && mpi::size(parent_) > 1) { - MPI_Comm node = MPI_COMM_NULL; + auto node = MPI_COMM_NULL; MPI_Comm_split_type(parent_.mpi, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &node); MPI_Comm_rank(node, &node_rank_); MPI_Comm_size(node, &node_size_); @@ -179,7 +182,8 @@ class PartitionGroup { #ifdef monoprop_ENABLE_MPI // A rank seeing 16 of 128 CPUs is equally "my own 16" and "eight of us share these 16": only the masks tell. auto classify_node_masks_(MPI_Comm node) -> void { - node_mask_ = NodeMask::Shared; + using enum monoprop::detail::partition::NodeMask; + node_mask_ = Shared; if (node_size_ <= 1) { report_placement_(nullptr, 0, "alone"); return; // nobody to collide with; the normal split already handles group_count == 1 @@ -193,7 +197,7 @@ class PartitionGroup { const bool disjoint = monoprop::detail::partition::masks_are_pairwise_disjoint(all.data(), static_cast(node_size_), kMaskWords); - node_mask_ = disjoint ? NodeMask::PerRank : NodeMask::Shared; + node_mask_ = disjoint ? PerRank : Shared; report_placement_(all.data(), static_cast(node_size_), disjoint ? "private" : "shared"); } #endif @@ -203,7 +207,7 @@ class PartitionGroup { * nullptr means no peers, so measure our own mask; the verdict is then "alone", which is NOT * evidence that a multi-rank launcher did the right thing. Reached only from the primary ctor, so * a clone does not re-emit -- the mask belongs to the process, not the object. */ - auto report_placement_(const uint64_t *masks, size_t peers, const char *verdict) -> void { + auto report_placement_(const uint64_t *masks, size_t peers, const char *verdict) const -> void { constexpr size_t kWords = monoprop::detail::partition::kAffinityMaskWords; std::array own{}; if (masks == nullptr && affinity_mask_words(own.data(), kWords)) { @@ -232,7 +236,7 @@ class PartitionGroup { #endif shm_ = std::make_unique(n_); } - auto comm_for_(int r) -> mpi::Comm { + auto comm_for_(int r) const -> mpi::Comm { #ifdef monoprop_ENABLE_MPI if (hyb_) { return mpi::Comm::make_hybrid(hyb_.get(), r); @@ -283,7 +287,7 @@ class PartitionGroup { const std::function *job = nullptr; { std::unique_lock lk(m_); - cv_start_.wait(lk, [&] { return stop_ || job_gen_ != seen; }); + cv_start_.wait(lk, [this, &seen] { return stop_ || job_gen_ != seen; }); if (stop_) { return; } @@ -314,14 +318,15 @@ class PartitionGroup { #ifdef monoprop_ENABLE_MPI std::unique_ptr hyb_; // set iff R > 1 #endif - std::vector>> partitions_; + std::vector> partitions_; std::vector errs_; std::vector cpusets_; std::vector masters_; // Job dispatch: the facade thread publishes one job and waits for all masters to complete it. std::mutex m_; - std::condition_variable cv_start_, cv_done_; + std::condition_variable cv_start_; + std::condition_variable cv_done_; const std::function *job_ = nullptr; unsigned job_gen_ = 0; int done_count_ = 0; @@ -329,26 +334,12 @@ class PartitionGroup { }; // One result per partition, indexed by partition rank. The slots are written from the owning master, so -// `body` must not touch the vector itself. Staged into a non-bit-packed `Slot` type: std::vector is -// the bit-packed specialization, so concurrent partition-master writes to different logical elements can -// tear the same underlying word (a data race) even though their indices are disjoint. -template > -auto collect_on_all(PartitionGroup &group, Body body) -> std::vector { - using Slot = std::conditional_t, std::uint8_t, R>; - std::vector staging(static_cast(group.partition_count())); - group.run_on_all([&](int r) { staging[static_cast(r)] = static_cast(body(r)); }); - if constexpr (std::is_same_v) { - return std::vector(staging.begin(), staging.end()); - } - else { - return staging; - } -} - -// collect_on_all over the partition propagators themselves: `body(partition)` on each partition's master. -template &>> -auto map_partitions(PartitionGroup &group, Body body) -> std::vector { - return collect_on_all(group, [&](int r) -> R { return body(group.partition(r)); }); +// `body` must not touch the vector itself -- see detail::staged_collect for what that rules out. +template > +auto collect_on_all(PartitionGroup &group, Body body) -> std::vector { + return detail::staged_collect(static_cast(group.partition_count()), [&group, &body](auto &&emit) { + group.run_on_all([&emit, &body](int r) { emit(r, body(r)); }); + }); } } // namespace detail::partition diff --git a/cpp/monoprop/detail/partition/StagedCollect.h b/cpp/monoprop/detail/partition/StagedCollect.h new file mode 100644 index 00000000..f8c9cf31 --- /dev/null +++ b/cpp/monoprop/detail/partition/StagedCollect.h @@ -0,0 +1,46 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace monoprop::detail { + +// One result per partition, indexed by partition rank, written concurrently from the partitions' own +// masters. `fan_out(emit)` runs `emit(rank, value)` once per rank, on whichever thread owns that rank. +// +// The staging vector exists for the bool case and only for it: std::vector is the bit-packed +// specialization, so concurrent writes to different logical elements can tear the same underlying word +// -- a data race even though the indices are disjoint. Everything else is written in place and moved out. +// +// Its own header, with no dependency beyond , because both fan-out paths need it and they must +// not see each other: MonomialPropagator reaches its partitions through a type-erased primitive +// precisely so the public header never sees PartitionGroup. +template +auto staged_collect(size_t n, FanOut &&fan_out) -> std::vector { + using Slot = std::conditional_t, std::uint8_t, R>; + std::vector staging(n); + fan_out([&staging](int r, R value) { staging[static_cast(r)] = static_cast(value); }); + if constexpr (std::is_same_v) { + return std::vector(staging.begin(), staging.end()); + } + else { + return staging; + } +} + +} // namespace monoprop::detail diff --git a/cpp/tests/AlgebraReference.h b/cpp/tests/AlgebraReference.h index 5c7615cd..945bf29a 100644 --- a/cpp/tests/AlgebraReference.h +++ b/cpp/tests/AlgebraReference.h @@ -17,26 +17,30 @@ // Majorana helpers the shipped library no longer calls, kept alive for tests/cpp/mpfunctions.cpp. #include +#include #include #include "monoprop/algebra/MajoranaAlgebra.h" namespace monoprop { -template -auto fermionic_to_binary_operator(const std::vector &op) -> MonomialList { - auto majorana_operator = MonomialList(op.size()); - std::transform(op.cbegin(), op.cend(), majorana_operator.begin(), indices_to_bitset); +inline auto fermionic_to_binary_operator(size_t num_modes, const std::vector &op) -> MonomialList { + MonomialList majorana_operator; + majorana_operator.reserve(op.size()); + // push_back, not a sized construction plus transform: sizing up front would fill with width-0 + // bitsets, and every slot is written here anyway. + std::ranges::transform(op, std::back_inserter(majorana_operator), [num_modes](const VecZ &term) { + return indices_to_bitset(term, 2 * num_modes); + }); return majorana_operator; } -template -auto get_multiplicative_phase(const Monomial &mono, - const Monomial &gen_mono, +auto get_multiplicative_phase(const MonomialLike auto &mono, + const MonomialLike auto &gen_mono, size_t mono_count, size_t gen_count, size_t overlap) -> int { - return interleave_phase(mono, gen_mono) * hermitian_phase(mono_count, gen_count, overlap); + return interleave_phase(mono, gen_mono) * hermitian_phase(mono_count, gen_count, overlap); } } // namespace monoprop diff --git a/cpp/tests/InlineWidths.h b/cpp/tests/InlineWidths.h new file mode 100644 index 00000000..dab4da91 --- /dev/null +++ b/cpp/tests/InlineWidths.h @@ -0,0 +1,43 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "monoprop/Bitset.h" + +namespace test_utils { + +// Runs `body` once per compile-time width in [1, Bitset::kInlineWords] (32..256 inline modes). +// +// kNarrowKernelWords controls which widths pick a specialized scan kernel. The kernels themselves must +// work for the full inline range. We stop at kInlineWords because larger widths spill to heap storage, +// which word kernels do not allow. +// +// Keep this width sweep defined in one place. Multiple test files rely on the same range, and duplicating +// this fold could let one copy drift without notice. +template +auto for_each_inline_width(std::index_sequence, auto &&body) -> void { + // +1 because W == 0 is not a width any word kernel accepts. + (body(std::integral_constant{}), ...); +} + +auto for_each_inline_width(auto &&body) -> void { + for_each_inline_width(std::make_index_sequence{}, + std::forward(body)); +} + +} // namespace test_utils diff --git a/cpp/tests/PauliTestOracle.cpp b/cpp/tests/PauliTestOracle.cpp new file mode 100644 index 00000000..415b391d --- /dev/null +++ b/cpp/tests/PauliTestOracle.cpp @@ -0,0 +1,241 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "PauliTestOracle.h" + +namespace pauli_oracle { + +namespace { +constexpr char LETTERS[4] = {'I', 'X', 'Y', 'Z'}; +} // namespace + +auto slots_of_string(const std::string &p) -> VecZ { + VecZ slots; + for (size_t q = 0; q < p.size(); ++q) { + switch (p[q]) { + case 'X': + slots.push_back(2 * q); + break; + case 'Y': + slots.push_back(2 * q + 1); + break; + case 'Z': + slots.push_back(2 * q); + slots.push_back(2 * q + 1); + break; + default: + break; // 'I' + } + } + return slots; +} + +auto native_bitset(size_t num_modes, const std::string &p) -> Bitset { + return indices_to_bitset(slots_of_string(p), 2 * num_modes); +} + +auto letter_from_bitset(const Bitset &mono, size_t q) -> char { + // Slots are MSb0 over the monomial's own width, which is the storage width and need not be + // 2*num_modes -- the propagator rounds a logical width up to a whole block. + const bool u = mono.test(mono.size() - 1 - 2 * q); // slot 2q + const bool v = mono.test(mono.size() - 2 - 2 * q); // slot 2q+1 + if (!u && !v) { + return 'I'; + } + if (u && !v) { + return 'X'; + } + if (!u && v) { + return 'Y'; + } + return 'Z'; +} + +auto pauli_to_fermi_indices(const std::string &pauli) -> VecZ { + std::vector acc; + bool flag_z = false; + for (int i = static_cast(pauli.size()) - 1; i >= 0; --i) { + const char p = pauli[static_cast(i)]; + const auto ii = static_cast(i); + if ((p == 'Z' && !flag_z) || (p == 'I' && flag_z)) { + acc.push_back(2 * ii + 1); + acc.push_back(2 * ii); + } + else if (p == 'X' && !flag_z) { + acc.push_back(2 * ii); + flag_z = true; + } + else if (p == 'X' && flag_z) { + acc.push_back(2 * ii + 1); + flag_z = false; + } + else if (p == 'Y' && !flag_z) { + acc.push_back(2 * ii + 1); + flag_z = true; + } + else if (p == 'Y' && flag_z) { + acc.push_back(2 * ii); + flag_z = false; + } + // (Z, flag_z) and (I, !flag_z): no-op + } + return VecZ(acc.rbegin(), acc.rend()); +} + +auto jw_bitset(size_t num_modes, const std::string &p) -> Bitset { + return indices_to_bitset(pauli_to_fermi_indices(p), 2 * num_modes); +} + +auto jw_basis(size_t num_modes, size_t n) -> MonomialList { + // The fill value carries the width: a sized MonomialList would otherwise hold width-0 bitsets, and + // the slots past 2*n are never assigned below yet still reach change_basis's XOR. + MonomialList basis(2 * num_modes, Bitset(2 * num_modes)); + for (size_t i = 0; i < n; ++i) { + VecZ z_str; + for (size_t z = 0; z < 2 * i; ++z) { + z_str.push_back(z); + } + VecZ even_vec = z_str; + even_vec.push_back(2 * i); + VecZ odd_vec = z_str; + odd_vec.push_back(2 * i + 1); + basis[2 * i] = indices_to_bitset(even_vec, 2 * num_modes); + basis[2 * i + 1] = indices_to_bitset(odd_vec, 2 * num_modes); + } + return basis; +} + +auto single_letter(char c) -> std::vector { + switch (c) { + case 'X': + return {cd(0, 0), cd(1, 0), cd(1, 0), cd(0, 0)}; + case 'Y': + return {cd(0, 0), cd(0, -1), cd(0, 1), cd(0, 0)}; + case 'Z': + return {cd(1, 0), cd(0, 0), cd(0, 0), cd(-1, 0)}; + default: + return {cd(1, 0), cd(0, 0), cd(0, 0), cd(1, 0)}; // I + } +} + +auto kron(const std::vector &a, size_t da, const std::vector &b, size_t db) -> std::vector { + const size_t d = da * db; + std::vector r(d * d, cd(0, 0)); + for (size_t i = 0; i < da; ++i) { + for (size_t j = 0; j < da; ++j) { + const cd aij = a[i * da + j]; + for (size_t k = 0; k < db; ++k) { + for (size_t l = 0; l < db; ++l) { + r[(i * db + k) * d + (j * db + l)] = aij * b[k * db + l]; + } + } + } + } + return r; +} + +auto matmul(const std::vector &a, const std::vector &b, size_t d) -> std::vector { + std::vector r(d * d, cd(0, 0)); + for (size_t i = 0; i < d; ++i) { + for (size_t k = 0; k < d; ++k) { + const cd aik = a[i * d + k]; + if (aik == cd(0, 0)) { + continue; + } + for (size_t j = 0; j < d; ++j) { + r[i * d + j] += aik * b[k * d + j]; + } + } + } + return r; +} + +auto matrix_from_string(const std::string &p) -> std::vector { + std::vector m = single_letter(p[0]); + size_t d = 2; + for (size_t q = 1; q < p.size(); ++q) { + m = kron(m, d, single_letter(p[q]), 2); + d *= 2; + } + return m; +} + +auto approx_equal(const std::vector &a, const std::vector &b, double tol) -> bool { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::abs(a[i] - b[i]) > tol) { + return false; + } + } + return true; +} + +auto scalar_mul(cd s, const std::vector &a) -> std::vector { + std::vector r(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + r[i] = s * a[i]; + } + return r; +} + +auto string_anticommutes(const std::string &a, const std::string &b) -> bool { + size_t local = 0; + for (size_t q = 0; q < a.size(); ++q) { + if (a[q] != 'I' && b[q] != 'I' && a[q] != b[q]) { + ++local; + } + } + return (local & 1U) != 0U; +} + +auto is_z_only(const std::string &p) -> bool { + for (char c : p) { + if (c == 'X' || c == 'Y') { + return false; + } + } + return true; +} + +auto all_strings(size_t n) -> std::vector { + std::vector out; + size_t total = 1; + for (size_t i = 0; i < n; ++i) { + total *= 4; + } + out.reserve(total); + for (size_t idx = 0; idx < total; ++idx) { + std::string s(n, 'I'); + size_t v = idx; + for (size_t q = 0; q < n; ++q) { + s[q] = LETTERS[v & 3U]; + v >>= 2U; + } + out.push_back(s); + } + return out; +} + +auto random_string(std::mt19937 &rng, size_t n) -> std::string { + std::uniform_int_distribution d(0, 3); + std::string s(n, 'I'); + for (size_t q = 0; q < n; ++q) { + s[q] = LETTERS[d(rng)]; + } + return s; +} + +} // namespace pauli_oracle diff --git a/cpp/tests/PauliTestOracle.h b/cpp/tests/PauliTestOracle.h index 64c21fdc..fcb366d8 100644 --- a/cpp/tests/PauliTestOracle.h +++ b/cpp/tests/PauliTestOracle.h @@ -17,10 +17,12 @@ // Independent Pauli reference oracle shared by the Pauli test files. Nothing here touches the // library under test beyond indices_to_bitset: the dense-matrix brute force and the JW image are // computed from first principles so the engine's inline kernels can be pinned against them. +// +// Definitions live in PauliTestOracle.cpp: five translation units include this header, so +// header-inline definitions would be five copies of an oracle that is never on a hot path. -#include #include -#include +#include #include #include #include @@ -34,237 +36,49 @@ namespace pauli_oracle { using namespace monoprop; using cd = std::complex; -inline constexpr char LETTERS[4] = {'I', 'X', 'Y', 'Z'}; - // Native symplectic-slot list for a Pauli string: X_q -> slot 2q, Y_q -> slot 2q+1, // Z_q -> {2q, 2q+1}. This is the format the propagator's initial_operator and // generators expect. -inline auto slots_of_string(const std::string &p) -> VecZ { - VecZ slots; - for (size_t q = 0; q < p.size(); ++q) { - switch (p[q]) { - case 'X': - slots.push_back(2 * q); - break; - case 'Y': - slots.push_back(2 * q + 1); - break; - case 'Z': - slots.push_back(2 * q); - slots.push_back(2 * q + 1); - break; - default: - break; // 'I' - } - } - return slots; -} +auto slots_of_string(const std::string &p) -> VecZ; -template -auto native_bitset(const std::string &p) -> Monomial { - return indices_to_bitset(slots_of_string(p)); -} +auto native_bitset(size_t num_modes, const std::string &p) -> Bitset; // Decode the single-qubit letter of qubit q from a native-encoded bitset // (MSb0 physical mapping): slot 2q is the x-plane bit, slot 2q+1 the z-plane bit. -template -auto letter_from_bitset(const Monomial &mono, size_t q) -> char { - const bool u = mono.test(2 * NumModes - 1 - 2 * q); // slot 2q - const bool v = mono.test(2 * NumModes - 2 - 2 * q); // slot 2q+1 - if (!u && !v) { - return 'I'; - } - if (u && !v) { - return 'X'; - } - if (!u && v) { - return 'Y'; - } - return 'Z'; -} +auto letter_from_bitset(const Bitset &mono, size_t q) -> char; // Faithful C++ port of _pauli_to_fermi (conversion_utils.py) -- indices only // (coeff dropped; the bitset only cares which Majorana modes are present). -inline auto pauli_to_fermi_indices(const std::string &pauli) -> VecZ { - std::vector acc; - bool flag_z = false; - for (int i = static_cast(pauli.size()) - 1; i >= 0; --i) { - const char p = pauli[static_cast(i)]; - const auto ii = static_cast(i); - if ((p == 'Z' && !flag_z) || (p == 'I' && flag_z)) { - acc.push_back(2 * ii + 1); - acc.push_back(2 * ii); - } - else if (p == 'X' && !flag_z) { - acc.push_back(2 * ii); - flag_z = true; - } - else if (p == 'X' && flag_z) { - acc.push_back(2 * ii + 1); - flag_z = false; - } - else if (p == 'Y' && !flag_z) { - acc.push_back(2 * ii + 1); - flag_z = true; - } - else if (p == 'Y' && flag_z) { - acc.push_back(2 * ii); - flag_z = false; - } - // (Z, flag_z) and (I, !flag_z): no-op - } - return VecZ(acc.rbegin(), acc.rend()); -} +auto pauli_to_fermi_indices(const std::string &pauli) -> VecZ; -template -auto jw_bitset(const std::string &p) -> Monomial { - return indices_to_bitset(pauli_to_fermi_indices(p)); -} +auto jw_bitset(size_t num_modes, const std::string &p) -> Bitset; -// jordan_wigner_basis_change(n) as a full-width (2*NumModes) basis so +// jordan_wigner_basis_change(n) as a full-width (2*num_modes) basis so // change_basis can index it by slot. -template -auto jw_basis(size_t n) -> MonomialList { - MonomialList basis(2 * NumModes); - for (size_t i = 0; i < n; ++i) { - VecZ z_str; - for (size_t z = 0; z < 2 * i; ++z) { - z_str.push_back(z); - } - VecZ even_vec = z_str; - even_vec.push_back(2 * i); - VecZ odd_vec = z_str; - odd_vec.push_back(2 * i + 1); - basis[2 * i] = indices_to_bitset(even_vec); - basis[2 * i + 1] = indices_to_bitset(odd_vec); - } - return basis; -} +auto jw_basis(size_t num_modes, size_t n) -> MonomialList; -inline auto single_letter(char c) -> std::vector { - switch (c) { - case 'X': - return {cd(0, 0), cd(1, 0), cd(1, 0), cd(0, 0)}; - case 'Y': - return {cd(0, 0), cd(0, -1), cd(0, 1), cd(0, 0)}; - case 'Z': - return {cd(1, 0), cd(0, 0), cd(0, 0), cd(-1, 0)}; - default: - return {cd(1, 0), cd(0, 0), cd(0, 0), cd(1, 0)}; // I - } -} +auto single_letter(char c) -> std::vector; // Kronecker product of A (da x da) and B (db x db); A is the more-significant factor. -inline auto kron(const std::vector &a, size_t da, const std::vector &b, size_t db) -> std::vector { - const size_t d = da * db; - std::vector r(d * d, cd(0, 0)); - for (size_t i = 0; i < da; ++i) { - for (size_t j = 0; j < da; ++j) { - const cd aij = a[i * da + j]; - for (size_t k = 0; k < db; ++k) { - for (size_t l = 0; l < db; ++l) { - r[(i * db + k) * d + (j * db + l)] = aij * b[k * db + l]; - } - } - } - } - return r; -} +auto kron(const std::vector &a, size_t da, const std::vector &b, size_t db) -> std::vector; -inline auto matmul(const std::vector &a, const std::vector &b, size_t d) -> std::vector { - std::vector r(d * d, cd(0, 0)); - for (size_t i = 0; i < d; ++i) { - for (size_t k = 0; k < d; ++k) { - const cd aik = a[i * d + k]; - if (aik == cd(0, 0)) { - continue; - } - for (size_t j = 0; j < d; ++j) { - r[i * d + j] += aik * b[k * d + j]; - } - } - } - return r; -} +auto matmul(const std::vector &a, const std::vector &b, size_t d) -> std::vector; // Dense matrix of a Pauli string (qubit 0 = most-significant tensor factor). -inline auto matrix_from_string(const std::string &p) -> std::vector { - std::vector m = single_letter(p[0]); - size_t d = 2; - for (size_t q = 1; q < p.size(); ++q) { - m = kron(m, d, single_letter(p[q]), 2); - d *= 2; - } - return m; -} +auto matrix_from_string(const std::string &p) -> std::vector; -inline auto approx_equal(const std::vector &a, const std::vector &b, double tol = 1e-9) -> bool { - if (a.size() != b.size()) { - return false; - } - for (size_t i = 0; i < a.size(); ++i) { - if (std::abs(a[i] - b[i]) > tol) { - return false; - } - } - return true; -} +auto approx_equal(const std::vector &a, const std::vector &b, double tol = 1e-9) -> bool; -inline auto scalar_mul(cd s, const std::vector &a) -> std::vector { - std::vector r(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - r[i] = s * a[i]; - } - return r; -} +auto scalar_mul(cd s, const std::vector &a) -> std::vector; // Local anticommutation from the strings alone: anticommute iff an odd number of // qubits carry two distinct non-identity letters. -inline auto string_anticommutes(const std::string &a, const std::string &b) -> bool { - size_t local = 0; - for (size_t q = 0; q < a.size(); ++q) { - if (a[q] != 'I' && b[q] != 'I' && a[q] != b[q]) { - ++local; - } - } - return (local & 1U) != 0U; -} +auto string_anticommutes(const std::string &a, const std::string &b) -> bool; -inline auto is_z_only(const std::string &p) -> bool { - for (char c : p) { - if (c == 'X' || c == 'Y') { - return false; - } - } - return true; -} +auto is_z_only(const std::string &p) -> bool; -inline auto all_strings(size_t n) -> std::vector { - std::vector out; - size_t total = 1; - for (size_t i = 0; i < n; ++i) { - total *= 4; - } - out.reserve(total); - for (size_t idx = 0; idx < total; ++idx) { - std::string s(n, 'I'); - size_t v = idx; - for (size_t q = 0; q < n; ++q) { - s[q] = LETTERS[v & 3U]; - v >>= 2U; - } - out.push_back(s); - } - return out; -} +auto all_strings(size_t n) -> std::vector; -inline auto random_string(std::mt19937 &rng, size_t n) -> std::string { - std::uniform_int_distribution d(0, 3); - std::string s(n, 'I'); - for (size_t q = 0; q < n; ++q) { - s[q] = LETTERS[d(rng)]; - } - return s; -} +auto random_string(std::mt19937 &rng, size_t n) -> std::string; } // namespace pauli_oracle diff --git a/cpp/tests/README.md b/cpp/tests/README.md index 6fc34490..30b113d1 100644 --- a/cpp/tests/README.md +++ b/cpp/tests/README.md @@ -13,6 +13,14 @@ discovers every Boost.Test case and registers it twice: `mpiexec -n ` for each rank in `monoprop_MPI_TEST_PROCS` (default `2`), registered when an MPI launcher is detected. +Both `serial` and `mpi` also get a `sparse-rows` variant per case/rank count +(`monoprop_ROW_STORE=sparse` forced), because the sparse backend has its own +wire format (`query_payload_words_for`'s stride, the escape tail) with no dense +counterpart, and it is the backend wide systems resolve to. The MPI sparse-rows +ranks are a separate list, `monoprop_MPI_SPARSE_ROWS_TEST_PROCS` (default `2`), +kept independent of `monoprop_MPI_TEST_PROCS` so widening dense rank coverage +does not silently multiply how many sparse-row `mpiexec` launches CI pays for. + Cases that need multiple ranks check `monoprop::mpi::size(MPI_COMM_WORLD)` and skip (with a message) when run with too few. @@ -60,7 +68,7 @@ name and cannot address suite-nested cases, tests use flat `LihFixture` = LiH/n=12), the `build_simulator`/`SimulatorConfig` helpers, expectation-value helpers, and the `near()` float comparison used by the equivalence suites. -- **`PauliTestOracle.h`**: independent Pauli reference oracle — native/JW +- **`PauliTestOracle.{h,cpp}`**: independent Pauli reference oracle — native/JW encoding (`slots_of_string`, `native_bitset`, `jw_basis`), dense Pauli-matrix brute force (`matrix_from_string`, `matmul`, ...), and string helpers. Shared by the Pauli algebra/build-layer tests and the equivalence suites. @@ -125,6 +133,10 @@ rank coverage: `-Dmonoprop_MPI_TEST_PROCS='1;2;4'`. To run a single case under MPI while debugging, invoke the binary directly: `mpirun -n 2 build/editable/Release/bin/monoprop_unit_tests.x --run_test=`. +The sparse-rows MPI variants (`monoprop_ROW_STORE=sparse`, label `sparse-rows`) +use their own rank list, `monoprop_MPI_SPARSE_ROWS_TEST_PROCS` (default `2`), +independent of `monoprop_MPI_TEST_PROCS`. + ## Adding New Tests 1. Add a `*.cpp` with flat `BOOST_AUTO_TEST_CASE`s (shared name prefix). diff --git a/cpp/tests/RandomMonomial.h b/cpp/tests/RandomMonomial.h new file mode 100644 index 00000000..38f6b00c --- /dev/null +++ b/cpp/tests/RandomMonomial.h @@ -0,0 +1,48 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "monoprop/Bitset.h" + +namespace test_utils { + +// A random monomial over `num_modes` modes occupying at most `max_slots` of them, each with a uniformly +// random non-empty code (one Majorana of the mode, the other, or the pair). +// +// One definition, deliberately: this is the input distribution of the whole randomized sparse/codes test +// surface, and what it biases toward -- paired slots in particular, which is what drives spills and +// product overflow -- decides what those tests actually cover. A per-file copy would let one of them be +// tuned and the rest silently left behind. Kept out of TestUtilities.h, which pulls in Boost.Test, +// MonomialPropagator and MPI; the files that want this want nothing else. +inline auto random_monomial(std::mt19937_64 &rng, size_t num_modes, size_t max_slots) -> monoprop::Bitset { + monoprop::Bitset mono(2 * num_modes); + const size_t occupied = rng() % (max_slots + 1); + for (size_t k = 0; k < occupied; ++k) { + const size_t mode = rng() % num_modes; + const auto code = 1U + static_cast(rng() % 3U); + if ((code & 1U) != 0U) { + mono.set(2 * mode); + } + if ((code & 2U) != 0U) { + mono.set((2 * mode) + 1); + } + } + return mono; +} + +} // namespace test_utils diff --git a/cpp/tests/TestData.h b/cpp/tests/TestData.h index 7fd0c22c..cc698313 100644 --- a/cpp/tests/TestData.h +++ b/cpp/tests/TestData.h @@ -36,4 +36,47 @@ struct CaseData { // Throws std::runtime_error if the fixture cannot be read or parsed. auto load_case(const std::filesystem::path& filename) -> CaseData; +// A monotone injection of a case's modes into the modes of a wider system -- the C++ twin of +// tests/cases.py's ModeEmbedding, carrying the same map. +// +// Relabelling modes monotonically is a canonical transformation: the map is strictly increasing, so a +// sorted Majorana index tuple stays sorted and no anticommutation sign appears, and the physics -- the +// reference expectation value included -- is the source problem's. That is how the suite reaches a +// storage width no checked-in fixture has, with no second reference calculation and no second fixture +// whose only difference from an existing one is a permutation. +struct ModeEmbedding { + size_t num_modes{0}; ///< Width of the embedding system. + monoprop::VecZ modes; ///< Where source mode m lands; strictly increasing and below num_modes. + + /// One Majorana index of the source system, in the embedding system. + [[nodiscard]] auto majorana(size_t index) const -> size_t { return (2 * modes[index / 2]) + (index % 2); } +}; + +/// `data` relabelled through `embedding`. Parameters, coefficients and actual_expval carry over as they +/// are; only mode-indexed data moves. +inline auto embed_case(const CaseData& data, const ModeEmbedding& embedding) -> CaseData { + const auto map_indices = [&](const monoprop::VecZ& indices) { + monoprop::VecZ out; + out.reserve(indices.size()); + for (const auto index : indices) { + out.push_back(embedding.majorana(index)); + } + return out; + }; + + CaseData out = data; + out.num_modes = embedding.num_modes; + for (auto& mode : out.initial_state) { + mode = embedding.modes.at(mode); + } + for (auto& mono : out.majoranas) { + mono = map_indices(mono); + } + out.hamiltonian.clear(); + for (const auto& [indices, coeff] : data.hamiltonian) { + out.hamiltonian.emplace(map_indices(indices), coeff); + } + return out; +} + } // namespace test_utils diff --git a/cpp/tests/TestOperator.h b/cpp/tests/TestOperator.h new file mode 100644 index 00000000..2d20e589 --- /dev/null +++ b/cpp/tests/TestOperator.h @@ -0,0 +1,44 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/operator/MPOperator.h" + +namespace test_utils { + +// An MPOperator holding `terms` whose rows are also *findable*: append_term writes a row and nothing +// else, so find()/find_batch see nothing until the hash index is populated, which only the +// insert_absent_terms path does. That sequence is the one correct incantation for "an operator a resolve +// can look terms up in", so it lives here rather than being copied into each test that needs one. +inline auto indexed_operator(size_t num_bits, + const monoprop::MonomialList &terms, + monoprop::Basis basis = monoprop::Basis::Majorana) -> monoprop::detail::MPOperator { + monoprop::detail::MPOperator op(num_bits); + op.basis = basis; + op.with_store([&](auto &rows) { + monoprop::detail::insert_absent_terms( + op, + rows, + terms.size(), + [&](size_t k) -> const monoprop::Bitset & { return terms[k]; }, + [&](size_t k, size_t base) { assign_row(rows, base + k, terms[k]); }); + }); + return op; +} + +} // namespace test_utils diff --git a/cpp/tests/TestPropagator.h b/cpp/tests/TestPropagator.h new file mode 100644 index 00000000..0284fe27 --- /dev/null +++ b/cpp/tests/TestPropagator.h @@ -0,0 +1,58 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/mpi/MPICompat.h" + +namespace test_utils { +using namespace monoprop; + +// Every propagator in this suite is built through here, so a change to the ctor's argument list is one +// edit rather than 26. +// +// num_modes is the system's width; the propagator rounds it up to a whole 32-mode block to get the width +// it stores monomials at. That width is not cosmetic -- indices are laid out MSb0, so it moves every bit +// position, which changes each monomial's hash, which changes owner routing, probe order, and therefore +// the order coefficients accumulate in. Expected values here are pinned to the rounded width. +inline auto make_propagator(size_t num_modes, + const OperatorDict& initial_operator, + unsigned int cutoff, + const VecZ& initial_state, + std::optional schrodinger_cutoff = std::nullopt, + mpi::Comm comm = MPI_COMM_SELF, + std::optional lower_atol = std::nullopt, + std::optional upper_atol = std::nullopt, + CutoffType cutoff_type = CutoffType::Length, + std::optional> basis_change = std::nullopt, + Basis basis = Basis::Majorana, + size_t partitions = 0) -> MonomialPropagator { + return MonomialPropagator(initial_operator, + cutoff, + initial_state, + num_modes, + schrodinger_cutoff, + comm, + lower_atol, + upper_atol, + cutoff_type, + basis_change, + basis, + partitions); +} +} // namespace test_utils diff --git a/cpp/tests/TestUtilities.h.in b/cpp/tests/TestUtilities.h.in index 37f590ed..c63c539c 100644 --- a/cpp/tests/TestUtilities.h.in +++ b/cpp/tests/TestUtilities.h.in @@ -32,6 +32,7 @@ #include #include "TestData.h" +#include "TestPropagator.h" #include "monoprop/MonomialPropagator.h" #include "monoprop/detail/mpi/MPICompat.h" @@ -70,7 +71,6 @@ static inline auto test_data_path() -> fs::path { return fs::path("@PROJECT_SOURCE_DIR@/tests/data"); } -template inline auto load_case_data(const std::string& filename) -> CaseData { const fs::path data_path = test_data_path() / filename; BOOST_REQUIRE_MESSAGE(fs::exists(data_path), "Missing msgpack data file: " << data_path); @@ -86,22 +86,22 @@ struct SimulatorConfig { std::optional> basis_change = std::nullopt; }; -template -inline auto build_simulator(const CaseData& data, const SimulatorConfig& cfg = {}) -> MonomialPropagator { - const auto cutoff = static_cast(2 * NumModes); - return MonomialPropagator(data.hamiltonian, - cutoff, - data.initial_state, - cfg.schrodinger_cutoff, - cfg.comm, - cfg.atol, - cfg.upper_atol, - cfg.cutoff_type, - cfg.basis_change); +inline auto build_simulator(size_t num_modes, const CaseData& data, const SimulatorConfig& cfg = {}) + -> MonomialPropagator { + const auto cutoff = static_cast(2 * num_modes); + return make_propagator(num_modes, + data.hamiltonian, + cutoff, + data.initial_state, + cfg.schrodinger_cutoff, + cfg.comm, + cfg.atol, + cfg.upper_atol, + cfg.cutoff_type, + cfg.basis_change); } -template -inline auto evaluate_expval(MonomialPropagator& sim, const CaseData& data, bool pare) -> double { +inline auto evaluate_expval(MonomialPropagator& sim, const CaseData& data, bool pare) -> double { sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const std::optional pare_threshold = pare ? std::optional{1e-10} : std::nullopt; auto expval_fn = sim.expectation_value_functional(pare_threshold); @@ -121,10 +121,12 @@ inline auto near(double lhs, double rhs, double atol = 1e-9, double rtol = kFpRt return std::abs(lhs - rhs) <= (atol + rtol * scale); } -template -inline auto test_evolve_build_graph(const CaseData& data, const SimulatorConfig& cfg, bool pare, double exact_expval) - -> void { - auto mp = build_simulator(data, cfg); +inline auto test_evolve_build_graph(size_t n_modes, + const CaseData& data, + const SimulatorConfig& cfg, + bool pare, + double exact_expval) -> void { + auto mp = build_simulator(n_modes, data, cfg); mp.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const std::optional pare_threshold = pare ? std::optional{1e-10} : std::nullopt; @@ -136,12 +138,12 @@ inline auto test_evolve_build_graph(const CaseData& data, const SimulatorConfig& } } -template -inline auto test_evolve_build_graph_with_coeffs(const CaseData& data, +inline auto test_evolve_build_graph_with_coeffs(size_t n_modes, + const CaseData& data, const SimulatorConfig& cfg, bool pare, double exact_expval) -> void { - auto mp = build_simulator(data, cfg); + auto mp = build_simulator(n_modes, data, cfg); // Coefficient-informed build; the seed is computed internally and the bare nullopt is gate_indices. mp.build_graph(data.majoranas, data.param_inds, data.gen_coeffs, std::nullopt, data.parameters); @@ -160,12 +162,12 @@ inline auto test_evolve_build_graph_with_coeffs(const CaseData& data, // only, so `cfg.schrodinger_cutoff` must be set: a Heisenberg build consumes each call back-to-front, // so a forward split is not equivalent to one call. Each call's `parameters` covers the prefix its own // mapping reaches, which is what the seeding guard demands of the second call. -template -inline auto test_evolve_build_graph_with_coeffs_extend(const CaseData& data, +inline auto test_evolve_build_graph_with_coeffs_extend(size_t n_modes, + const CaseData& data, const SimulatorConfig& cfg, bool pare, double exact_expval) -> void { - auto mp = build_simulator(data, cfg); + auto mp = build_simulator(n_modes, data, cfg); const size_t k = data.majoranas.size() / 2; const auto slice_z = [](const VecZ& v, size_t lo, size_t hi) { @@ -217,7 +219,7 @@ struct ExampleDataFix { struct LihFixture { static constexpr size_t n_modes = 12; CaseData data; - LihFixture() : data(load_case_data("lih_fermionic_spin_exact.msgpack")) {} + LihFixture() : data(load_case_data("lih_fermionic_spin_exact.msgpack")) {} }; inline constexpr std::array ds_pare_values{false, true}; diff --git a/cpp/tests/bitset_tests.cpp b/cpp/tests/bitset_tests.cpp index 6a3bac26..f42d49c5 100644 --- a/cpp/tests/bitset_tests.cpp +++ b/cpp/tests/bitset_tests.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Bitset.h in isolation (single-word and multi-word) against a std::bitset oracle, so a regression -// in the hand-rolled shift / scan / mask surfaces here rather than as a distant energy drift. +// Bitset.h in isolation (inline and heap-spilled widths) against a std::bitset oracle, so a +// regression in the hand-rolled shift / scan / mask / trampoline surfaces here rather than as a +// distant energy drift. N stays a *template* parameter of the test helpers purely so std::bitset +// (the oracle) can be spelled; the Bitset under test is always constructed with a runtime width. #include @@ -29,8 +31,8 @@ using monoprop::Bitset; namespace { template -auto make_pair(const std::vector &positions) -> std::pair, std::bitset> { - Bitset bs; +auto make_pair(const std::vector &positions) -> std::pair> { + Bitset bs(N); std::bitset ref; for (size_t p : positions) { bs.set(p); @@ -40,7 +42,8 @@ auto make_pair(const std::vector &positions) -> std::pair, std } template -auto expect_equal(const Bitset &bs, const std::bitset &ref) -> void { +auto expect_equal(const Bitset &bs, const std::bitset &ref) -> void { + BOOST_TEST(bs.size() == N); for (size_t i = 0; i < N; ++i) { BOOST_TEST(bs.test(i) == ref.test(i), "bit " << i); } @@ -49,12 +52,12 @@ auto expect_equal(const Bitset &bs, const std::bitset &ref) -> void { } // namespace -// The ctor masks off bits beyond NumBits (kTopMask), so a partial top word never leaks stray high bits. +// The ctor masks off bits beyond the requested width, so a partial top word never leaks stray high bits. BOOST_AUTO_TEST_CASE(bitset_ctor_sanitizes_top) { - const Bitset<10> b(0xFFFFULL); + const Bitset b(10, 0xFFFFULL); BOOST_TEST(b.count() == 10U); BOOST_TEST(b.word(0) == 0x3FFULL); - const Bitset<64> full(~uint64_t{0}); + const Bitset full(64, ~uint64_t{0}); BOOST_TEST(full.count() == 64U); } @@ -80,16 +83,35 @@ BOOST_AUTO_TEST_CASE(bitset_count_and_parity_and_cross_word) { BOOST_TEST(!c.parity_and(d)); } +// fused_xor must agree with the composed operator^ / count_and it replaces in the hot path +// (TermProduct.h's emit_term_products) -- same operands, same three quantities, one pass instead of two. +BOOST_AUTO_TEST_CASE(bitset_fused_xor_matches_composed_ops) { + auto [a, ra] = make_pair<192>({1, 63, 64, 130, 191}); + auto [b, rb] = make_pair<192>({63, 64, 65, 130}); + const auto fused = a.fused_xor(b); + expect_equal<192>(fused.result, ra ^ rb); + BOOST_TEST(fused.overlap == a.count_and(b)); + BOOST_TEST(fused.result_count == (ra ^ rb).count()); + + // Single-word path (num_words() == 1) takes the same loop body, just one iteration. + auto [c, rc] = make_pair<64>({0, 7, 31, 63}); + auto [d, rd] = make_pair<64>({7, 8, 31}); + const auto fused_sw = c.fused_xor(d); + expect_equal<64>(fused_sw.result, rc ^ rd); + BOOST_TEST(fused_sw.overlap == c.count_and(d)); + BOOST_TEST(fused_sw.result_count == (rc ^ rd).count()); +} + BOOST_AUTO_TEST_CASE(bitset_not_respects_top_mask) { - BOOST_TEST((~Bitset<100>{}).count() == 100U); - BOOST_TEST((~Bitset<64>{}).count() == 64U); - BOOST_TEST((~Bitset<10>{}).count() == 10U); + BOOST_TEST((~Bitset(100)).count() == 100U); + BOOST_TEST((~Bitset(64)).count() == 64U); + BOOST_TEST((~Bitset(10)).count() == 10U); auto [bs, ref] = make_pair<100>({3, 70, 99}); BOOST_TEST((~~bs) == bs); (void)ref; } -// Shift amounts cover exact word multiples, sub-word crossings, and >= NumBits (which must zero the set). +// Shift amounts cover exact word multiples, sub-word crossings, and >= size() (which must zero the set). BOOST_AUTO_TEST_CASE(bitset_shift_right_cross_word) { const std::vector pos{0, 5, 63, 64, 65, 130, 191}; for (size_t s : {size_t{0}, @@ -107,7 +129,7 @@ BOOST_AUTO_TEST_CASE(bitset_shift_right_cross_word) { const std::bitset<192> expected = ref >> s; expect_equal<192>(bs, expected); } - // Single-word path (kNumWords == 1) takes a separate branch. + // Single-word path (num_words() == 1) takes a separate branch. auto [bs, ref] = make_pair<64>({0, 7, 31, 63}); bs >>= 8; expect_equal<64>(bs, ref >> 8); @@ -121,8 +143,8 @@ BOOST_AUTO_TEST_CASE(bitset_find_first_next_chain) { BOOST_TEST(bs.find_next(63) == 64U); BOOST_TEST(bs.find_next(64) == 130U); BOOST_TEST(bs.find_next(130) == 191U); - BOOST_TEST(bs.find_next(191) == 192U); // past the last set bit -> NumBits - BOOST_TEST(Bitset<192>{}.find_first() == 192U); + BOOST_TEST(bs.find_next(191) == 192U); // past the last set bit -> size() + BOOST_TEST(Bitset(192).find_first() == 192U); // Single-word find_next branch. auto [sb, sref] = make_pair<64>({0, 40}); (void)sref; @@ -133,14 +155,14 @@ BOOST_AUTO_TEST_CASE(bitset_find_first_next_chain) { // The multi-word hash must depend on which word carries a bit (the +i mix guard), and be deterministic. BOOST_AUTO_TEST_CASE(bitset_splitmix_hash_position_sensitive) { - Bitset<128> low; + Bitset low(128); low.set(0); - Bitset<128> high; + Bitset high(128); high.set(64); // bit 0 of word 1 — same intra-word position as `low`'s bit - const std::hash> h; + const std::hash h; BOOST_TEST(h(low) != h(high)); BOOST_TEST(h(low) == h(low)); - Bitset<128> low_copy; + Bitset low_copy(128); low_copy.set(0); BOOST_TEST(h(low) == h(low_copy)); } @@ -165,5 +187,105 @@ BOOST_AUTO_TEST_CASE(bitset_random_differential) { expect_equal(a >> s, ra >> s); BOOST_TEST(a.count_and(b) == (ra & rb).count()); BOOST_TEST((a == b) == (ra == rb)); + const auto fused = a.fused_xor(b); + expect_equal(fused.result, ra ^ rb); + BOOST_TEST(fused.overlap == a.count_and(b)); + BOOST_TEST(fused.result_count == (ra ^ rb).count()); } } + +// kInlineWords == 8 (512 bits): the trampoline's own boundary. 512 is the last inline width, 576 the +// first spilled one -- both must agree bit-for-bit with the oracle and with each other's operations. +BOOST_AUTO_TEST_CASE(bitset_trampoline_inline_spill_boundary) { + std::mt19937_64 rng(0xB0DA51ULL); + for (size_t n : {size_t{511}, size_t{512}, size_t{513}, size_t{576}, size_t{1024}, size_t{4096}}) { + std::uniform_int_distribution bit(0, n - 1); + std::vector pa; + std::vector pb; + for (int k = 0; k < 20; ++k) { + pa.push_back(bit(rng)); + pb.push_back(bit(rng)); + } + Bitset a(n); + Bitset b(n); + std::vector ra(n, false); + std::vector rb(n, false); + for (size_t p : pa) { + a.set(p); + ra[p] = true; + } + for (size_t p : pb) { + b.set(p); + rb[p] = true; + } + BOOST_TEST(a.num_words() == (n + 63) / 64); + BOOST_TEST(a.size() == n); + + size_t expected_and = 0; + size_t expected_xor_count = 0; + for (size_t i = 0; i < n; ++i) { + expected_and += static_cast(ra[i] && rb[i]); + expected_xor_count += static_cast(ra[i] != rb[i]); + } + BOOST_TEST(a.count_and(b) == expected_and); + + const auto x = a ^ b; + BOOST_TEST(x.count() == expected_xor_count); + for (size_t i = 0; i < n; ++i) { + BOOST_TEST(x.test(i) == (ra[i] != rb[i]), "n=" << n << " bit " << i); + } + + const auto fused = a.fused_xor(b); + BOOST_TEST(fused.overlap == expected_and); + BOOST_TEST(fused.result_count == expected_xor_count); + BOOST_TEST((fused.result == x)); + } +} + +// A spilled Bitset (n > 512 bits) must copy deeply -- mutating a copy must not alias the original's +// heap buffer. +BOOST_AUTO_TEST_CASE(bitset_spilled_copy_is_independent) { + Bitset a(1024); + a.set(1000); + Bitset b = a; // copy + b.set(5); + BOOST_TEST(a.test(5) == false); + BOOST_TEST(b.test(5) == true); + BOOST_TEST(a.test(1000) == true); + BOOST_TEST(b.test(1000) == true); + BOOST_TEST(a.count() == 1U); + BOOST_TEST(b.count() == 2U); +} + +// find_first/find_next must walk past the inline/spill boundary and across many spilled words. +BOOST_AUTO_TEST_CASE(bitset_spilled_find_chain) { + Bitset bs(2048); + bs.set(0); + bs.set(511); + bs.set(512); + bs.set(1000); + bs.set(2047); + BOOST_TEST(bs.find_first() == 0U); + BOOST_TEST(bs.find_next(0) == 511U); + BOOST_TEST(bs.find_next(511) == 512U); + BOOST_TEST(bs.find_next(512) == 1000U); + BOOST_TEST(bs.find_next(1000) == 2047U); + BOOST_TEST(bs.find_next(2047) == 2048U); +} + +// Equality must be symmetric even across widths. A width-0 bitset used to compare equal to everything +// while nothing compared equal to it, which in a hash map is silent corruption rather than a crash. +BOOST_AUTO_TEST_CASE(bitset_equality_is_symmetric_across_widths) { + const Bitset zero; + const Bitset narrow(64, 0xdeadbeefULL); + const Bitset wide(256, 0xdeadbeefULL); + + BOOST_TEST(!(zero == narrow)); + BOOST_TEST(!(narrow == zero)); + // Same words, different widths: still distinct. + BOOST_TEST(!(narrow == wide)); + BOOST_TEST(!(wide == narrow)); + // Same width, same words: equal both ways. + BOOST_TEST((wide == Bitset(256, 0xdeadbeefULL))); + BOOST_TEST((Bitset(256, 0xdeadbeefULL) == wide)); +} diff --git a/cpp/tests/boost-test.cmake b/cpp/tests/boost-test.cmake index 7d0d52c9..b8f960cd 100644 --- a/cpp/tests/boost-test.cmake +++ b/cpp/tests/boost-test.cmake @@ -4,6 +4,12 @@ set( CACHE STRING "Semicolon-separated list of ranks for MPI test variants" ) +set( + monoprop_MPI_SPARSE_ROWS_TEST_PROCS + "2" + CACHE STRING + "Semicolon-separated list of ranks for the sparse-row-backend MPI test variants (kept separate from monoprop_MPI_TEST_PROCS so growing dense-backend MPI coverage does not silently multiply how many sparse-row mpiexec launches CI pays for)" +) set(_monoprop_mpiexec "${MPIEXEC_EXECUTABLE}") if(NOT _monoprop_mpiexec) @@ -71,6 +77,7 @@ function(discover_tests TARGET) "TEST_LIST=${_TEST_LIST}" -D "CTEST_FILE=${ctest_tests_file}" -D "TEST_ENABLE_MPI_VARIANTS=${_enable_mpi_variants}" -D "TEST_MPI_NUMPROCS=${monoprop_MPI_TEST_PROCS}" -D + "TEST_MPI_SPARSE_ROWS_NUMPROCS=${monoprop_MPI_SPARSE_ROWS_TEST_PROCS}" -D "MPIEXEC_EXECUTABLE=${_monoprop_mpiexec}" -D "MPIEXEC_NUMPROC_FLAG=${_monoprop_mpiexec_numproc_flag}" -D "MPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" -D diff --git a/cpp/tests/boostAddTests.cmake b/cpp/tests/boostAddTests.cmake index 74b5ff00..7faaae49 100644 --- a/cpp/tests/boostAddTests.cmake +++ b/cpp/tests/boostAddTests.cmake @@ -4,6 +4,9 @@ endif() if(NOT DEFINED TEST_MPI_NUMPROCS) set(TEST_MPI_NUMPROCS "2") endif() +if(NOT DEFINED TEST_MPI_SPARSE_ROWS_NUMPROCS) + set(TEST_MPI_SPARSE_ROWS_NUMPROCS "2") +endif() if(TEST_ENABLE_MPI_VARIANTS AND NOT MPIEXEC_EXECUTABLE) message( WARNING @@ -37,6 +40,28 @@ if(TEST_ENABLE_MPI_VARIANTS) list(REMOVE_DUPLICATES _mpi_ranks) endif() +# Validated the same way as _mpi_ranks above, but kept in its own list (TEST_MPI_SPARSE_ROWS_NUMPROCS) +# rather than reusing _mpi_ranks: growing dense-backend rank coverage must not silently multiply how +# many sparse-row mpiexec launches CI pays for. +set(_mpi_sparse_ranks) +if(TEST_ENABLE_MPI_VARIANTS) + if("${TEST_MPI_SPARSE_ROWS_NUMPROCS}" STREQUAL "") + set(TEST_MPI_SPARSE_ROWS_NUMPROCS 2) + endif() + + foreach(_rank IN LISTS TEST_MPI_SPARSE_ROWS_NUMPROCS) + if(NOT _rank MATCHES "^[1-9][0-9]*$") + message( + FATAL_ERROR + "Invalid MPI rank '${_rank}' in TEST_MPI_SPARSE_ROWS_NUMPROCS='${TEST_MPI_SPARSE_ROWS_NUMPROCS}'. Use positive integers." + ) + endif() + endforeach() + + set(_mpi_sparse_ranks ${TEST_MPI_SPARSE_ROWS_NUMPROCS}) + list(REMOVE_DUPLICATES _mpi_sparse_ranks) +endif() + set(extra_args ${TEST_EXTRA_ARGS}) set(properties ${TEST_PROPERTIES}) set(serial_env ${TEST_SERIAL_ENVIRONMENT}) @@ -209,6 +234,28 @@ foreach(LINE ${LINES}) ENVIRONMENT ${serial_env} ) + # Run the same case again with the support-form row backend forced. The suite is below + # SparseRowStore::preferred_for_modes()'s crossover, so the automatic choice would compile + # that backend but never run it, even though it is the one used for wide systems. Running each + # case separately makes any divergence easy to identify. + # + # Keep serial_env for the same reason as the variant above: this is another world-size-1 run + # of the same case, so without it half of `-L serial` would pay the MPI_Init setup cost that + # the other half avoids. + register_variant("${test}_sparse_rows" + COMMAND + "${TEST_EXECUTABLE}" + "--run_test=${test}" + "--report_level=detailed" + "--catch_system_errors=yes" + ${extra_args} + LABELS + serial + sparse-rows + ENVIRONMENT + ${serial_env} + "monoprop_ROW_STORE=sparse" + ) endif() endforeach() @@ -247,6 +294,47 @@ if(TEST_ENABLE_MPI_VARIANTS AND MPIEXEC_EXECUTABLE) "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" ) endforeach() + + # MPI counterpart of the "_sparse_rows" serial variant above: the sparse backend is not just an + # alternate local layout, it changes the wire format (query_payload_words_for's per-backend + # stride, the escape tail, kOverflowLane, append_escape_tail have no dense counterpart), and it + # is the backend wide (MPI-scale) systems actually resolve to -- so it needs its own multi-rank + # coverage, not just the single-rank one above. Runs over _mpi_sparse_ranks, not _mpi_ranks, so + # it stays cheap by default regardless of how wide the dense rank list grows. + foreach(_mpi_rank IN LISTS _mpi_sparse_ranks) + set(mpi_cmd "${MPIEXEC_EXECUTABLE}") + list( + APPEND mpi_cmd + "${MPIEXEC_NUMPROC_FLAG}" + "${_mpi_rank}" + ) + if(MPIEXEC_PREFLAGS) + list(APPEND mpi_cmd ${MPIEXEC_PREFLAGS}) + endif() + list( + APPEND mpi_cmd + "${TEST_EXECUTABLE}" + "--report_level=detailed" + "--catch_system_errors=yes" + ${extra_args} + ) + if(MPIEXEC_POSTFLAGS) + list(APPEND mpi_cmd ${MPIEXEC_POSTFLAGS}) + endif() + + register_variant("${TEST_TARGET}_mpi_${_mpi_rank}_sparse_rows" + COMMAND + ${mpi_cmd} + LABELS + mpi + "mpi-${_mpi_rank}" + sparse-rows + ENVIRONMENT + "OMPI_ALLOW_RUN_AS_ROOT=1" + "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" + "monoprop_ROW_STORE=sparse" + ) + endforeach() endif() # Create a list of all discovered tests, which users may use to e.g. set diff --git a/cpp/tests/build_graph_tests.cpp b/cpp/tests/build_graph_tests.cpp index 83e36d2b..7793d962 100644 --- a/cpp/tests/build_graph_tests.cpp +++ b/cpp/tests/build_graph_tests.cpp @@ -33,7 +33,7 @@ BOOST_DATA_TEST_CASE_F(ExampleDataFix, .cutoff_type = cutoff_type, .basis_change = basis_change, }; - test_evolve_build_graph(data, cfg, pare, data.actual_expval); + test_evolve_build_graph(n_modes, data, cfg, pare, data.actual_expval); } BOOST_DATA_TEST_CASE_F(ExampleDataFix, @@ -47,7 +47,7 @@ BOOST_DATA_TEST_CASE_F(ExampleDataFix, .cutoff_type = cutoff_type, .basis_change = basis_change, }; - test_evolve_build_graph_with_coeffs(data, cfg, pare, data.actual_expval); + test_evolve_build_graph_with_coeffs(n_modes, data, cfg, pare, data.actual_expval); } // Schrodinger-only by construction; the reason is on test_evolve_build_graph_with_coeffs_extend. @@ -58,16 +58,17 @@ BOOST_DATA_TEST_CASE_F(ExampleDataFix, build_graph_with_coeffs_extend_cases, bda .cutoff_type = cutoff_type, .basis_change = basis_change, }; - test_evolve_build_graph_with_coeffs_extend(data, cfg, pare, data.actual_expval); + test_evolve_build_graph_with_coeffs_extend(n_modes, data, cfg, pare, data.actual_expval); } // graph_size().first counts cos-scaled non-endpoints, recomputed from the operator's inverted index. BOOST_AUTO_TEST_CASE(graph_size_reports_real_cosine_only_count) { constexpr size_t N = 8; - const auto data = test_utils::load_case_data("random_exact.msgpack"); + const auto data = test_utils::load_case_data("random_exact.msgpack"); const auto sized = [&](unsigned int cutoff) { - auto sim = MonomialPropagator(data.hamiltonian, cutoff, data.initial_state, std::nullopt, MPI_COMM_SELF); + auto sim = + test_utils::make_propagator(N, data.hamiltonian, cutoff, data.initial_state, std::nullopt, MPI_COMM_SELF); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); return sim.graph_size(); }; diff --git a/cpp/tests/codes_algebra_tests.cpp b/cpp/tests/codes_algebra_tests.cpp new file mode 100644 index 00000000..ee6bdb0b --- /dev/null +++ b/cpp/tests/codes_algebra_tests.cpp @@ -0,0 +1,289 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The differential test between CodesAlgebra.h and the dense implementations it must replace. Every +// function is checked to agree *exactly* -- these are integer and sign quantities, so there is no +// tolerance to spend -- over real fixture monomials and over randomized rows, at storage widths both +// equal to and wider than the logical width. Making the codes form the default is gated on this. + +#include + +#include +#include +#include +#include +#include +#include + +#include "monoprop/MonomialPropagator.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/CodesAlgebra.h" +#include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/algebra/PauliAlgebra.h" + +#include "TestData.h" +#include "TestUtilities.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +// Rebuild a dense monomial from a row's mode lanes and an arbitrary codes word, so a codes-side +// transform (pair_swap) can be compared against its dense counterpart. sparse_row_to_bitset is the +// store's own materialization, which is the point: re-implementing it here would leave this oracle +// agreeing with a slot convention the store no longer uses. Only valid where the substituted codes word +// has the same occupancy as the row's own, which is the case for every transform here. +auto to_bitset(const SparseRow &row, RowCodes codes, size_t num_bits) -> Bitset { + return sparse_row_to_bitset(SparseRow{.modes = row.modes, .codes = codes}, num_bits); +} + +// Which outcomes the comparison actually reached. Every branch of every ported function must be +// exercised by the inputs, or agreement is vacuous -- an interleave phase stubbed to `return 1` agrees +// with the dense version on any input set that happens to contain only even permutations. +struct Seen { + bool paired = false; + bool unpaired = false; + bool y_letters = false; + bool phase_plus = false; + bool phase_minus = false; + bool anticommutes = false; + bool commutes = false; + bool cutoff_kept = false; + bool cutoff_dropped = false; + + auto operator|=(const Seen &o) -> Seen & { + paired |= o.paired; + unpaired |= o.unpaired; + y_letters |= o.y_letters; + phase_plus |= o.phase_plus; + phase_minus |= o.phase_minus; + anticommutes |= o.anticommutes; + commutes |= o.commutes; + cutoff_kept |= o.cutoff_kept; + cutoff_dropped |= o.cutoff_dropped; + return *this; + } +}; + +auto require_discriminating(const Seen &seen) -> void { + BOOST_TEST(seen.paired); + BOOST_TEST(seen.unpaired); + BOOST_TEST(seen.y_letters); + BOOST_TEST(seen.phase_plus); + BOOST_TEST(seen.phase_minus); + BOOST_TEST(seen.anticommutes); + BOOST_TEST(seen.commutes); + BOOST_TEST(seen.cutoff_kept); + BOOST_TEST(seen.cutoff_dropped); +} + +// Every single-row function at once, against the dense version of each. +auto check_row(const Bitset &mono, const SparseRow &row, size_t logical_num_modes, Seen &seen) -> void { + const size_t num_bits = mono.size(); + const size_t inactive_prefix = (num_bits / 2) - logical_num_modes; + + const auto dense_sums = cutoff_sums(mono, CutoffMasks::make(num_bits, logical_num_modes)); + const auto codes_sums = codes_cutoff_sums(row, inactive_prefix); + BOOST_TEST(codes_sums.or_sum == dense_sums.or_sum); + BOOST_TEST(codes_sums.popcount_sum == dense_sums.popcount_sum); + BOOST_TEST(codes_sums.xor_sum == dense_sums.xor_sum); + + // Both sides of each cutoff's two branches: a cutoff below the term's measure exercises the + // fully-paired escape, one above it the plain comparison. + for (const unsigned int cutoff : {0U, 1U, 2U, 4U, 8U, 64U}) { + const bool kept = codes_length_cutoff(row, cutoff, inactive_prefix); + BOOST_TEST(kept == length_cutoff(mono, cutoff, CutoffMasks::make(num_bits, logical_num_modes))); + BOOST_TEST(codes_support_cutoff(row, cutoff, inactive_prefix) + == support_cutoff(mono, cutoff, CutoffMasks::make(num_bits, logical_num_modes))); + seen.cutoff_kept |= kept; + seen.cutoff_dropped |= !kept; + } + + const bool paired = codes_is_paired(row.codes); + BOOST_TEST(paired == is_paired(mono)); + seen.paired |= paired; + seen.unpaired |= !paired; + + const size_t y = codes_pauli_y_count(row.codes); + BOOST_TEST(y == pauli_y_count(mono)); + seen.y_letters |= y > 0; + + BOOST_TEST((to_bitset(row, codes_pair_swap(row.codes), num_bits) == pair_swap(mono))); +} + +// Encode monomials into one store and hand back both the store and the dense originals. All rows are +// pushed before any view is taken: a view borrows the store's arrays, so growth would dangle it. +struct Encoded { + std::vector dense; + SparseRowStore store; + + explicit Encoded(size_t num_bits) : store(num_bits, SparseRowStore::kMaxSlots) {} + + auto add(const Bitset &mono) -> void { + dense.push_back(mono); + store.push_back(mono); + } +}; + +auto check_all(Encoded &enc, size_t logical_num_modes) -> Seen { + Seen seen; + BOOST_REQUIRE(enc.dense.size() == enc.store.size()); + for (size_t i = 0; i < enc.dense.size(); ++i) { + BOOST_REQUIRE_MESSAGE(!enc.store.spilled(i), "row " << i << " spilled; the algebra needs a codes word"); + check_row(enc.dense[i], enc.store.view(i), logical_num_modes, seen); + } + // The two-row functions, over every ordered pair for small sets and a stride otherwise: they are + // O(n^2) in the row count and the fixtures carry hundreds of terms. + const size_t n = enc.dense.size(); + const size_t stride = n > 24 ? (n / 24) + 1 : 1; + for (size_t i = 0; i < n; i += stride) { + for (size_t k = 0; k < n; k += stride) { + const auto maj = enc.store.view(i); + const auto gen = enc.store.view(k); + const int phase = codes_interleave_phase(maj, gen); + BOOST_TEST(phase == interleave_phase(enc.dense[i], enc.dense[k])); + const bool anti = codes_pauli_anticommutes(maj, gen); + BOOST_TEST(anti == pauli_anticommutes(enc.dense[i], enc.dense[k])); + seen.phase_plus |= phase > 0; + seen.phase_minus |= phase < 0; + seen.anticommutes |= anti; + seen.commutes |= !anti; + } + } + return seen; +} + +// The fixtures' Hamiltonian keys and generator index lists are the real-world monomials: Hermitian +// Majorana products, so the set includes fully paired rows, which are the inputs both cutoffs treat +// specially. +auto check_fixture(const std::string &name, size_t storage_num_modes) -> Seen { + const auto data = test_utils::load_case_data(name); + BOOST_REQUIRE(data.num_modes > 0); + BOOST_REQUIRE(storage_num_modes >= data.num_modes); + const size_t num_bits = 2 * storage_num_modes; + const size_t max_index = 2 * data.num_modes; + + Encoded enc(num_bits); + for (const auto &[inds, coeff] : data.hamiltonian) { + if (inds.size() > SparseRowStore::kMaxSlots) { + continue; // would spill; the store's own tests cover that path + } + enc.add(indices_to_bitset_checked(inds, max_index, num_bits)); + } + for (const auto &inds : data.majoranas) { + if (inds.size() > SparseRowStore::kMaxSlots) { + continue; + } + enc.add(indices_to_bitset_checked(inds, max_index, num_bits)); + } + BOOST_REQUIRE_MESSAGE(enc.dense.size() > 1, "fixture " << name << " yielded no monomials to compare"); + return check_all(enc, data.num_modes); +} + +} // namespace + +// Whole register: storage width equals the logical width, so every mode is active and the codes form +// takes its zero-prefix path. +BOOST_AUTO_TEST_CASE(codes_algebra_matches_dense_on_fixtures_whole_register) { + Seen seen; + for (const std::string name : {"random_exact.msgpack", + "lih_fermionic_spin_exact.msgpack", + "S0_8e8o_majoranic_c6.msgpack", + "majorana_lattice_layer_30.msgpack"}) { + const auto data = test_utils::load_case_data(name); + seen |= check_fixture(name, data.num_modes); + } + require_discriminating(seen); +} + +// The production layout: storage rounds up to whole 32-mode blocks, so the logical modes occupy the top +// of the register and the low physical modes are inactive. This is the case cutoff_sums applies +// active_bit_offset for, and the one the codes form has to reproduce by dropping a slot prefix. +BOOST_AUTO_TEST_CASE(codes_algebra_matches_dense_on_fixtures_padded_storage) { + Seen seen; + for (const std::string name : {"random_exact.msgpack", + "lih_fermionic_spin_exact.msgpack", + "S0_8e8o_majoranic_c6.msgpack", + "majorana_lattice_layer_30.msgpack"}) { + const auto data = test_utils::load_case_data(name); + size_t storage = monoprop::detail::storage_modes_for(data.num_modes); + if (storage == data.num_modes) { + // A mode count that is already a whole block leaves no inactive prefix, which is the case + // above. One more block is still a legal storage width -- the C++ suite passes such widths + // explicitly -- and gives this case something to exercise. + storage += 32; + } + seen |= check_fixture(name, storage); + } + require_discriminating(seen); +} + +// Randomized rows reach occupancies and code patterns the fixtures do not: single-position modes in +// every combination, rows at the slot capacity, empty rows, and inactive modes actually populated -- +// which a propagator never produces but the dense functions accept, so the two must still agree. +BOOST_AUTO_TEST_CASE(codes_algebra_matches_dense_on_randomized_rows) { + std::mt19937_64 rng(20260812U); + Seen seen; + for (const size_t storage_num_modes : {32U, 64U, 128U}) { + for (const size_t logical_num_modes : {storage_num_modes, storage_num_modes / 2, storage_num_modes - 3}) { + const size_t num_bits = 2 * storage_num_modes; + Encoded enc(num_bits); + for (size_t trial = 0; trial < 120; ++trial) { + Bitset mono(num_bits); + const size_t occupied = rng() % (SparseRowStore::kMaxSlots + 1); + // Every fourth row is forced fully paired: that is the branch both cutoffs short-circuit + // on and the only input is_paired accepts. + const bool force_paired = (trial % 4) == 0; + for (size_t k = 0; k < occupied; ++k) { + const size_t mode = rng() % storage_num_modes; + const unsigned int code = force_paired ? 0b11U : 1U + static_cast(rng() % 3U); + if ((code & 1U) != 0U) { + mono.set(2 * mode); + } + if ((code & 2U) != 0U) { + mono.set((2 * mode) + 1); + } + } + enc.add(mono); + } + seen |= check_all(enc, logical_num_modes); + } + } + require_discriminating(seen); +} + +// The identities in the header, spelled out on hand-built words so a regression names the broken one. +BOOST_AUTO_TEST_CASE(codes_algebra_word_identities) { + // Slots 0..2 = 0b11, 0b10, 0b01: one paired mode, one upper-only, one lower-only. + constexpr RowCodes codes = 0b01'10'11ULL; + BOOST_TEST(row_slot_count(codes) == 3U); + const auto sums = codes_cutoff_sums(codes); + BOOST_TEST(sums.or_sum == 3U); // n + BOOST_TEST(sums.popcount_sum == 4U); // n + d, d = 1 + BOOST_TEST(sums.xor_sum == 2U); // n - d + BOOST_TEST(!codes_is_paired(codes)); + BOOST_TEST(codes_is_paired(0b11'11ULL)); + BOOST_TEST(codes_is_paired(0U)); // an empty row is vacuously paired, as dense is_paired agrees + BOOST_TEST(codes_pauli_y_count(codes) == 1U); + BOOST_TEST(codes_pair_swap(codes) == 0b10'01'11ULL); + BOOST_TEST(codes_pair_swap(codes_pair_swap(codes)) == codes); // an involution + + BOOST_TEST(codes_popcount_below(codes, 0U) == 0U); + BOOST_TEST(codes_popcount_below(codes, 1U) == 2U); + BOOST_TEST(codes_popcount_below(codes, 2U) == 3U); + BOOST_TEST(codes_popcount_below(codes, 3U) == 4U); + // Past the last slot the answer is the whole word, and the shift that would express it is undefined. + BOOST_TEST(codes_popcount_below(codes, SparseRowStore::kMaxSlots) == 4U); +} diff --git a/cpp/tests/codes_product_tests.cpp b/cpp/tests/codes_product_tests.cpp new file mode 100644 index 00000000..7d97eff9 --- /dev/null +++ b/cpp/tests/codes_product_tests.cpp @@ -0,0 +1,257 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The scan's per-term kernel in support form, against the dense one it must replace. What +// emit_term_products computes per term is the product M(+)G, the overlap popcount(M&G), and the basis +// rotation sign; this asserts all three agree exactly, for both algebras, on real generators from the +// fixtures and on randomized rows -- including the capacity overflow, which must be reported rather than +// silently truncating a mode list. + +#include + +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/algebra/CodesAlgebra.h" +#include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/algebra/PauliAlgebra.h" + +#include "RandomMonomial.h" +#include "TestData.h" +#include "TestUtilities.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +// A row plus the lane storage behind it, so a test can hold several at once. The store's own rows borrow +// its arrays; these do not, which is what lets a product row be built and then compared. +struct OwnedRow { + std::vector lanes; + RowCodes codes = 0; + + explicit OwnedRow(size_t capacity) : lanes(capacity, 0) {} + + [[nodiscard]] auto view() const -> SparseRow { return SparseRow{lanes.data(), codes}; } + + static auto encode(const Bitset &mono, size_t capacity) -> OwnedRow { + OwnedRow row(capacity); + size_t used = 0; + for_each_mode_slot(mono, [&](size_t mode, unsigned int code) { + BOOST_REQUIRE_MESSAGE(used < capacity, "test row exceeded its capacity"); + row.lanes[used] = static_cast(mode); + row.codes |= static_cast(code) << (2 * used); + ++used; + }); + return row; + } +}; + +struct Seen { + bool cancelled_a_mode = false; // a mode present in both, cancelling to nothing + bool nonzero_overlap = false; + bool zero_overlap = false; + bool majorana_minus = false; + bool majorana_plus = false; + bool pauli_minus = false; + bool pauli_plus = false; + + auto operator|=(const Seen &o) -> Seen & { + cancelled_a_mode |= o.cancelled_a_mode; + nonzero_overlap |= o.nonzero_overlap; + zero_overlap |= o.zero_overlap; + majorana_minus |= o.majorana_minus; + majorana_plus |= o.majorana_plus; + pauli_minus |= o.pauli_minus; + pauli_plus |= o.pauli_plus; + return *this; + } +}; + +// One term against one generator, every quantity emit_term_products would produce. +auto check_product(const Bitset &mono, const Bitset &gen, size_t capacity, Seen &seen) -> void { + const size_t num_bits = mono.size(); + const auto mono_row = OwnedRow::encode(mono, capacity); + const auto gen_row = OwnedRow::encode(gen, capacity); + + // The dense reference, exactly as the scan computes it. + Bitset dense_product(num_bits); + const auto fused = mono.fused_xor_into(gen, dense_product); + + std::vector out_lanes(capacity, 0); + const auto product = sparse_toggle(mono_row.view(), gen_row.view(), std::span(out_lanes)); + BOOST_REQUIRE(!product.overflowed); + + const SparseRow product_row{out_lanes.data(), product.codes}; + BOOST_TEST(product.overlap == fused.overlap); + BOOST_TEST(product.num_slots == row_slot_count(product.codes)); + BOOST_TEST((sparse_row_to_bitset(product_row, num_bits) == dense_product)); + + // The two rotation signs. Majorana's dense form goes through the per-layer interleave mask, which is + // the hot path the sparse walk replaces, so compare against that and not only against + // interleave_phase. + const auto majorana_ctx = MajoranaAlgebra::make_gen_context(gen); + const int dense_majorana = MajoranaAlgebra::rotation_sign(majorana_ctx, mono, dense_product); + const int sparse_majorana = codes_interleave_phase(mono_row.view(), gen_row.view()); + BOOST_TEST(sparse_majorana == dense_majorana); + + const auto pauli_ctx = PauliAlgebra::make_gen_context(gen); + const int dense_pauli = PauliAlgebra::rotation_sign(pauli_ctx, mono, dense_product); + const int sparse_pauli = codes_pauli_rotation_sign(mono_row.view(), gen_row.view()); + BOOST_TEST(sparse_pauli == dense_pauli); + + seen.nonzero_overlap |= fused.overlap > 0; + seen.zero_overlap |= fused.overlap == 0; + seen.majorana_minus |= dense_majorana < 0; + seen.majorana_plus |= dense_majorana > 0; + seen.pauli_minus |= dense_pauli < 0; + seen.pauli_plus |= dense_pauli > 0; + // A cancelling mode is the case a naive union would get wrong: fewer product slots than the union of + // the two inputs' modes. + size_t shared_cancelling = 0; + for_each_mode_slot(mono, [&](size_t mode, unsigned int code) { + const auto row = gen_row.view(); + for (size_t j = 0; j < row.num_slots(); ++j) { + if (row.mode(j) == mode && row.code(j) == code) { + ++shared_cancelling; + } + } + }); + seen.cancelled_a_mode |= shared_cancelling > 0; +} + +} // namespace + +// Randomized terms against randomized generators. Generators are drawn from the same distribution and +// deliberately overlap the terms, since a disjoint generator exercises neither the overlap count nor the +// cancelling-mode branch. +BOOST_AUTO_TEST_CASE(codes_product_matches_dense_on_randomized_rows) { + std::mt19937_64 rng(20260812U); + Seen seen; + for (const size_t num_modes : {32U, 64U, 300U}) { + for (size_t trial = 0; trial < 400; ++trial) { + const auto mono = test_utils::random_monomial(rng, num_modes, 6); + const auto gen = test_utils::random_monomial(rng, num_modes, 4); + check_product(mono, gen, SparseRowStore::kMaxSlots, seen); + } + } + BOOST_TEST(seen.cancelled_a_mode); + BOOST_TEST(seen.nonzero_overlap); + BOOST_TEST(seen.zero_overlap); + BOOST_TEST(seen.majorana_minus); + BOOST_TEST(seen.majorana_plus); + BOOST_TEST(seen.pauli_minus); + BOOST_TEST(seen.pauli_plus); +} + +// Small mode counts, so terms and generators collide constantly: nearly every product goes through the +// equal-mode branch, and many modes cancel outright. +BOOST_AUTO_TEST_CASE(codes_product_matches_dense_under_heavy_overlap) { + std::mt19937_64 rng(4242U); + Seen seen; + for (size_t trial = 0; trial < 2000; ++trial) { + const auto mono = test_utils::random_monomial(rng, 6, 6); + const auto gen = test_utils::random_monomial(rng, 6, 6); + check_product(mono, gen, SparseRowStore::kMaxSlots, seen); + } + BOOST_TEST(seen.cancelled_a_mode); + BOOST_TEST(seen.nonzero_overlap); + BOOST_TEST(seen.majorana_minus); + BOOST_TEST(seen.pauli_minus); +} + +// Real generators and real terms: the fixtures' Majorana generator list against their Hamiltonian keys. +BOOST_AUTO_TEST_CASE(codes_product_matches_dense_on_fixture_generators) { + Seen seen; + size_t pairs = 0; + for (const std::string name : {"random_exact.msgpack", "lih_fermionic_spin_exact.msgpack"}) { + const auto data = test_utils::load_case_data(name); + const size_t num_bits = 2 * data.num_modes; + const size_t max_index = 2 * data.num_modes; + + std::vector terms; + for (const auto &[inds, coeff] : data.hamiltonian) { + if (inds.size() <= 12) { + terms.push_back(indices_to_bitset_checked(inds, max_index, num_bits)); + } + } + std::vector gens; + for (const auto &inds : data.majoranas) { + if (inds.size() <= 12) { + gens.push_back(indices_to_bitset_checked(inds, max_index, num_bits)); + } + } + BOOST_REQUIRE(!terms.empty()); + BOOST_REQUIRE(!gens.empty()); + + const size_t stride = terms.size() > 40 ? (terms.size() / 40) + 1 : 1; + for (size_t i = 0; i < terms.size(); i += stride) { + for (const auto &gen : gens) { + check_product(terms[i], gen, SparseRowStore::kMaxSlots, seen); + ++pairs; + } + } + } + BOOST_TEST(pairs > 100U); + BOOST_TEST(seen.nonzero_overlap); + BOOST_TEST(seen.majorana_plus); + BOOST_TEST(seen.pauli_plus); +} + +// The product occupies up to the term's modes plus the generator's, which is why a scratch row is sized +// max_mode_bound() + generator locality. Past that the answer must be "overflowed", never a truncated +// mode list beside a plausible codes word -- that combination is what made a Stage 3 capacity bug read +// as a speedup. +BOOST_AUTO_TEST_CASE(codes_product_reports_capacity_overflow) { + constexpr size_t kNumBits = 64; + Bitset mono(kNumBits); + Bitset gen(kNumBits); + for (const size_t mode : {0U, 1U, 2U}) { // three disjoint modes each + mono.set(2 * mode); + } + for (const size_t mode : {10U, 11U, 12U}) { + gen.set(2 * mode); + } + const auto mono_row = OwnedRow::encode(mono, 8); + const auto gen_row = OwnedRow::encode(gen, 8); + + // Six modes in the product, so five lanes is one short and six is exactly enough. + for (const size_t capacity : {1U, 3U, 5U}) { + std::vector lanes(capacity, 0); + const auto product = sparse_toggle(mono_row.view(), gen_row.view(), std::span(lanes)); + BOOST_TEST(product.overflowed); + BOOST_TEST(product.codes == 0U); + BOOST_TEST(product.num_slots == 0U); + } + std::vector lanes(6, 0); + const auto product = sparse_toggle(mono_row.view(), gen_row.view(), std::span(lanes)); + BOOST_TEST(!product.overflowed); + BOOST_TEST(product.num_slots == 6U); + BOOST_TEST(product.overlap == 0U); + + // A cancelling term needs *fewer* lanes than the union, so capacity is about the product and not + // about the inputs: gen against itself is empty. + std::vector same(1, 0); + const auto cancelled = sparse_toggle(gen_row.view(), gen_row.view(), std::span(same)); + BOOST_TEST(!cancelled.overflowed); + BOOST_TEST(cancelled.num_slots == 0U); + BOOST_TEST(cancelled.codes == 0U); + BOOST_TEST(cancelled.overlap == 3U); +} diff --git a/cpp/tests/combined_recompute_equivalence.cpp b/cpp/tests/combined_recompute_equivalence.cpp index 9cc31f56..6def5051 100644 --- a/cpp/tests/combined_recompute_equivalence.cpp +++ b/cpp/tests/combined_recompute_equivalence.cpp @@ -37,27 +37,24 @@ constexpr size_t kNumModes = 8; // These oracles cover the Majorana fold only; the Pauli J(G) fold generator has no equivalence test yet. constexpr auto kBasis = Basis::Majorana; -template -auto generator_of(const LayerTraversal &layer) -> Monomial { - Monomial gen{}; +auto generator_of(size_t num_modes, const LayerTraversal &layer) -> Bitset { + Bitset gen(2 * num_modes); const auto &gw = layer.generator_words(); std::memcpy(gen.data(), gw.data(), gw.size() * sizeof(uint64_t)); return gen; } // Reference oracle (test-only): replay a materialised FoldCache buffer, which the live path never does. -template -void scale_cos_cached(const monoprop::detail::FoldCache &p, double *coeff, double cos_val) { +void scale_cos_cached(const monoprop::detail::FoldCache &p, double *coeff, double cos_val) { const size_t mask_words = p.fold.mask_words; for (size_t wi = 0; wi < mask_words; ++wi) { - monoprop::detail::for_each_cos_index(wi * 64, monoprop::detail::fold_word(p, wi), [&](size_t i) { + monoprop::detail::for_each_cos_index(wi * 64, monoprop::detail::fold_word(p, wi), [&](size_t i) { coeff[i] *= cos_val; }); } } -template -double accumulate_cos_cached(const monoprop::detail::FoldCache &p, +double accumulate_cos_cached(const monoprop::detail::FoldCache &p, double *state, double *ham, double cos_val, @@ -65,7 +62,7 @@ double accumulate_cos_cached(const monoprop::detail::FoldCache &p, const size_t mask_words = p.fold.mask_words; double loc = 0.0; for (size_t wi = 0; wi < mask_words; ++wi) { - monoprop::detail::for_each_cos_index(wi * 64, monoprop::detail::fold_word(p, wi), [&](size_t i) { + monoprop::detail::for_each_cos_index(wi * 64, monoprop::detail::fold_word(p, wi), [&](size_t i) { loc += state[i] * ham[i]; ham[i] *= sec_val; state[i] *= cos_val; @@ -79,9 +76,9 @@ double accumulate_cos_cached(const monoprop::detail::FoldCache &p, // scale: coeff[i] *= cos over the layer's cosine index set — a pure per-index scatter, so the two // paths must produce byte-identical arrays. BOOST_AUTO_TEST_CASE(combined_scale_cache_equals_recompute) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(kNumModes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const auto &inverted_index = sim.mp_op().inverted_index(); @@ -102,18 +99,18 @@ BOOST_AUTO_TEST_CASE(combined_scale_cache_equals_recompute) { if (layer.generator_words().empty()) { continue; } - const auto gen = generator_of(layer); + const auto gen = generator_of(kNumModes, layer); if (gen.count() % 2 != 0) { ++odd_layers; } - auto prepared = monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), kBasis); - auto recipe = monoprop::detail::make_lazy_fold(inverted_index, gen, layer.scaled_count(), kBasis); + auto prepared = monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), kBasis); + auto recipe = monoprop::detail::make_lazy_fold(inverted_index, gen, layer.scaled_count(), kBasis); std::vector a = baseline; std::vector b = baseline; - scale_cos_cached(prepared, a.data(), cos_val); - monoprop::detail::scale_cos_lazy(inverted_index, recipe, b.data(), cos_val); + scale_cos_cached(prepared, a.data(), cos_val); + monoprop::detail::scale_cos_lazy(inverted_index, recipe, b.data(), cos_val); BOOST_TEST_INFO("layer " << li); BOOST_TEST(std::memcmp(a.data(), b.data(), n * sizeof(double)) == 0); @@ -125,9 +122,9 @@ BOOST_AUTO_TEST_CASE(combined_scale_cache_equals_recompute) { // accumulate: the per-index state/ham mutations must be byte-identical; the returned reduction may be // summed in a different order, so it is compared within a tight fp tolerance. BOOST_AUTO_TEST_CASE(combined_accumulate_cache_equals_recompute) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(kNumModes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const auto &inverted_index = sim.mp_op().inverted_index(); @@ -149,19 +146,15 @@ BOOST_AUTO_TEST_CASE(combined_accumulate_cache_equals_recompute) { if (layer.generator_words().empty()) { continue; } - const auto gen = generator_of(layer); - auto prepared = monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), kBasis); - auto recipe = monoprop::detail::make_lazy_fold(inverted_index, gen, layer.scaled_count(), kBasis); + const auto gen = generator_of(kNumModes, layer); + auto prepared = monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), kBasis); + auto recipe = monoprop::detail::make_lazy_fold(inverted_index, gen, layer.scaled_count(), kBasis); std::vector sa = state0, ha = ham0; std::vector sb = state0, hb = ham0; - const double ea = accumulate_cos_cached(prepared, sa.data(), ha.data(), cos_val, sec_val); - const double eb = monoprop::detail::accumulate_cos_lazy(inverted_index, - recipe, - sb.data(), - hb.data(), - cos_val, - sec_val); + const double ea = accumulate_cos_cached(prepared, sa.data(), ha.data(), cos_val, sec_val); + const double eb = + monoprop::detail::accumulate_cos_lazy(inverted_index, recipe, sb.data(), hb.data(), cos_val, sec_val); BOOST_TEST_INFO("layer " << li); BOOST_TEST(std::memcmp(sa.data(), sb.data(), n * sizeof(double)) == 0); @@ -177,9 +170,9 @@ BOOST_AUTO_TEST_CASE(combined_accumulate_cache_equals_recompute) { // -- so a run that ignored the record would differ by value rather than by trust. The record covers every // index, so those outside the cosine set, which the kernel never visits, are exercised too. BOOST_AUTO_TEST_CASE(a_recorded_coefficient_leaves_the_sum_undivided) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(kNumModes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const auto &inverted_index = sim.mp_op().inverted_index(); @@ -209,33 +202,31 @@ BOOST_AUTO_TEST_CASE(a_recorded_coefficient_leaves_the_sum_undivided) { if (layer.generator_words().empty()) { continue; } - const auto gen = generator_of(layer); - auto recipe = - monoprop::detail::make_lazy_fold(inverted_index, gen, layer.scaled_count(), kBasis); - auto prepared = - monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), kBasis); - if (monoprop::detail::fold_popcount(prepared) == 0) { + const auto gen = generator_of(kNumModes, layer); + auto recipe = monoprop::detail::make_lazy_fold(inverted_index, gen, layer.scaled_count(), kBasis); + auto prepared = monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), kBasis); + if (monoprop::detail::fold_popcount(prepared) == 0) { continue; } ++exercised_layers; // Sum over the cosine set of state * the recorded ham, with no scaling anywhere. std::vector ref_state = state0, ref_ham = recorded; - const double expected = monoprop::detail::accumulate_cos_lazy(inverted_index, - recipe, - ref_state.data(), - ref_ham.data(), - 1.0, - 1.0); + const double expected = monoprop::detail::accumulate_cos_lazy(inverted_index, + recipe, + ref_state.data(), + ref_ham.data(), + 1.0, + 1.0); std::vector state = state0, ham = polluted; monoprop::detail::predivide_cos_record(ham.data(), record, cos_val); - double got = monoprop::detail::accumulate_cos_lazy(inverted_index, - recipe, - state.data(), - ham.data(), - cos_val, - sec_val); + double got = monoprop::detail::accumulate_cos_lazy(inverted_index, + recipe, + state.data(), + ham.data(), + cos_val, + sec_val); got *= sec_val; monoprop::detail::restore_cos_record(ham.data(), record); @@ -246,12 +237,12 @@ BOOST_AUTO_TEST_CASE(a_recorded_coefficient_leaves_the_sum_undivided) { // Without the record the pollution shows through, so the checks above are not vacuous. std::vector bare_state = state0, bare_ham = polluted; - const double bare = monoprop::detail::accumulate_cos_lazy(inverted_index, - recipe, - bare_state.data(), - bare_ham.data(), - cos_val, - sec_val); + const double bare = monoprop::detail::accumulate_cos_lazy(inverted_index, + recipe, + bare_state.data(), + bare_ham.data(), + cos_val, + sec_val); BOOST_TEST(std::abs((bare * sec_val) - expected) > 1e-9 * (1.0 + std::abs(expected))); } BOOST_TEST(exercised_layers > 0u); @@ -274,7 +265,7 @@ BOOST_AUTO_TEST_CASE(a_cosine_near_one_leaves_the_state_unchanged) { // Lives here because it re-runs the same recompute machinery exercised above. BOOST_FIXTURE_TEST_CASE(snapshot_invariance_repeated_evaluation, ExampleDataFix) { SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(n_modes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto fn = sim.expectation_value_functional(); @@ -290,9 +281,9 @@ BOOST_FIXTURE_TEST_CASE(snapshot_invariance_repeated_evaluation, ExampleDataFix) // so it must hold no pointer into that buffer. Pins both halves — that the buffer really does move // under growth, and that a fold built before the growth still folds like a FoldCache built after it. BOOST_AUTO_TEST_CASE(lazy_fold_survives_operator_growth) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(kNumModes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); // Find an odd-|G| layer: row_parity_ is only consulted for those (Pauli and even |G| never touch it). @@ -300,7 +291,7 @@ BOOST_AUTO_TEST_CASE(lazy_fold_survives_operator_growth) { size_t odd_layer = graph.layers(); for (size_t li = 0; li < graph.layers(); ++li) { const auto layer = graph.get_layer_traversal(li); - if (!layer.generator_words().empty() && generator_of(layer).count() % 2 != 0) { + if (!layer.generator_words().empty() && generator_of(kNumModes, layer).count() % 2 != 0) { odd_layer = li; break; } @@ -308,12 +299,12 @@ BOOST_AUTO_TEST_CASE(lazy_fold_survives_operator_growth) { BOOST_REQUIRE(odd_layer < graph.layers()); const auto layer = graph.get_layer_traversal(odd_layer); - const auto gen = generator_of(layer); + const auto gen = generator_of(kNumModes, layer); const auto scaled_count = layer.scaled_count(); const uint64_t *before = sim.mp_op().inverted_index().row_parity_words(); BOOST_REQUIRE(before != nullptr); - auto recipe = monoprop::detail::make_lazy_fold(sim.mp_op().inverted_index(), gen, scaled_count, kBasis); + auto recipe = monoprop::detail::make_lazy_fold(sim.mp_op().inverted_index(), gen, scaled_count, kBasis); // Grow the operator, forcing the index and its row parity onto fresh storage. sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); @@ -328,12 +319,11 @@ BOOST_AUTO_TEST_CASE(lazy_fold_survives_operator_growth) { } const double cos_val = 0.6234; - auto prepared = - monoprop::detail::make_fold_cache(sim.mp_op().inverted_index(), gen, scaled_count, kBasis); + auto prepared = monoprop::detail::make_fold_cache(sim.mp_op().inverted_index(), gen, scaled_count, kBasis); std::vector expected = baseline; std::vector actual = baseline; - scale_cos_cached(prepared, expected.data(), cos_val); - monoprop::detail::scale_cos_lazy(sim.mp_op().inverted_index(), recipe, actual.data(), cos_val); + scale_cos_cached(prepared, expected.data(), cos_val); + monoprop::detail::scale_cos_lazy(sim.mp_op().inverted_index(), recipe, actual.data(), cos_val); BOOST_TEST(std::memcmp(expected.data(), actual.data(), n * sizeof(double)) == 0); } diff --git a/cpp/tests/ctor_validation_tests.cpp b/cpp/tests/ctor_validation_tests.cpp index 2eddcd19..a890cb75 100644 --- a/cpp/tests/ctor_validation_tests.cpp +++ b/cpp/tests/ctor_validation_tests.cpp @@ -32,7 +32,7 @@ using test_utils::SimulatorConfig; namespace { constexpr size_t N = 8; -using MP = MonomialPropagator; +using MP = MonomialPropagator; // Construct with the full argument list; individual cases vary just the field(s) under test. auto make(const OperatorDict &op, @@ -41,19 +41,19 @@ auto make(const OperatorDict &op, std::optional upper_atol = std::nullopt, CutoffType cutoff_type = CutoffType::Length, std::optional> basis_change = std::nullopt, - size_t logical_num_modes = N, + std::optional num_modes = std::nullopt, Basis basis = Basis::Majorana) -> MP { - return MP(op, - cutoff, - VecZ{}, - std::nullopt, - MPI_COMM_SELF, - lower_atol, - upper_atol, - cutoff_type, - basis_change, - logical_num_modes, - basis); + return test_utils::make_propagator(num_modes.value_or(N), + op, + cutoff, + VecZ{}, + std::nullopt, + MPI_COMM_SELF, + lower_atol, + upper_atol, + cutoff_type, + basis_change, + basis); } } // namespace @@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(ctor_accepts_valid_config) { BOOST_CHECK_NO_THROW(make(OperatorDict{})); } -BOOST_AUTO_TEST_CASE(ctor_logical_num_modes_out_of_range_throws) { +BOOST_AUTO_TEST_CASE(ctor_zero_num_modes_throws) { BOOST_CHECK_THROW(make(OperatorDict{}, 2 * N, std::nullopt, @@ -70,14 +70,34 @@ BOOST_AUTO_TEST_CASE(ctor_logical_num_modes_out_of_range_throws) { std::nullopt, /*logical=*/0), std::runtime_error); - BOOST_CHECK_THROW(make(OperatorDict{}, - 2 * N, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - /*logical=*/N + 1), - std::runtime_error); +} + +// The storage-width rule. Rounding keeps the hash index's probe +// layout aligned across nearby system sizes; the one-block floor keeps a small system off a partly +// populated word. Both are observable, since the width is part of every monomial's hash. +BOOST_AUTO_TEST_CASE(storage_modes_for_rounds_up_to_a_whole_block_with_a_floor) { + BOOST_TEST(monoprop::detail::storage_modes_for(1) == 32U); + BOOST_TEST(monoprop::detail::storage_modes_for(31) == 32U); + BOOST_TEST(monoprop::detail::storage_modes_for(32) == 32U); + BOOST_TEST(monoprop::detail::storage_modes_for(33) == 64U); + BOOST_TEST(monoprop::detail::storage_modes_for(250) == 256U); + // No ceiling: this used to be a compile-time template argument bounded by monoprop_MAX_NUM_MODES. + BOOST_TEST(monoprop::detail::storage_modes_for(4096) == 4096U); + BOOST_TEST(monoprop::detail::storage_modes_for(4097) == 4128U); +} + +BOOST_AUTO_TEST_CASE(ctor_storage_width_is_the_rounding_of_the_logical_width) { + // The storage width is not settable: it is whatever storage_modes_for() makes of num_modes, so two + // propagators over the same system cannot end up hashing monomials at different widths. N == 8 + // rounds to one 32-mode block. + const MonomialPropagator rounded(OperatorDict{}, 2 * N, VecZ{}, /*num_modes=*/N, std::nullopt, MPI_COMM_SELF); + BOOST_TEST(rounded.num_modes() == N); + BOOST_TEST(rounded.storage_num_modes() == 32U); + + // A narrower logical width still stores at the same block; it only narrows what the system means. + auto narrow = make(OperatorDict{}, 2 * N, std::nullopt, std::nullopt, CutoffType::Length, std::nullopt, 4); + BOOST_TEST(narrow.num_modes() == 4U); + BOOST_TEST(narrow.storage_num_modes() == 32U); } BOOST_AUTO_TEST_CASE(ctor_pauli_requires_support_cutoff_throws) { @@ -107,13 +127,13 @@ BOOST_AUTO_TEST_CASE(ctor_operator_index_out_of_range_throws) { BOOST_CHECK_THROW(make(op), std::runtime_error); } -// A gate generator index outside the system must throw, not underflow 2*NumModes-1-index into an +// A gate generator index outside the system must throw, not underflow 2*num_modes-1-index into an // out-of-bounds Bitset::set. BOOST_AUTO_TEST_CASE(build_graph_generator_index_out_of_range_throws) { OperatorDict op; op[VecZ{0, 1}] = std::complex(0.0, 1.0); auto sim = make(op); - // 2*logical_num_modes == 16, so slot 20 is outside this system. + // 2*num_modes == 16, so slot 20 is outside this system. BOOST_CHECK_THROW(sim.build_graph({VecZ{20, 21}}, VecZ{0}, VecD{1.0}), std::runtime_error); BOOST_CHECK_NO_THROW(sim.build_graph({VecZ{0, 3}}, VecZ{0}, VecD{1.0})); } @@ -158,7 +178,7 @@ BOOST_AUTO_TEST_CASE(setters_enforce_the_constructor_invariants) { BOOST_CHECK_THROW(pauli.update_basis_change(std::vector(2 * N, VecZ{0})), std::invalid_argument); auto majorana = make(OperatorDict{}); - // Too few rows: regenerate_cutoff_fn_ indexes [0, 2*logical_num_modes) unconditionally. + // Too few rows: regenerate_cutoff_fn_ indexes [0, 2*num_modes) unconditionally. BOOST_CHECK_THROW(majorana.update_basis_change(std::vector{VecZ{0}}), std::invalid_argument); BOOST_CHECK_THROW(majorana.update_basis_change(std::vector(2 * N, VecZ{2 * N})), std::runtime_error); std::vector identity(2 * N); @@ -169,7 +189,7 @@ BOOST_AUTO_TEST_CASE(setters_enforce_the_constructor_invariants) { } BOOST_FIXTURE_TEST_CASE(propagate_on_nonempty_graph_throws, ExampleDataFix) { - auto sim = build_simulator(data, SimulatorConfig{}); + auto sim = build_simulator(n_modes, data, SimulatorConfig{}); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); BOOST_REQUIRE(sim.graph_layers() > 0); BOOST_CHECK_THROW(sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters), @@ -178,7 +198,7 @@ BOOST_FIXTURE_TEST_CASE(propagate_on_nonempty_graph_throws, ExampleDataFix) { // Pins MPGraph::get_layer's checked_layer_offset throw site. BOOST_FIXTURE_TEST_CASE(graph_get_layer_out_of_range_throws, ExampleDataFix) { - auto sim = build_simulator(data, SimulatorConfig{}); + auto sim = build_simulator(n_modes, data, SimulatorConfig{}); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const auto &graph = sim.graph(); const size_t n_layers = graph.layers(); diff --git a/cpp/tests/evolution_detail_tests.cpp b/cpp/tests/evolution_detail_tests.cpp index 91f2662b..db06c66c 100644 --- a/cpp/tests/evolution_detail_tests.cpp +++ b/cpp/tests/evolution_detail_tests.cpp @@ -23,6 +23,7 @@ #include #include "monoprop/TypeAliases.h" +#include "monoprop/Utilities.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" @@ -30,12 +31,16 @@ #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" +#include "TestOperator.h" + using namespace monoprop; using monoprop::detail::CutoffContext; using monoprop::detail::MatchedEpochSet; namespace { +constexpr size_t kNumBits = 2 * 8; + // resolve_range_ touches only wants_values and self_hit, so the engine drives without the cross-rank sink surface. struct RecordingSink { static constexpr bool wants_values = false; @@ -43,15 +48,8 @@ struct RecordingSink { auto self_hit(size_t src, size_t found, int /*phase*/, double /*v_src*/) -> void { hits.emplace_back(src, found); } }; -// append_term writes a row only; find_batch needs the hash index, which insert_absent_terms populates. -auto indexed_op(const std::vector> &terms) -> detail::MPOperator<8> { - detail::MPOperator<8> op; - detail::insert_absent_terms<8>( - op, - terms.size(), - [&](size_t k) -> const Monomial<8> & { return terms[k]; }, - [&](size_t k, size_t base) { assign_row<8>(*op.store, base + k, terms[k]); }); - return op; +auto indexed_op(const MonomialList &terms) -> detail::MPOperator { + return test_utils::indexed_operator(kNumBits, terms); } } // namespace @@ -135,11 +133,11 @@ BOOST_AUTO_TEST_CASE(matched_epoch_stamp_wrap_reached_by_gate_count) { // A self-resolve hit whose index the store only grew into after construction is a real hit -- it must reach // the sink -- but it is outside the matched set, whose array is sized to combined_size. BOOST_AUTO_TEST_CASE(self_resolve_mark_bounded_by_combined_size) { - std::vector> terms; + MonomialList terms; for (size_t i = 0; i < 6; ++i) { - terms.push_back(indices_to_bitset<8>({i, i + 8})); + terms.push_back(indices_to_bitset({i, i + 8}, kNumBits)); } - detail::MPOperator<8> op = indexed_op(terms); + detail::MPOperator op = indexed_op(terms); const size_t combined_size = 4; // rows 4 and 5 stand for terms this layer inserted after construction MatchedEpochSet matched; @@ -147,27 +145,35 @@ BOOST_AUTO_TEST_CASE(self_resolve_mark_bounded_by_combined_size) { // than past the end of epoch_, where it would be silent undefined behaviour. matched.begin_gate(op.size()); - detail::LayerBuildEngine<8, RecordingSink> eng(op, - mpi::Comm{}, - /*R_=*/1, - /*my_rank_=*/0, - matched, - combined_size, - RecordingSink{}); - detail::query_push<8>(eng.queries_r[0], terms[1], 1); - detail::query_push<8>(eng.queries_r[0], terms[5], -1); - eng.src_idx_r[0] = {0, 2}; - - eng.resolve_self_queries(/*is_leader_pass=*/true); - - // Both keys are in the store, so both resolve as hits and neither may be deferred as a miss. - BOOST_TEST(eng.deferred_self_misses.empty()); - BOOST_TEST_REQUIRE(eng.sink.hits.size() == 2U); - BOOST_TEST(eng.sink.hits[0].second == 1U); - BOOST_TEST(eng.sink.hits[1].second == 5U); - BOOST_TEST(matched.is_marked(1)); - BOOST_TEST(!matched.is_marked(4)); - BOOST_TEST(!matched.is_marked(5)); + op.with_store([&](auto &store) { + const size_t capacity = 0; // no generator here, so no sparse record to size + detail::LayerBuildEngine> eng( + op, + store, + mpi::Comm{}, + /*R_=*/1, + /*my_rank_=*/0, + matched, + combined_size, + detail::query_payload_words_for(store, capacity), + capacity, + RecordingSink{}); + eng.queries_r[0] = detail::query_buffer(); + detail::query_push(eng.queries_r[0], terms[1], 1); + detail::query_push(eng.queries_r[0], terms[5], -1); + eng.src_idx_r[0] = {0, 2}; + + eng.resolve_self_queries(/*is_leader_pass=*/true); + + // Both keys are in the store, so both resolve as hits and neither may be deferred as a miss. + BOOST_TEST(eng.deferred_self_misses.empty()); + BOOST_TEST_REQUIRE(eng.sink.hits.size() == 2U); + BOOST_TEST(eng.sink.hits[0].second == 1U); + BOOST_TEST(eng.sink.hits[1].second == 5U); + BOOST_TEST(matched.is_marked(1)); + BOOST_TEST(!matched.is_marked(4)); + BOOST_TEST(!matched.is_marked(5)); + }); } BOOST_AUTO_TEST_CASE(cutoff_context_abs_coeff_for) { diff --git a/cpp/tests/exact_upper_atol_rescue.cpp b/cpp/tests/exact_upper_atol_rescue.cpp index 3243d965..334e495b 100644 --- a/cpp/tests/exact_upper_atol_rescue.cpp +++ b/cpp/tests/exact_upper_atol_rescue.cpp @@ -34,22 +34,21 @@ constexpr double kEnergyAtol = 1e-9; enum class CommMode { Self, World }; -// build_simulator cannot express this: it hardcodes cutoff = 2*NumModes. -template -auto build_zero_cutoff_full_rescue(const CaseData& data, MPI_Comm comm) -> MonomialPropagator { - return MonomialPropagator(data.hamiltonian, - /*cutoff=*/0U, - data.initial_state, - /*schrodinger_cutoff=*/std::nullopt, - comm, - /*atol=*/std::nullopt, - /*upper_atol=*/std::optional{0.0}, - CutoffType::Length, - /*basis_change=*/std::nullopt); +// build_simulator cannot express this: it hardcodes cutoff = 2*num_modes. +auto build_zero_cutoff_full_rescue(size_t num_modes, const CaseData& data, MPI_Comm comm) -> MonomialPropagator { + return test_utils::make_propagator(num_modes, + data.hamiltonian, + /*cutoff=*/0U, + data.initial_state, + /*schrodinger_cutoff=*/std::nullopt, + comm, + /*atol=*/std::nullopt, + /*upper_atol=*/std::optional{0.0}, + CutoffType::Length, + /*basis_change=*/std::nullopt); } -template -auto evaluate_zero_cutoff_full_rescue_energy(MonomialPropagator& simulator, const CaseData& data) -> double { +auto evaluate_zero_cutoff_full_rescue_energy(MonomialPropagator& simulator, const CaseData& data) -> double { simulator.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); auto energy_fn = simulator.expectation_value_functional(std::nullopt); return energy_fn(VecD{}); @@ -58,16 +57,16 @@ auto evaluate_zero_cutoff_full_rescue_energy(MonomialPropagator& simul } // namespace // One test per (fixture, comm) so a failure pinpoints the configuration. -#define MAKE_ZERO_CUTOFF_RESCUE_TEST(NAME, FixtureType, CommToken) \ - BOOST_FIXTURE_TEST_CASE(NAME##_##CommToken, FixtureType) { \ - MPI_Comm comm = (CommMode::CommToken == CommMode::Self) ? MPI_COMM_SELF : MPI_COMM_WORLD; \ - if (CommMode::CommToken == CommMode::World && mpi::size(comm) == 1) { \ - BOOST_TEST_MESSAGE("Skipping multi-rank scenario for " #NAME " (world size=1)"); \ - return; \ - } \ - auto simulator = build_zero_cutoff_full_rescue(data, comm); \ - const double energy = evaluate_zero_cutoff_full_rescue_energy(simulator, data); \ - BOOST_CHECK_SMALL(std::abs(energy - data.actual_expval), kEnergyAtol); \ +#define MAKE_ZERO_CUTOFF_RESCUE_TEST(NAME, FixtureType, CommToken) \ + BOOST_FIXTURE_TEST_CASE(NAME##_##CommToken, FixtureType) { \ + MPI_Comm comm = (CommMode::CommToken == CommMode::Self) ? MPI_COMM_SELF : MPI_COMM_WORLD; \ + if (CommMode::CommToken == CommMode::World && mpi::size(comm) == 1) { \ + BOOST_TEST_MESSAGE("Skipping multi-rank scenario for " #NAME " (world size=1)"); \ + return; \ + } \ + auto simulator = build_zero_cutoff_full_rescue(FixtureType::n_modes, data, comm); \ + const double energy = evaluate_zero_cutoff_full_rescue_energy(simulator, data); \ + BOOST_CHECK_SMALL(std::abs(energy - data.actual_expval), kEnergyAtol); \ } MAKE_ZERO_CUTOFF_RESCUE_TEST(zero_cutoff_upper_atol_zero_is_exact, ExampleDataFix, Self) diff --git a/cpp/tests/fused_cos_sweep_tests.cpp b/cpp/tests/fused_cos_sweep_tests.cpp index a8d36ea2..bcaa5959 100644 --- a/cpp/tests/fused_cos_sweep_tests.cpp +++ b/cpp/tests/fused_cos_sweep_tests.cpp @@ -31,25 +31,23 @@ using namespace monoprop; constexpr double kAgreeAtol = 1e-12; constexpr double kExactAtol = 1e-9; -template -auto inplace_energy(const CaseData &data, const SimulatorConfig &cfg) -> double { - auto sim = build_simulator(data, cfg); +auto inplace_energy(size_t num_modes, const CaseData &data, const SimulatorConfig &cfg) -> double { + auto sim = build_simulator(num_modes, data, cfg); sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); auto fn = sim.expectation_value_functional(std::nullopt); return fn(VecD{}); } -template -auto graph_energy(const CaseData &data, const SimulatorConfig &cfg) -> double { - auto sim = build_simulator(data, cfg); +auto graph_energy(size_t num_modes, const CaseData &data, const SimulatorConfig &cfg) -> double { + auto sim = build_simulator(num_modes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto fn = sim.expectation_value_functional(std::nullopt); return fn(data.parameters); } void check_agreement(const CaseData &data, const SimulatorConfig &cfg, const char *label) { - const double inplace = inplace_energy(data, cfg); - const double graph = graph_energy(data, cfg); + const double inplace = inplace_energy(ExampleDataFix::n_modes, data, cfg); + const double graph = graph_energy(ExampleDataFix::n_modes, data, cfg); BOOST_TEST_CONTEXT(label << " inplace=" << inplace << " graph=" << graph) { BOOST_CHECK_SMALL(inplace - graph, kAgreeAtol); BOOST_CHECK_SMALL(inplace - data.actual_expval, kExactAtol); diff --git a/cpp/tests/fused_query_codec_tests.cpp b/cpp/tests/fused_query_codec_tests.cpp index 72f76763..420e372a 100644 --- a/cpp/tests/fused_query_codec_tests.cpp +++ b/cpp/tests/fused_query_codec_tests.cpp @@ -30,18 +30,28 @@ namespace { using namespace monoprop; using monoprop::detail::build_fused_query_value; -using monoprop::detail::kQueryWords; -using monoprop::detail::kQueryWordsFused; +using monoprop::detail::kQueryHeaderWords; +using monoprop::detail::query_buffer; using monoprop::detail::query_push; using monoprop::detail::query_read; +using monoprop::detail::query_record_count; using monoprop::detail::query_value; +using monoprop::detail::query_words; +using monoprop::detail::query_words_fused; -constexpr size_t kModes = 8; // 2*kModes = 16 majorana bits, one 64-bit word +constexpr size_t kModes = 8; // 2*kModes = 16 majorana bits, one 64-bit word +constexpr size_t kBits = 2 * kModes; // a monomial's width is data, so every one here is built at it +constexpr size_t kWordsPerMono = (kBits + 63) / 64; + +// query_value takes the word count; wrap it so the assertions below stay readable. +auto query_value_at(const VecZ &buf, size_t q) -> double { + return query_value(buf, q, kWordsPerMono); +} // A deterministic, distinct majorana bit pattern per record index. -auto make_mono(size_t r) -> Monomial { - Monomial m; - for (size_t b = 0; b < 2 * kModes; ++b) { +auto make_mono(size_t r) -> Bitset { + Bitset m(kBits); + for (size_t b = 0; b < kBits; ++b) { if (((r * 2654435761u + b * 40503u) & 3u) == 0u) { m.set(b); } @@ -62,26 +72,30 @@ BOOST_AUTO_TEST_CASE(fused_record_roundtrip_exact) { }; const size_t nq = values.size(); - VecZ plain; - std::vector> monos(nq); + VecZ plain = query_buffer(); + // Sized up front and assigned into, so the fill value has to carry the width. + MonomialList monos(nq, Bitset(kBits)); for (size_t r = 0; r < nq; ++r) { monos[r] = make_mono(r); - query_push(plain, monos[r], phases[r]); + query_push(plain, monos[r], phases[r]); } - BOOST_REQUIRE_EQUAL(plain.size(), nq * kQueryWords); + BOOST_REQUIRE_EQUAL(plain.size(), kQueryHeaderWords + nq * query_words(kWordsPerMono)); + BOOST_REQUIRE_EQUAL(query_record_count(plain), nq); VecZ fused; - build_fused_query_value(plain, values, fused); - BOOST_REQUIRE_EQUAL(fused.size(), nq * kQueryWordsFused); + build_fused_query_value(plain, values, fused, kWordsPerMono); + BOOST_REQUIRE_EQUAL(fused.size(), kQueryHeaderWords + nq * query_words_fused(kWordsPerMono)); + // The count survives the re-layout: it is what the resolver reads records off, at either stride. + BOOST_REQUIRE_EQUAL(query_record_count(fused), nq); for (size_t q = 0; q < nq; ++q) { - Monomial m_out; + Bitset m_out(kBits); // pre-sized: the reader copies mono_out.num_words() words into it int ph_out = 0; - query_read>(fused, q, m_out, ph_out); + query_read(fused, q, query_words_fused(kWordsPerMono), m_out, ph_out); BOOST_CHECK(m_out == monos[q]); BOOST_CHECK_EQUAL(ph_out, phases[q]); // Compare the raw payload, so -0.0 and denormals stay distinguished from 0.0. - const double v_out = query_value(fused, q); + const double v_out = query_value_at(fused, q); BOOST_CHECK(std::memcmp(&v_out, &values[q], sizeof(double)) == 0); } } @@ -89,37 +103,44 @@ BOOST_AUTO_TEST_CASE(fused_record_roundtrip_exact) { // Reusing `out` across calls must leak no stale words: capacity is a high-water mark, size is exact. // This is the reuse pattern LayerBuildEngine::combined_qv_ relies on gate to gate. BOOST_AUTO_TEST_CASE(fused_buffer_reuse_shrinks_logical_size) { - VecZ plain_big; + VecZ plain_big = query_buffer(); std::vector vbig; for (size_t r = 0; r < 32; ++r) { - query_push(plain_big, make_mono(r), (r % 2 == 0) ? 1 : -1); + query_push(plain_big, make_mono(r), (r % 2 == 0) ? 1 : -1); vbig.push_back(static_cast(r) * 1.5 - 7.0); } VecZ out; - build_fused_query_value(plain_big, vbig, out); + build_fused_query_value(plain_big, vbig, out, kWordsPerMono); const size_t cap_after_big = out.capacity(); - VecZ plain_small; + VecZ plain_small = query_buffer(); std::vector vsmall = {42.0, -42.0, 0.25}; for (size_t r = 0; r < vsmall.size(); ++r) { - query_push(plain_small, make_mono(100 + r), 1); + query_push(plain_small, make_mono(100 + r), 1); } - build_fused_query_value(plain_small, vsmall, out); - BOOST_CHECK_EQUAL(out.size(), vsmall.size() * kQueryWordsFused); + build_fused_query_value(plain_small, vsmall, out, kWordsPerMono); + BOOST_CHECK_EQUAL(out.size(), kQueryHeaderWords + vsmall.size() * query_words_fused(kWordsPerMono)); BOOST_CHECK_GE(out.capacity(), cap_after_big); for (size_t q = 0; q < vsmall.size(); ++q) { - const double v_out = query_value(out, q); + const double v_out = query_value_at(out, q); BOOST_CHECK(std::memcmp(&v_out, &vsmall[q], sizeof(double)) == 0); } } -// Empty input arises for the self slot, which resolve_self_queries clears before the exchange. +// Empty input arises for the self slot, which resolve_self_queries clears before the exchange -- not even a +// header, which is why an under-length buffer has to read as no stream rather than as a malformed one. BOOST_AUTO_TEST_CASE(fused_empty_input) { VecZ empty; std::vector no_values; VecZ out{1, 2, 3}; // pre-dirtied; build must clear it - build_fused_query_value(empty, no_values, out); + build_fused_query_value(empty, no_values, out, kWordsPerMono); BOOST_CHECK(out.empty()); + + // A stream that was created but never pushed to is a header alone, and re-lays out as one. + VecZ header_only = query_buffer(); + build_fused_query_value(header_only, no_values, out, kWordsPerMono); + BOOST_CHECK_EQUAL(out.size(), kQueryHeaderWords); + BOOST_CHECK_EQUAL(query_record_count(out), 0U); } } // namespace diff --git a/cpp/tests/gate_boundaries.cpp b/cpp/tests/gate_boundaries.cpp index 0fa4b601..54d71d58 100644 --- a/cpp/tests/gate_boundaries.cpp +++ b/cpp/tests/gate_boundaries.cpp @@ -19,6 +19,7 @@ #include #include +#include "TestPropagator.h" #include "monoprop/MonomialPropagator.h" #include "monoprop/detail/mpi/MPICompat.h" @@ -28,19 +29,20 @@ namespace { constexpr size_t kModes = 2; -auto make_sim() -> MonomialPropagator { +auto make_sim() -> MonomialPropagator { OperatorDict ham; ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; VecZ initial_state{0, 1}; - return MonomialPropagator(ham, - 2 * kModes, - initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt); + return test_utils::make_propagator(kModes, + ham, + 2 * kModes, + initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt); } } // namespace diff --git a/cpp/tests/inverted_index_tests.cpp b/cpp/tests/inverted_index_tests.cpp index b20e27b7..421579f8 100644 --- a/cpp/tests/inverted_index_tests.cpp +++ b/cpp/tests/inverted_index_tests.cpp @@ -26,17 +26,18 @@ // Internals of the even-parity scan inverted index: the tiered column store, the lazily-built // per-row parity(|M|) bitmap, and the fill order. Rows are read through the backend-agnostic -// for_each_row_position accessor, so a plain std::vector> stands in for the store. +// for_each_row_position accessor, so a plain MonomialList stands in for the store. using namespace monoprop; using namespace monoprop::detail; namespace { constexpr size_t N = 32; // 2N = 64 majorana columns -using Sc = InvertedIndex; -using MSet = Monomial; +using Sc = InvertedIndex; +constexpr size_t kCols = 2 * N; // InvertedIndex is runtime-sized: one column per bit position +using MSet = Bitset; MSet bs(const VecZ &r) { - return indices_to_bitset(r); + return indices_to_bitset(r, kCols); } // indices_to_bitset maps mode m to bit 2N-1-m; columns are indexed by raw bit position. constexpr size_t col_of(size_t mode) { @@ -70,7 +71,7 @@ BOOST_AUTO_TEST_CASE(inverted_index_row_parity_matches_popcount) { bs({5}), bs({4, 5, 6, 7}), }; - Sc sc; + Sc sc(kCols); sc.rebuild(op); BOOST_TEST(sc.rows() == op.size()); @@ -103,9 +104,9 @@ BOOST_AUTO_TEST_CASE(inverted_index_promotes_column_at_density_crossover) { else { pos.push_back(2); // mode 2 set in 118 rows -> DENSE (keeps every row non-empty) } - op.push_back(indices_to_bitset(pos)); + op.push_back(indices_to_bitset(pos, kCols)); } - Sc sc; + Sc sc(kCols); sc.rebuild(op); BOOST_TEST(sc.rows() == kR); BOOST_TEST(sc.column_is_dense(col_of(0))); // 10/128 >= 1/64 @@ -118,15 +119,16 @@ BOOST_AUTO_TEST_CASE(inverted_index_promotes_column_at_density_crossover) { // combine_columns_block's lower_bound relies on. BOOST_AUTO_TEST_CASE(inverted_index_fill_yields_ascending_sparse_rows) { constexpr size_t M = 64; // 2M = 128 columns - using ScW = InvertedIndex; + using ScW = InvertedIndex; + constexpr size_t kColsW = 2 * M; constexpr size_t kR = 16'385; // large operator, many sparse columns - std::vector> op; + MonomialList op; op.reserve(kR); for (size_t i = 0; i < kR; ++i) { // One mode per row over 128 columns: 128 hits each, and 128*64 < 16385, so all stay sparse. - op.push_back(indices_to_bitset({i % 128})); + op.push_back(indices_to_bitset({i % 128}, kColsW)); } - ScW sc; + ScW sc(kColsW); sc.rebuild(op); BOOST_TEST(sc.rows() == kR); @@ -168,10 +170,10 @@ BOOST_AUTO_TEST_CASE(inverted_index_append_rows_matches_rebuild) { op.push_back(bs(pos)); } - Sc full; + Sc full(kCols); full.rebuild(op); - Sc inc; + Sc inc(kCols); inc.rebuild(std::vector(op.begin(), op.begin() + 64)); // Force the lazy parity bitmap to exist before the append, which is the branch that extends it. static_cast(inc.row_parity_words()); @@ -179,7 +181,7 @@ BOOST_AUTO_TEST_CASE(inverted_index_append_rows_matches_rebuild) { inc.append_rows(op, 164, kR - 164); BOOST_REQUIRE_EQUAL(inc.rows(), full.rows()); - for (size_t c = 0; c < Sc::kNumColumns; ++c) { + for (size_t c = 0; c < full.num_columns(); ++c) { BOOST_TEST_CONTEXT("column " << c) { BOOST_TEST(rows_of(inc, c) == rows_of(full, c), boost::test_tools::per_element()); } @@ -225,7 +227,7 @@ BOOST_AUTO_TEST_CASE(combine_columns_block_folds_dense_and_sparse_identically) { } op.push_back(bs(pos)); } - Sc sc; + Sc sc(kCols); sc.rebuild(op); BOOST_REQUIRE(sc.column_is_dense(col_of(0))); BOOST_REQUIRE(sc.column_is_dense(col_of(1))); @@ -243,21 +245,50 @@ BOOST_AUTO_TEST_CASE(combine_columns_block_folds_dense_and_sparse_identically) { } std::vector whole(words, 0xdeadbeefULL); // pre-dirtied: the kernel seeds, never accumulates - combine_columns_block(sc, cols, whole.data(), 0, words); + combine_columns_block(sc, cols, whole.data(), 0, words); BOOST_TEST(whole == expected, boost::test_tools::per_element()); std::vector pieced(words, 0xdeadbeefULL); for (size_t w = 0; w < words; ++w) { - combine_columns_block(sc, cols, pieced.data() + w, w, w + 1); + combine_columns_block(sc, cols, pieced.data() + w, w, w + 1); } BOOST_TEST(pieced == expected, boost::test_tools::per_element()); // Sparse-only column list: no dense column to memcpy from, so this is the memset seed path. const size_t sparse_col = col_of(2); std::vector sparse_fold(words, 0xdeadbeefULL); - combine_columns_block(sc, std::span(&sparse_col, 1), sparse_fold.data(), 0, words); + combine_columns_block(sc, std::span(&sparse_col, 1), sparse_fold.data(), 0, words); std::vector sparse_expected(words, 0); sparse_expected[5 >> 6] |= uint64_t{1} << (5 & 63U); sparse_expected[200 >> 6] |= uint64_t{1} << (200 & 63U); BOOST_TEST(sparse_fold == sparse_expected, boost::test_tools::per_element()); } + +// The column vector is the index's only width-driven cost, and the width sweep behind Stage 6 step 2 +// reads it off memory_bytes(), so it has to be inside that number rather than alongside it: an empty +// index over a wide register holds one Column per bit position and no payload at all. +BOOST_AUTO_TEST_CASE(inverted_index_memory_counts_the_column_vector) { + constexpr size_t kWideCols = 4096; + Sc wide(kWideCols); + BOOST_TEST(wide.columns_bytes() >= kWideCols * sizeof(Sc::Column)); + // No rows yet, so the column vector is the whole of it. + BOOST_TEST(wide.memory_bytes() == wide.columns_bytes()); + const auto tiers = wide.tier_memory_bytes(); + BOOST_TEST(tiers[0] == 0U); + BOOST_TEST(tiers[1] == 0U); + + Sc narrow(kWideCols / 4); + BOOST_TEST(narrow.columns_bytes() * 4 == wide.columns_bytes()); + + // With payload, memory_bytes() is the columns plus both tiers -- nothing double-counted, nothing + // left out (row_parity_ is unbuilt here, and is the only other term). + MonomialList op; + for (size_t r = 0; r < 200; ++r) { + op.push_back(indices_to_bitset({r % 7, 3}, kWideCols)); + } + Sc filled(kWideCols); + filled.rebuild(op); + const auto filled_tiers = filled.tier_memory_bytes(); + BOOST_TEST(filled.memory_bytes() == filled.columns_bytes() + filled_tiers[0] + filled_tiers[1]); + BOOST_TEST(filled_tiers[0] + filled_tiers[1] > 0U); +} diff --git a/cpp/tests/link_export_probe/link_export_probe.cpp b/cpp/tests/link_export_probe/link_export_probe.cpp index 0f110797..843fc6e6 100644 --- a/cpp/tests/link_export_probe/link_export_probe.cpp +++ b/cpp/tests/link_export_probe/link_export_probe.cpp @@ -16,29 +16,24 @@ // plain OBJECT library), this target links against the installed "monoprop" SHARED target, exactly as // an external find_package(monoprop CONFIG) consumer would -- so it crosses the same hidden-visibility // boundary. cpp/tests/CMakeLists.txt is compiled with "-Wl,--no-undefined" / "-Wl,-undefined,error" so -// the link step itself fails, reporting every symbol MonomialPropagator's public template -// chain references but that the shared library does not export. +// the link step itself fails, reporting every symbol MonomialPropagator's public chain references but +// that the shared library does not export. // -// Explicit class-template instantiation alone is not sufficient: it emits every member function's -// object code (including their calls into detail/** free functions), which is what the linker checks, -// but it does not run any of it. main() below actually drives both chains implicated by the bug report: +// The width is a constructor argument rather than a template parameter, so there is no explicit +// instantiation to force member emission with -- the reachable set is whatever main() calls. It +// therefore drives both chains implicated by the bug report end to end: // (a) the graph-building / Schrodinger path (detail/graph_encoding/MPGraphEncodingStorage.h), via // build_graph(), graph_memory_usage(), and expectation_value_and_gradient(). // (b) the partition path (detail/partition/CpuTopology.h), via a partitions > 1 construction, which is // the only way to make PartitionGroup actually place and pin partition-worker threads. -#include "monoprop/MonomialPropagator.h" -#include "monoprop/detail/mpi/MPICompat.h" - #include -#include #include +#include #include -// Forces every member function of MonomialPropagator to be compiled for these two -// representative widths, regardless of which ones main() below happens to call. -template class monoprop::MonomialPropagator<2>; -template class monoprop::MonomialPropagator<6>; +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/mpi/MPICompat.h" namespace { @@ -47,20 +42,20 @@ using namespace monoprop; // Drives Engine::finish() -> LayerBuildSink::finalize() -> build_layer_storage_unified(), plus the // graph-memory and gradient accessors that read the resulting PackedCrossRankStorage / exchange layout. auto run_graph_build_chain() -> void { - constexpr size_t kModes = 2; OperatorDict ham; ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; const VecZ initial_state{0, 1}; - MonomialPropagator sim(ham, - 2 * kModes, - initial_state, - /*schrodinger_cutoff=*/std::optional{4U}, - MPI_COMM_SELF, - /*lower_atol=*/std::nullopt, - /*upper_atol=*/std::nullopt, - CutoffType::Length, - /*basis_change=*/std::nullopt); + MonomialPropagator sim(ham, + 6, + initial_state, + 4, + /*schrodinger_cutoff=*/std::optional{4U}, + MPI_COMM_SELF, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::nullopt, + CutoffType::Length, + /*basis_change=*/std::nullopt); const std::vector monos{{0}, {1}, {2}}; sim.build_graph(monos, VecZ{0, 1, 2}, VecD{1.0, 1.0, 1.0}); @@ -68,8 +63,8 @@ auto run_graph_build_chain() -> void { const auto mem = sim.graph_memory_usage(); const auto [value, grad] = sim.expectation_value_and_gradient(VecD{0.1, 0.2, 0.3}); - std::fprintf(stderr, - "[link_export_probe] graph chain: layers=%zu cross_rank_bytes=%zu value=%f grad_size=%zu\n", + std::println(stderr, + "[link_export_probe] graph chain: layers={} cross_rank_bytes={} value={} grad_size={}", sim.graph_layers(), mem.cross_rank_bytes, value, @@ -80,24 +75,23 @@ auto run_graph_build_chain() -> void { // enumerate_physical_cores / affinity_mask_words / summarize_masks / format_place_line / partition_cpusets / // pin_this_thread all run for real (not merely compiled) on the master threads it spawns. auto run_partition_chain() -> void { - constexpr size_t kModes = 6; OperatorDict ham; ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; - MonomialPropagator sim(ham, - 2 * kModes, - VecZ{0, 1}, - /*schrodinger_cutoff=*/std::nullopt, - MPI_COMM_SELF, - /*lower_atol=*/std::nullopt, - /*upper_atol=*/std::nullopt, - CutoffType::Length, - /*basis_change=*/std::nullopt, - /*logical_num_modes=*/kModes, - Basis::Majorana, - /*partitions=*/2); - - std::fprintf(stderr, "[link_export_probe] partition chain: size=%zu\n", sim.size()); + MonomialPropagator sim(ham, + 6, + VecZ{0, 1}, + 12, + /*schrodinger_cutoff=*/std::nullopt, + MPI_COMM_SELF, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::nullopt, + CutoffType::Length, + /*basis_change=*/std::nullopt, + Basis::Majorana, + /*partitions=*/2); + + std::println(stderr, "[link_export_probe] partition chain: size={}", sim.size()); } } // namespace @@ -105,6 +99,6 @@ auto run_partition_chain() -> void { auto main() -> int { run_graph_build_chain(); run_partition_chain(); - std::fprintf(stderr, "[link_export_probe] OK\n"); + std::println(stderr, "[link_export_probe] OK"); return 0; } diff --git a/cpp/tests/majorana_cutoff_tests.cpp b/cpp/tests/majorana_cutoff_tests.cpp index 50f8632f..35495a9e 100644 --- a/cpp/tests/majorana_cutoff_tests.cpp +++ b/cpp/tests/majorana_cutoff_tests.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Sets are built directly in raw-bit space (Monomial::set) so the "fully paired" condition is +// Sets are built directly in raw-bit space (Bitset::set) so the "fully paired" condition is // unambiguous: a pair is raw bits (2k, 2k+1). #include @@ -31,47 +31,47 @@ using cd = std::complex; // Raw bits {0,1} and {4,5} are two complete pairs. BOOST_AUTO_TEST_CASE(majorana_cutoff_paired_kept_unconditionally) { constexpr size_t N = 32; - Monomial paired; + Bitset paired(2 * N); paired.set(0); paired.set(1); paired.set(4); paired.set(5); - BOOST_TEST(is_paired(paired)); - BOOST_TEST(length_cutoff(paired, 0)); - BOOST_TEST(length_cutoff(paired, 2)); - BOOST_TEST(support_cutoff(paired, 0)); + BOOST_TEST(is_paired(paired)); + BOOST_TEST(length_cutoff(paired, 0)); + BOOST_TEST(length_cutoff(paired, 2)); + BOOST_TEST(support_cutoff(paired, 0)); } // An unpaired set of length 3 (raw bits {0,2,4}: each even bit lacks its odd partner). BOOST_AUTO_TEST_CASE(majorana_cutoff_length_and_support_thresholds) { constexpr size_t N = 32; - Monomial unpaired; + Bitset unpaired(2 * N); unpaired.set(0); unpaired.set(2); unpaired.set(4); - BOOST_TEST(!is_paired(unpaired)); + BOOST_TEST(!is_paired(unpaired)); // length = popcount = 3; support (distinct orbitals) = 3 here. - BOOST_TEST(length_cutoff(unpaired, 3)); - BOOST_TEST(!length_cutoff(unpaired, 2)); - BOOST_TEST(support_cutoff(unpaired, 3)); - BOOST_TEST(!support_cutoff(unpaired, 2)); + BOOST_TEST(length_cutoff(unpaired, 3)); + BOOST_TEST(!length_cutoff(unpaired, 2)); + BOOST_TEST(support_cutoff(unpaired, 3)); + BOOST_TEST(!support_cutoff(unpaired, 2)); // support <= length always, so passing length implies passing support at the same cutoff. std::mt19937_64 rng(0x50FA11ULL); std::uniform_int_distribution bit(0, 2 * N - 1); for (int trial = 0; trial < 400; ++trial) { - Monomial m; + Bitset m(2 * N); for (int k = 0; k < 5; ++k) { m.set(bit(rng)); } for (unsigned int c : {0U, 1U, 2U, 3U}) { - if (length_cutoff(m, c)) { - BOOST_TEST(support_cutoff(m, c)); + if (length_cutoff(m, c)) { + BOOST_TEST(support_cutoff(m, c)); } } - BOOST_TEST(length_cutoff(m, 2 * N)); - BOOST_TEST(length_cutoff(m, 0) == is_paired(m)); + BOOST_TEST(length_cutoff(m, 2 * N)); + BOOST_TEST(length_cutoff(m, 0) == is_paired(m)); } } @@ -80,61 +80,65 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_length_and_support_thresholds) { BOOST_AUTO_TEST_CASE(majorana_cutoff_logical_num_modes_masks_prefix_single_word) { constexpr size_t N = 32; constexpr size_t logical = 6; // active window = raw bits [2*(32-6), 64) = [52, 64) - Monomial prefix_only; + Bitset prefix_only(2 * N); prefix_only.set(0); // lone unpaired bit, inside the inactive prefix // Active window is empty -> treated as fully paired -> kept even at cutoff 0. - BOOST_TEST(length_cutoff(prefix_only, 0, logical)); - BOOST_TEST(support_cutoff(prefix_only, 0, logical)); + BOOST_TEST(length_cutoff(prefix_only, 0, logical)); + BOOST_TEST(support_cutoff(prefix_only, 0, logical)); // Over the whole register the lone bit is unpaired and exceeds cutoff 0 -> dropped. - BOOST_TEST(!length_cutoff(prefix_only, 0, N)); - BOOST_TEST(!length_cutoff(prefix_only, 0)); // whole-register overload + BOOST_TEST(!length_cutoff(prefix_only, 0, N)); + BOOST_TEST(!length_cutoff(prefix_only, 0)); // whole-register overload - Monomial active_bit; + Bitset active_bit(2 * N); active_bit.set(52); - BOOST_TEST(!length_cutoff(active_bit, 0, logical)); - Monomial active_pair; + BOOST_TEST(!length_cutoff(active_bit, 0, logical)); + Bitset active_pair(2 * N); active_pair.set(52); active_pair.set(53); - BOOST_TEST(length_cutoff(active_pair, 0, logical)); + BOOST_TEST(length_cutoff(active_pair, 0, logical)); } BOOST_AUTO_TEST_CASE(majorana_cutoff_logical_num_modes_masks_prefix_multi_word) { constexpr size_t N = 96; constexpr size_t logical = 90; // active window = raw bits [2*(96-90), 192) = [12, 192) - Monomial prefix_only; + Bitset prefix_only(2 * N); prefix_only.set(4); // lone unpaired bit in the inactive prefix - BOOST_TEST(length_cutoff(prefix_only, 0, logical)); // active window empty -> kept - BOOST_TEST(!length_cutoff(prefix_only, 0, N)); // whole register -> dropped + BOOST_TEST(length_cutoff(prefix_only, 0, logical)); // active window empty -> kept + BOOST_TEST(!length_cutoff(prefix_only, 0, N)); // whole register -> dropped } BOOST_AUTO_TEST_CASE(majorana_cutoff_evaluator_dispatch_and_popcount) { constexpr size_t N = 32; - CutoffFn length_fn = detail::LengthCutoff{.cutoff = 3}; - detail::CutoffEvaluator length_ev(length_fn); + // Both the logical width and the storage width must be stated: the functor precomputes its masks + // from them. + CutoffFn length_fn = detail::LengthCutoff{3, N, 2 * N}; + detail::CutoffEvaluator length_ev(length_fn); BOOST_TEST((length_ev.length_cutoff() != nullptr)); BOOST_TEST((length_ev.support_cutoff() == nullptr)); BOOST_REQUIRE(length_ev.max_slot_bound().has_value()); // A length cutoff counts set bits directly, so the slot bound IS the cutoff. BOOST_TEST(length_ev.max_slot_bound().value() == 3U); - CutoffFn support_fn = detail::SupportCutoff{.cutoff = 2}; - detail::CutoffEvaluator support_ev(support_fn); + CutoffFn support_fn = detail::SupportCutoff{2, N, 2 * N}; + detail::CutoffEvaluator support_ev(support_fn); BOOST_TEST((support_ev.length_cutoff() == nullptr)); BOOST_TEST((support_ev.support_cutoff() != nullptr)); // A support cutoff counts modes/qubits and each spans two slots, so the slot bound doubles. BOOST_TEST(support_ev.max_slot_bound().value() == 4U); - CutoffFn opaque_fn = [](const Monomial &) { return true; }; - detail::CutoffEvaluator opaque_ev(opaque_fn); + // A lambda, so neither target<>() probe matches and the evaluator falls back to calling through + // the std::function -- the same path cutoff_function_basis_change deliberately takes. + CutoffFn opaque_fn = [](const Bitset &) { return true; }; + detail::CutoffEvaluator opaque_ev(opaque_fn); BOOST_TEST((opaque_ev.length_cutoff() == nullptr)); BOOST_TEST((opaque_ev.support_cutoff() == nullptr)); BOOST_TEST(!opaque_ev.max_slot_bound().has_value()); // passes_with_popcount: pc <= cutoff short-circuits to true; otherwise it equals a direct eval. - Monomial unpaired; // length 4, not paired + Bitset unpaired(2 * N); // length 4, not paired unpaired.set(0); unpaired.set(2); unpaired.set(4); @@ -143,7 +147,7 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_evaluator_dispatch_and_popcount) { BOOST_TEST(!length_ev.passes_with_popcount(unpaired, 4)); BOOST_TEST(length_ev.passes_with_popcount(unpaired, 4) == length_ev(unpaired)); - Monomial paired; // pc>cutoff but paired -> direct eval keeps it + Bitset paired(2 * N); // pc>cutoff but paired -> direct eval keeps it paired.set(0); paired.set(1); paired.set(2); @@ -158,14 +162,14 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_interleave_phase_mask_cross_check) { std::mt19937_64 rng(0xABCDEF01ULL + N); std::uniform_int_distribution bit(0, 2 * N - 1); for (int trial = 0; trial < 500; ++trial) { - Monomial m; - Monomial g; + Bitset m(2 * N); + Bitset g(2 * N); for (int k = 0; k < 6; ++k) { m.set(bit(rng)); g.set(bit(rng)); } - const int reference = interleave_phase(m, g); - const auto w = interleave_phase_mask(g); + const int reference = interleave_phase(m, g); + const auto w = interleave_phase_mask(g); const int masked = m.parity_and(w) ? -1 : 1; BOOST_TEST(reference == masked); } @@ -176,19 +180,19 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_interleave_phase_mask_cross_check) { BOOST_AUTO_TEST_CASE(majorana_cutoff_encode_decode_coeff) { constexpr size_t N = 32; - Monomial mono; + Bitset mono(2 * N); mono.set(0); mono.set(3); mono.set(6); for (double r : {1.0, -2.5, 0.0, 7.25}) { - const cd hermitian = decode_coeff(cd(r, 0.0), mono); // r * hermitian_coefficient(mono) - BOOST_TEST(encode_coeff(hermitian, mono) == r); + const cd hermitian = decode_coeff(cd(r, 0.0), mono); // r * hermitian_coefficient(mono) + BOOST_TEST(encode_coeff(hermitian, mono) == r); } // Multiply by i to break Hermiticity: the encoded value then has a nonzero imaginary part. - const cd non_hermitian = decode_coeff(cd(1.0, 0.0), mono) * cd(0.0, 1.0); - BOOST_CHECK_THROW(encode_coeff(non_hermitian, mono), std::runtime_error); + const cd non_hermitian = decode_coeff(cd(1.0, 0.0), mono) * cd(0.0, 1.0); + BOOST_CHECK_THROW(encode_coeff(non_hermitian, mono), std::runtime_error); } // max_ones counts pairs, so it saturates at logical_num_modes, not at the bit count 2*logical_num_modes. @@ -197,12 +201,12 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_paired_op_saturates_at_one_pair_per_mode) { constexpr size_t N = 32; constexpr size_t kLogical = 4; - const auto full = generate_paired_op(kLogical, kLogical); + const auto full = generate_paired_op(kLogical, kLogical, 2 * N); // Every subset of the kLogical pairs, so 2^kLogical monomials. BOOST_TEST(full.size() == (size_t{1} << kLogical)); for (const size_t over : {kLogical + 1, 2 * kLogical, 2 * kLogical + 3}) { - const auto clamped = generate_paired_op(over, kLogical); + const auto clamped = generate_paired_op(over, kLogical, 2 * N); BOOST_TEST(clamped.size() == full.size()); for (size_t i = 0; i < full.size(); ++i) { BOOST_TEST(clamped[i] == full[i]); diff --git a/cpp/tests/mp_operator_tests.cpp b/cpp/tests/mp_operator_tests.cpp index 2c822c38..a21fea91 100644 --- a/cpp/tests/mp_operator_tests.cpp +++ b/cpp/tests/mp_operator_tests.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// White-box tests for detail::MPOperator built directly (append_term / init_op_map / basis) rather +// White-box tests for detail::MPOperator built directly (append_term / init_op_map / basis) rather // than through a simulator, with the algebra primitives (is_paired, algebra_state_phase, // encode_pauli_coeff) as the oracle. They pin composition -- incremental scoring, slot placement, the // init-map drain, the picture/basis branches -- not the phase math (majorana_cutoff_tests.cpp). @@ -31,39 +31,44 @@ #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" +#include "TestOperator.h" +#include "TestPropagator.h" + using namespace monoprop; using cd = std::complex; namespace { -// Build an MPOperator whose store rows are also indexed (findable). append_term writes a row only; -// find() needs the hash index, which only the insert_absent_terms path populates. -auto build_indexed_op(const std::vector> &terms, Basis basis = Basis::Majorana) -> detail::MPOperator<8> { - detail::MPOperator<8> op; - op.basis = basis; - detail::insert_absent_terms<8>( - op, - terms.size(), - [&](size_t k) -> const Monomial<8> & { return terms[k]; }, - [&](size_t k, size_t base) { assign_row<8>(*op.store, base + k, terms[k]); }); - return op; +// These oracles are all written against 8 modes, so every MPOperator here is built at that storage +// width. MPOperator carries the width as data now, so it is passed rather than named as a template +// argument. +constexpr size_t kNumBits = 2 * 8; + +// Every monomial here is built at kNumBits; bs() is the one spelling of that, so the term literals +// below stay index lists. +auto bs(const VecZ &inds) -> Bitset { + return indices_to_bitset(inds, kNumBits); +} + +auto build_indexed_op(const MonomialList &terms, Basis basis = Basis::Majorana) -> detail::MPOperator { + return test_utils::indexed_operator(kNumBits, terms, basis); } // Independent expected state vector: score paired rows with the basis' state phase, 0 otherwise. -auto expected_state(detail::MPOperator<8> &op, Basis basis, const VecZ &initial_state) -> VecD { - const auto state_mask = initial_state_mask<8>(initial_state); +auto expected_state(detail::MPOperator &op, Basis basis, const VecZ &initial_state) -> VecD { + const auto state_mask = initial_state_mask(initial_state, kNumBits); VecD expected(op.size(), 0.0); for (size_t i = 0; i < op.size(); ++i) { - const auto row = materialize_row<8>(*op.store, i); - if (is_paired<8>(row)) { - expected[i] = algebra_state_phase<8>(basis, row, state_mask); + const auto row = op.with_store([&](const auto &rows) { return materialize_row(rows, i); }); + if (is_paired(row)) { + expected[i] = algebra_state_phase(basis, row, state_mask); } } return expected; } // Independent expected sparse state: ascending rows that score nonzero, and their phases. -auto expected_sparse_state(detail::MPOperator<8> &op, Basis basis, const VecZ &initial_state) -> std::pair { +auto expected_sparse_state(detail::MPOperator &op, Basis basis, const VecZ &initial_state) -> std::pair { const auto dense = expected_state(op, basis, initial_state); std::pair expected; for (size_t i = 0; i < dense.size(); ++i) { @@ -75,8 +80,7 @@ auto expected_sparse_state(detail::MPOperator<8> &op, Basis basis, const VecZ &i return expected; } -auto sparse_state_equals(const detail::MPOperator<8>::SparseState &sparse, const std::pair &expected) - -> bool { +auto sparse_state_equals(const detail::MPOperator::SparseState &sparse, const std::pair &expected) -> bool { return std::ranges::equal(sparse.rows, expected.first, {}, [](TermIndex r) { return static_cast(r); }) && std::ranges::equal(sparse.values, expected.second); } @@ -86,15 +90,15 @@ auto sparse_state_equals(const detail::MPOperator<8>::SparseState &sparse, const BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_paired_terms_majorana_and_pauli) { const VecZ initial_state = {0, 1}; // occupied modes for (const Basis basis : {Basis::Majorana, Basis::Pauli}) { - detail::MPOperator<8> op; + detail::MPOperator op(kNumBits); op.basis = basis; op.initial_state = initial_state; - Monomial<8> identity; // empty -> paired - Monomial<8> paired_mode0; // raw bits {0,1} -> mode 0 paired + Bitset identity(kNumBits); // empty -> paired + Bitset paired_mode0(kNumBits); // raw bits {0,1} -> mode 0 paired paired_mode0.set(0); paired_mode0.set(1); - Monomial<8> unpaired; // raw bit {0} only -> not paired + Bitset unpaired(kNumBits); // raw bit {0} only -> not paired unpaired.set(0); op.append_term(identity); @@ -122,10 +126,10 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_paired_terms_majorana_and_paul BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_only_new_terms_incrementally) { const VecZ initial_state = {0}; - detail::MPOperator<8> op; + detail::MPOperator op(kNumBits); op.initial_state = initial_state; - Monomial<8> a; + Bitset a(kNumBits); a.set(0); a.set(1); // paired op.append_term(a); @@ -138,7 +142,7 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_only_new_terms_incrementally) // already-scored row, so this value has to survive. op.state_coeffs[0] = 7.5; - Monomial<8> b; + Bitset b(kNumBits); b.set(2); b.set(3); // paired op.append_term(b); @@ -158,11 +162,11 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_state_scores_only_new_terms_incrementally) } BOOST_AUTO_TEST_CASE(mp_operator_get_operator_drains_present_terms_from_init_map) { - const auto a = indices_to_bitset<8>({0, 1}); - const auto b = indices_to_bitset<8>({2, 3}); + const auto a = bs({0, 1}); + const auto b = bs({2, 3}); auto op = build_indexed_op({a, b}); - const auto absent = indices_to_bitset<8>({4, 5}); + const auto absent = bs({4, 5}); op.init_op_map[a] = 3.0; // present in store -> should land on row 0 and be erased op.init_op_map[absent] = 9.0; // absent from store -> stays pending @@ -179,15 +183,15 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_operator_drains_present_terms_from_init_map // erase/clear leave bucket_count(), so init_operator_bytes must fall, not just the entry count. BOOST_AUTO_TEST_CASE(mp_operator_get_operator_releases_init_map_when_fully_bound) { - const auto a = indices_to_bitset<8>({0, 1}); - const auto b = indices_to_bitset<8>({2, 3}); + const auto a = bs({0, 1}); + const auto b = bs({2, 3}); auto op = build_indexed_op({a, b}); op.init_op_map.reserve(4096); // buckets far in excess of the two live entries op.init_op_map[a] = 3.0; op.init_op_map[b] = 5.0; - const auto before = detail::estimate_memory_usage<8>(op); + const auto before = detail::estimate_memory_usage(op); BOOST_REQUIRE_EQUAL(before.init_operator_entries, 2U); const VecD &coeffs = op.get_operator(); @@ -195,7 +199,7 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_operator_releases_init_map_when_fully_bound BOOST_CHECK_EQUAL(coeffs[0], 3.0); BOOST_CHECK_EQUAL(coeffs[1], 5.0); - const auto after = detail::estimate_memory_usage<8>(op); + const auto after = detail::estimate_memory_usage(op); BOOST_CHECK_EQUAL(after.init_operator_entries, 0U); BOOST_CHECK_LT(after.init_operator_bytes, before.init_operator_bytes); BOOST_CHECK_EQUAL(op.init_op_map.bucket_count(), 0U); // released, not merely shrunk to the minimum @@ -203,18 +207,18 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_operator_releases_init_map_when_fully_bound // Partial bind: the pending entry survives with its value and the bucket array shrinks to the remainder. BOOST_AUTO_TEST_CASE(mp_operator_get_operator_shrinks_init_map_when_partially_bound) { - const auto a = indices_to_bitset<8>({0, 1}); - const auto b = indices_to_bitset<8>({2, 3}); - const auto absent = indices_to_bitset<8>({4, 5}); + const auto a = bs({0, 1}); + const auto b = bs({2, 3}); + const auto absent = bs({4, 5}); auto op = build_indexed_op({a, b}); op.init_op_map.reserve(4096); op.init_op_map[a] = 3.0; op.init_op_map[absent] = 9.0; - const auto before = detail::estimate_memory_usage<8>(op); + const auto before = detail::estimate_memory_usage(op); (void)op.get_operator(); - const auto after = detail::estimate_memory_usage<8>(op); + const auto after = detail::estimate_memory_usage(op); BOOST_REQUIRE_EQUAL(after.init_operator_entries, 1U); const auto found = op.init_op_map.find(absent); @@ -226,21 +230,21 @@ BOOST_AUTO_TEST_CASE(mp_operator_get_operator_shrinks_init_map_when_partially_bo // Nothing bound: the map is left exactly as it was, buckets included. BOOST_AUTO_TEST_CASE(mp_operator_get_operator_keeps_init_map_when_nothing_bound) { - auto op = build_indexed_op({indices_to_bitset<8>({0, 1})}); + auto op = build_indexed_op({bs({0, 1})}); - const auto absent = indices_to_bitset<8>({4, 5}); + const auto absent = bs({4, 5}); op.init_op_map[absent] = 9.0; (void)op.get_operator(); - const auto after = detail::estimate_memory_usage<8>(op); + const auto after = detail::estimate_memory_usage(op); BOOST_CHECK_EQUAL(after.init_operator_entries, 1U); BOOST_CHECK(op.init_op_map.find(absent) != op.init_op_map.end()); } BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_heisenberg_branches_pauli) { - const auto present = indices_to_bitset<8>({0, 2}); + const auto present = bs({0, 2}); auto op = build_indexed_op({present}, Basis::Pauli); // row 0 indexed - op.init_op_map[indices_to_bitset<8>({4, 6})] = 0.0; // seed a pending term + op.init_op_map[bs({4, 6})] = 0.0; // seed a pending term OperatorDict dict; dict[VecZ{0, 2}] = cd(1.5, 0.0); // present in store -> row coeff @@ -249,13 +253,13 @@ BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_heisenberg_branches_pau const auto grad = op.update_initial_operator(dict, /*schrodinger=*/false); BOOST_REQUIRE_EQUAL(op.op_coeffs.size(), 1U); BOOST_CHECK_EQUAL(op.op_coeffs[0], encode_pauli_coeff(cd(1.5, 0.0))); // Pauli encode path - BOOST_CHECK(op.init_op_map.find(indices_to_bitset<8>({4, 6})) != op.init_op_map.end()); + BOOST_CHECK(op.init_op_map.find(bs({4, 6})) != op.init_op_map.end()); BOOST_CHECK(op.init_op_map.find(present) == op.init_op_map.end()); BOOST_CHECK_EQUAL(grad.first.size(), 2U); // every supplied term recorded in the grad arrays } BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_heisenberg_rejects_absent_term) { - auto op = build_indexed_op({indices_to_bitset<8>({0, 2})}, Basis::Pauli); + auto op = build_indexed_op({bs({0, 2})}, Basis::Pauli); OperatorDict dict; dict[VecZ{1, 3, 5}] = cd(1.0, 0.0); // absent from both store and init_op_map @@ -263,10 +267,10 @@ BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_heisenberg_rejects_abse } BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_schrodinger_admits_absent_term) { - auto op = build_indexed_op({indices_to_bitset<8>({0, 2})}, Basis::Pauli); + auto op = build_indexed_op({bs({0, 2})}, Basis::Pauli); OperatorDict dict; - const auto fresh = indices_to_bitset<8>({1, 3, 5}); + const auto fresh = bs({1, 3, 5}); dict[VecZ{1, 3, 5}] = cd(4.0, 0.0); op.update_initial_operator(dict, /*schrodinger=*/true); BOOST_CHECK(op.init_op_map.find(fresh) != op.init_op_map.end()); @@ -275,75 +279,143 @@ BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_schrodinger_admits_abse BOOST_AUTO_TEST_CASE(mp_operator_update_initial_operator_majorana_encode_identity_term) { // The Majorana codec divides by the term's hermitian phase, which is 1 for the identity term, so // a real coefficient round-trips as itself without tripping the non-Hermitian guard. - const Monomial<8> identity; // empty + const Bitset identity(kNumBits); // empty auto op = build_indexed_op({identity}); // basis defaults to Majorana OperatorDict dict; dict[VecZ{}] = cd(2.75, 0.0); op.update_initial_operator(dict, /*schrodinger=*/false); BOOST_REQUIRE_EQUAL(op.op_coeffs.size(), 1U); - BOOST_CHECK_EQUAL(op.op_coeffs[0], algebra_encode_coeff<8>(Basis::Majorana, cd(2.75, 0.0), identity)); + BOOST_CHECK_EQUAL(op.op_coeffs[0], algebra_encode_coeff(Basis::Majorana, cd(2.75, 0.0), identity)); BOOST_CHECK_EQUAL(op.op_coeffs[0], 2.75); } BOOST_AUTO_TEST_CASE(mp_operator_insert_absent_terms_grows_and_indexes) { - const auto e0 = indices_to_bitset<8>({0, 1}); - const auto e1 = indices_to_bitset<8>({2, 3}); + const auto e0 = bs({0, 1}); + const auto e1 = bs({2, 3}); auto op = build_indexed_op({e0, e1}); - const std::vector> fresh = {indices_to_bitset<8>({4, 5}), - indices_to_bitset<8>({6, 7}), - indices_to_bitset<8>({0, 3})}; + const MonomialList fresh = {bs({4, 5}), bs({6, 7}), bs({0, 3})}; - const size_t base = detail::insert_absent_terms<8>( - op, - fresh.size(), - [&](size_t k) -> const Monomial<8> & { return fresh[k]; }, - [&](size_t k, size_t b) { assign_row<8>(*op.store, b + k, fresh[k]); }); + const size_t base = op.with_store([&](auto &rows) { + return detail::insert_absent_terms( + op, + rows, + fresh.size(), + [&](size_t k) -> const Bitset & { return fresh[k]; }, + [&](size_t k, size_t b) { assign_row(rows, b + k, fresh[k]); }); + }); BOOST_CHECK_EQUAL(base, 2U); BOOST_CHECK_EQUAL(op.size(), 5U); for (const auto &f : fresh) { - BOOST_CHECK(op.store->find(f).has_value()); + BOOST_CHECK(op.find(f).has_value()); } - BOOST_CHECK(op.store->find(e0).has_value()); // existing rows intact - BOOST_CHECK(op.store->find(e1).has_value()); + BOOST_CHECK(op.find(e0).has_value()); // existing rows intact + BOOST_CHECK(op.find(e1).has_value()); } BOOST_AUTO_TEST_CASE(mp_operator_append_term_after_materialization_rebuilds_inverted_index) { - detail::MPOperator<8> op; - op.append_term(indices_to_bitset<8>({0, 1})); + detail::MPOperator op(kNumBits); + op.append_term(bs({0, 1})); BOOST_CHECK_EQUAL(op.inverted_index().rows(), 1U); // materializes the index (rows == size) // append_term does not sync the index; the next inverted_index() sees rows() != store size and // rebuilds against the grown store. - op.append_term(indices_to_bitset<8>({2, 3})); + op.append_term(bs({2, 3})); BOOST_CHECK_EQUAL(op.inverted_index().rows(), 2U); } BOOST_AUTO_TEST_CASE(mp_operator_estimate_memory_usage_tracks_inverted_index_presence) { - detail::MPOperator<8> op; - op.append_term(indices_to_bitset<8>({0, 1})); - op.append_term(indices_to_bitset<8>({2, 3})); + detail::MPOperator op(kNumBits); + op.append_term(bs({0, 1})); + op.append_term(bs({2, 3})); - const auto before = detail::estimate_memory_usage<8>(op); + const auto before = detail::estimate_memory_usage(op); BOOST_CHECK_GT(before.total_bytes(), 0U); BOOST_CHECK_GT(before.operator_terms_bytes, 0U); BOOST_CHECK_EQUAL(before.inverted_index_bytes, 0U); // absent arm (void)op.inverted_index(); - const auto after = detail::estimate_memory_usage<8>(op); + const auto after = detail::estimate_memory_usage(op); BOOST_CHECK_GT(after.inverted_index_bytes, 0U); // present arm } +// get_operator() drains init_op_map as each pending term appears as a row, and a flat map keeps its slot +// array across erases -- so a propagator would otherwise carry an empty map sized for the whole initial +// operator for its whole life. The release is a shrink-to-fit, so a partial drain keeps a table for the +// terms left in it and a full drain gives the array back outright. +BOOST_AUTO_TEST_CASE(mp_operator_drained_init_map_gives_its_slot_array_back) { + detail::MPOperator op(kNumBits); + // Two pending terms, only one of which is a findable row -- append_term writes a row, index_term is + // what makes find() see it -- so the drain is partial and nothing is released. + op.append_term(bs({0, 1})); + op.index_term(bs({0, 1}), 0); + op.init_op_map[bs({0, 1})] = 1.5; + op.init_op_map[bs({2, 3})] = 2.5; + const size_t buckets_when_pending = op.init_op_map.bucket_count(); + + (void)op.get_operator(); + BOOST_CHECK_EQUAL(op.init_op_map.size(), 1U); // the unmaterialized term stays + BOOST_CHECK_LE(op.init_op_map.bucket_count(), buckets_when_pending); + BOOST_CHECK_GE(op.init_op_map.bucket_count(), 1U); + BOOST_CHECK_EQUAL(detail::estimate_memory_usage(op).init_operator_entries, 1U); + BOOST_CHECK_EQUAL(op.op_coeffs[0], 1.5); + + // Now the second term becomes a findable row too, so the map drains fully and hands the array back. + op.append_term(bs({2, 3})); + op.index_term(bs({2, 3}), 1); + (void)op.get_operator(); + BOOST_CHECK(op.init_op_map.empty()); + BOOST_CHECK_LT(op.init_op_map.bucket_count(), buckets_when_pending); + const auto drained = detail::estimate_memory_usage(op); + BOOST_CHECK_EQUAL(drained.init_operator_entries, 0U); + BOOST_CHECK_LT(drained.init_operator_bytes, sizeof(MonomialMap) + 64U); + BOOST_CHECK_EQUAL(op.op_coeffs[1], 2.5); +} + +// A monomial wider than Bitset's inline capacity (8 words / 256 modes) owns its words on the heap, which +// the slot array does not contain. Nothing else in the suite runs a *container* of spilled monomials, so +// this also covers insert / find / erase and the move-on-rehash a growing flat map performs on them. +BOOST_AUTO_TEST_CASE(mp_operator_init_map_accounts_for_spilled_keys) { + constexpr size_t kWideBits = 2 * 512; // 16 words: spilled + detail::MPOperator wide(kWideBits); + MonomialList keys; + for (size_t k = 0; k < 64; ++k) { + // Distinct, and spread so no two share a word pattern. + keys.push_back(indices_to_bitset({k, 200 + k, 900 + k}, kWideBits)); + wide.init_op_map[keys.back()] = static_cast(k); + } + BOOST_REQUIRE_EQUAL(wide.init_op_map.size(), 64U); + // Survived every rehash on the way to 64 entries: each key still finds its own coefficient. + for (size_t k = 0; k < keys.size(); ++k) { + const auto found = wide.init_op_map.find(keys[k]); + BOOST_REQUIRE(found != wide.init_op_map.end()); + BOOST_CHECK_EQUAL(found->second, static_cast(k)); + } + + const auto b = detail::estimate_memory_usage(wide); + const size_t slots_only = detail::unordered_flat_map_storage_bytes(wide.init_op_map); + const size_t spilled = 64 * 16 * sizeof(uint64_t); + BOOST_CHECK_EQUAL(b.init_operator_bytes, slots_only + spilled); + BOOST_CHECK_EQUAL(b.init_operator_entries, 64U); + + // An inline-width map of the same shape owns nothing outside its slots, which is why an accounting + // that skips heap_bytes() looks right until someone runs past 8 words. + detail::MPOperator narrow(kNumBits); + narrow.init_op_map[bs({0, 1})] = 1.0; + BOOST_CHECK_EQUAL(detail::estimate_memory_usage(narrow).init_operator_bytes, + detail::unordered_flat_map_storage_bytes(narrow.init_op_map)); +} + // matched_scratch_bytes is summed by total_bytes() and accumulated by operator+= for the facade's sum. BOOST_AUTO_TEST_CASE(mp_operator_breakdown_counts_matched_scratch_in_total_and_sum) { - detail::MPOperatorMemoryBreakdown<8> acc; + detail::MPOperatorMemoryBreakdown acc; acc.op_coeffs_bytes = 100; acc.matched_scratch_bytes = 7; BOOST_CHECK_EQUAL(acc.total_bytes(), 107U); - detail::MPOperatorMemoryBreakdown<8> other; + detail::MPOperatorMemoryBreakdown other; other.op_coeffs_bytes = 20; other.matched_scratch_bytes = 3; @@ -352,8 +424,8 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_counts_matched_scratch_in_total_and_s BOOST_CHECK_EQUAL(acc.total_bytes(), 130U); // An operator on its own has no stamp array to report. - auto bare = build_indexed_op({indices_to_bitset<8>({0, 1})}); - BOOST_CHECK_EQUAL(detail::estimate_memory_usage<8>(bare).matched_scratch_bytes, 0U); + auto bare = build_indexed_op({bs({0, 1})}); + BOOST_CHECK_EQUAL(detail::estimate_memory_usage(bare).matched_scratch_bytes, 0U); } // epoch_ is empty until the first begin_gate, so this must apply a gate before the bytes can be nonzero. @@ -362,15 +434,7 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_matched_scratch_nonzero_after_a_gate) OperatorDict ham; ham[VecZ{0, 1}] = cd{0.0, 1.0}; VecZ initial_state{0, 1}; - auto sim = MonomialPropagator(ham, - 2 * kModes, - initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt); + auto sim = test_utils::make_propagator(kModes, ham, 2 * kModes, initial_state); BOOST_CHECK_EQUAL(sim.operator_memory_usage().matched_scratch_bytes, 0U); // no gate applied yet const std::vector monos{{0}}; @@ -386,12 +450,12 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_matched_scratch_nonzero_after_a_gate) // init_operator_entries is a count: accumulated by operator+= but never summed into total_bytes(). BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_init_operator_entries_out_of_total) { - detail::MPOperatorMemoryBreakdown<8> acc; + detail::MPOperatorMemoryBreakdown acc; acc.op_coeffs_bytes = 100; acc.init_operator_entries = 2; BOOST_CHECK_EQUAL(acc.total_bytes(), 100U); - detail::MPOperatorMemoryBreakdown<8> other; + detail::MPOperatorMemoryBreakdown other; other.op_coeffs_bytes = 20; other.init_operator_entries = 5; @@ -401,19 +465,62 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_init_operator_entries_out_of_to } BOOST_AUTO_TEST_CASE(mp_operator_copy_constructor_clones_store_and_coeffs) { - auto op = build_indexed_op({indices_to_bitset<8>({0, 1}), indices_to_bitset<8>({2, 3})}); + auto op = build_indexed_op({bs({0, 1}), bs({2, 3})}); op.initial_state = {0}; (void)op.sparse_state(); - detail::MPOperator<8> copy(op); // deep copy via clone() + detail::MPOperator copy(op); // deep copy via clone() BOOST_CHECK_EQUAL(copy.size(), op.size()); BOOST_CHECK_EQUAL(copy.state_scored_rows_, op.state_scored_rows_); BOOST_CHECK(copy.state_rows_ == op.state_rows_); BOOST_CHECK(copy.state_vals_ == op.state_vals_); BOOST_CHECK(copy.materialize_state() == op.materialize_state()); - BOOST_CHECK(copy.store->find(indices_to_bitset<8>({0, 1})).has_value()); + BOOST_CHECK(copy.find(bs({0, 1})).has_value()); // Mutating the copy must not touch the original (independent stores). - copy.append_term(indices_to_bitset<8>({4, 5})); + copy.append_term(bs({4, 5})); BOOST_CHECK_EQUAL(op.size(), 2U); BOOST_CHECK_EQUAL(copy.size(), 3U); } + +// resize_store() is what update_cutoff() calls when the cutoff-derived row-width bound moves: unlike +// set_store(), it must keep existing rows (and their index, since op_coeffs/state_rows_/the graph key +// off it) rather than installing an empty store. Both backends, since row_width() and resize_store() +// must dispatch to whichever is live without the caller knowing which. +BOOST_AUTO_TEST_CASE(mp_operator_resize_store_migrates_existing_rows_dense) { + detail::MPOperator op(kNumBits); + op.set_store(std::make_unique(kNumBits, 2)); // row 1 (3 positions) spills + op.append_term(bs({0, 1})); + op.index_term(bs({0, 1}), 0); + op.append_term(bs({2, 3, 4})); + op.index_term(bs({2, 3, 4}), 1); + BOOST_REQUIRE(!op.rows_are_sparse()); + BOOST_REQUIRE_EQUAL(op.row_width(), 2U); + + op.resize_store(4); // wide enough that row 1 no longer needs to spill + BOOST_CHECK_EQUAL(op.row_width(), 4U); + BOOST_CHECK_EQUAL(op.size(), 2U); + BOOST_CHECK((op.find(bs({0, 1})).has_value())); + BOOST_CHECK_EQUAL(*op.find(bs({0, 1})), 0U); + BOOST_CHECK((op.find(bs({2, 3, 4})).has_value())); + BOOST_CHECK_EQUAL(*op.find(bs({2, 3, 4})), 1U); +} + +BOOST_AUTO_TEST_CASE(mp_operator_resize_store_migrates_existing_rows_sparse) { + detail::MPOperator op(kNumBits); + op.set_store(std::make_unique(kNumBits, 2)); + op.append_term(bs({0, 1})); + op.index_term(bs({0, 1}), 0); + op.append_term(bs({2, 3, 4})); + op.index_term(bs({2, 3, 4}), 1); + BOOST_REQUIRE(op.rows_are_sparse()); + BOOST_REQUIRE_EQUAL(op.row_width(), 2U); + + op.resize_store(4); + BOOST_CHECK(op.rows_are_sparse()); + BOOST_CHECK_EQUAL(op.row_width(), 4U); + BOOST_CHECK_EQUAL(op.size(), 2U); + BOOST_CHECK((op.find(bs({0, 1})).has_value())); + BOOST_CHECK_EQUAL(*op.find(bs({0, 1})), 0U); + BOOST_CHECK((op.find(bs({2, 3, 4})).has_value())); + BOOST_CHECK_EQUAL(*op.find(bs({2, 3, 4})), 1U); +} diff --git a/cpp/tests/mpfunctions.cpp b/cpp/tests/mpfunctions.cpp index b4ac2727..81d1e319 100644 --- a/cpp/tests/mpfunctions.cpp +++ b/cpp/tests/mpfunctions.cpp @@ -37,12 +37,13 @@ static std::vector ds_input_indices_to_bitset_test = { {5}, // Single index set {4, 7} // Two indices set }; -static std::vector> ds_output_indices_to_bitset_test = { +// A monomial carries its width, so each pattern is widened here rather than by an element type. +static MonomialList ds_output_indices_to_bitset_test = { - {0b11110000}, // Full indices set - {0b00000000}, // No indices set, empty majorana - {0b00000100}, // Single index set - {0b00001001} // Two indices set + Bitset(2 * NumQubits, 0b11110000), // Full indices set + Bitset(2 * NumQubits, 0b00000000), // No indices set, empty majorana + Bitset(2 * NumQubits, 0b00000100), // Single index set + Bitset(2 * NumQubits, 0b00001001) // Two indices set }; @@ -50,14 +51,14 @@ BOOST_DATA_TEST_CASE(indices_to_bitset_test, bdata::make(ds_input_indices_to_bitset_test) ^ ds_output_indices_to_bitset_test, input_indices, expected_bitset) { - auto bitset = indices_to_bitset(input_indices); + auto bitset = indices_to_bitset(input_indices, 2 * NumQubits); BOOST_CHECK(bitset == expected_bitset); } -static std::vector> ds_input_bitset_to_indices_test = { - 0b00000000, // fully paired - 0b00011000, // 2 slots - 0b10101010 // 4 slots +static MonomialList ds_input_bitset_to_indices_test = { + Bitset(2 * NumQubits, 0b00000000), // fully paired + Bitset(2 * NumQubits, 0b00011000), // 2 slots + Bitset(2 * NumQubits, 0b10101010) // 4 slots }; static std::vector ds_cutoff_values = { 4, @@ -76,50 +77,53 @@ BOOST_DATA_TEST_CASE(length_cutoff_test, input_bitset, cutoff, expected_result) { - auto result = length_cutoff(input_bitset, cutoff); + auto result = length_cutoff(input_bitset, cutoff); BOOST_CHECK(result == expected_result); } BOOST_AUTO_TEST_CASE(test_fermionic_to_binary_operator_empty) { std::vector empty_operator; - auto result = fermionic_to_binary_operator(empty_operator); + auto result = fermionic_to_binary_operator(NumQubits, empty_operator); BOOST_CHECK(result.empty()); } BOOST_AUTO_TEST_CASE(test_fermionic_to_binary_operator_single_term) { std::vector single_term_operator = {{0, 1, 2}}; - auto result = fermionic_to_binary_operator(single_term_operator); + auto result = fermionic_to_binary_operator(NumQubits, single_term_operator); BOOST_CHECK(result.size() == 1); - BOOST_CHECK(result[0] == 0b11100000); + BOOST_CHECK(result[0] == Bitset(2 * NumQubits, 0b11100000)); } BOOST_AUTO_TEST_CASE(test_fermionic_to_binary_operator_multiple_terms) { std::vector multi_term_operator = {{0, 1}, {2, 3}}; - auto result = fermionic_to_binary_operator(multi_term_operator); + auto result = fermionic_to_binary_operator(NumQubits, multi_term_operator); BOOST_CHECK(result.size() == 2); - BOOST_CHECK(result[0] == 0b11000000); - BOOST_CHECK(result[1] == 0b00110000); + BOOST_CHECK(result[0] == Bitset(2 * NumQubits, 0b11000000)); + BOOST_CHECK(result[1] == Bitset(2 * NumQubits, 0b00110000)); } constexpr size_t NumQubits2 = 2; -static std::vector, int>> ds_get_multiplicative_phase = {{{0b0001}, 0}, - {{0b0101}, -1}, - {{0b1001}, 1}}; +static std::vector> ds_get_multiplicative_phase = {{Bitset(2 * NumQubits2, 0b0001), 0}, + {Bitset(2 * NumQubits2, 0b0101), -1}, + {Bitset(2 * NumQubits2, 0b1001), 1}}; BOOST_DATA_TEST_CASE(get_multiplicative_phase_test, bdata::make(ds_get_multiplicative_phase), test_pair) { auto [majorana_set, expected_phase] = test_pair; VecZ gen_vec = {0, 1}; - auto gen_bitset = indices_to_bitset(gen_vec); + auto gen_bitset = indices_to_bitset(gen_vec, 2 * NumQubits2); auto mono_count = majorana_set.count(); auto gen_count = gen_bitset.count(); auto overlap = (majorana_set & gen_bitset).count(); - auto result = get_multiplicative_phase(majorana_set, gen_bitset, mono_count, gen_count, overlap); + auto result = get_multiplicative_phase(majorana_set, gen_bitset, mono_count, gen_count, overlap); BOOST_CHECK(result == expected_phase); } struct IS_FULLY_PAIRED_TEST_CASE { VecZ inds; - MonomialList op_terms; + // Bit patterns, not monomials: a MonomialList element is a runtime-width Bitset, whose one-argument + // constructor takes a *width*, so bare literals here would silently mean something else. The test + // body widens each to 2 * NumQubits2, leaving the patterns below readable as patterns. + std::vector op_terms; VecZ expected_result; std::string test_name; @@ -134,19 +138,24 @@ static std::vector ds_is_fully_paired_test = { {{0, 1, 2, 3, 4, 5, 6}, {0b0001, 0b0011, 0b1000, 0b0101, 0b1100, 0b0110, 0b1110}, {1, 4}, "Partially paired"}}; BOOST_DATA_TEST_CASE(is_fully_paired_test, bdata::make(ds_is_fully_paired_test), test_case) { - auto result = is_fully_paired(test_case.inds, test_case.op_terms); + MonomialList op_terms; + op_terms.reserve(test_case.op_terms.size()); + for (const auto bits : test_case.op_terms) { + op_terms.emplace_back(2 * NumQubits2, bits); + } + auto result = is_fully_paired(test_case.inds, op_terms, 2 * NumQubits2); BOOST_CHECK(std::is_permutation(result.cbegin(), result.cend(), test_case.expected_result.cbegin())); } BOOST_AUTO_TEST_CASE(bit_flipping_utilities) { - auto val1 = even_bits<10, LSb0>(); - auto val2 = odd_bits<10, LSb0>(); - auto val3 = even_bits<10, MSb0>(); - auto val4 = odd_bits<10, MSb0>(); - BOOST_TEST(val1 == 0b0101010101); - BOOST_TEST(val2 == 0b1010101010); - BOOST_TEST(val3 == 0b1010101010); - BOOST_TEST(val4 == 0b0101010101); + auto val1 = even_bits(10); + auto val2 = odd_bits(10); + auto val3 = even_bits(10); + auto val4 = odd_bits(10); + BOOST_TEST((val1 == Bitset(10, 0b0101010101ULL))); + BOOST_TEST((val2 == Bitset(10, 0b1010101010ULL))); + BOOST_TEST((val3 == Bitset(10, 0b1010101010ULL))); + BOOST_TEST((val4 == Bitset(10, 0b0101010101ULL))); } // The evaluation functional carries the reference state sparsely, so every EvalState operation has to @@ -262,12 +271,12 @@ BOOST_AUTO_TEST_CASE(eval_state_indices_above_matches_the_dense_scan) { // graph and on the pared one, in both pictures. BOOST_AUTO_TEST_CASE(sparse_energy_matches_the_dense_gradient_value_bit_exactly) { constexpr size_t kNumModes = 8; - const auto data = test_utils::load_case_data("random_exact.msgpack"); + const auto data = test_utils::load_case_data("random_exact.msgpack"); for (const auto schrodinger_cutoff : {std::optional{}, std::optional{4}}) { BOOST_TEST_CONTEXT("schrodinger_cutoff = " << (schrodinger_cutoff ? "4" : "none")) { test_utils::SimulatorConfig cfg{.schrodinger_cutoff = schrodinger_cutoff, .comm = MPI_COMM_SELF}; - auto sim = test_utils::build_simulator(data, cfg); + auto sim = test_utils::build_simulator(kNumModes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); BOOST_CHECK_EQUAL(sim.expectation_value(data.parameters), @@ -287,11 +296,15 @@ BOOST_AUTO_TEST_CASE(sparse_energy_matches_the_dense_gradient_value_bit_exactly) // interleaving gradient calls must each keep reproducing their isolated value exactly. BOOST_AUTO_TEST_CASE(interleaved_gradients_do_not_share_scratch_state) { constexpr size_t kNumModes = 8; - const auto data = test_utils::load_case_data("random_exact.msgpack"); + const auto data = test_utils::load_case_data("random_exact.msgpack"); auto build = [&data](unsigned int cutoff) { - auto sim = - MonomialPropagator(data.hamiltonian, cutoff, data.initial_state, std::nullopt, MPI_COMM_SELF); + auto sim = test_utils::make_propagator(kNumModes, + data.hamiltonian, + cutoff, + data.initial_state, + std::nullopt, + MPI_COMM_SELF); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); return sim; }; diff --git a/cpp/tests/mpi_distributed_layer_equivalence.cpp b/cpp/tests/mpi_distributed_layer_equivalence.cpp index 152eb2b7..3a9945cb 100644 --- a/cpp/tests/mpi_distributed_layer_equivalence.cpp +++ b/cpp/tests/mpi_distributed_layer_equivalence.cpp @@ -43,19 +43,20 @@ struct TestInputs { }; auto load_inputs() -> TestInputs { - return {load_case_data("random_exact.msgpack")}; + return {load_case_data("random_exact.msgpack")}; } auto run_energy(const TestInputs& inputs, MPI_Comm comm) -> double { - MonomialPropagator sim(inputs.data.hamiltonian, - kCutoff, - inputs.data.initial_state, - std::nullopt, - comm, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt); + auto sim = test_utils::make_propagator(kNumModes, + inputs.data.hamiltonian, + kCutoff, + inputs.data.initial_state, + std::nullopt, + comm, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt); sim.build_graph(inputs.data.majoranas, inputs.data.param_inds, inputs.data.gen_coeffs); auto fn = sim.expectation_value_functional(); return fn(inputs.data.parameters); @@ -81,15 +82,16 @@ BOOST_AUTO_TEST_CASE(gradient_rank_count_within_fp_tolerance) { const auto& inputs = load_inputs(); auto run_gradient = [&](MPI_Comm comm) -> VecD { - MonomialPropagator sim(inputs.data.hamiltonian, - kCutoff, - inputs.data.initial_state, - std::nullopt, - comm, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt); + auto sim = test_utils::make_propagator(kNumModes, + inputs.data.hamiltonian, + kCutoff, + inputs.data.initial_state, + std::nullopt, + comm, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt); sim.build_graph(inputs.data.majoranas, inputs.data.param_inds, inputs.data.gen_coeffs); auto fn = sim.expectation_value_and_gradient_functional(); return fn(inputs.data.parameters).second; @@ -115,17 +117,17 @@ auto run_pauli_energy(MPI_Comm comm) -> double { OperatorDict init; init[slots_of_string("ZIIIII")] = std::complex(1.0, 0.0); init[slots_of_string("IIZZII")] = std::complex(0.5, 0.0); - MonomialPropagator sim(init, - kPauliQ, - VecZ{}, - std::nullopt, - comm, - 1e-12, - std::nullopt, - CutoffType::Support, - std::nullopt, - kPauliQ, - Basis::Pauli); + auto sim = test_utils::make_propagator(kPauliQ, + init, + kPauliQ, + VecZ{}, + std::nullopt, + comm, + 1e-12, + std::nullopt, + CutoffType::Support, + std::nullopt, + Basis::Pauli); std::vector gens; VecZ pmap; VecD gcoeffs; @@ -166,18 +168,18 @@ BOOST_AUTO_TEST_CASE(pauli_rank_count_energy_within_fp_tolerance) { // exercising the hybrid transport end to end. auto run_energy_partitioned(const TestInputs& inputs, MPI_Comm comm, size_t partitions) -> std::pair { - MonomialPropagator sim(inputs.data.hamiltonian, - kCutoff, - inputs.data.initial_state, - std::nullopt, - comm, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - partitions); + auto sim = test_utils::make_propagator(kNumModes, + inputs.data.hamiltonian, + kCutoff, + inputs.data.initial_state, + std::nullopt, + comm, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt, + Basis::Majorana, + partitions); sim.build_graph(inputs.data.majoranas, inputs.data.param_inds, inputs.data.gen_coeffs); auto fn = sim.expectation_value_functional(); const double e = fn(inputs.data.parameters); diff --git a/cpp/tests/mpi_fresh_insert_equivalence.cpp b/cpp/tests/mpi_fresh_insert_equivalence.cpp index 2973131b..43aa119d 100644 --- a/cpp/tests/mpi_fresh_insert_equivalence.cpp +++ b/cpp/tests/mpi_fresh_insert_equivalence.cpp @@ -38,17 +38,17 @@ using pauli_oracle::slots_of_string; // schrodinger_cutoff engages the picture; a low structural cutoff plus the upper_atol = 0 rescue // forces most partners to be fresh inserts, so the miss arm runs on nearly every partner. -template -auto run_schrodinger_majorana(const CaseData& data, MPI_Comm comm) -> double { - MonomialPropagator sim(data.hamiltonian, - /*cutoff=*/2U, - data.initial_state, - /*schrodinger_cutoff=*/std::optional{4U}, - comm, - /*lower_atol=*/std::nullopt, - /*upper_atol=*/std::optional{0.0}, - CutoffType::Length, - /*basis_change=*/std::nullopt); +auto run_schrodinger_majorana(size_t num_modes, const CaseData& data, MPI_Comm comm) -> double { + auto sim = test_utils::make_propagator(num_modes, + data.hamiltonian, + /*cutoff=*/2U, + data.initial_state, + /*schrodinger_cutoff=*/std::optional{4U}, + comm, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::optional{0.0}, + CutoffType::Length, + /*basis_change=*/std::nullopt); sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); auto energy_fn = sim.expectation_value_functional(std::nullopt); return energy_fn(VecD{}); @@ -59,8 +59,8 @@ BOOST_FIXTURE_TEST_CASE(mpi_fresh_insert_schrodinger_majorana_serial_world_equiv BOOST_TEST_MESSAGE("Skipping Schrödinger Majorana fresh-insert equivalence (world size = 1)."); return; } - const double e_serial = run_schrodinger_majorana(data, MPI_COMM_SELF); - const double e_world = run_schrodinger_majorana(data, MPI_COMM_WORLD); + const double e_serial = run_schrodinger_majorana(ExampleDataFix::n_modes, data, MPI_COMM_SELF); + const double e_world = run_schrodinger_majorana(ExampleDataFix::n_modes, data, MPI_COMM_WORLD); BOOST_TEST_MESSAGE("schrodinger majorana serial=" << e_serial << " world=" << e_world); BOOST_TEST(near(e_serial, e_world)); } @@ -73,17 +73,17 @@ auto run_schrodinger_pauli(MPI_Comm comm) -> double { OperatorDict init; init[slots_of_string("ZIIIII")] = std::complex(1.0, 0.0); init[slots_of_string("IIZZII")] = std::complex(0.5, 0.0); - MonomialPropagator sim(init, - /*cutoff=*/2U, - VecZ{}, - /*schrodinger_cutoff=*/std::optional{4U}, - comm, - /*lower_atol=*/std::nullopt, - /*upper_atol=*/std::optional{0.0}, - CutoffType::Support, - /*basis_change=*/std::nullopt, - kPauliQ, - Basis::Pauli); + auto sim = test_utils::make_propagator(kPauliQ, + init, + /*cutoff=*/2U, + VecZ{}, + /*schrodinger_cutoff=*/std::optional{4U}, + comm, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::optional{0.0}, + CutoffType::Support, + /*basis_change=*/std::nullopt, + Basis::Pauli); std::vector gens; VecZ pmap; VecD gcoeffs; diff --git a/cpp/tests/mpi_pare.cpp b/cpp/tests/mpi_pare.cpp index 0e57f162..56dde967 100644 --- a/cpp/tests/mpi_pare.cpp +++ b/cpp/tests/mpi_pare.cpp @@ -27,18 +27,18 @@ BOOST_AUTO_TEST_CASE(multi_rank_pare_expval_is_finite) { return; } - constexpr size_t NumModes = 8; + constexpr size_t kNumModes = 8; constexpr double kExpvalAtol = 1e-9; - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_WORLD}; - auto baseline_sim = build_simulator(data, cfg); + auto baseline_sim = build_simulator(kNumModes, data, cfg); const double baseline_expval = evaluate_expval(baseline_sim, data, false); BOOST_CHECK_SMALL(std::abs(baseline_expval - data.actual_expval), kExpvalAtol); - auto pared_sim = build_simulator(data, cfg); + auto pared_sim = build_simulator(kNumModes, data, cfg); const double pared_expval = evaluate_expval(pared_sim, data, true); BOOST_TEST_CONTEXT("world_size=" << world_size) { diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 8372e88f..9fbe12e4 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -33,12 +33,12 @@ BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_range_and_hash_mod) { for (int k = 0; k < 4; ++k) { inds.push_back(slot(rng)); } - const auto mono = indices_to_bitset(inds); + const auto mono = indices_to_bitset(inds, 2 * N); for (size_t n_ranks : {size_t{1}, size_t{2}, size_t{3}, size_t{7}}) { - const size_t r = find_rank(mono, n_ranks); + const size_t r = find_rank(mono, n_ranks); BOOST_TEST(r < n_ranks); - BOOST_TEST(r == monomial_hash(mono) % n_ranks); - BOOST_TEST(r == find_rank(mono, n_ranks)); // deterministic + BOOST_TEST(r == monomial_hash(mono) % n_ranks); + BOOST_TEST(r == find_rank(mono, n_ranks)); // deterministic } } } @@ -46,30 +46,32 @@ BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_range_and_hash_mod) { // n_ranks == 0 is degenerate: owner is rank 0, not a modulo by zero. BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_zero_ranks) { constexpr size_t N = 32; - const auto mono = indices_to_bitset(VecZ{0, 3, 5}); - BOOST_TEST(find_rank(mono, 0) == 0U); + const auto mono = indices_to_bitset(VecZ{0, 3, 5}, 2 * N); + BOOST_TEST(find_rank(mono, 0) == 0U); } BOOST_AUTO_TEST_CASE(mpi_utils_monomial_words_roundtrip) { constexpr size_t N = 96; // 2N = 192 bits -> 3 words - const auto a = indices_to_bitset(VecZ{0, 1, 100, 191}); - const auto b = indices_to_bitset(VecZ{5}); - const auto c = indices_to_bitset(VecZ{}); + const auto a = indices_to_bitset(VecZ{0, 1, 100, 191}, 2 * N); + const auto b = indices_to_bitset(VecZ{5}, 2 * N); + const auto c = indices_to_bitset(VecZ{}, 2 * N); + // The record width comes off the monomial now, not a kWords constant. + const size_t kW = a.num_words(); VecZ buf; - mpi_detail::append_monomial_words(a, buf); - mpi_detail::append_monomial_words(b, buf); - mpi_detail::append_monomial_words(c, buf); - BOOST_REQUIRE(buf.size() == 3 * mpi_detail::kWords); + mpi_detail::append_monomial_words(a, buf); + mpi_detail::append_monomial_words(b, buf); + mpi_detail::append_monomial_words(c, buf); + BOOST_REQUIRE(buf.size() == 3 * kW); - BOOST_TEST((mpi_detail::read_monomial_from_words(buf, 0) == a)); - BOOST_TEST((mpi_detail::read_monomial_from_words(buf, mpi_detail::kWords) == b)); - BOOST_TEST((mpi_detail::read_monomial_from_words(buf, 2 * mpi_detail::kWords) == c)); + BOOST_TEST((mpi_detail::read_monomial_from_words(buf, 0, 2 * N) == a)); + BOOST_TEST((mpi_detail::read_monomial_from_words(buf, kW, 2 * N) == b)); + BOOST_TEST((mpi_detail::read_monomial_from_words(buf, 2 * kW, 2 * N) == c)); constexpr size_t M = 32; - const auto d = indices_to_bitset(VecZ{2, 40, 63}); + const auto d = indices_to_bitset(VecZ{2, 40, 63}, 2 * M); VecZ sbuf; - mpi_detail::append_monomial_words(d, sbuf); - BOOST_REQUIRE(sbuf.size() == mpi_detail::kWords); - BOOST_TEST((mpi_detail::read_monomial_from_words(sbuf, 0) == d)); + mpi_detail::append_monomial_words(d, sbuf); + BOOST_REQUIRE(sbuf.size() == d.num_words()); + BOOST_TEST((mpi_detail::read_monomial_from_words(sbuf, 0, 2 * M) == d)); } diff --git a/cpp/tests/operator_index_tests.cpp b/cpp/tests/operator_index_tests.cpp index b88bad78..52c8f468 100644 --- a/cpp/tests/operator_index_tests.cpp +++ b/cpp/tests/operator_index_tests.cpp @@ -40,8 +40,9 @@ BOOST_AUTO_TEST_CASE(operator_index_term_index_width_matches_build) { namespace { constexpr size_t N = 32; -using Store = OperatorIndex; -using MSet = Monomial; +using Store = OperatorIndex; +constexpr size_t kBits = 2 * N; // the store is runtime-width now +using MSet = Bitset; // Owners hold the store by unique_ptr and share stable pointers into it, so it must stay // non-copyable and non-movable; clone() is the only deep copy. @@ -49,12 +50,12 @@ static_assert(!std::is_move_constructible_v, "OperatorIndex must remain n static_assert(!std::is_copy_constructible_v, "OperatorIndex must remain non-copyable"); MSet bs(const VecZ &r) { - return indices_to_bitset(r); + return indices_to_bitset(r, kBits); } } // namespace BOOST_AUTO_TEST_CASE(rows_roundtrip_dense_popcount_positions) { - Store s; + Store s(kBits); s.push_back(bs({0, 3, 5})); s.push_back(bs({1, 2})); BOOST_TEST(s.size() == 2u); @@ -64,14 +65,14 @@ BOOST_AUTO_TEST_CASE(rows_roundtrip_dense_popcount_positions) { std::vector pos; s.for_each_position(0, [&](size_t b) { pos.push_back(b); }); BOOST_TEST(pos.size() == 3u); - // for_each_position yields raw bit positions (ascending). indices_to_bitset<32>({0,3,5}) + // for_each_position yields raw bit positions (ascending). indices_to_bitset({0,3,5}, 64) // sets bits at 2*32-1-0=63, 2*32-1-3=60, 2*32-1-5=58, so find_first gives 58 first. BOOST_TEST(pos[0] == 58u); BOOST_TEST(pos[2] == 63u); } BOOST_AUTO_TEST_CASE(index_emplace_then_find_roundtrip) { - Store s; + Store s(kBits); s.push_back(bs({0, 3, 5})); s.emplace(bs({0, 3, 5}), 0); s.push_back(bs({1, 2})); @@ -83,7 +84,7 @@ BOOST_AUTO_TEST_CASE(index_emplace_then_find_roundtrip) { } BOOST_AUTO_TEST_CASE(width_is_a_construction_invariant) { - Store s(4); // stride = 1 + 4, fixed at construction + Store s(kBits, 4); // stride = 1 + 4, fixed at construction s.push_back(bs({0, 2, 4, 6})); // a 4-position row fits inline at width 4 s.reserve(20); // capacity only -- width/stride are never touched by reserve BOOST_TEST(s.popcount(0) == 4u); @@ -91,14 +92,14 @@ BOOST_AUTO_TEST_CASE(width_is_a_construction_invariant) { } BOOST_AUTO_TEST_CASE(overflow_is_lossless_above_width) { - Store s(2); // width 2; a 3-position row must overflow + Store s(kBits, 2); // width 2; a 3-position row must overflow s.push_back(bs({0, 1, 2})); BOOST_TEST(s.popcount(0) == 3u); // popcount recovered from the overflow map BOOST_TEST((s.row(0) == bs({0, 1, 2}))); } BOOST_AUTO_TEST_CASE(index_survives_rehash_in_place) { - Store a; + Store a(kBits); // 64 distinct rows (positions i and (i+7)%62) force at least one rehash of the in-place index. for (int i = 0; i < 64; ++i) { a.push_back(bs({static_cast(i % 62), static_cast((i + 7) % 62)})); @@ -110,7 +111,7 @@ BOOST_AUTO_TEST_CASE(index_survives_rehash_in_place) { } BOOST_AUTO_TEST_CASE(clone_is_deep_and_independent) { - Store a(4); // non-default width must carry over + Store a(kBits, 4); // non-default width must carry over a.push_back(bs({0, 3, 5})); a.emplace(bs({0, 3, 5}), 0); a.push_back(bs({1, 2})); @@ -137,7 +138,7 @@ BOOST_AUTO_TEST_CASE(clone_is_deep_and_independent) { } BOOST_AUTO_TEST_CASE(clone_preserves_overflow_rows) { - Store a(2); // width 2; a 3-position row overflows losslessly + Store a(kBits, 2); // width 2; a 3-position row overflows losslessly a.push_back(bs({0, 1, 2})); a.emplace(bs({0, 1, 2}), 0); @@ -147,12 +148,58 @@ BOOST_AUTO_TEST_CASE(clone_preserves_overflow_rows) { BOOST_TEST(*b->find(bs({0, 1, 2})) == 0u); } +// resized() is the migration update_cutoff() relies on: every row keeps its index, its content and its +// findability, whichever way the width moved -- including rows that cross the overflow boundary in +// either direction, since set() re-decides that per row rather than trusting the old classification. +BOOST_AUTO_TEST_CASE(resized_preserves_index_content_and_find_when_widening) { + Store a(kBits, 2); // width 2: the 3-position row below starts in overflow_ + a.push_back(bs({0, 1})); + a.emplace(bs({0, 1}), 0); + a.push_back(bs({0, 1, 2})); + a.emplace(bs({0, 1, 2}), 1); + + auto b = a.resized(4); // now fits inline at both rows + BOOST_TEST(b->inline_width() == 4u); + BOOST_TEST(b->size() == 2u); + BOOST_TEST((b->row(0) == bs({0, 1}))); + BOOST_TEST((b->row(1) == bs({0, 1, 2}))); + BOOST_TEST(b->popcount(1) == 3u); + auto f0 = b->find(bs({0, 1})); + auto f1 = b->find(bs({0, 1, 2})); + BOOST_TEST(f0.has_value()); + BOOST_TEST(*f0 == 0u); + BOOST_TEST(f1.has_value()); + BOOST_TEST(*f1 == 1u); + + // Independent of the source: mutating a after the fact must not reach b. + a.set(0, bs({5, 6})); + BOOST_TEST((b->row(0) == bs({0, 1}))); +} + +BOOST_AUTO_TEST_CASE(resized_preserves_index_content_and_find_when_narrowing) { + Store a(kBits, 4); // width 4: both rows below fit inline + a.push_back(bs({0, 1})); + a.emplace(bs({0, 1}), 0); + a.push_back(bs({0, 1, 2})); + a.emplace(bs({0, 1, 2}), 1); + + auto b = a.resized(2); // row 1 must now spill to overflow_ + BOOST_TEST(b->inline_width() == 2u); + BOOST_TEST(b->size() == 2u); + BOOST_TEST((b->row(0) == bs({0, 1}))); + BOOST_TEST((b->row(1) == bs({0, 1, 2}))); // recovered from overflow_, popcount unaffected + BOOST_TEST(b->popcount(1) == 3u); + auto f1 = b->find(bs({0, 1, 2})); + BOOST_TEST(f1.has_value()); + BOOST_TEST(*f1 == 1u); +} + // find_batch (the group-prefetch pipelined lookup) must be semantically identical to n independent // find() calls. The query mix below spans several G=16 groups plus a short tail and interleaves // present and absent keys, so every branch but the h32-collision fallback runs; that one needs a // real 32-bit hash collision, but the equivalence assertion pins it whichever path a key takes. BOOST_AUTO_TEST_CASE(find_batch_matches_scalar_find) { - Store s; + Store s(kBits); constexpr size_t kRows = 200; // > 12 groups of G=16 // (i/60, 4 + i%60) is a bijection for i < 240 over the disjoint ranges {0..3} and {4..63}. for (size_t i = 0; i < kRows; ++i) { @@ -187,7 +234,7 @@ BOOST_AUTO_TEST_CASE(find_batch_matches_scalar_find) { // Pins find_batch's partition.count == 0 early-out. BOOST_AUTO_TEST_CASE(find_batch_on_empty_store_is_all_missing) { - Store s; + Store s(kBits); const std::array keys{bs({0, 3}), bs({1, 2}), bs({4, 5, 6})}; std::array out{0, 0, 0}; s.find_batch(keys.data(), keys.size(), out.data()); @@ -195,3 +242,78 @@ BOOST_AUTO_TEST_CASE(find_batch_on_empty_store_is_all_missing) { BOOST_TEST(out[i] == Store::kNotFound); } } + +// The row payload width is chosen from num_bits: one byte per slot while a bit position fits one, two +// above that. The row array is the operator's largest, and rows are payload -- never a hash input, never +// serialized -- so a widening here changes no term and no energy and a baseline diff cannot see it. This +// is the footprint gate. memory_bytes() - slack_bytes() is the *used* part of the array, which makes the +// figure exact instead of allocator-dependent. +BOOST_AUTO_TEST_CASE(row_slot_width_follows_the_position_count) { + constexpr size_t kInline = 6; + constexpr size_t kRows = 500; + constexpr size_t kNarrowBits = Store::kNarrowPositions; + constexpr size_t kWideBits = Store::kNarrowPositions + 2; + + Store narrow(kNarrowBits, kInline); + Store wide(kWideBits, kInline); + for (size_t i = 0; i < kRows; ++i) { + // One bit per row: any popcount <= kInline works, but staying at 1 keeps every row off the + // overflow side-map, whose bytes are counted separately and would blur the comparison. + narrow.push_back(Bitset(kNarrowBits, uint64_t{1} << (i % 64))); + wide.push_back(Bitset(kWideBits, uint64_t{1} << (i % 64))); + } + + const size_t slots = kRows * (1 + kInline); + BOOST_TEST(narrow.memory_bytes() - narrow.slack_bytes() == slots); + BOOST_TEST(wide.memory_bytes() - wide.slack_bytes() == 2 * slots); +} + +// The narrow overflow marker is 255, which is also a legal bit position at 256 positions. A row holding +// it must read back as a position: the marker only ever occupies slot 0. +BOOST_AUTO_TEST_CASE(a_narrow_row_holds_the_marker_valued_position) { + constexpr size_t kNarrowBits = Store::kNarrowPositions; + Store s(kNarrowBits, 4); + Bitset m(kNarrowBits); + m.set(0); + m.set(kNarrowBits - 1); + s.push_back(m); + s.emplace(m, 0); + + BOOST_TEST(s.popcount(0) == 2u); + BOOST_TEST((s.row(0) == m)); + BOOST_TEST(s.find(m).has_value()); + BOOST_TEST(*s.find(m) == 0u); + std::vector pos; + s.for_each_position(0, [&](size_t b) { pos.push_back(b); }); + BOOST_TEST(pos.size() == 2u); + BOOST_TEST(pos[0] == 0u); + BOOST_TEST(pos[1] == kNarrowBits - 1); +} + +// The marker is per-width, so the lossless spill has to be exercised on both sides of the crossover. +BOOST_AUTO_TEST_CASE(overflow_spills_at_both_row_widths) { + for (const size_t bits : {Store::kNarrowPositions, Store::kNarrowPositions + 2}) { + Store s(bits, 2); + Bitset m(bits); + m.set(0); + m.set(1); + m.set(bits - 1); // popcount 3, above the inline width of 2 + s.push_back(m); + s.emplace(m, 0); + + BOOST_TEST(s.popcount(0) == 3u); + BOOST_TEST((s.row(0) == m)); + BOOST_TEST(*s.find(m) == 0u); + // A round trip through clone() too: it copies both row arrays and the side-map. + const auto c = s.clone(); + BOOST_TEST(c->popcount(0) == 3u); + BOOST_TEST((c->row(0) == m)); + } +} + +// A store past kMaxPositions has no row width that can hold its positions, and says so rather than +// truncating a position into a plausible-looking row. +BOOST_AUTO_TEST_CASE(a_width_past_the_row_payload_bound_throws) { + BOOST_CHECK_THROW(Store(Store::kMaxPositions + 2), OperatorIndexWidthUnsupported); + BOOST_CHECK_NO_THROW(Store{Store::kMaxPositions}); +} diff --git a/cpp/tests/pare_graph_tests.cpp b/cpp/tests/pare_graph_tests.cpp index f92e9bc8..e234857c 100644 --- a/cpp/tests/pare_graph_tests.cpp +++ b/cpp/tests/pare_graph_tests.cpp @@ -36,27 +36,24 @@ constexpr size_t kNumModes = 8; // Mirrors the streaming provider the pare functional uses: fold the operator's inverted index, // truncated to each layer's scaled_count. -template -auto recompute_cos(const monoprop::detail::InvertedIndex &inverted_index, const LayerTraversal &layer) +auto recompute_cos(size_t num_modes, const monoprop::detail::InvertedIndex &inverted_index, const LayerTraversal &layer) -> CosMask { - Monomial gen{}; + Bitset gen(2 * num_modes); const auto &gw = layer.generator_words(); std::memcpy(gen.data(), gw.data(), gw.size() * sizeof(uint64_t)); - const auto combined = monoprop::detail::make_fold_cache(inverted_index, - gen, - layer.scaled_count(), - monoprop::Basis::Majorana); - return monoprop::detail::fold_to_cos_mask(combined); + const auto combined = + monoprop::detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), monoprop::Basis::Majorana); + return monoprop::detail::fold_to_cos_mask(combined); } } // namespace // The streaming pare sweep must engage pruned_cos on exactly the layers whose cos loses an index. BOOST_AUTO_TEST_CASE(pare_graph_emits_expected_layer_kinds) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(kNumModes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const auto &graph = sim.graph(); @@ -82,7 +79,7 @@ BOOST_AUTO_TEST_CASE(pare_graph_emits_expected_layer_kinds) { const uint64_t synth_bit = uint64_t{1} << (synth_index & 63U); auto provider = [&](size_t i) -> CosMask { - CosMask cos = recompute_cos(inverted_index, graph.get_layer_traversal(i)); + CosMask cos = recompute_cos(kNumModes, inverted_index, graph.get_layer_traversal(i)); if (i == marked_layer) { bool merged = false; for (auto &b : cos.blocks) { @@ -143,10 +140,10 @@ BOOST_AUTO_TEST_CASE(pare_graph_emits_expected_layer_kinds) { } BOOST_AUTO_TEST_CASE(pare_graph_energy_matches_unpared) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim_full = build_simulator(data, cfg); + auto sim_full = build_simulator(kNumModes, data, cfg); sim_full.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto ev_full = sim_full.expectation_value_functional(std::nullopt); const double e_full = ev_full(data.parameters); @@ -154,13 +151,13 @@ BOOST_AUTO_TEST_CASE(pare_graph_energy_matches_unpared) { // The pared and unpared replays reduce in an unpinned accumulation order, so they differ by a // few ULP (~1e-18 here) run-to-run: exact == is the wrong assertion. The tolerance is far tighter // than any real pruning effect but comfortably above that reorder noise. - auto sim_tiny = build_simulator(data, cfg); + auto sim_tiny = build_simulator(kNumModes, data, cfg); sim_tiny.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto ev_tiny = sim_tiny.expectation_value_functional(std::optional{1e-12}); const double e_tiny = ev_tiny(data.parameters); BOOST_CHECK_SMALL(std::abs(e_full - e_tiny), 1e-12); - auto sim_real = build_simulator(data, cfg); + auto sim_real = build_simulator(kNumModes, data, cfg); sim_real.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto ev_real = sim_real.expectation_value_functional(std::optional{1e-10}); const double e_real = ev_real(data.parameters); diff --git a/cpp/tests/partition_equivalence_tests.cpp b/cpp/tests/partition_equivalence_tests.cpp index 9c8a984e..19783b7c 100644 --- a/cpp/tests/partition_equivalence_tests.cpp +++ b/cpp/tests/partition_equivalence_tests.cpp @@ -41,23 +41,23 @@ constexpr size_t kNumModes = 8; constexpr unsigned int kCutoff = 4; auto majorana_sim(const CaseData &data, size_t partitions, std::optional lower_atol = std::nullopt) - -> MonomialPropagator { - return MonomialPropagator(data.hamiltonian, - kCutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - lower_atol, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - partitions); + -> MonomialPropagator { + return test_utils::make_propagator(kNumModes, + data.hamiltonian, + kCutoff, + data.initial_state, + std::nullopt, + MPI_COMM_SELF, + lower_atol, + std::nullopt, + CutoffType::Length, + std::nullopt, + Basis::Majorana, + partitions); } BOOST_AUTO_TEST_CASE(partition_majorana_energy_matches_across_partition_counts) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto ref = majorana_sim(data, 1); ref.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const double e1 = ref.expectation_value(data.parameters); @@ -75,7 +75,7 @@ BOOST_AUTO_TEST_CASE(partition_majorana_energy_matches_across_partition_counts) } BOOST_AUTO_TEST_CASE(partition_majorana_gradient_matches_across_partition_counts) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto ref = majorana_sim(data, 1); ref.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const auto g1 = ref.expectation_value_and_gradient(data.parameters).second; @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(partition_majorana_gradient_matches_across_partition_counts } BOOST_AUTO_TEST_CASE(partition_majorana_propagate_then_expectation_matches) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto run = [&](size_t S) { auto sim = majorana_sim(data, S); sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); @@ -113,7 +113,7 @@ BOOST_AUTO_TEST_CASE(partition_majorana_propagate_then_expectation_matches) { // Two independent S=4 runs are bit-identical: ShmComm sums in ascending rank order and each partition is // deterministic, so a given partition count has no run-to-run jitter. BOOST_AUTO_TEST_CASE(partition_energy_is_deterministic) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto energy_s4 = [&] { auto sim = majorana_sim(data, 4); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); @@ -127,7 +127,7 @@ BOOST_AUTO_TEST_CASE(partition_energy_is_deterministic) { // the multiset: the partitions are disjoint and cover every term. Sorting both sides is the only // comparison the API's contract supports — see the note on contract_partially(). BOOST_AUTO_TEST_CASE(partition_contract_partially_matches_as_a_multiset) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto sorted_coeffs = [&](size_t S) { auto sim = majorana_sim(data, S); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); @@ -150,12 +150,12 @@ BOOST_AUTO_TEST_CASE(partition_contract_partially_matches_as_a_multiset) { // The raw per-partition accessors have no facade reading: the facade's own graph_/mp_op_ are never // populated, so returning them would hand a C++ consumer empty state that looks valid. BOOST_AUTO_TEST_CASE(partition_raw_accessors_reject_a_facade) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto sim = majorana_sim(data, 4); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); BOOST_CHECK_THROW(static_cast(sim.graph()), std::runtime_error); BOOST_CHECK_THROW(static_cast(sim.mp_op()), std::runtime_error); - BOOST_CHECK_THROW(static_cast(sim.indexing()), std::runtime_error); + BOOST_CHECK_THROW(static_cast(sim.num_local_terms()), std::runtime_error); BOOST_CHECK_THROW(static_cast(sim.graph_data()), std::runtime_error); auto solo = majorana_sim(data, 1); @@ -171,21 +171,21 @@ BOOST_AUTO_TEST_CASE(partition_raw_accessors_reject_a_facade) { // oracle must itself differ from the wide run, else the last assertion would hold vacuously. BOOST_AUTO_TEST_CASE(partition_setters_reach_every_partition) { constexpr size_t kLihModes = LihFixture::n_modes; - const auto data = load_case_data("lih_fermionic_spin_exact.msgpack"); + const auto data = load_case_data("lih_fermionic_spin_exact.msgpack"); auto build = [&](unsigned int cutoff, unsigned int updated_cutoff) { - MonomialPropagator sim(data.hamiltonian, - cutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kLihModes, - Basis::Majorana, - /*partitions=*/4); + auto sim = test_utils::make_propagator(kLihModes, + data.hamiltonian, + cutoff, + data.initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt, + Basis::Majorana, + /*partitions=*/4); if (cutoff != updated_cutoff) { sim.update_cutoff(updated_cutoff); BOOST_CHECK_EQUAL(sim.cutoff(), updated_cutoff); @@ -201,13 +201,48 @@ BOOST_AUTO_TEST_CASE(partition_setters_reach_every_partition) { BOOST_CHECK_EQUAL(n_tight, build(2 * kLihModes, 4)); } +// update_cutoff() resizes the live row store in place (MPOperator::resize_store) rather than dropping +// it, so the initial fill's rows -- inserted before the setter runs -- must migrate to the new width +// with their content and index intact. Both propagators below end up at the same cutoff before +// build_graph, so the row-store width at construction time (single-partition single-rank data, no width +// past kMaxInlinePositions the migration could disagree over the encoding of) is the only difference: +// if migration ever corrupted a row or its index, the initial fill's coefficients would be wrong for +// every layer built from them, not just the resized one. +BOOST_AUTO_TEST_CASE(update_cutoff_after_initial_fill_matches_direct_construction) { + const auto data = load_case_data("random_exact.msgpack"); + + auto ref = majorana_sim(data, 1); + ref.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + const double e_ref = ref.expectation_value(data.parameters); + + // Constructed at cutoff 1 -- the narrowest row-store width available -- so every initial term with + // more than one occupied mode starts in the dense backend's overflow map, then widened to kCutoff + // before the graph is built at all. + auto sim = test_utils::make_propagator(kNumModes, + data.hamiltonian, + /*cutoff=*/1U, + data.initial_state, + std::nullopt, + MPI_COMM_SELF); + sim.update_cutoff(kCutoff); + BOOST_REQUIRE_EQUAL(sim.cutoff(), kCutoff); + BOOST_REQUIRE_EQUAL(sim.size(), ref.size()); // the initial fill survived the resize term-for-term + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + const double e_sim = sim.expectation_value(data.parameters); + + BOOST_TEST_CONTEXT("e_ref=" << e_ref << " e_sim=" << e_sim) { + BOOST_TEST(near(e_ref, e_sim)); + } + BOOST_CHECK_EQUAL(ref.size(), sim.size()); +} + BOOST_AUTO_TEST_CASE(partition_deep_copy_matches) { - const auto data = load_case_data("random_exact.msgpack"); + const auto data = load_case_data("random_exact.msgpack"); auto sim = majorana_sim(data, 4); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const double e = sim.expectation_value(data.parameters); - MonomialPropagator copy(sim); // clones the partition group (fresh threads + ShmComm) + MonomialPropagator copy(sim); // clones the partition group (fresh threads + ShmComm) const double e_copy = copy.expectation_value(data.parameters); BOOST_CHECK_EQUAL(e, e_copy); BOOST_CHECK_EQUAL(sim.size(), copy.size()); @@ -215,23 +250,23 @@ BOOST_AUTO_TEST_CASE(partition_deep_copy_matches) { constexpr size_t kNq = 6; -auto pauli_sim(const std::map &obs, size_t partitions) -> MonomialPropagator { +auto pauli_sim(const std::map &obs, size_t partitions) -> MonomialPropagator { OperatorDict init; for (const auto &[p, c] : obs) { init[slots_of_string(p)] = std::complex(c, 0.0); } - return MonomialPropagator(init, - /*cutoff=*/kNq, - /*initial_state=*/{}, - std::nullopt, - MPI_COMM_SELF, - /*lower_atol=*/1e-12, - std::nullopt, - CutoffType::Support, - std::nullopt, - kNq, - Basis::Pauli, - partitions); + return test_utils::make_propagator(kNq, + init, + /*cutoff=*/kNq, + /*initial_state=*/{}, + std::nullopt, + MPI_COMM_SELF, + /*lower_atol=*/1e-12, + std::nullopt, + CutoffType::Support, + std::nullopt, + Basis::Pauli, + partitions); } auto run_pauli_energy(size_t partitions) -> std::pair { @@ -281,37 +316,37 @@ BOOST_AUTO_TEST_CASE(partition_pauli_energy_matches_across_partition_counts) { // master threads before building the partitions on them (first-touch locality), so the unwind has to join // already-started threads. Every MonomialPropagator ctor validation reaches this path. BOOST_AUTO_TEST_CASE(partition_factory_exception_propagates_without_terminate) { - const auto data = load_case_data("random_exact.msgpack"); - // logical_num_modes = 0 is rejected by each partition's own constructor, on its own master thread. - BOOST_CHECK_THROW(MonomialPropagator(data.hamiltonian, - kCutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - /*logical_num_modes=*/0, - Basis::Majorana, - /*partitions=*/4), + const auto data = load_case_data("random_exact.msgpack"); + // num_modes = 0 is rejected by each partition's own constructor, on its own master thread. + BOOST_CHECK_THROW(test_utils::make_propagator(/*num_modes=*/0, + data.hamiltonian, + kCutoff, + data.initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt, + Basis::Majorana, + /*partitions=*/4), std::runtime_error); // An out-of-range operator index takes the same path, and the group stays usable afterwards. auto bad_op = data.hamiltonian; bad_op[VecZ{2 * kNumModes}] = std::complex(1.0, 0.0); - BOOST_CHECK_THROW(MonomialPropagator(bad_op, - kCutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - /*partitions=*/4), + BOOST_CHECK_THROW(test_utils::make_propagator(kNumModes, + bad_op, + kCutoff, + data.initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Length, + std::nullopt, + Basis::Majorana, + /*partitions=*/4), std::runtime_error); BOOST_CHECK_NO_THROW(majorana_sim(data, 4)); } diff --git a/cpp/tests/partition_group_clone_tests.cpp b/cpp/tests/partition_group_clone_tests.cpp index 7781879a..2912d222 100644 --- a/cpp/tests/partition_group_clone_tests.cpp +++ b/cpp/tests/partition_group_clone_tests.cpp @@ -36,10 +36,9 @@ constexpr unsigned int kCutoff = 4; // A toy subclass exercising both extension points: `child_factory` builds more of itself, and // `clone_()` overrides the base's default (which would otherwise slice a copy down to Base). -template -class DerivedPropagator : public MonomialPropagator { +class DerivedPropagator : public MonomialPropagator { public: - using Base = MonomialPropagator; + using Base = MonomialPropagator; DerivedPropagator(const OperatorDict &initial_operator, unsigned int cutoff, @@ -49,16 +48,16 @@ class DerivedPropagator : public MonomialPropagator { : Base(initial_operator, cutoff, initial_state, + kNumModes, std::nullopt, comm, std::nullopt, std::nullopt, CutoffType::Length, std::nullopt, - NumModes, Basis::Majorana, partitions, - typename Base::PartitionChildFactory{[=](mpi::Comm partition_comm) -> std::unique_ptr { + Base::PartitionChildFactory{[=](mpi::Comm partition_comm) -> std::unique_ptr { return std::make_unique(initial_operator, cutoff, initial_state, @@ -90,8 +89,8 @@ class DerivedPropagator : public MonomialPropagator { } // namespace BOOST_AUTO_TEST_CASE(partition_facade_children_are_derived_type) { - const auto data = load_case_data("random_exact.msgpack"); - DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/4); + const auto data = load_case_data("random_exact.msgpack"); + DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/4); BOOST_TEST(sim.is_facade()); BOOST_TEST(sim.children_are_all_derived()); @@ -99,10 +98,10 @@ BOOST_AUTO_TEST_CASE(partition_facade_children_are_derived_type) { } BOOST_AUTO_TEST_CASE(partition_facade_copy_children_are_derived_type) { - const auto data = load_case_data("random_exact.msgpack"); - DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/4); + const auto data = load_case_data("random_exact.msgpack"); + DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/4); - DerivedPropagator copy(sim); // exercises PartitionGroup's copy ctor -> clone_() + DerivedPropagator copy(sim); // exercises PartitionGroup's copy ctor -> clone_() BOOST_TEST(copy.is_facade()); BOOST_TEST(copy.children_are_all_derived()); } diff --git a/cpp/tests/pauli_algebra_tests.cpp b/cpp/tests/pauli_algebra_tests.cpp index 170cf631..51b1bc9a 100644 --- a/cpp/tests/pauli_algebra_tests.cpp +++ b/cpp/tests/pauli_algebra_tests.cpp @@ -35,11 +35,10 @@ namespace { // the header's primitives (detail::pauli_uv, detail::mod4, pauli_y_count). // Qubit Pauli weight = number of non-identity single-qubit letters = or_sum = |x | z|. -template -[[nodiscard]] auto pauli_weight(const Monomial &p) -> size_t { - constexpr auto e_mask = pauli_even_mask(); +[[nodiscard]] auto pauli_weight(size_t num_modes, const Bitset &p) -> size_t { + const auto e_mask = pauli_even_mask(2 * num_modes); size_t weight = 0; - for (size_t w = 0; w < Monomial::num_words(); ++w) { + for (size_t w = 0; w < p.num_words(); ++w) { const auto [v, u] = detail::pauli_uv(p.word(w), e_mask.word(w)); weight += static_cast(std::popcount(v | u)); } @@ -49,15 +48,14 @@ template // The mod-4 exponent of the product-phase i^e for A*B, with A the left operand. // e = yA + yB - yR + 2*(zA . xB) (mod 4), R = A ^ B. // e is odd iff A,B anticommute (phase = +/- i); even iff they commute (phase = +/- 1). -template -[[nodiscard]] auto product_phase_exponent(const Monomial &a, const Monomial &b) -> int { - constexpr auto e_mask = pauli_even_mask(); +[[nodiscard]] auto product_phase_exponent(size_t num_modes, const Bitset &a, const Bitset &b) -> int { + const auto e_mask = pauli_even_mask(2 * num_modes); const auto r = a ^ b; - const long y_a = static_cast(pauli_y_count(a)); - const long y_b = static_cast(pauli_y_count(b)); - const long y_r = static_cast(pauli_y_count(r)); + const long y_a = static_cast(pauli_y_count(a)); + const long y_b = static_cast(pauli_y_count(b)); + const long y_r = static_cast(pauli_y_count(r)); long cross = 0; // zA . xB = popcount(v-plane(A) & x-plane(B)) - for (size_t w = 0; w < Monomial::num_words(); ++w) { + for (size_t w = 0; w < a.num_words(); ++w) { const uint64_t e = e_mask.word(w); const uint64_t z_a = a.word(w) & e; // v-plane of A const auto [v_b, u_b] = detail::pauli_uv(b.word(w), e); @@ -68,17 +66,14 @@ template } // Product phase phi (unit modulus) such that A*B = phi * (A ^ B), A the left operand. -template -[[nodiscard]] auto pauli_product_phase(const Monomial &a, const Monomial &b) - -> std::complex { - return POWERS_OF_I[product_phase_exponent(a, b)]; +[[nodiscard]] auto pauli_product_phase(size_t num_modes, const Bitset &a, const Bitset &b) -> std::complex { + return POWERS_OF_I[product_phase_exponent(num_modes, a, b)]; } // Emit sign +/-1 such that A*B = sign * i * (A ^ B), valid when A,B anticommute (exponent e odd). // The raw product sign; pauli_rotation_sign returns exactly -pauli_emit_sign_antic. -template -[[nodiscard]] auto pauli_emit_sign_antic(const Monomial &a, const Monomial &b) -> int { - return product_phase_exponent(a, b) == 1 ? 1 : -1; +[[nodiscard]] auto pauli_emit_sign_antic(size_t num_modes, const Bitset &a, const Bitset &b) -> int { + return product_phase_exponent(num_modes, a, b) == 1 ? 1 : -1; } } // namespace @@ -88,11 +83,11 @@ BOOST_AUTO_TEST_CASE(pauli_algebra_pair_swap_and_anticommutation) { for (size_t n : {size_t{1}, size_t{2}}) { for (const auto &pa : all_strings(n)) { - const auto a = native_bitset(pa); - BOOST_TEST((pair_swap(pair_swap(a)) == a)); + const auto a = native_bitset(N, pa); + BOOST_TEST((pair_swap(pair_swap(a)) == a)); for (const auto &pb : all_strings(n)) { - const auto b = native_bitset(pb); - const bool antic = pauli_anticommutes(a, b); + const auto b = native_bitset(N, pb); + const bool antic = pauli_anticommutes(a, b); BOOST_TEST(antic == string_anticommutes(pa, pb)); // Independent dense-matrix check: anticommute iff AB == -BA. const auto ma = matrix_from_string(pa); @@ -111,20 +106,20 @@ BOOST_AUTO_TEST_CASE(pauli_algebra_pair_swap_and_anticommutation) { const size_t n = 1 + (rng() % 6); const auto pa = random_string(rng, n); const auto pb = random_string(rng, n); - const auto a = native_bitset(pa); - const auto b = native_bitset(pb); - BOOST_TEST((pair_swap(pair_swap(a)) == a)); - BOOST_TEST(pauli_anticommutes(a, b) == string_anticommutes(pa, pb)); + const auto a = native_bitset(N, pa); + const auto b = native_bitset(N, pb); + BOOST_TEST((pair_swap(pair_swap(a)) == a)); + BOOST_TEST(pauli_anticommutes(a, b) == string_anticommutes(pa, pb)); } constexpr size_t NW = 40; // 2N = 80 bits -> 2 words: exercises multiword kernels. for (size_t trial = 0; trial < 2000; ++trial) { const auto pa = random_string(rng, NW); const auto pb = random_string(rng, NW); - const auto a = native_bitset(pa); - const auto b = native_bitset(pb); - BOOST_TEST((pair_swap(pair_swap(a)) == a)); - BOOST_TEST(pauli_anticommutes(a, b) == string_anticommutes(pa, pb)); + const auto a = native_bitset(NW, pa); + const auto b = native_bitset(NW, pb); + BOOST_TEST((pair_swap(pair_swap(a)) == a)); + BOOST_TEST(pauli_anticommutes(a, b) == string_anticommutes(pa, pb)); } } @@ -132,17 +127,17 @@ BOOST_AUTO_TEST_CASE(pauli_algebra_pair_swap_and_anticommutation) { BOOST_AUTO_TEST_CASE(pauli_algebra_encoding_is_jw_image) { constexpr size_t N = 8; for (size_t n : {size_t{1}, size_t{2}}) { - const auto basis = jw_basis(n); + const auto basis = jw_basis(N, n); for (const auto &p : all_strings(n)) { - BOOST_TEST((native_bitset(p) == change_basis(jw_bitset(p), basis))); + BOOST_TEST((native_bitset(N, p) == change_basis(jw_bitset(N, p), basis))); } } std::mt19937 rng(0x1234ABCDU); for (size_t trial = 0; trial < 4000; ++trial) { const size_t n = 1 + (rng() % 6); - const auto basis = jw_basis(n); + const auto basis = jw_basis(N, n); const auto p = random_string(rng, n); - BOOST_TEST((native_bitset(p) == change_basis(jw_bitset(p), basis))); + BOOST_TEST((native_bitset(N, p) == change_basis(jw_bitset(N, p), basis))); } } @@ -152,32 +147,32 @@ BOOST_AUTO_TEST_CASE(pauli_algebra_product_phase_vs_brute_force) { const size_t d = size_t{1} << n; const auto strs = all_strings(n); for (const auto &pa : strs) { - const auto a = native_bitset(pa); + const auto a = native_bitset(N, pa); const auto ma = matrix_from_string(pa); for (const auto &pb : strs) { - const auto b = native_bitset(pb); + const auto b = native_bitset(N, pb); const auto r = a ^ b; std::string pr(n, 'I'); for (size_t q = 0; q < n; ++q) { - pr[q] = letter_from_bitset(r, q); + pr[q] = letter_from_bitset(r, q); } const auto mr = matrix_from_string(pr); const auto mb = matrix_from_string(pb); const auto ab = matmul(ma, mb, d); - const cd phi = pauli_product_phase(a, b); + const cd phi = pauli_product_phase(N, a, b); BOOST_TEST(std::abs(std::abs(phi) - 1.0) < 1e-12); BOOST_TEST(approx_equal(ab, scalar_mul(phi, mr))); - if (pauli_anticommutes(a, b)) { - const int sign = pauli_emit_sign_antic(a, b); + if (pauli_anticommutes(a, b)) { + const int sign = pauli_emit_sign_antic(N, a, b); BOOST_TEST((sign == 1 || sign == -1)); // A*B = sign * i * R for anticommuting Hermitian Paulis. BOOST_TEST(approx_equal(ab, scalar_mul(cd(0, static_cast(sign)), mr))); // Hot kernel returns the rotation sign = negated raw emit sign. - const auto ctx = make_pauli_gen_context(b); - BOOST_TEST(pauli_rotation_sign(ctx, a, r) == -sign); + const auto ctx = make_pauli_gen_context(b); + BOOST_TEST(pauli_rotation_sign(ctx, a, r) == -sign); } } } @@ -187,39 +182,39 @@ BOOST_AUTO_TEST_CASE(pauli_algebra_product_phase_vs_brute_force) { constexpr size_t NW = 40; std::mt19937 rng(0xBEEF01U); for (size_t trial = 0; trial < 3000; ++trial) { - const auto a = native_bitset(random_string(rng, NW)); - const auto b = native_bitset(random_string(rng, NW)); - const auto ctx = make_pauli_gen_context(b); - BOOST_TEST(pauli_rotation_sign(ctx, a, a ^ b) == -pauli_emit_sign_antic(a, b)); + const auto a = native_bitset(NW, random_string(rng, NW)); + const auto b = native_bitset(NW, random_string(rng, NW)); + const auto ctx = make_pauli_gen_context(b); + BOOST_TEST(pauli_rotation_sign(ctx, a, a ^ b) == -pauli_emit_sign_antic(NW, a, b)); } } BOOST_AUTO_TEST_CASE(pauli_algebra_cutoff_and_weight_equivalence) { constexpr size_t N = 32; // single word (2N = 64) constexpr size_t logical = 6; - const auto basis = jw_basis(logical); + const auto basis = jw_basis(N, logical); std::mt19937 rng(0x0DDBALLU); for (size_t trial = 0; trial < 3000; ++trial) { const auto p = random_string(rng, logical); // P on the low qubits 0..logical-1 - const auto native = native_bitset(p); - const auto via_jw = change_basis(jw_bitset(p), basis); + const auto native = native_bitset(N, p); + const auto via_jw = change_basis(jw_bitset(N, p), basis); BOOST_TEST((native == via_jw)); for (unsigned int c : {0U, 1U, 2U, 3U, 6U}) { - BOOST_TEST(support_cutoff(native, c, logical) == support_cutoff(via_jw, c, logical)); - // Also exercise the whole-register (logical == NumModes) code path. - BOOST_TEST(support_cutoff(native, c) == support_cutoff(via_jw, c)); + BOOST_TEST(support_cutoff(native, c, logical) == support_cutoff(via_jw, c, logical)); + // Also exercise the whole-register (logical == num_modes) code path. + BOOST_TEST(support_cutoff(native, c) == support_cutoff(via_jw, c)); } size_t true_weight = 0; for (char ch : p) { true_weight += (ch != 'I') ? 1 : 0; } - BOOST_TEST(pauli_weight(native) == true_weight); + BOOST_TEST(pauli_weight(N, native) == true_weight); // is_paired (support_cutoff's xor_sum == 0) detects exactly the Z-only strings. - BOOST_TEST(is_paired(native) == is_z_only(p)); + BOOST_TEST(is_paired(native) == is_z_only(p)); } } @@ -240,21 +235,21 @@ BOOST_AUTO_TEST_CASE(pauli_algebra_state_phase) { occupied_slots.push_back(2 * q + 1); // z-plane bit of qubit q (even physical bit) } } - const auto state_mask = indices_to_bitset(occupied_slots); + const auto state_mask = indices_to_bitset(occupied_slots, 2 * N); // Z-only Pauli: pauli_state_phase must match (-1)^{|Z ∩ occupied|} and dense . std::string pz(n, 'I'); for (size_t q = 0; q < n; ++q) { pz[q] = use_z(rng) ? 'Z' : 'I'; } - const auto z_mono = native_bitset(pz); + const auto z_mono = native_bitset(N, pz); int expected = 1; for (size_t q = 0; q < n; ++q) { if (pz[q] == 'Z' && b[q] != 0) { expected = -expected; } } - const double phase = pauli_state_phase(z_mono, state_mask); + const double phase = pauli_state_phase(z_mono, state_mask); BOOST_TEST(phase == static_cast(expected)); const size_t d = size_t{1} << n; diff --git a/cpp/tests/pauli_build_layer_tests.cpp b/cpp/tests/pauli_build_layer_tests.cpp index af069fcd..18b2d42d 100644 --- a/cpp/tests/pauli_build_layer_tests.cpp +++ b/cpp/tests/pauli_build_layer_tests.cpp @@ -26,6 +26,7 @@ #include #include "PauliTestOracle.h" +#include "TestPropagator.h" #include "monoprop/MonomialPropagator.h" #include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/algebra/PauliAlgebra.h" @@ -37,9 +38,8 @@ using namespace pauli_oracle; namespace { // JW basis-change table (basis vector -> slot indices) for the basis_change parameter. -template -auto jw_basis_indices(size_t n) -> std::vector { - std::vector table(2 * NumModes); +auto jw_basis_indices(size_t num_modes, size_t n) -> std::vector { + std::vector table(2 * num_modes); for (size_t i = 0; i < n; ++i) { VecZ z_str; for (size_t z = 0; z < 2 * i; ++z) { @@ -53,41 +53,40 @@ auto jw_basis_indices(size_t n) -> std::vector { table[2 * i + 1] = odd_vec; } // Inactive high modes map to themselves (identity). - for (size_t s = 2 * n; s < 2 * NumModes; ++s) { + for (size_t s = 2 * n; s < 2 * num_modes; ++s) { table[s] = VecZ{s}; } return table; } -template -auto build_pauli_sim(const std::map &obs, +auto build_pauli_sim(size_t num_modes, + const std::map &obs, unsigned int cutoff, std::optional schrodinger_cutoff = std::nullopt, const VecZ &initial_state = {}, - std::optional lower_atol = std::nullopt) -> MonomialPropagator { + std::optional lower_atol = std::nullopt) -> MonomialPropagator { OperatorDict init; for (const auto &[p, c] : obs) { init[slots_of_string(p)] = cd(c, 0.0); } - return MonomialPropagator(init, - cutoff, - initial_state, - schrodinger_cutoff, - MPI_COMM_SELF, - lower_atol, - std::nullopt, - CutoffType::Support, - std::nullopt, - N, - Basis::Pauli); + return test_utils::make_propagator(num_modes, + init, + cutoff, + initial_state, + schrodinger_cutoff, + MPI_COMM_SELF, + lower_atol, + std::nullopt, + CutoffType::Support, + std::nullopt, + Basis::Pauli); } -template -auto dense_operator(MonomialPropagator &mp) -> std::vector { - const size_t d = size_t{1} << N; +auto dense_operator(size_t num_modes, MonomialPropagator &mp) -> std::vector { + const size_t d = size_t{1} << num_modes; std::vector m(d * d, cd(0, 0)); const auto &coeffs = mp.mp_op().get_operator(); - mp.indexing().for_each([&](const Monomial &mono, size_t idx) { + mp.for_each_term([&](const auto &mono, size_t idx) { if (idx >= coeffs.size()) { return; } @@ -95,9 +94,9 @@ auto dense_operator(MonomialPropagator &mp) -> std::vector { if (c == 0.0) { return; } - std::string s(N, 'I'); - for (size_t q = 0; q < N; ++q) { - s[q] = letter_from_bitset(mono, q); + std::string s(num_modes, 'I'); + for (size_t q = 0; q < num_modes; ++q) { + s[q] = letter_from_bitset(mono, q); } const auto pm = matrix_from_string(s); for (size_t k = 0; k < d * d; ++k) { @@ -107,9 +106,8 @@ auto dense_operator(MonomialPropagator &mp) -> std::vector { return m; } -template -auto dense_observable(const std::map &obs) -> std::vector { - const size_t d = size_t{1} << N; +auto dense_observable(size_t num_modes, const std::map &obs) -> std::vector { + const size_t d = size_t{1} << num_modes; std::vector m(d * d, cd(0, 0)); for (const auto &[p, c] : obs) { const auto pm = matrix_from_string(p); @@ -122,15 +120,17 @@ auto dense_observable(const std::map &obs) -> std::vector -auto check_pauli_gate(const std::map &obs, const std::string &gstr, double g, double theta) - -> void { - auto mp = build_pauli_sim(obs, /*cutoff=*/2 * N); +auto check_pauli_gate(size_t num_modes, + const std::map &obs, + const std::string &gstr, + double g, + double theta) -> void { + auto mp = build_pauli_sim(num_modes, obs, /*cutoff=*/static_cast(2 * num_modes)); mp.propagate({slots_of_string(gstr)}, VecZ{0}, VecD{g}, VecD{theta}); - const auto engine = dense_operator(mp); + const auto engine = dense_operator(num_modes, mp); - const size_t d = size_t{1} << N; - const auto O = dense_observable(obs); + const size_t d = size_t{1} << num_modes; + const auto O = dense_observable(num_modes, obs); const auto G = matrix_from_string(gstr); const double param = g * theta; const double cs = std::cos(param); @@ -146,7 +146,7 @@ auto check_pauli_gate(const std::map &obs, const std::strin } } const auto ref = matmul(matmul(Ud, O, d), U, d); - BOOST_TEST_CONTEXT("N=" << N << " G=" << gstr << " g=" << g << " theta=" << theta) { + BOOST_TEST_CONTEXT("num_modes=" << num_modes << " G=" << gstr << " g=" << g << " theta=" << theta) { BOOST_TEST(approx_equal(engine, ref)); } } @@ -229,28 +229,28 @@ auto jw_gate_arrays(const PauliCircuit &c) -> std::pair, VecD> // The JW-image Majorana propagator for the same physical observable; the JW basis change makes its // Support cutoff measure Pauli weight, matching the native arm. -template -auto build_jw_sim(const std::map &obs, +auto build_jw_sim(size_t num_modes, + const std::map &obs, unsigned int cutoff, std::optional schrodinger_cutoff = std::nullopt, const VecZ &initial_state = {}, - std::optional lower_atol = std::nullopt) -> MonomialPropagator { + std::optional lower_atol = std::nullopt) -> MonomialPropagator { OperatorDict init; for (const auto &[p, c] : obs) { const auto [idx, jw] = pauli_to_fermi_full(p); init[idx] = jw * cd(c, 0.0); } - return MonomialPropagator(init, - cutoff, - initial_state, - schrodinger_cutoff, - MPI_COMM_SELF, - lower_atol, - std::nullopt, - CutoffType::Support, - jw_basis_indices(N), - N, - Basis::Majorana); + return test_utils::make_propagator(num_modes, + init, + cutoff, + initial_state, + schrodinger_cutoff, + MPI_COMM_SELF, + lower_atol, + std::nullopt, + CutoffType::Support, + jw_basis_indices(num_modes, num_modes), + Basis::Majorana); } } // namespace @@ -259,13 +259,13 @@ auto build_jw_sim(const std::map &obs, BOOST_AUTO_TEST_CASE(pauli_build_layer_dense_matrix_ground_truth) { const std::map o2{{"XY", 0.5}, {"ZZ", -0.3}, {"YX", 0.7}, {"IZ", 0.2}, {"YY", -0.15}}; for (double th : {0.37, 0.8, 1.3, -0.6}) { - check_pauli_gate<2>(o2, "XX", 1.0, th); - check_pauli_gate<2>(o2, "ZZ", 0.9, th); - check_pauli_gate<2>(o2, "XY", 1.1, th); - check_pauli_gate<2>(o2, "YZ", 0.5, th); - check_pauli_gate<2>(o2, "XI", 1.0, th); - check_pauli_gate<2>(o2, "IY", 1.0, th); - check_pauli_gate<2>(o2, "ZI", 0.7, th); + check_pauli_gate(2, o2, "XX", 1.0, th); + check_pauli_gate(2, o2, "ZZ", 0.9, th); + check_pauli_gate(2, o2, "XY", 1.1, th); + check_pauli_gate(2, o2, "YZ", 0.5, th); + check_pauli_gate(2, o2, "XI", 1.0, th); + check_pauli_gate(2, o2, "IY", 1.0, th); + check_pauli_gate(2, o2, "ZI", 0.7, th); } const std::map o3{{"XYZ", 0.4}, @@ -275,11 +275,11 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_dense_matrix_ground_truth) { {"ZZI", -0.35}, {"XXX", 0.2}}; for (double th : {0.41, -0.9, 1.05}) { - check_pauli_gate<3>(o3, "XZI", 1.0, th); - check_pauli_gate<3>(o3, "YIY", 0.8, th); - check_pauli_gate<3>(o3, "ZZZ", 0.6, th); - check_pauli_gate<3>(o3, "IXY", 1.0, th); - check_pauli_gate<3>(o3, "YYZ", 0.5, th); + check_pauli_gate(3, o3, "XZI", 1.0, th); + check_pauli_gate(3, o3, "YIY", 0.8, th); + check_pauli_gate(3, o3, "ZZZ", 0.6, th); + check_pauli_gate(3, o3, "IXY", 1.0, th); + check_pauli_gate(3, o3, "YYZ", 0.5, th); } std::mt19937 rng(0xB0A710U); @@ -303,14 +303,13 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_dense_matrix_ground_truth) { } const double g = 0.5 + coeff(rng); const double th = coeff(rng) * 1.5; - check_pauli_gate<4>(o4, gstr, g, th); + check_pauli_gate(4, o4, gstr, g, th); } } // Heisenberg ⟨b|O_evolved|b⟩ against the initial state, after a contract-immediately propagate: // core + Σ state·op. -template -auto heisenberg_expval(MonomialPropagator &sim) -> double { +auto heisenberg_expval(MonomialPropagator &sim) -> double { const VecD st = sim.mp_op().materialize_state(); const auto &op = sim.mp_op().get_operator(); double s = 0.0; @@ -355,8 +354,8 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_jw_isomorphism) { {std::optional(5), 3, std::optional(1e-6), "schrodinger-lower-atol"}, }; for (const auto &cf : cfgs) { - auto nat = build_pauli_sim(obs, cf.cutoff, cf.sch, initial_state, cf.atol); - auto jw = build_jw_sim(obs, cf.cutoff, cf.sch, initial_state, cf.atol); + auto nat = build_pauli_sim(N, obs, cf.cutoff, cf.sch, initial_state, cf.atol); + auto jw = build_jw_sim(N, obs, cf.cutoff, cf.sch, initial_state, cf.atol); nat.build_graph(nat_monos, circ.param_map, nat_gcs); jw.build_graph(jw_majs, circ.param_map, jw_gcs); const double en = nat.expectation_value(circ.params); @@ -376,8 +375,8 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_jw_isomorphism) { pre.params.assign(circ.params.begin(), circ.params.begin() + static_cast(k)); const auto [nm, ng] = native_gate_arrays(pre); const auto [jm, jg] = jw_gate_arrays(pre); - auto nat = build_pauli_sim(obs, 3); - auto jw = build_jw_sim(obs, 3); + auto nat = build_pauli_sim(N, obs, 3); + auto jw = build_jw_sim(N, obs, 3); nat.propagate(nm, pre.param_map, ng, pre.params); jw.propagate(jm, pre.param_map, jg, pre.params); BOOST_TEST_CONTEXT("prefix k=" << k) { @@ -397,17 +396,19 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_replay_fold_consumers) { // Direct fold guard: for a single odd-popcount X gate, graph_data's fold-recomputed cos set must // equal the terms anticommuting with X in the Pauli sense. { - auto mp = build_pauli_sim(obs, 3); + auto mp = build_pauli_sim(N, obs, 3); mp.build_graph({slots_of_string("XII")}, VecZ{0}, VecD{1.0}); const auto layers = mp.graph_data(); BOOST_TEST_REQUIRE(layers.size() == 1U); const VecZ &cos_inds = std::get<0>(layers[0]); std::set got(cos_inds.begin(), cos_inds.end()); - const auto Gb = indices_to_bitset(slots_of_string("XII")); + // The generator has to be as wide as the terms it is compared against, which is the + // propagator's storage width, not 2 * N. + const auto Gb = indices_to_bitset(slots_of_string("XII"), 2 * mp.storage_num_modes()); std::set expected; (void)mp.mp_op().get_operator(); // materialize the store size - mp.indexing().for_each([&](const Monomial &mono, size_t idx) { - if (pauli_anticommutes(mono, Gb)) { + mp.for_each_term([&](const auto &mono, size_t idx) { + if (pauli_anticommutes(mono, Gb)) { expected.insert(idx); } }); @@ -423,17 +424,17 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_replay_fold_consumers) { const auto [jw_majs, jw_gcs] = jw_gate_arrays(circ); // (a) fused contract-immediately propagate. - auto prop = build_pauli_sim(obs, 3, std::nullopt, initial_state); + auto prop = build_pauli_sim(N, obs, 3, std::nullopt, initial_state); prop.propagate(nat_monos, circ.param_map, nat_gcs, circ.params); - const double e_prop = heisenberg_expval(prop); + const double e_prop = heisenberg_expval(prop); // (b) graph build + functional (expectation_value recomputes the cos from the fold). - auto grp = build_pauli_sim(obs, 3, std::nullopt, initial_state); + auto grp = build_pauli_sim(N, obs, 3, std::nullopt, initial_state); grp.build_graph(nat_monos, circ.param_map, nat_gcs); const double e_graph = grp.expectation_value(circ.params); // (c) contract_partially (evolve_operator_with_recompute — the same fold path, non-inplace). - auto ctr = build_pauli_sim(obs, 3, std::nullopt, initial_state); + auto ctr = build_pauli_sim(N, obs, 3, std::nullopt, initial_state); ctr.build_graph(nat_monos, circ.param_map, nat_gcs); const auto evolved = ctr.contract_partially(circ.params, /*inplace=*/false); const VecD st = ctr.mp_op().materialize_state(); @@ -444,7 +445,7 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_replay_fold_consumers) { const double e_contract = ctr.core_term() + s; // (d) JW-image Majorana reference. - auto jw = build_jw_sim(obs, 3, std::nullopt, initial_state); + auto jw = build_jw_sim(N, obs, 3, std::nullopt, initial_state); jw.build_graph(jw_majs, circ.param_map, jw_gcs); const double e_jw = jw.expectation_value(circ.params); diff --git a/cpp/tests/row_accessor_tests.cpp b/cpp/tests/row_accessor_tests.cpp index 123426a5..2a8f5cbd 100644 --- a/cpp/tests/row_accessor_tests.cpp +++ b/cpp/tests/row_accessor_tests.cpp @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// The dense-vector and packed OperatorIndex backends must agree through every RowAccess.h accessor. +// The dense-vector, packed OperatorIndex and sparse SparseRowStore backends must agree through every +// RowAccess.h accessor. #include @@ -22,69 +23,97 @@ #include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/detail/operator/OperatorIndex.h" #include "monoprop/detail/operator/RowAccess.h" +#include "monoprop/detail/operator/SparseRowStore.h" using namespace monoprop; namespace { -template auto positions_of(const auto &backend, size_t i) -> std::vector { std::vector out; - for_each_row_position(backend, i, [&](size_t b) { out.push_back(b); }); + for_each_row_position(backend, i, [&](size_t b) { out.push_back(b); }); return out; } -template -auto check_backends_agree(const std::vector> &raw_rows) -> void { - std::vector> dense; - detail::OperatorIndex packed; +// slots is the sparse backend's per-row mode capacity: pass one below a row's occupied-mode count to +// drive that row down the overflow path, which must stay invisible through the accessors. +auto check_backends_agree(size_t num_modes, const std::vector> &raw_rows, size_t slots = 8) + -> void { + MonomialList dense; + detail::OperatorIndex packed(2 * num_modes); + detail::SparseRowStore sparse(2 * num_modes, slots); for (const auto &bits : raw_rows) { - Monomial m; + Bitset m(2 * num_modes); for (size_t b : bits) { m.set(b); } dense.push_back(m); packed.push_back(m); + sparse.push_back(m); } BOOST_REQUIRE(packed.size() == dense.size()); + BOOST_REQUIRE(sparse.size() == dense.size()); for (size_t i = 0; i < dense.size(); ++i) { - BOOST_TEST((materialize_row(dense, i) == materialize_row(packed, i))); - BOOST_TEST(row_popcount(dense, i) == row_popcount(packed, i)); - BOOST_TEST(row_popcount(dense, i) == materialize_row(dense, i).count()); - BOOST_TEST(positions_of(dense, i) == positions_of(packed, i)); + BOOST_TEST((materialize_row(dense, i) == materialize_row(packed, i))); + BOOST_TEST((materialize_row(dense, i) == materialize_row(sparse, i))); + BOOST_TEST(row_popcount(dense, i) == row_popcount(packed, i)); + BOOST_TEST(row_popcount(dense, i) == row_popcount(sparse, i)); + BOOST_TEST(row_popcount(dense, i) == materialize_row(dense, i).count()); + BOOST_TEST(positions_of(dense, i) == positions_of(packed, i)); + BOOST_TEST(positions_of(dense, i) == positions_of(sparse, i)); } } } // namespace BOOST_AUTO_TEST_CASE(row_accessor_backends_agree_single_word) { - check_backends_agree<32>({{0, 3, 5}, {1, 2}, {}, {63}, {0, 1, 2, 3, 62, 63}}); + check_backends_agree(32, {{0, 3, 5}, {1, 2}, {}, {63}, {0, 1, 2, 3, 62, 63}}); } BOOST_AUTO_TEST_CASE(row_accessor_backends_agree_multi_word) { - check_backends_agree<96>({{0, 64, 191}, {5, 63, 64, 65}, {}, {128, 190}}); + check_backends_agree(96, {{0, 64, 191}, {5, 63, 64, 65}, {}, {128, 190}}); +} + +// Four occupied modes against a two-slot capacity: the first two rows spill, the empty row and the +// one-mode row do not, so the same store serves both kinds. +BOOST_AUTO_TEST_CASE(row_accessor_backends_agree_sparse_overflow) { + check_backends_agree(32, {{0, 3, 5, 8, 20, 21}, {1, 2, 40, 41, 62, 63}, {}, {10, 11}}, 2); } BOOST_AUTO_TEST_CASE(row_accessor_assign_row_overwrites) { constexpr size_t N = 32; - std::vector> dense; - detail::OperatorIndex packed; - Monomial original; + MonomialList dense; + detail::OperatorIndex packed(2 * N); + // Two slots, and the original occupies three modes: the row starts spilled and the overwrite must + // pull it back inline rather than leaving the stale side-map entry to shadow it. + detail::SparseRowStore sparse(2 * N, 2); + Bitset original(2 * N); original.set(1); original.set(2); + original.set(40); + original.set(41); + original.set(60); dense.push_back(original); packed.push_back(original); + sparse.push_back(original); + BOOST_TEST(sparse.spilled(0)); - Monomial replacement; + // Three set bits over two modes, so it fits the sparse store's two slots. + Bitset replacement(2 * N); replacement.set(10); - replacement.set(20); + replacement.set(11); replacement.set(30); - assign_row(dense, 0, replacement); - assign_row(packed, 0, replacement); + assign_row(dense, 0, replacement); + assign_row(packed, 0, replacement); + assign_row(sparse, 0, replacement); - BOOST_TEST((materialize_row(dense, 0) == replacement)); - BOOST_TEST((materialize_row(packed, 0) == replacement)); - BOOST_TEST(row_popcount(packed, 0) == 3U); - BOOST_TEST(positions_of(dense, 0) == positions_of(packed, 0)); + BOOST_TEST((materialize_row(dense, 0) == replacement)); + BOOST_TEST((materialize_row(packed, 0) == replacement)); + BOOST_TEST((materialize_row(sparse, 0) == replacement)); + BOOST_TEST(row_popcount(packed, 0) == 3U); + BOOST_TEST(row_popcount(sparse, 0) == 3U); + BOOST_TEST(!sparse.spilled(0)); + BOOST_TEST(positions_of(dense, 0) == positions_of(packed, 0)); + BOOST_TEST(positions_of(dense, 0) == positions_of(sparse, 0)); } diff --git a/cpp/tests/row_store_selection_tests.cpp b/cpp/tests/row_store_selection_tests.cpp new file mode 100644 index 00000000..ab6531d2 --- /dev/null +++ b/cpp/tests/row_store_selection_tests.cpp @@ -0,0 +1,94 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Which row backend a propagator ends up on, and that monoprop_ROW_STORE is wired to it. +// +// This is the guard against a false negative in the rest of the suite: every case runs a second time +// under monoprop_ROW_STORE=sparse (see cpp/tests/boostAddTests.cmake), and every fixture is far below +// the automatic crossover -- so if the variable reached nothing, those 246 extra passes would be the +// dense backend passing twice and nobody would notice. + +#include + +#include + +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/EnvConfig.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +#include "TestData.h" +#include "TestUtilities.h" + +using namespace monoprop; + +namespace { + +constexpr size_t kNumModes = 8; + +auto small_propagator() -> MonomialPropagator { + const auto data = test_utils::load_case_data("random_exact.msgpack"); + return test_utils::build_simulator(kNumModes, data); +} + +} // namespace + +// The propagator's backend must be what the environment asked for -- and, unset, what the crossover +// says. Both arms are live: the default ctest variant takes the first, the sparse-rows variant the +// second, so this case is the one that fails if the variable is ignored. +BOOST_AUTO_TEST_CASE(row_store_selection_follows_the_environment) { + const auto propagator = small_propagator(); + const size_t storage_modes = propagator.storage_num_modes(); + // A fixture-sized system: below every shipped crossover, so `auto` must pick dense here. If this + // ever fails, the suite's sparse coverage has stopped being a second configuration. + BOOST_REQUIRE(!monoprop::detail::SparseRowStore::preferred_for_modes(storage_modes)); + + // Dereferenced: an unrecognized value is nullopt, and the propagator above would have thrown on it + // before this line -- see the rejects-an-unrecognized-value case below. + BOOST_REQUIRE(config::get().row_store.has_value()); + switch (*config::get().row_store) { + case config::RowStore::Sparse: + BOOST_TEST(propagator.rows_are_sparse()); + break; + case config::RowStore::Dense: + case config::RowStore::Auto: + BOOST_TEST(!propagator.rows_are_sparse()); + break; + } +} + +// The automatic rule, independent of any propagator: the crossover is a whole 32-mode block, which is +// what storage_modes_for() produces, so no storage width can straddle it. +BOOST_AUTO_TEST_CASE(row_store_auto_crossover_is_a_whole_storage_block) { + constexpr size_t kMin = monoprop::detail::SparseRowStore::kMinModes; + BOOST_TEST(kMin % 32 == 0U); + BOOST_TEST(!monoprop::detail::SparseRowStore::preferred_for_modes(kMin - 1)); + BOOST_TEST(monoprop::detail::SparseRowStore::preferred_for_modes(kMin)); + BOOST_TEST(monoprop::detail::SparseRowStore::preferred_for_modes(kMin + 32)); + BOOST_TEST(monoprop::detail::storage_modes_for(kMin) == kMin); +} + +// An unrecognized value must be rejected, not silently treated as `auto`: the whole reason to set the +// variable is to know which backend ran, and a typo that fell back would read as a passing run of a +// configuration that never happened. Parsed here rather than through config::get(), which caches the +// process environment once and so cannot be re-read per case. +BOOST_AUTO_TEST_CASE(row_store_env_parses_only_the_three_values) { + BOOST_TEST((config::detail::parse_row_store(nullptr) == config::RowStore::Auto)); + BOOST_TEST((config::detail::parse_row_store("") == config::RowStore::Auto)); + BOOST_TEST((config::detail::parse_row_store("auto") == config::RowStore::Auto)); + BOOST_TEST((config::detail::parse_row_store("dense") == config::RowStore::Dense)); + BOOST_TEST((config::detail::parse_row_store("sparse") == config::RowStore::Sparse)); + for (const char *bad : {"Sparse", "SPARSE", "spars", "sparse ", "1", "on", "packed"}) { + BOOST_TEST(!config::detail::parse_row_store(bad).has_value()); + } +} diff --git a/cpp/tests/simulator_copy_tests.cpp b/cpp/tests/simulator_copy_tests.cpp index be79c160..925cd393 100644 --- a/cpp/tests/simulator_copy_tests.cpp +++ b/cpp/tests/simulator_copy_tests.cpp @@ -22,19 +22,19 @@ // Copy-constructing a simulator must produce a fully independent deep copy -- the mechanism behind // Python __deepcopy__. The operator store is non-copyable, so the copy rebuilds it via clone() and -// find()/indexing() have to work on the copy's own rows. The MPI communicator handle is shared. +// find()/for_each_term() have to work on the copy's own rows. The MPI communicator handle is shared. using namespace test_utils; using namespace monoprop; // Copy assignment is deliberately deleted: the unique_ptr-owned store needs no assignment. -static_assert(std::is_copy_constructible_v>, "simulator must be copyable"); -static_assert(std::is_move_constructible_v>, "simulator must stay movable"); -static_assert(!std::is_copy_assignable_v>, "copy assignment stays deleted"); +static_assert(std::is_copy_constructible_v, "simulator must be copyable"); +static_assert(std::is_move_constructible_v, "simulator must stay movable"); +static_assert(!std::is_copy_assignable_v, "copy assignment stays deleted"); BOOST_FIXTURE_TEST_CASE(copy_constructed_simulator_matches_energy, ExampleDataFix) { SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(n_modes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto copy = sim; @@ -49,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE(copy_constructed_simulator_matches_energy, ExampleDataFi BOOST_FIXTURE_TEST_CASE(copy_is_independent_of_source, ExampleDataFix) { SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(n_modes, data, cfg); auto copy = sim; BOOST_TEST(copy.graph_layers() == 0u); @@ -69,7 +69,7 @@ BOOST_FIXTURE_TEST_CASE(copy_is_independent_of_source, ExampleDataFix) { // core references. BOOST_FIXTURE_TEST_CASE(copy_graph_survives_other_being_contracted_and_destroyed, ExampleDataFix) { SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto original = build_simulator(data, cfg); + auto original = build_simulator(n_modes, data, cfg); original.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); const size_t layers_before = original.graph_layers(); @@ -92,16 +92,15 @@ BOOST_FIXTURE_TEST_CASE(copy_graph_survives_other_being_contracted_and_destroyed BOOST_FIXTURE_TEST_CASE(copy_constructed_simulator_index_valid, ExampleDataFix) { SimulatorConfig cfg{.comm = MPI_COMM_SELF}; - auto sim = build_simulator(data, cfg); + auto sim = build_simulator(n_modes, data, cfg); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); auto copy = sim; - const auto &idx = copy.indexing(); - BOOST_TEST(idx.size() == sim.indexing().size()); + BOOST_TEST(copy.num_local_terms() == sim.num_local_terms()); bool all_found = true; - idx.for_each([&](const auto &mono, size_t i) { - const auto f = idx.find(mono); + copy.for_each_term([&](const auto &mono, size_t i) { + const auto f = copy.mp_op().find(mono); if (!f || *f != i) { all_found = false; } diff --git a/cpp/tests/sparse_index_tests.cpp b/cpp/tests/sparse_index_tests.cpp new file mode 100644 index 00000000..40b88b3b --- /dev/null +++ b/cpp/tests/sparse_index_tests.cpp @@ -0,0 +1,295 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// SparseRowStore's keyless index must answer lookups exactly as OperatorIndex's does -- same hits, same +// misses, same insert-or-no-op -- because Stage 6 swaps one for the other. The hash *value* differs by +// design (that is the documented re-baseline); nothing else may. + +#include + +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +#include "RandomMonomial.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +// Distinct rows only: bulk_insert and the emplace-is-idempotent check both require it, and a duplicate +// would make "found at index i" ambiguous. +struct RowSet { + std::vector rows; + std::set> seen; + + auto add(const Bitset &mono) -> bool { + if (!seen.insert(positions(mono)).second) { + return false; + } + rows.push_back(mono); + return true; + } + [[nodiscard]] auto contains(const Bitset &mono) const -> bool { return seen.contains(positions(mono)); } + + static auto positions(const Bitset &mono) -> std::vector { + std::vector out; + for (size_t b = mono.find_first(); b < mono.size(); b = mono.find_next(b)) { + out.push_back(b); + } + return out; + } +}; + +} // namespace + +// A spilled row has no codes word, so it can only be hashed by walking the dense monomial. That walk +// and the row walk are two producers of one hash, and if they ever disagreed a spilled row would become +// unfindable -- silently, and only for the ~0.07% of rows that spill. +BOOST_AUTO_TEST_CASE(sparse_index_hash_agrees_between_row_and_monomial) { + std::mt19937_64 rng(20260812U); + size_t compared = 0; + for (const size_t num_modes : {32U, 64U, 300U}) { + SparseRowStore store(2 * num_modes, SparseRowStore::kMaxSlots); + std::vector rows; + for (size_t t = 0; t < 200; ++t) { + rows.push_back(test_utils::random_monomial(rng, num_modes, SparseRowStore::kMaxSlots)); + store.push_back(rows.back()); + } + for (size_t i = 0; i < rows.size(); ++i) { + BOOST_REQUIRE(!store.spilled(i)); + BOOST_TEST(sparse_row_hash(store.view(i)) == sparse_row_hash(rows[i])); + ++compared; + } + } + BOOST_TEST(compared == 600U); +} + +// The row capacity is a tuning parameter. If it leaked into the hash, changing it would move probe order +// and MPI owner routing, so two stores tuned differently must agree on every row they can both hold. +BOOST_AUTO_TEST_CASE(sparse_index_hash_is_independent_of_row_capacity) { + std::mt19937_64 rng(4321U); + SparseRowStore narrow(64, 8); + SparseRowStore wide(64, SparseRowStore::kMaxSlots); + size_t compared = 0; + for (size_t t = 0; t < 300; ++t) { + const auto mono = test_utils::random_monomial(rng, 32, 8); + narrow.push_back(mono); + wide.push_back(mono); + const size_t i = narrow.size() - 1; + if (narrow.spilled(i)) { + continue; + } + BOOST_TEST(sparse_row_hash(narrow.view(i)) == sparse_row_hash(wide.view(i))); + ++compared; + } + BOOST_TEST(compared > 200U); +} + +// The lookup contract, against OperatorIndex as the oracle, at three widths and three row capacities so +// that spilled and inline rows both occur. +BOOST_AUTO_TEST_CASE(sparse_index_lookups_match_the_packed_backend) { + std::mt19937_64 rng(99U); + size_t spilled_rows = 0; + size_t misses = 0; + for (const size_t num_modes : {32U, 64U, 300U}) { + const size_t num_bits = 2 * num_modes; + for (const size_t slots : {4U, 8U, 32U}) { + SparseRowStore sparse(num_bits, slots); + OperatorIndex packed(num_bits); + RowSet set; + for (size_t t = 0; t < 300; ++t) { + const auto mono = test_utils::random_monomial(rng, num_modes, 12); + if (!set.add(mono)) { + continue; + } + const size_t i = sparse.size(); + sparse.push_back(mono); + packed.push_back(mono); + sparse.emplace(mono, i); + packed.emplace(mono, i); + spilled_rows += sparse.spilled(i) ? 1 : 0; + } + BOOST_REQUIRE(sparse.size() == set.rows.size()); + + for (size_t i = 0; i < set.rows.size(); ++i) { + const auto by_mono = sparse.find(set.rows[i]); + BOOST_REQUIRE(by_mono.has_value()); + BOOST_TEST(*by_mono == i); + BOOST_TEST(*by_mono == packed.find(set.rows[i]).value()); + if (!sparse.spilled(i)) { + const auto by_row = sparse.find(sparse.view(i)); + BOOST_REQUIRE(by_row.has_value()); + BOOST_TEST(*by_row == i); + } + } + + // Absent keys must miss, not land on a hash neighbour. + for (size_t t = 0; t < 300; ++t) { + Bitset mono(num_bits); + for (size_t k = 0; k < 1 + (rng() % 6); ++k) { + mono.set(rng() % num_bits); + } + if (set.contains(mono)) { + continue; + } + BOOST_TEST(!sparse.find(mono).has_value()); + BOOST_TEST(!packed.find(mono).has_value()); + ++misses; + } + + // emplace is insert-or-no-op, so replaying every key must add no slot. + const size_t indexed = sparse.indexed_count(); + for (size_t i = 0; i < set.rows.size(); ++i) { + sparse.emplace(set.rows[i], i); + } + BOOST_TEST(sparse.indexed_count() == indexed); + + const auto copy = sparse.clone(); + BOOST_TEST(copy->indexed_count() == indexed); + for (size_t i = 0; i < set.rows.size(); ++i) { + const auto found = copy->find(set.rows[i]); + BOOST_REQUIRE(found.has_value()); + BOOST_TEST(*found == i); + } + } + } + // Both row kinds have to have occurred, or the spill path above was never taken. + BOOST_TEST(spilled_rows > 0U); + BOOST_TEST(misses > 100U); +} + +// find_batch is a pipelined re-implementation of find, so it needs its own equality check -- against +// find, for both key forms, over a query list mixing hits and misses. +BOOST_AUTO_TEST_CASE(sparse_index_find_batch_matches_find) { + std::mt19937_64 rng(555U); + constexpr size_t kNumBits = 128; + SparseRowStore store(kNumBits, 8); + RowSet set; + for (size_t t = 0; t < 400; ++t) { + const auto mono = test_utils::random_monomial(rng, 64, 10); + if (!set.add(mono)) { + continue; + } + const size_t i = store.size(); + store.push_back(mono); + store.emplace(mono, i); + } + + // More than one group of 16, and interleaved absentees so a group holds both. + std::vector queries; + for (size_t i = 0; i < set.rows.size(); ++i) { + queries.push_back(set.rows[i]); + if (i % 3 == 0) { + Bitset absent(kNumBits); + absent.set(rng() % kNumBits); + if (!set.contains(absent)) { + queries.push_back(absent); + } + } + } + BOOST_REQUIRE(queries.size() > 16U); + + std::vector batched(queries.size()); + store.find_batch(queries.data(), queries.size(), batched.data()); + for (size_t j = 0; j < queries.size(); ++j) { + const auto scalar = store.find(queries[j]); + BOOST_TEST(batched[j] == (scalar ? *scalar : SparseRowStore::kNotFound)); + } + + // The row-key form, which is what the scan will hand it. + std::vector row_queries; + std::vector expected; + for (size_t i = 0; i < store.size(); ++i) { + if (!store.spilled(i)) { + row_queries.push_back(store.view(i)); + expected.push_back(i); + } + } + BOOST_REQUIRE(row_queries.size() > 16U); + std::vector row_batched(row_queries.size()); + store.find_batch(row_queries.data(), row_queries.size(), row_batched.data()); + BOOST_TEST(row_batched == expected, boost::test_tools::per_element()); +} + +// bulk_insert skips the duplicate probe, so it is only correct on provably distinct keys; what it must +// still produce is an index that finds every one of them. +BOOST_AUTO_TEST_CASE(sparse_index_bulk_insert_indexes_every_row) { + std::mt19937_64 rng(777U); + constexpr size_t kNumBits = 64; + SparseRowStore store(kNumBits, 8); + RowSet set; + for (size_t t = 0; t < 200; ++t) { + const auto mono = test_utils::random_monomial(rng, 32, 6); + if (set.add(mono)) { + store.push_back(mono); + } + } + store.bulk_insert(store.size(), 0, [&](size_t k) { return set.rows[k]; }); + BOOST_TEST(store.indexed_count() == store.size()); + for (size_t i = 0; i < set.rows.size(); ++i) { + const auto found = store.find(set.rows[i]); + BOOST_REQUIRE(found.has_value()); + BOOST_TEST(*found == i); + } +} + +// The sparse hash is a different function from the dense one, which is exactly why the store swap needs +// a re-baseline. Pinned so that "the results moved" is never a surprise, and so a future change that +// accidentally reunified them would be noticed. +BOOST_AUTO_TEST_CASE(sparse_index_hash_differs_from_the_dense_hash) { + std::mt19937_64 rng(8888U); + size_t differing = 0; + size_t total = 0; + for (size_t t = 0; t < 200; ++t) { + const auto mono = test_utils::random_monomial(rng, 32, 8); + if (mono.count() == 0) { + continue; + } + differing += sparse_row_hash(mono) != monomial_hash(mono) ? 1 : 0; + ++total; + } + BOOST_REQUIRE(total > 100U); + BOOST_TEST(differing == total); +} + +// A pre-filter that collided often would still be correct but would degrade every probe into a lane +// compare. Over distinct rows the 32-bit folds should be near-injective. +BOOST_AUTO_TEST_CASE(sparse_index_hash_folds_are_near_injective) { + std::mt19937_64 rng(31337U); + constexpr size_t kNumBits = 128; + SparseRowStore store(kNumBits, SparseRowStore::kMaxSlots); + RowSet set; + for (size_t t = 0; t < 4000; ++t) { + const auto mono = test_utils::random_monomial(rng, 64, 10); + if (set.add(mono)) { + store.push_back(mono); + } + } + std::set folds; + for (const auto &mono : set.rows) { + const size_t full = sparse_row_hash(mono); + folds.insert(static_cast(full ^ (static_cast(full) >> 32))); + } + BOOST_REQUIRE(set.rows.size() > 3000U); + // Birthday-bound expectation for n draws from 2^32 is ~n^2/2^33 collisions, i.e. under 1 for n=4000; + // allowing 4 keeps this from being flaky while still failing on a hash that structurally collides. + BOOST_TEST(set.rows.size() - folds.size() <= 4U); +} diff --git a/cpp/tests/sparse_row_store_tests.cpp b/cpp/tests/sparse_row_store_tests.cpp new file mode 100644 index 00000000..a5045993 --- /dev/null +++ b/cpp/tests/sparse_row_store_tests.cpp @@ -0,0 +1,526 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// SparseRowStore's own invariants. The three-way agreement with the dense and packed backends through +// the TypeAliases.h accessors lives in row_accessor_tests.cpp; what is checked here is the part that +// has no counterpart in the other backends -- the codes word, and the row sizing that feeds it. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +#include "RandomMonomial.h" + +using namespace monoprop; +using namespace monoprop::detail; + +// Interchangeable with OperatorIndex means the same ownership rules, so the store cannot be silently +// copied out of MPOperator's unique_ptr. +static_assert(!std::is_move_constructible_v, "SparseRowStore must remain non-movable"); +static_assert(!std::is_copy_constructible_v, "SparseRowStore must remain non-copyable"); + +namespace { + +// The Stage 5 identities, spelled out here against the dense cutoff_sums so the port has something to +// be differentially tested against before any of it is written. +auto sums_from_codes(RowCodes codes) -> CutoffSums { + const auto n = static_cast(std::popcount(row_occupied_bits(codes))); + const auto d = static_cast(std::popcount(row_paired_bits(codes))); + return {n - d, n + d, n}; +} + +// Owning lanes plus codes, so a test can hold a row the way the scan's scratch does. The lanes come out +// ascending and contiguous from slot 0, which is what the representation requires of every producer. +struct RowBuffer { + std::vector lanes; + RowCodes codes = 0; + + [[nodiscard]] auto view() const -> SparseRow { return SparseRow{lanes.data(), codes}; } +}; + +// Only for rows that fit one codes word (<= kRowMaxSlots slots) -- the shape a SparseRow can hold at all. +auto row_of(const Bitset &mono) -> RowBuffer { + RowBuffer out; + for_each_mode_slot(mono, [&](size_t mode, unsigned int code) { + out.codes |= static_cast(code) << (2 * out.lanes.size()); + out.lanes.push_back(static_cast(mode)); + }); + return out; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(sparse_row_store_codes_encode_slot_pairs) { + constexpr size_t kNumBits = 64; + SparseRowStore store(kNumBits, 8); + + // Modes 1 (both positions), 4 (upper only) and 9 (lower only), so the codes word must read + // 0b11, 0b10, 0b01 from slot 0 up. + Bitset mono(kNumBits); + mono.set(2); + mono.set(3); + mono.set(9); + mono.set(18); + store.push_back(mono); + + BOOST_TEST(!store.spilled(0)); + // Slot 0 = mode 1 (0b11) in bits 0-1, slot 1 = mode 4 (0b10) in bits 2-3, slot 2 = mode 9 (0b01) + // in bits 4-5. + BOOST_TEST(store.codes(0) == 0b01'10'11ULL); + BOOST_TEST(store.slot_count(0) == 3U); + BOOST_TEST(store.popcount(0) == 4U); + + std::vector modes; + std::vector codes; + store.for_each_slot(0, [&](size_t mode, unsigned int code) { + modes.push_back(mode); + codes.push_back(code); + }); + BOOST_REQUIRE(modes.size() == 3U); + BOOST_TEST(modes == (std::vector{1U, 4U, 9U}), boost::test_tools::per_element()); + BOOST_TEST(codes == (std::vector{0b11U, 0b10U, 0b01U}), boost::test_tools::per_element()); +} + +BOOST_AUTO_TEST_CASE(sparse_row_store_empty_row_has_empty_codes) { + SparseRowStore store(64, 4); + store.push_back(Bitset(64)); + BOOST_TEST(!store.spilled(0)); + BOOST_TEST(store.codes(0) == 0U); + BOOST_TEST(store.slot_count(0) == 0U); + BOOST_TEST(store.popcount(0) == 0U); + BOOST_TEST(store.row(0) == Bitset(64)); +} + +// The identities the whole support form rests on: or_sum = n, popcount_sum = n + d, xor_sum = n - d, +// against the dense cutoff_sums over the full storage window. Randomized rather than enumerated +// because what can break them is a particular occupancy pattern, not a particular width. +BOOST_AUTO_TEST_CASE(sparse_row_store_codes_reproduce_cutoff_sums) { + std::mt19937_64 rng(20260812U); + for (const size_t num_modes : {32U, 64U, 128U, 512U}) { + const size_t num_bits = 2 * num_modes; + // Capacity above any row built below, so nothing spills and every row exercises the codes path. + SparseRowStore store(num_bits, SparseRowStore::kMaxSlots); + const auto masks = CutoffMasks::make(num_bits, num_modes); + for (size_t trial = 0; trial < 200; ++trial) { + Bitset mono(num_bits); + const size_t occupied = rng() % (SparseRowStore::kMaxSlots + 1); + for (size_t k = 0; k < occupied; ++k) { + const size_t mode = rng() % num_modes; + // 1..3 so the mode is genuinely occupied, and all three codes appear. + const unsigned int code = 1U + static_cast(rng() % 3U); + if ((code & 1U) != 0U) { + mono.set(2 * mode); + } + if ((code & 2U) != 0U) { + mono.set((2 * mode) + 1); + } + } + store.push_back(mono); + const size_t i = store.size() - 1; + BOOST_REQUIRE(!store.spilled(i)); + + const auto dense = cutoff_sums(mono, masks); + const auto sparse = sums_from_codes(store.codes(i)); + BOOST_TEST(sparse.or_sum == dense.or_sum); + BOOST_TEST(sparse.popcount_sum == dense.popcount_sum); + BOOST_TEST(sparse.xor_sum == dense.xor_sum); + BOOST_TEST(store.slot_count(i) == dense.or_sum); + BOOST_TEST(store.popcount(i) == dense.popcount_sum); + } + } +} + +// Spilled rows have no codes word, so the two measures must still come off the dense monomial. +BOOST_AUTO_TEST_CASE(sparse_row_store_spilled_rows_report_the_same_measures) { + constexpr size_t kNumBits = 128; + SparseRowStore store(kNumBits, 2); + Bitset mono(kNumBits); + for (const size_t b : {0U, 1U, 4U, 20U, 21U, 99U}) { // modes 0 (paired), 2, 10 (paired), 49 + mono.set(b); + } + store.push_back(mono); + + BOOST_TEST(store.spilled(0)); + BOOST_TEST(store.row(0) == mono); + const auto dense = cutoff_sums(mono, CutoffMasks::make(kNumBits, kNumBits / 2)); + BOOST_TEST(store.slot_count(0) == dense.or_sum); + BOOST_TEST(store.popcount(0) == dense.popcount_sum); +} + +BOOST_AUTO_TEST_CASE(sparse_row_store_clone_preserves_rows_and_spills) { + constexpr size_t kNumBits = 64; + SparseRowStore store(kNumBits, 2); + Bitset inline_row(kNumBits); + inline_row.set(4); + inline_row.set(5); + Bitset spilled_row(kNumBits); + for (const size_t b : {0U, 6U, 10U, 30U}) { + spilled_row.set(b); + } + store.push_back(inline_row); + store.push_back(spilled_row); + + const auto copy = store.clone(); + BOOST_REQUIRE(copy->size() == 2U); + BOOST_TEST(copy->num_bits() == kNumBits); + BOOST_TEST(copy->slots_per_row() == 2U); + BOOST_TEST(!copy->spilled(0)); + BOOST_TEST(copy->codes(0) == store.codes(0)); + BOOST_TEST(copy->row(0) == inline_row); + BOOST_TEST(copy->spilled(1)); + BOOST_TEST(copy->row(1) == spilled_row); +} + +// resized() is the migration update_cutoff() relies on: every row keeps its index and its content, +// whichever way the width moved -- including a row that crosses the overflow boundary, since set() +// re-decides that per row rather than trusting the old classification. +BOOST_AUTO_TEST_CASE(sparse_row_store_resized_preserves_rows_when_widening) { + constexpr size_t kNumBits = 64; + SparseRowStore store(kNumBits, 2); // width 2: the 3-slot row below starts spilled + Bitset inline_row(kNumBits); + inline_row.set(4); + inline_row.set(5); + Bitset spilled_row(kNumBits); + for (const size_t b : {0U, 6U, 10U, 30U}) { + spilled_row.set(b); + } + store.push_back(inline_row); + store.push_back(spilled_row); + + const auto wide = store.resized(4); // now fits inline + BOOST_REQUIRE(wide->size() == 2U); + BOOST_TEST(wide->slots_per_row() == 4U); + BOOST_TEST(!wide->spilled(0)); + BOOST_TEST(wide->row(0) == inline_row); + BOOST_TEST(!wide->spilled(1)); + BOOST_TEST(wide->row(1) == spilled_row); + + // Independent of the source: mutating store after the fact must not reach wide. + store.set(0, Bitset(kNumBits)); + BOOST_TEST(wide->row(0) == inline_row); +} + +BOOST_AUTO_TEST_CASE(sparse_row_store_resized_preserves_rows_when_narrowing) { + constexpr size_t kNumBits = 64; + SparseRowStore store(kNumBits, 4); // width 4: both rows below fit inline + Bitset a(kNumBits); + a.set(4); + a.set(5); + Bitset b(kNumBits); + for (const size_t bit : {0U, 6U, 10U, 30U}) { + b.set(bit); + } + store.push_back(a); + store.push_back(b); + BOOST_REQUIRE(!store.spilled(1)); + + const auto narrow = store.resized(2); // row 1 must now spill + BOOST_REQUIRE(narrow->size() == 2U); + BOOST_TEST(narrow->slots_per_row() == 2U); + BOOST_TEST(!narrow->spilled(0)); + BOOST_TEST(narrow->row(0) == a); + BOOST_TEST(narrow->spilled(1)); + BOOST_TEST(narrow->row(1) == b); +} + +// K comes from the cutoff in modes, and is the same number for both cutoff kinds -- halving the slot +// bound for a support cutoff would truncate rows a length cutoff of the same size admits. +BOOST_AUTO_TEST_CASE(sparse_row_store_slots_come_from_the_cutoff_in_modes) { + constexpr size_t kNumModes = 32; + constexpr size_t kNumBits = 2 * kNumModes; + + const CutoffFn length = LengthCutoff(6U, kNumModes, kNumBits); + const CutoffFn support = SupportCutoff(6U, kNumModes, kNumBits); + BOOST_TEST(CutoffEvaluator(length).max_slot_bound().value() == 6U); + BOOST_TEST(CutoffEvaluator(support).max_slot_bound().value() == 12U); + BOOST_TEST(CutoffEvaluator(length).max_mode_bound().value() == 6U); + BOOST_TEST(CutoffEvaluator(support).max_mode_bound().value() == 6U); + + BOOST_TEST(SparseRowStore::slots_for_bound(6U) == 6U); + // A bound past one codes word clamps rather than throwing: the excess rows spill. + BOOST_TEST(SparseRowStore::slots_for_bound(100U) == SparseRowStore::kMaxSlots); + BOOST_TEST(SparseRowStore::slots_for_bound(0U) == 1U); +} + +BOOST_AUTO_TEST_CASE(sparse_row_store_rejects_widths_past_the_lane_markers) { + BOOST_CHECK_NO_THROW((SparseRowStore(2 * SparseRowStore::kMaxModes, 4))); + BOOST_CHECK_THROW((SparseRowStore(2 * (SparseRowStore::kMaxModes + 1), 4)), SparseRowStoreUnsupported); +} + +// The switch rule itself: a build-time constant, because the crossover follows the target ISA rather +// than anything known at run time. The values are the measured crossovers, so what this pins is that +// the CMake default reached the compiler at all -- a missing definition would silently fall back. +BOOST_AUTO_TEST_CASE(sparse_row_store_preference_threshold_matches_the_build) { + static_assert(SparseRowStore::kMinModes > 0, "the sparse crossover must be a positive mode count"); + BOOST_TEST(!SparseRowStore::preferred_for_modes(SparseRowStore::kMinModes - 1)); + BOOST_TEST(SparseRowStore::preferred_for_modes(SparseRowStore::kMinModes)); + BOOST_TEST(SparseRowStore::preferred_for_modes(SparseRowStore::kMinModes + 1)); + // 32 modes is where the Stage 3 gate had sparse 1.9x behind dense even on baseline x86-64; no + // build should be switching there. + BOOST_TEST(!SparseRowStore::preferred_for_modes(32U)); +} + +// The row form of set() against the dense one, which is the only definition of what it must produce. +// Both fill the same row of two stores built alike; every observable of the two must agree, including +// the hash the table keys on -- a row written one way has to be findable by a key written the other. +BOOST_AUTO_TEST_CASE(sparse_row_store_row_form_set_matches_the_dense_one) { + std::mt19937_64 rng(20260813U); + for (const size_t num_modes : {32U, 96U, 512U}) { + SparseRowStore dense_written(2 * num_modes, SparseRowStore::kMaxSlots); + SparseRowStore row_written(2 * num_modes, SparseRowStore::kMaxSlots); + for (size_t trial = 0; trial < 200; ++trial) { + const Bitset mono = test_utils::random_monomial(rng, num_modes, SparseRowStore::kMaxSlots); + const RowBuffer row = row_of(mono); + + dense_written.push_back(mono); + const size_t i = row_written.grow_rows_geometric(1); + row_written.set(i, row.view()); + + BOOST_REQUIRE(i == dense_written.size() - 1); + BOOST_TEST(row_written.spilled(i) == dense_written.spilled(i)); + BOOST_TEST(row_written.codes(i) == dense_written.codes(i)); + BOOST_TEST(row_written.slot_count(i) == dense_written.slot_count(i)); + BOOST_TEST(row_written.popcount(i) == dense_written.popcount(i)); + BOOST_TEST(row_written.row(i) == mono); + // The three key forms are interchangeable only if they hash alike; the table's probe order + // (and so MPI owner routing) is downstream of this. + BOOST_TEST(sparse_row_hash(row.view()) == sparse_row_hash(mono)); + BOOST_TEST(sparse_row_hash(SparseRowKey{.row = row.view()}) == sparse_row_hash(mono)); + BOOST_TEST(sparse_row_hash(SparseRowKey{.spilled = &mono}) == sparse_row_hash(mono)); + } + // Written rows are findable by either form, whichever way they went in. + for (size_t i = 0; i < row_written.size(); ++i) { + const Bitset mono = row_written.row(i); + const RowBuffer row = row_of(mono); + row_written.emplace(row.view(), i); + } + for (size_t i = 0; i < row_written.size(); ++i) { + const Bitset mono = row_written.row(i); + const RowBuffer row = row_of(mono); + const auto by_mono = row_written.find(mono); + const auto by_row = row_written.find(row.view()); + const auto by_key = row_written.find(SparseRowKey{.row = row.view()}); + BOOST_REQUIRE(by_mono.has_value()); + BOOST_TEST((by_row == by_mono)); + BOOST_TEST((by_key == by_mono)); + // Distinct monomials may repeat across trials, so the found row must equal this one rather + // than be this index. + BOOST_TEST(row_written.row(*by_mono) == mono); + } + } +} + +// A row wider than the store's capacity has to spill, exactly as the dense set() spills it. The capacity +// is sized from the cutoff and a fully paired term escapes the cutoff, so this arm is reachable by +// construction and cannot be sized away. +BOOST_AUTO_TEST_CASE(sparse_row_store_row_form_set_spills_a_row_past_the_capacity) { + constexpr size_t kNumBits = 128; + SparseRowStore store(kNumBits, 2); + Bitset mono(kNumBits); + for (const size_t b : {0U, 1U, 4U, 20U, 21U, 99U}) { // modes 0 (paired), 2, 10 (paired), 49 + mono.set(b); + } + const RowBuffer row = row_of(mono); + BOOST_REQUIRE(row.lanes.size() == 4U); + + const size_t i = store.grow_rows_geometric(1); + store.set(i, row.view()); + BOOST_TEST(store.spilled(i)); + BOOST_TEST(store.row(i) == mono); + BOOST_TEST(store.slot_count(i) == 4U); + BOOST_TEST(store.popcount(i) == 6U); + + // A spilled row is found by the dense key; the row key finds it through either shape it carries. + store.emplace(mono, i); + BOOST_TEST((store.find(mono) == std::optional(i))); + BOOST_TEST((store.find(SparseRowKey{.spilled = &mono}) == std::optional(i))); + BOOST_TEST((store.find(SparseRowKey{.row = row.view()}) == std::optional(i))); +} + +// Rows are overwritten in place by the miss inserts, so a row must not inherit the previous occupant's +// spill -- in either direction. +BOOST_AUTO_TEST_CASE(sparse_row_store_row_form_set_clears_a_stale_spill) { + constexpr size_t kNumBits = 64; + SparseRowStore store(kNumBits, 2); + Bitset wide(kNumBits); + for (const size_t b : {0U, 6U, 10U, 30U}) { + wide.set(b); + } + Bitset narrow(kNumBits); + narrow.set(4); + narrow.set(5); + + const size_t i = store.grow_rows_geometric(1); + store.set(i, row_of(wide).view()); + BOOST_REQUIRE(store.spilled(i)); + + store.set(i, row_of(narrow).view()); + BOOST_TEST(!store.spilled(i)); + BOOST_TEST(store.row(i) == narrow); + BOOST_TEST(store.codes(i) == 0b11ULL); // one slot, mode 2, both positions + + store.set(i, row_of(wide).view()); + BOOST_TEST(store.spilled(i)); + BOOST_TEST(store.row(i) == wide); + + // The empty row is the case the lane padding exists for: it writes no lane of its own, so without the + // pad the previous occupant's overflow marker survives in lane 0 and the row reads as spilled while + // its side-map entry is gone. + store.set(i, row_of(Bitset(kNumBits)).view()); + BOOST_TEST(!store.spilled(i)); + BOOST_TEST(store.codes(i) == 0U); + BOOST_TEST(store.slot_count(i) == 0U); + BOOST_TEST(store.row(i) == Bitset(kNumBits)); +} + +// A batch of keys is homogeneous by type but not by shape: the spilled ones are what a query record +// escapes to. find_batch must resolve both, in one pass, against a store holding both kinds of row. +BOOST_AUTO_TEST_CASE(sparse_row_key_batch_resolves_both_shapes) { + constexpr size_t kNumBits = 128; + SparseRowStore store(kNumBits, 3); + std::mt19937_64 rng(20260814U); + + std::vector monos; + for (size_t k = 0; k < 64; ++k) { + // Up to 6 slots against a capacity of 3, so roughly half the rows spill. + Bitset mono = test_utils::random_monomial(rng, kNumBits / 2, 6); + if (std::find(monos.begin(), monos.end(), mono) != monos.end()) { + continue; + } + store.push_back(mono); + store.emplace(mono, store.size() - 1); + monos.push_back(std::move(mono)); + } + BOOST_REQUIRE(monos.size() > 8U); + + std::vector rows; + std::vector keys; + rows.reserve(monos.size()); + keys.reserve(monos.size()); + for (const auto &mono : monos) { + rows.push_back(row_of(mono)); + } + size_t spilled_keys = 0; + for (size_t k = 0; k < monos.size(); ++k) { + // The shape a record would have carried: within the capacity it stays a row, past it the record + // escapes to the dense monomial. + if (rows[k].lanes.size() > store.slots_per_row()) { + keys.push_back(SparseRowKey{.spilled = &monos[k]}); + ++spilled_keys; + } + else { + keys.push_back(SparseRowKey{.row = rows[k].view()}); + } + } + BOOST_REQUIRE(spilled_keys > 0U); + BOOST_REQUIRE(spilled_keys < monos.size()); + + std::vector found(keys.size(), SparseRowStore::kNotFound); + store.find_batch(keys.data(), keys.size(), found.data()); + for (size_t k = 0; k < keys.size(); ++k) { + BOOST_REQUIRE(found[k] < store.size()); + BOOST_TEST(store.row(found[k]) == monos[k]); + } + + // An absent key must miss through either shape. + Bitset absent(kNumBits); + absent.set(6); + absent.set(7); + absent.set(120); + while (std::find(monos.begin(), monos.end(), absent) != monos.end()) { + absent.set(9); + } + const RowBuffer absent_row = row_of(absent); + BOOST_TEST(!store.find(SparseRowKey{.row = absent_row.view()}).has_value()); + BOOST_TEST(!store.find(SparseRowKey{.spilled = &absent}).has_value()); +} + +// The codes array's element width follows slots_per_row_, since a codes word only ever sets bits below +// 2 * slots_per_row_. The row array is the operator's largest, and rows are payload -- never a hash +// input, never serialized -- so a narrowing here changes no term and no energy and a baseline diff +// cannot see it. This is the footprint gate, the support-form counterpart to +// row_slot_width_follows_the_position_count in operator_index_tests.cpp. +// memory_bytes() - slack_bytes() is the *used* part of the arrays, which makes the figure exact rather +// than allocator-dependent. +BOOST_AUTO_TEST_CASE(codes_width_follows_the_slot_count) { + constexpr size_t kNumBits = 256; + constexpr size_t kRows = 500; + // 2 bytes of codes at up to 8 slots, 4 up to 16, 8 above -- with the mode lanes constant per slot. + const std::array, 3> kCases{{{8, 2}, {16, 4}, {17, 8}}}; + + for (const auto &[slots, codes_bytes] : kCases) { + SparseRowStore store(kNumBits, slots); + for (size_t i = 0; i < kRows; ++i) { + // One occupied mode per row: any slot count <= slots works, but staying at one keeps every + // row off the overflow side-map, whose bytes are counted separately and would blur this. + Bitset mono(kNumBits); + mono.set(2 * (i % (kNumBits / 2))); + store.push_back(mono); + } + const size_t expected = kRows * ((slots * sizeof(RowMode)) + codes_bytes); + BOOST_TEST(store.memory_bytes() - store.slack_bytes() == expected); + } +} + +// The narrowed storage must be invisible above the seam: a store at each codes width has to hold and +// return the same rows, hash them the same way and find them the same way. +BOOST_AUTO_TEST_CASE(a_narrowed_codes_word_reads_back_unchanged) { + constexpr size_t kNumBits = 256; + std::mt19937_64 rng(20260828); + + std::vector monos; + for (size_t i = 0; i < 200; ++i) { + // At most 8 occupied modes, so every row fits the narrowest store's slots and none spills. + Bitset mono(kNumBits); + for (size_t k = 0; k < 8; ++k) { + mono.set(rng() % kNumBits); + } + if (std::find(monos.begin(), monos.end(), mono) == monos.end()) { + monos.push_back(mono); + } + } + + SparseRowStore narrow(kNumBits, 8); + SparseRowStore wide(kNumBits, SparseRowStore::kMaxSlots); + for (const auto &mono : monos) { + narrow.push_back(mono); + wide.push_back(mono); + narrow.emplace(mono, narrow.size() - 1); + wide.emplace(mono, wide.size() - 1); + } + BOOST_REQUIRE(narrow.memory_bytes() < wide.memory_bytes()); + + for (size_t i = 0; i < monos.size(); ++i) { + BOOST_REQUIRE(!narrow.spilled(i)); + BOOST_TEST(narrow.codes(i) == wide.codes(i)); + BOOST_TEST(narrow.row(i) == monos[i]); + BOOST_TEST(narrow.popcount(i) == wide.popcount(i)); + BOOST_TEST(narrow.slot_count(i) == wide.slot_count(i)); + BOOST_TEST(sparse_row_hash(narrow.view(i)) == sparse_row_hash(wide.view(i))); + BOOST_TEST(narrow.find(monos[i]).value() == i); + BOOST_TEST(narrow.find(narrow.view(i)).value() == i); + } +} diff --git a/cpp/tests/sparse_wire_tests.cpp b/cpp/tests/sparse_wire_tests.cpp new file mode 100644 index 00000000..b67a84b7 --- /dev/null +++ b/cpp/tests/sparse_wire_tests.cpp @@ -0,0 +1,534 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The support-form query record: a row must survive the wire exactly, and the record must keep the dense +// record's shape so the stride arithmetic, alltoallv counts and phase/value readers work unchanged. +// +// Plus the two things that shape carries with it -- the header word every buffer opens with, and the tail a +// query too wide for any fixed-stride sparse record escapes to. + +#include + +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +struct OwnedRow { + std::vector lanes; + RowCodes codes = 0; + + [[nodiscard]] auto view() const -> SparseRow { return SparseRow{lanes.data(), codes}; } +}; + +// The four batch/codec cases below key their assertions off a per-record shape, so a default-constructed +// placeholder stands in for the escaped records' absent row. +static_assert(std::is_default_constructible_v); + +auto random_row(std::mt19937_64 &rng, size_t num_modes, size_t capacity) -> OwnedRow { + OwnedRow row{std::vector(capacity, 0), 0}; + // Ascending distinct modes, which is what a real row is; a random unsorted list would not be one. + std::vector modes; + for (size_t m = 0; m < num_modes && modes.size() < capacity; ++m) { + if ((rng() % 4) == 0) { + modes.push_back(m); + } + } + for (size_t j = 0; j < modes.size(); ++j) { + row.lanes[j] = static_cast(modes[j]); + row.codes |= static_cast(1U + (rng() % 3U)) << (2 * j); + } + return row; +} + +} // namespace + +// A record is lane words, then codes, then phase -- so the phase sits at offset `payload`, exactly where +// the dense reader expects it for a payload of that many words. That is the property that lets the record +// machinery stay untouched, so it is pinned rather than left implicit. +BOOST_AUTO_TEST_CASE(sparse_wire_record_keeps_the_dense_record_shape) { + BOOST_TEST(sparse_lane_words(1U) == 1U); + BOOST_TEST(sparse_lane_words(4U) == 1U); + BOOST_TEST(sparse_lane_words(5U) == 2U); + BOOST_TEST(sparse_lane_words(12U) == 3U); + BOOST_TEST(sparse_lane_words(32U) == 8U); + BOOST_TEST(sparse_payload_words(12U) == 4U); + BOOST_TEST(query_words(sparse_payload_words(12U)) == 5U); + BOOST_TEST(query_words_fused(sparse_payload_words(12U)) == 6U); + + // A 12-slot row rides 5 words where a 1024-mode monomial needs 33, and the two are equal at 128 modes + // -- which is roughly where the crossover puts the sparse backend in a wheel build anyway. + BOOST_TEST(query_words(sparse_payload_words(12U)) < query_words(1024U / 32U)); + BOOST_TEST(query_words(sparse_payload_words(12U)) == query_words(128U / 32U)); + + constexpr size_t kCapacity = 12; + VecZ buf = query_buffer(); + OwnedRow row{std::vector(kCapacity, 0), 0}; + row.lanes[0] = 7; + row.codes = 0b11ULL; + sparse_query_push(buf, row.view(), kCapacity, -1); + BOOST_REQUIRE(buf.size() == kQueryHeaderWords + query_words(sparse_payload_words(kCapacity))); + BOOST_TEST(query_record_count(buf) == 1U); + // query_phase reads the phase off the payload width alone, with no idea what the payload holds. + BOOST_TEST(query_phase(buf, 0, sparse_payload_words(kCapacity)) == -1); +} + +BOOST_AUTO_TEST_CASE(sparse_wire_round_trips_rows_exactly) { + std::mt19937_64 rng(20260812U); + size_t full_rows = 0; + size_t empty_rows = 0; + for (const size_t capacity : {1U, 4U, 5U, 12U, 32U}) { + for (const size_t num_modes : {32U, 64U, 1024U, 32000U}) { + const size_t stride = query_words(sparse_payload_words(capacity)); + VecZ buf = query_buffer(); + std::vector rows; + for (size_t t = 0; t < 40; ++t) { + // Every fifth record is the empty row: at these mode counts the random generator would + // essentially never produce one, and an empty row is the record whose lanes are all + // padding. + rows.push_back((t % 5) == 0 ? OwnedRow{std::vector(capacity, 0), 0} + : random_row(rng, num_modes, capacity)); + // Phases are +-1 on the real path; both must survive the unsigned round-trip. + sparse_query_push(buf, rows.back().view(), capacity, (t % 2) == 0 ? 1 : -1); + } + BOOST_REQUIRE(buf.size() == kQueryHeaderWords + (rows.size() * stride)); + BOOST_REQUIRE(query_record_count(buf) == rows.size()); + + for (size_t q = 0; q < rows.size(); ++q) { + std::vector lanes(capacity, 0xEEEE); // poisoned, so an unwritten lane shows up + RowCodes codes = 0; + int phase = 0; + sparse_query_read(buf, q, stride, capacity, lanes.data(), codes, phase); + BOOST_TEST(codes == rows[q].codes); + BOOST_TEST(phase == ((q % 2) == 0 ? 1 : -1)); + const size_t n = row_slot_count(codes); + BOOST_REQUIRE(n == rows[q].view().num_slots()); + for (size_t j = 0; j < n; ++j) { + BOOST_TEST(lanes[j] == rows[q].lanes[j]); + } + full_rows += n == capacity ? 1 : 0; + empty_rows += n == 0 ? 1 : 0; + } + } + } + // Both edges of the capacity have to have occurred, or the packing was never pushed to its bounds. + BOOST_TEST(full_rows > 0U); + BOOST_TEST(empty_rows > 0U); +} + +// Mode indices are packed four to a word, so a lane must not bleed into its neighbours. The largest mode +// a store admits is kMaxModes - 1, which is also the widest lane value. +BOOST_AUTO_TEST_CASE(sparse_wire_packs_lanes_without_bleeding) { + constexpr size_t kCapacity = 8; // two lane words, so the boundary between them is exercised + const size_t stride = query_words(sparse_payload_words(kCapacity)); + const auto top = static_cast(SparseRowStore::kMaxModes - 1); + + OwnedRow row{std::vector(kCapacity, 0), 0}; + // Ascending, and straddling the 4-lane word boundary with extreme values on both sides of it. + const std::vector modes{0, 1, top - 2, top - 1, top, 0, 0, 0}; + for (size_t j = 0; j < 5; ++j) { + row.lanes[j] = modes[j]; + row.codes |= RowCodes{0b11} << (2 * j); + } + + VecZ buf = query_buffer(); + sparse_query_push(buf, row.view(), kCapacity, 1); + std::vector lanes(kCapacity, 0); + RowCodes codes = 0; + int phase = 0; + sparse_query_read(buf, 0, stride, kCapacity, lanes.data(), codes, phase); + BOOST_TEST(codes == row.codes); + for (size_t j = 0; j < 5; ++j) { + BOOST_TEST(lanes[j] == modes[j]); + } +} + +// Records are read by position out of one flat buffer, so a short row must still occupy a full stride and +// must not be able to see the previous record's lanes. +BOOST_AUTO_TEST_CASE(sparse_wire_short_rows_keep_the_full_stride) { + constexpr size_t kCapacity = 12; + const size_t stride = query_words(sparse_payload_words(kCapacity)); + VecZ buf = query_buffer(); + + OwnedRow full{std::vector(kCapacity, 0), 0}; + for (size_t j = 0; j < kCapacity; ++j) { + full.lanes[j] = static_cast(100 + j); + full.codes |= RowCodes{0b11} << (2 * j); + } + OwnedRow empty{std::vector(kCapacity, 0), 0}; + + sparse_query_push(buf, full.view(), kCapacity, 1); + sparse_query_push(buf, empty.view(), kCapacity, -1); + BOOST_REQUIRE(buf.size() == kQueryHeaderWords + (2 * stride)); + + std::vector lanes(kCapacity, 0xEEEE); + RowCodes codes = 0xDEAD; + int phase = 0; + sparse_query_read(buf, 1, stride, kCapacity, lanes.data(), codes, phase); + BOOST_TEST(codes == 0U); + BOOST_TEST(phase == -1); + BOOST_TEST(row_slot_count(codes) == 0U); + // Nothing was written into lanes, so the poison is still there -- the empty row cannot have inherited + // the previous record's modes. + BOOST_TEST(lanes[0] == 0xEEEE); +} + +// --- the query key batches ------------------------------------------------------------------------- +// +// Both resolve paths fill one of these from wire records and hand it to find_batch contiguously. They are +// grow-only and reused across layers, which is the shape the measurements asked for, and that reuse is +// where the two ways to get it wrong live: a stale element read before being overwritten, and -- for the +// sparse batch, whose keys are views -- a view left pointing into storage that growth moved. + +// Each store is queried in the form it keys its rows by, so a resolve never converts one into the other. +static_assert(std::is_same_v::type, DenseQueryKeys>); +static_assert(std::is_same_v::type, SparseQueryKeys>); + +BOOST_AUTO_TEST_CASE(query_keys_dense_batch_round_trips_records) { + constexpr size_t kNumBits = 128; + const size_t stride = query_words(kNumBits / 64); + VecZ buf = query_buffer(); + std::vector monos; + for (size_t t = 0; t < 40; ++t) { + Bitset mono(kNumBits); + mono.set(t); + mono.set(kNumBits - 1 - t); + monos.push_back(mono); + query_push(buf, mono, (t % 2) == 0 ? 1 : -1); + } + + BOOST_TEST(query_record_count(buf) == monos.size()); + DenseQueryKeys keys; + keys.configure(kNumBits, /*capacity=*/0); + keys.ensure(monos.size()); + keys.begin_batch(); + for (size_t q = 0; q < monos.size(); ++q) { + BOOST_TEST(keys.read_record(buf, q, stride, q) == ((q % 2) == 0 ? 1 : -1)); + } + for (size_t q = 0; q < monos.size(); ++q) { + BOOST_TEST((keys[q] == monos[q])); + BOOST_TEST((keys.data()[q] == monos[q])); + } +} + +BOOST_AUTO_TEST_CASE(query_keys_sparse_batch_survives_growth) { + constexpr size_t kCapacity = 12; + const size_t stride = query_words(sparse_payload_words(kCapacity)); + std::mt19937_64 rng(20260812U); + VecZ buf = query_buffer(); + std::vector rows; + for (size_t t = 0; t < 64; ++t) { + rows.push_back(random_row(rng, 1024, kCapacity)); + sparse_query_push(buf, rows.back().view(), kCapacity, (t % 2) == 0 ? 1 : -1); + } + + SparseQueryKeys keys; + keys.configure(/*num_bits=*/2048, kCapacity); + + // Fill a small prefix, then grow: the lane array reallocates, so every view has to be rebuilt. If only + // the new tail were, the prefix read back below would be reading freed storage. + keys.ensure(8); + for (size_t q = 0; q < 8; ++q) { + BOOST_TEST(keys.read_record(buf, q, stride, q) == ((q % 2) == 0 ? 1 : -1)); + } + keys.ensure(rows.size()); + // Checked as a pointer invariant rather than by reading the prefix and hoping it looks wrong: a view + // left over from before the growth points into freed storage, which is undefined behaviour and might + // read back plausibly. Every view must address this batch's current lane array at its own stride. + for (size_t q = 0; q < rows.size(); ++q) { + BOOST_TEST(keys.data()[q].row.modes == keys.data()[0].row.modes + (q * kCapacity)); + } + for (size_t q = 8; q < rows.size(); ++q) { + BOOST_TEST(keys.read_record(buf, q, stride, q) == ((q % 2) == 0 ? 1 : -1)); + } + + for (size_t q = 0; q < 8; ++q) { + // Re-read the prefix records so the prefix slots are written again after the growth; what is being + // checked is that the view still addresses this batch's own lanes. + BOOST_TEST(keys.read_record(buf, q, stride, q) == ((q % 2) == 0 ? 1 : -1)); + } + for (size_t q = 0; q < rows.size(); ++q) { + const auto &key = keys.data()[q]; + BOOST_REQUIRE(!key.is_spilled()); + BOOST_TEST(key.row.codes == rows[q].codes); + const size_t n = key.row.num_slots(); + BOOST_REQUIRE(n == rows[q].view().num_slots()); + for (size_t j = 0; j < n; ++j) { + BOOST_TEST(key.row.mode(j) == rows[q].view().mode(j)); + } + } +} + +// configure() with a different extent must drop the storage: a thread servicing two propagators of +// different widths would otherwise write a wide record into a narrow element. +BOOST_AUTO_TEST_CASE(query_keys_reconfigure_resizes_the_elements) { + DenseQueryKeys dense; + dense.configure(64, 0); + dense.ensure(4); + BOOST_TEST(dense[0].size() == 64U); + dense.configure(256, 0); + dense.ensure(4); + BOOST_TEST(dense[0].size() == 256U); + + SparseQueryKeys sparse; + sparse.configure(64, 4); + sparse.ensure(4); + const auto *narrow_base = sparse.data()[0].row.modes; + const auto *narrow_next = sparse.data()[1].row.modes; + BOOST_TEST(narrow_next - narrow_base == 4); + sparse.configure(64, 12); + sparse.ensure(4); + BOOST_TEST(sparse.data()[1].row.modes - sparse.data()[0].row.modes == 12); +} + +// --- the escape tail --------------------------------------------------------------------------------- +// +// A query is M ⊕ G and a fully paired product escapes the cutoff, so a query's support is unbounded and no +// fixed-stride sparse record can hold every one. The escaped record keeps its place and its stride and +// names a tail entry instead, which is what leaves the engine's offsets, counts and compaction as plain +// arithmetic. + +namespace { + +// The scan's shape: records into one buffer, escape monomials into another, concatenated once pushing is +// done -- a record cannot be appended after the tail has started. +struct WireBuffers { + VecZ records = query_buffer(); + VecZ escapes; + + auto finish() -> VecZ { + VecZ out = records; + out.insert(out.end(), escapes.begin(), escapes.end()); + return out; + } +}; + +auto wide_monomial(size_t num_bits, size_t seed) -> Bitset { + Bitset mono(num_bits); + // Fully paired and far wider than any record capacity below: the shape that escapes the cutoff and so + // cannot be sized away. + for (size_t mode = seed % 3; mode < (num_bits / 2); mode += 3) { + mono.set(2 * mode); + mono.set((2 * mode) + 1); + } + return mono; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(sparse_wire_escaped_records_carry_their_monomial_in_the_tail) { + constexpr size_t kCapacity = 8; + constexpr size_t kNumBits = 512; + const size_t stride = query_words(sparse_payload_words(kCapacity)); + std::mt19937_64 rng(20260813U); + + WireBuffers wire; + std::vector rows; + std::vector escaped; + std::vector is_escape; + for (size_t t = 0; t < 24; ++t) { + // Every third record escapes, so escaped and plain records interleave -- the ordering the tail + // indices have to survive. + if (t % 3 == 2) { + escaped.push_back(wide_monomial(kNumBits, t)); + sparse_query_push_escape(wire.records, wire.escapes, escaped.back(), kCapacity, (t % 2) == 0 ? 1 : -1); + rows.emplace_back(); + is_escape.push_back(true); + continue; + } + rows.push_back(random_row(rng, kNumBits / 2, kCapacity)); + sparse_query_push(wire.records, rows.back().view(), kCapacity, (t % 2) == 0 ? 1 : -1); + is_escape.push_back(false); + } + const VecZ buf = wire.finish(); + + // The header counts records, escaped ones included; the tail sits right after them. + BOOST_REQUIRE(query_record_count(buf) == is_escape.size()); + BOOST_REQUIRE(query_tail_offset(buf, stride) == kQueryHeaderWords + (is_escape.size() * stride)); + BOOST_REQUIRE(buf.size() == query_tail_offset(buf, stride) + (escaped.size() * (kNumBits / 64))); + + SparseQueryKeys keys; + keys.configure(kNumBits, kCapacity); + keys.ensure(is_escape.size()); + keys.begin_batch(); + for (size_t q = 0; q < is_escape.size(); ++q) { + BOOST_TEST(keys.read_record(buf, q, stride, q) == ((q % 2) == 0 ? 1 : -1)); + } + + size_t next_escape = 0; + for (size_t q = 0; q < is_escape.size(); ++q) { + const auto &key = keys[q]; + BOOST_REQUIRE(key.is_spilled() == is_escape[q]); + if (is_escape[q]) { + // The monomial has to arrive bit-for-bit: it is the only form this query exists in. + BOOST_TEST((*key.spilled == escaped[next_escape])); + ++next_escape; + continue; + } + BOOST_TEST(key.row.codes == rows[q].codes); + for (size_t j = 0; j < key.row.num_slots(); ++j) { + BOOST_TEST(key.row.mode(j) == rows[q].view().mode(j)); + } + } + BOOST_TEST(next_escape == escaped.size()); +} + +// The fused sink widens every record by a value word, which moves the tail. The escape indices survive +// because they name a position *within* the tail rather than an offset into the buffer -- so the same +// records read back at the fused stride. +BOOST_AUTO_TEST_CASE(sparse_wire_escape_indices_survive_the_fused_relayout) { + constexpr size_t kCapacity = 6; + constexpr size_t kNumBits = 256; + const size_t payload = sparse_payload_words(kCapacity); + std::mt19937_64 rng(20260814U); + + WireBuffers wire; + std::vector escaped; + std::vector values; + std::vector is_escape; + for (size_t t = 0; t < 12; ++t) { + if (t % 2 == 0) { + escaped.push_back(wide_monomial(kNumBits, t)); + sparse_query_push_escape(wire.records, wire.escapes, escaped.back(), kCapacity, 1); + is_escape.push_back(true); + } + else { + const OwnedRow row = random_row(rng, kNumBits / 2, kCapacity); + sparse_query_push(wire.records, row.view(), kCapacity, -1); + is_escape.push_back(false); + } + values.push_back(static_cast(t) * 0.5 - 3.0); + } + const VecZ plain = wire.finish(); + + VecZ fused; + build_fused_query_value(plain, values, fused, payload); + BOOST_REQUIRE(query_record_count(fused) == is_escape.size()); + BOOST_REQUIRE(fused.size() + == kQueryHeaderWords + (is_escape.size() * query_words_fused(payload)) + + (escaped.size() * (kNumBits / 64))); + + SparseQueryKeys keys; + keys.configure(kNumBits, kCapacity); + keys.ensure(is_escape.size()); + keys.begin_batch(); + size_t next_escape = 0; + for (size_t q = 0; q < is_escape.size(); ++q) { + BOOST_TEST(keys.read_record(fused, q, query_words_fused(payload), q) == (is_escape[q] ? 1 : -1)); + BOOST_TEST(query_value(fused, q, payload) == values[q]); + BOOST_REQUIRE(keys[q].is_spilled() == is_escape[q]); + if (is_escape[q]) { + BOOST_TEST((*keys[q].spilled == escaped[next_escape])); + ++next_escape; + } + } + BOOST_TEST(next_escape == escaped.size()); +} + +// Deferred self-misses are inserted after both resolve passes, by which time the batch has been refilled +// many times over -- so a key that must survive that has to be retained, and retaining it has to copy. +BOOST_AUTO_TEST_CASE(query_keys_retained_survive_a_refill) { + constexpr size_t kCapacity = 6; + constexpr size_t kNumBits = 256; + const size_t stride = query_words(sparse_payload_words(kCapacity)); + std::mt19937_64 rng(20260815U); + + WireBuffers wire; + std::vector rows; + std::vector escaped; + std::vector is_escape; + for (size_t t = 0; t < 8; ++t) { + if (t % 4 == 3) { + escaped.push_back(wide_monomial(kNumBits, t)); + sparse_query_push_escape(wire.records, wire.escapes, escaped.back(), kCapacity, 1); + rows.emplace_back(); + is_escape.push_back(true); + continue; + } + rows.push_back(random_row(rng, kNumBits / 2, kCapacity)); + sparse_query_push(wire.records, rows.back().view(), kCapacity, 1); + is_escape.push_back(false); + } + const VecZ buf = wire.finish(); + + SparseQueryKeys keys; + keys.configure(kNumBits, kCapacity); + keys.ensure(2); + + // Read the records two at a time into the same two slots, retaining every key -- the resolve path's + // batching, with kResolveBatch of 2. + std::vector handles; + for (size_t q = 0; q < is_escape.size(); q += 2) { + keys.begin_batch(); + for (size_t j = 0; j < 2; ++j) { + (void)keys.read_record(buf, q + j, stride, j); + } + for (size_t j = 0; j < 2; ++j) { + handles.push_back(keys.retain(j)); + } + } + + size_t next_escape = 0; + for (size_t q = 0; q < handles.size(); ++q) { + const auto key = keys.retained(handles[q]); + BOOST_REQUIRE(key.is_spilled() == is_escape[q]); + if (is_escape[q]) { + BOOST_TEST((*key.spilled == escaped[next_escape])); + ++next_escape; + continue; + } + BOOST_TEST(key.row.codes == rows[q].codes); + for (size_t j = 0; j < key.row.num_slots(); ++j) { + BOOST_TEST(key.row.mode(j) == rows[q].view().mode(j)); + } + } + BOOST_TEST(next_escape == escaped.size()); + + // The dense batch owes the same guarantee, and its keys are whole monomials. + VecZ dense_buf = query_buffer(); + std::vector monos; + for (size_t t = 0; t < 8; ++t) { + Bitset mono(kNumBits); + mono.set(t); + mono.set(kNumBits - 1 - t); + monos.push_back(mono); + query_push(dense_buf, mono, 1); + } + DenseQueryKeys dense; + dense.configure(kNumBits, 0); + dense.ensure(2); + std::vector dense_handles; + for (size_t q = 0; q < monos.size(); q += 2) { + dense.begin_batch(); + for (size_t j = 0; j < 2; ++j) { + (void)dense.read_record(dense_buf, q + j, query_words(kNumBits / 64), j); + } + for (size_t j = 0; j < 2; ++j) { + dense_handles.push_back(dense.retain(j)); + } + } + for (size_t q = 0; q < dense_handles.size(); ++q) { + BOOST_TEST((dense.retained(dense_handles[q]) == monos[q])); + } +} diff --git a/cpp/tests/store_interchange_tests.cpp b/cpp/tests/store_interchange_tests.cpp new file mode 100644 index 00000000..1d4e1157 --- /dev/null +++ b/cpp/tests/store_interchange_tests.cpp @@ -0,0 +1,217 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Which of the operator store's consumers actually care which store they are given. The answer decides how +// much Stage 6's swap has to touch, so it is asserted rather than reasoned about: +// +// InvertedIndex -- no: it reads rows only through the TypeAliases.h accessors, so both stores build the +// same columns, the same parity words and the same tiering. +// MonomialMap -- no: keyed by Bitset, and Bitset survives (the Stage 3 gate retained the dense +// backend), so init_op_map keeps its key type and looks up through the store's +// Bitset-key find. +// for_each -- no, once SparseRowStore offers OperatorIndex's fn(monomial, row_index) signature. + +#include + +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/operator/InvertedIndex.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +#include "RandomMonomial.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +// Every observable of a built index, so a difference cannot hide in a field the test forgot. +auto columns_agree(const InvertedIndex &a, const InvertedIndex &b) -> bool { + if (a.num_columns() != b.num_columns() || a.rows() != b.rows() || a.words() != b.words()) { + return false; + } + for (size_t c = 0; c < a.num_columns(); ++c) { + if (a.column_is_dense(c) != b.column_is_dense(c)) { + return false; + } + if (a.column_is_dense(c)) { + for (size_t w = 0; w < a.words(); ++w) { + if (a.dense_column_data(c)[w] != b.dense_column_data(c)[w]) { + return false; + } + } + } + else if (a.sparse_column_rows(c) != b.sparse_column_rows(c)) { + return false; + } + } + for (size_t w = 0; w < a.words(); ++w) { + if (a.row_parity_words()[w] != b.row_parity_words()[w]) { + return false; + } + } + return a.tier_memory_bytes() == b.tier_memory_bytes(); +} + +} // namespace + +BOOST_AUTO_TEST_CASE(store_interchange_inverted_index_is_store_agnostic) { + std::mt19937_64 rng(20260812U); + for (const size_t num_modes : {32U, 64U}) { + const size_t num_bits = 2 * num_modes; + OperatorIndex packed(num_bits); + // Capacity 4 on purpose, so a good share of rows spill and the index reads them through the + // accessors' overflow path rather than off a codes word. + SparseRowStore sparse(num_bits, 4); + size_t spilled = 0; + for (size_t t = 0; t < 400; ++t) { + const auto mono = test_utils::random_monomial(rng, num_modes, 8); + packed.push_back(mono); + sparse.push_back(mono); + spilled += sparse.spilled(sparse.size() - 1) ? 1 : 0; + } + BOOST_TEST(spilled > 0U); + + InvertedIndex from_packed(num_bits); + InvertedIndex from_sparse(num_bits); + from_packed.rebuild(packed); + from_sparse.rebuild(sparse); + BOOST_TEST(columns_agree(from_packed, from_sparse)); + + // append_rows is the incremental path evolution actually takes, so it needs its own comparison -- + // and a like-for-like one. An appended index and a rebuilt index legitimately differ in tiering: + // rebuild() counts every column's postings up front and pre-promotes, while append_rows can only + // promote as rows arrive. That is InvertedIndex's own behaviour and says nothing about the store, + // so what is compared here is append-vs-append. + InvertedIndex appended_sparse(num_bits); + InvertedIndex appended_packed(num_bits); + appended_sparse.rebuild(sparse); + appended_packed.rebuild(packed); + for (size_t t = 0; t < 50; ++t) { + const auto mono = test_utils::random_monomial(rng, num_modes, 8); + const size_t base = sparse.size(); + packed.push_back(mono); + sparse.push_back(mono); + appended_sparse.append_rows(sparse, base, 1); + appended_packed.append_rows(packed, base, 1); + } + BOOST_TEST(columns_agree(appended_sparse, appended_packed)); + } +} + +// The plan expected MonomialMap to need variable-length keys, on the assumption that the dense Bitset +// would be deleted. The Stage 3 gate retained it, so the map keeps its key type and this path needs no +// change at all -- what makes that true is the store's Bitset-key find, which is asserted here. +BOOST_AUTO_TEST_CASE(store_interchange_monomial_map_keys_still_resolve) { + std::mt19937_64 rng(1234U); + constexpr size_t kNumModes = 32; + constexpr size_t kNumBits = 2 * kNumModes; + OperatorIndex packed(kNumBits); + SparseRowStore sparse(kNumBits, 4); + + MonomialMap pending; + std::vector stored; + for (size_t t = 0; t < 200; ++t) { + const auto mono = test_utils::random_monomial(rng, kNumModes, 8); + // One row per distinct monomial: emplace is insert-or-no-op, so a duplicate would resolve to the + // first row holding it and the coefficient below would name the wrong index. + if (sparse.find(mono).has_value()) { + continue; + } + const size_t i = sparse.size(); + packed.push_back(mono); + sparse.push_back(mono); + packed.emplace(mono, i); + sparse.emplace(mono, i); + stored.push_back(mono); + pending[mono] = static_cast(i); + } + // Terms the store does not hold, which MPOperator::get_operator must leave pending. + size_t absent = 0; + for (size_t t = 0; t < 200; ++t) { + Bitset mono(kNumBits); + for (size_t k = 0; k < 1 + (rng() % 5); ++k) { + mono.set(rng() % kNumBits); + } + // emplace, not [], and count only what it inserted: the same absent monomial can be drawn twice, + // and assigning would leave the map smaller than the count. + if (!sparse.find(mono).has_value() && pending.emplace(mono, -1.0).second) { + ++absent; + } + } + BOOST_TEST(absent > 0U); + + // get_operator's loop, run against both stores: a key present in the store drains to its row, one + // absent stays. Both stores must agree on which is which, and on the row. + size_t drained = 0; + size_t retained = 0; + for (const auto &[mono, coeff] : pending) { + const auto in_sparse = sparse.find(mono); + const auto in_packed = packed.find(mono); + BOOST_TEST(in_sparse.has_value() == in_packed.has_value()); + if (in_sparse) { + BOOST_TEST(*in_sparse == *in_packed); + BOOST_TEST(coeff == static_cast(*in_sparse)); + ++drained; + } + else { + ++retained; + } + } + BOOST_TEST(drained == stored.size()); + BOOST_TEST(retained == absent); +} + +// evolved_operator_terms iterates the index, so both stores must offer the same signature and -- since the +// shared RowHashTable fixes slot order for a given insertion sequence -- visit the same row indices in the +// same order. The monomials come out equal even though the two stores hash differently, because that order +// is the *table's*, and both tables saw the same sequence of (index, hash) pairs from their own hash. +BOOST_AUTO_TEST_CASE(store_interchange_for_each_visits_every_row) { + std::mt19937_64 rng(4321U); + constexpr size_t kNumBits = 64; + OperatorIndex packed(kNumBits); + SparseRowStore sparse(kNumBits, 4); + std::vector stored; + for (size_t t = 0; t < 200; ++t) { + const auto mono = test_utils::random_monomial(rng, 32, 8); + if (sparse.find(mono).has_value()) { + continue; // one row per distinct monomial, so an index maps to one term + } + const size_t i = sparse.size(); + packed.push_back(mono); + sparse.push_back(mono); + packed.emplace(mono, i); + sparse.emplace(mono, i); + stored.push_back(mono); + } + + std::map from_packed; + std::map from_sparse; + packed.for_each([&](const auto &mono, size_t idx) { from_packed.emplace(idx, mono); }); + sparse.for_each([&](const auto &mono, size_t idx) { from_sparse.emplace(idx, mono); }); + + BOOST_REQUIRE(from_packed.size() == stored.size()); + BOOST_REQUIRE(from_sparse.size() == stored.size()); + for (size_t i = 0; i < stored.size(); ++i) { + BOOST_TEST((from_packed.at(i) == stored[i])); + BOOST_TEST((from_sparse.at(i) == stored[i])); + } +} diff --git a/cpp/tests/term_product_tests.cpp b/cpp/tests/term_product_tests.cpp new file mode 100644 index 00000000..80865b58 --- /dev/null +++ b/cpp/tests/term_product_tests.cpp @@ -0,0 +1,636 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The scan's per-term kernel, sparse against dense, through the interface the scan actually calls: +// product() -> passes() -> owner() -> push(). codes_algebra_tests.cpp and codes_product_tests.cpp already +// pin the pieces against their dense counterparts; what this adds is the *composition* -- that +// SparseTermProducts routes a term to the right one of them, and that its three fallbacks (a spilled store +// row, a product past the scratch capacity, a cutoff with no codes form) produce the dense answer rather +// than a wrong one. +// +// This is the gate on Stage 6's store swap: with MPOperator::Store still OperatorIndex the sparse kernel +// is unreachable from the library, so nothing else would instantiate it. +// +// The width-bound dense kernel, DenseTermProductsW, is the third answer to the same questions and +// is compared here for the same reason: it restates four of the five off raw words with the storage word +// count fixed at compile time, so nothing but a differential test can tell a restatement that agrees +// from one that merely runs. Its cases live at the bottom of the file; WordKernel's own primitives +// are pinned separately in word_kernel_tests.cpp. + +#include + +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/detail/evolution/layer_build/TermProduct.h" +#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/operator/SparseRowStore.h" + +#include "InlineWidths.h" +#include "RandomMonomial.h" +#include "TestData.h" +#include "TestUtilities.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +// Which branches a run exercised. Every case asserts on these: a fallback that silently swallowed every +// term would otherwise pass by comparing the dense kernel against itself. +struct Seen { + size_t sparse_terms = 0; + size_t fallback_terms = 0; + size_t cutoff_passed = 0; + size_t cutoff_failed = 0; + size_t sparse_cutoff_failed = 0; // the codes cutoff said no, not the dense one + size_t row_records = 0; // survivors pushed as a support-form record + size_t escaped_records = 0; // survivors pushed as an escape plus a tail entry +}; + +// A fully paired term of `modes` modes. These are the rows that make support unbounded -- xor_sum == 0 +// escapes the cutoff -- so they are what drives a store row to spill and a product to overflow. +auto paired_term(size_t num_modes, size_t modes) -> Bitset { + Bitset mono(2 * num_modes); + for (size_t m = 0; m < modes; ++m) { + mono.set(2 * m); + mono.set((2 * m) + 1); + } + return mono; +} + +// One term against one generator through both kernels, comparing every answer the scan reads. The two +// stores must hold the same monomial at index i, which the caller guarantees by inserting in lockstep. +template +auto check_term(DenseTermProducts &dense, + SparseTermProducts &sparse_kernel, + const OperatorIndex &packed, + const SparseRowStore &sparse_store, + size_t i, + size_t gen_pop, + Seen &seen) -> void { + const size_t mono_pop = packed.popcount(i); + BOOST_REQUIRE(sparse_store.popcount(i) == mono_pop); + + const auto reference = dense.product(packed, i); + const auto candidate = sparse_kernel.product(sparse_store, i); + BOOST_TEST(candidate.overlap == reference.overlap); + BOOST_TEST(candidate.phase_factor == reference.phase_factor); + + const size_t new_pop = mono_pop + gen_pop - (2 * reference.overlap); + const bool reference_passes = dense.passes(new_pop); + const bool candidate_passes = sparse_kernel.passes(new_pop); + BOOST_TEST(candidate_passes == reference_passes); + + // Both remaining answers are read only for a surviving term, so ask them only there. owner() is still + // the dense hash on both sides, because owner routing is monomial_hash everywhere. + if (reference_passes) { + for (const size_t rank_count : {2U, 3U, 8U}) { + BOOST_TEST(sparse_kernel.owner(rank_count) == dense.owner(rank_count)); + } + + // The two records no longer agree byte for byte -- that is what the support form is for -- so what + // is compared is what a resolver reads back out of them: the same key, and the same phase. + const size_t num_bits = packed.num_bits(); + const size_t capacity = sparse_kernel.record_capacity(); + BOOST_TEST(sparse_kernel.record_words() == query_words(sparse_payload_words(capacity))); + + VecZ unused_escapes; + VecZ reference_record = query_buffer(); + dense.push(QueryOut{reference_record, unused_escapes}, -1); + BOOST_TEST(unused_escapes.empty()); // a dense record has nowhere to escape to and never needs one + + VecZ candidate_record = query_buffer(); + VecZ candidate_escapes; + sparse_kernel.push(QueryOut{candidate_record, candidate_escapes}, -1); + const bool escaped = !candidate_escapes.empty(); + BOOST_TEST(escaped == sparse_kernel.fell_back()); + append_escape_tail(candidate_record, candidate_escapes); + seen.row_records += escaped ? 0 : 1; + seen.escaped_records += escaped ? 1 : 0; + + DenseQueryKeys dense_keys; + dense_keys.configure(num_bits, 0); + dense_keys.ensure(1); + dense_keys.begin_batch(); + BOOST_TEST(dense_keys.read_record(reference_record, 0, dense.record_words(), 0) == -1); + + SparseQueryKeys sparse_keys; + sparse_keys.configure(num_bits, capacity); + sparse_keys.ensure(1); + sparse_keys.begin_batch(); + BOOST_TEST(sparse_keys.read_record(candidate_record, 0, sparse_kernel.record_words(), 0) == -1); + BOOST_TEST(sparse_keys[0].is_spilled() == escaped); + BOOST_TEST((key_monomial(sparse_keys[0], num_bits) == dense_keys[0])); + // And the store agrees with the key either way, which is what the resolve's find_batch rests on. + BOOST_TEST(sparse_row_hash(sparse_keys[0]) == sparse_row_hash(dense_keys[0])); + } + + if (sparse_kernel.fell_back()) { + ++seen.fallback_terms; + } + else { + ++seen.sparse_terms; + seen.sparse_cutoff_failed += reference_passes ? 0 : 1; + } + seen.cutoff_passed += reference_passes ? 1 : 0; + seen.cutoff_failed += reference_passes ? 0 : 1; +} + +// Every term of `terms` against every generator of `gens`, over one algebra and one cutoff. +template +auto sweep(const std::vector &terms, + const std::vector &gens, + const CutoffFn &cutoff_fn, + size_t sparse_slots, + Seen &seen) -> void { + BOOST_REQUIRE(!terms.empty()); + const size_t num_bits = terms.front().size(); + OperatorIndex packed(num_bits); + SparseRowStore sparse_store(num_bits, sparse_slots); + for (const auto &mono : terms) { + packed.push_back(mono); + sparse_store.push_back(mono); + } + + const CutoffEvaluator cutoff_eval{cutoff_fn}; + for (const auto &gen : gens) { + DenseTermProducts dense(gen, cutoff_eval); + SparseTermProducts sparse_kernel(gen, cutoff_eval); + const size_t gen_pop = gen.count(); + for (size_t i = 0; i < terms.size(); ++i) { + check_term(dense, sparse_kernel, packed, sparse_store, i, gen_pop, seen); + } + } +} + +} // namespace + +// Randomized terms and generators at three widths, both algebras, both cutoff kinds. Slots are generous +// enough that no row spills, so what is under test is the sparse path itself. +BOOST_AUTO_TEST_CASE(term_product_sparse_kernel_matches_dense_on_randomized_rows) { + std::mt19937_64 rng(20260812U); + Seen seen; + for (const size_t num_modes : {32U, 64U, 300U}) { + std::vector terms; + for (size_t t = 0; t < 150; ++t) { + terms.push_back(test_utils::random_monomial(rng, num_modes, 6)); + } + std::vector gens; + for (size_t t = 0; t < 8; ++t) { + gens.push_back(test_utils::random_monomial(rng, num_modes, 4)); + } + for (const unsigned int cutoff : {4U, 8U}) { + const auto length = cutoff_function(CutoffType::Length, cutoff, num_modes, 2 * num_modes); + const auto support = cutoff_function(CutoffType::Support, cutoff, num_modes, 2 * num_modes); + sweep(terms, gens, length, SparseRowStore::kMaxSlots, seen); + sweep(terms, gens, support, SparseRowStore::kMaxSlots, seen); + sweep(terms, gens, support, SparseRowStore::kMaxSlots, seen); + sweep(terms, gens, length, SparseRowStore::kMaxSlots, seen); + } + } + // Most terms take the sparse path, and some do not: the terms are drawn up to 6 modes wide against a + // cutoff of 4 or 8, and a term above the bound overflows a capacity that is sized from that bound. + // Which is the real shape of the thing -- a stored row exceeds the bound whenever it is fully paired. + BOOST_TEST(seen.sparse_terms > 20000U); + BOOST_TEST(seen.fallback_terms > 0U); + BOOST_TEST(seen.cutoff_passed > 0U); + // The point of the codes cutoff is rejecting a product without materializing it, so a run where + // nothing was rejected would not have tested it. + BOOST_TEST(seen.sparse_cutoff_failed > 0U); + // Every survivor here was pushed as a row: an overflowing product is wider than the cutoff's bound, so + // unless it is fully paired the cutoff rejects it before a record is ever asked for. A *surviving* + // escape needs a fully paired product, which has a case of its own below. + BOOST_TEST(seen.row_records > 0U); + BOOST_TEST(seen.escaped_records == 0U); +} + +// Real terms and generators: the fixtures' Hamiltonian keys against their Majorana generator list, which +// is where the products and the ordering signs are the ones the engine actually computes. +BOOST_AUTO_TEST_CASE(term_product_sparse_kernel_matches_dense_on_fixture_generators) { + Seen seen; + for (const std::string name : {"random_exact.msgpack", "lih_fermionic_spin_exact.msgpack"}) { + const auto data = test_utils::load_case_data(name); + const size_t num_bits = 2 * data.num_modes; + const size_t max_index = 2 * data.num_modes; + + std::vector terms; + for (const auto &[inds, coeff] : data.hamiltonian) { + terms.push_back(indices_to_bitset_checked(inds, max_index, num_bits)); + } + std::vector gens; + for (const auto &inds : data.majoranas) { + gens.push_back(indices_to_bitset_checked(inds, max_index, num_bits)); + } + BOOST_REQUIRE(!terms.empty()); + BOOST_REQUIRE(!gens.empty()); + // A stride, not the whole cross product: the fixtures are large and the pairs are homogeneous. + std::vector sampled_gens; + const size_t stride = gens.size() > 12 ? (gens.size() / 12) + 1 : 1; + for (size_t g = 0; g < gens.size(); g += stride) { + sampled_gens.push_back(gens[g]); + } + + for (const unsigned int cutoff : {4U, 6U}) { + sweep(terms, + sampled_gens, + cutoff_function(CutoffType::Length, cutoff, data.num_modes, num_bits), + SparseRowStore::kMaxSlots, + seen); + sweep(terms, + sampled_gens, + cutoff_function(CutoffType::Support, cutoff, data.num_modes, num_bits), + SparseRowStore::kMaxSlots, + seen); + } + } + // No claim that nothing fell back: a wide fixture term against a wide generator legitimately overflows + // a capacity of cutoff + |G|, and that case is covered on its own below. + BOOST_TEST(seen.sparse_terms > 100U); + BOOST_TEST(seen.cutoff_passed > 0U); + BOOST_TEST(seen.sparse_cutoff_failed > 0U); +} + +// A store row with no view: the kernel must take the dense path for that term alone and keep taking the +// sparse one for the rest. Rows are a mix, so both happen in the same gate. +BOOST_AUTO_TEST_CASE(term_product_falls_back_on_a_spilled_row) { + std::mt19937_64 rng(99U); + constexpr size_t kNumModes = 64; + std::vector terms; + for (size_t t = 0; t < 60; ++t) { + terms.push_back(test_utils::random_monomial(rng, kNumModes, 3)); + terms.push_back(paired_term(kNumModes, 9)); // 9 modes > the 4 slots below + } + std::vector gens{test_utils::random_monomial(rng, kNumModes, 2), + test_utils::random_monomial(rng, kNumModes, 4)}; + + Seen seen; + sweep(terms, gens, cutoff_function(CutoffType::Length, 6, kNumModes, 2 * kNumModes), 4, seen); + sweep(terms, gens, cutoff_function(CutoffType::Support, 6, kNumModes, 2 * kNumModes), 4, seen); + BOOST_TEST(seen.fallback_terms > 0U); + BOOST_TEST(seen.sparse_terms > 0U); +} + +// A product past the scratch capacity, with the store row itself perfectly representable: capacity is +// max_mode_bound() + the generator's locality, so a fully paired row well above the cutoff overflows it. +// sparse_toggle reports that rather than truncating, and the kernel must then answer densely. +BOOST_AUTO_TEST_CASE(term_product_falls_back_on_a_capacity_overflow) { + std::mt19937_64 rng(1010U); + constexpr size_t kNumModes = 64; + std::vector terms; + for (size_t t = 0; t < 40; ++t) { + terms.push_back(test_utils::random_monomial(rng, kNumModes, 3)); + // 12 modes: inside the 20-slot store rows below, past a capacity of 4 + |G|. + terms.push_back(paired_term(kNumModes, 12)); + } + std::vector gens{test_utils::random_monomial(rng, kNumModes, 2)}; + + Seen seen; + sweep(terms, gens, cutoff_function(CutoffType::Length, 4, kNumModes, 2 * kNumModes), 20, seen); + sweep(terms, gens, cutoff_function(CutoffType::Support, 4, kNumModes, 2 * kNumModes), 20, seen); + BOOST_TEST(seen.fallback_terms > 0U); + BOOST_TEST(seen.sparse_terms > 0U); + BOOST_TEST(seen.cutoff_passed > 0U); +} + +// The escape's own case, and the reason no capacity can settle the question: a fully paired product is kept +// unconditionally (xor_sum == 0), so a product both wider than the record and kept by the cutoff exists by +// construction. The generator is fully paired on modes the term already holds, so the product is the term +// minus those modes -- still fully paired, still far wider than a capacity of bound + |G|. +BOOST_AUTO_TEST_CASE(term_product_escapes_a_surviving_product_no_record_can_hold) { + constexpr size_t kNumModes = 64; + const std::vector terms{paired_term(kNumModes, 12)}; + Bitset gen(2 * kNumModes); + for (const size_t m : {3U, 4U}) { + gen.set(2 * m); + gen.set((2 * m) + 1); + } + const std::vector gens{gen}; + + Seen seen; + sweep(terms, gens, cutoff_function(CutoffType::Length, 4, kNumModes, 2 * kNumModes), 20, seen); + sweep(terms, gens, cutoff_function(CutoffType::Support, 4, kNumModes, 2 * kNumModes), 20, seen); + BOOST_TEST(seen.fallback_terms == 2U); + BOOST_TEST(seen.sparse_terms == 0U); + BOOST_TEST(seen.cutoff_passed == 2U); + BOOST_TEST(seen.escaped_records == 2U); + BOOST_TEST(seen.row_records == 0U); +} + +// A logical width narrower than the storage width, which is what storage_modes_for() produces for any +// mode count that is not a whole 32-mode block. The inactive modes are the low ones, so the codes cutoff +// drops them as a prefix of the ascending slots where the dense one shifts the whole register -- and the +// terms here deliberately occupy modes on both sides of that boundary. +BOOST_AUTO_TEST_CASE(term_product_sparse_kernel_matches_dense_in_a_narrow_active_window) { + std::mt19937_64 rng(7070U); + constexpr size_t kStorageModes = 64; + constexpr size_t kLogicalModes = 50; // an inactive prefix of 14 modes + std::vector terms; + for (size_t t = 0; t < 120; ++t) { + terms.push_back(test_utils::random_monomial(rng, kStorageModes, 6)); + } + std::vector gens{test_utils::random_monomial(rng, kStorageModes, 3), + test_utils::random_monomial(rng, kStorageModes, 2)}; + + Seen seen; + sweep(terms, + gens, + cutoff_function(CutoffType::Length, 4, kLogicalModes, 2 * kStorageModes), + SparseRowStore::kMaxSlots, + seen); + sweep(terms, + gens, + cutoff_function(CutoffType::Support, 4, kLogicalModes, 2 * kStorageModes), + SparseRowStore::kMaxSlots, + seen); + BOOST_TEST(seen.sparse_terms > 0U); + BOOST_TEST(seen.cutoff_passed > 0U); + BOOST_TEST(seen.sparse_cutoff_failed > 0U); +} + +// A cutoff that is neither concrete functor -- the basis-change form, which is a lambda on purpose -- has +// no codes counterpart, so the whole gate falls back. Asserted because the alternative to falling back is +// answering with the wrong cutoff, which no other test would catch. +BOOST_AUTO_TEST_CASE(term_product_falls_back_when_the_cutoff_has_no_codes_form) { + std::mt19937_64 rng(2020U); + constexpr size_t kNumModes = 32; + std::vector terms; + for (size_t t = 0; t < 80; ++t) { + terms.push_back(test_utils::random_monomial(rng, kNumModes, 5)); + } + std::vector gens{test_utils::random_monomial(rng, kNumModes, 3)}; + + // The identity basis, so the predicate is an ordinary length cutoff wrapped in a lambda: the answers + // must still match, and every term must have gone the dense way to produce them. + MonomialList basis; + for (size_t b = 0; b < 2 * kNumModes; ++b) { + Bitset single(2 * kNumModes); + single.set(b); + basis.push_back(single); + } + const auto wrapped = cutoff_function_basis_change(CutoffType::Length, 4, basis, kNumModes); + BOOST_REQUIRE(CutoffEvaluator{wrapped}.length_cutoff() == nullptr); + + Seen seen; + sweep(terms, gens, wrapped, SparseRowStore::kMaxSlots, seen); + BOOST_TEST(seen.sparse_terms == 0U); + BOOST_TEST(seen.fallback_terms == terms.size()); +} + +// A generator wider than one codes word cannot be encoded as a row at all, so the gate falls back +// wholesale. Exotic, but the branch exists and an unencodable generator must not be silently truncated +// into a *different* generator. +BOOST_AUTO_TEST_CASE(term_product_falls_back_on_a_generator_past_one_codes_word) { + std::mt19937_64 rng(3030U); + constexpr size_t kNumModes = 128; + std::vector terms; + for (size_t t = 0; t < 40; ++t) { + terms.push_back(test_utils::random_monomial(rng, kNumModes, 4)); + } + std::vector gens{paired_term(kNumModes, SparseRowStore::kMaxSlots + 5)}; + + Seen seen; + sweep(terms, gens, cutoff_function(CutoffType::Length, 6, kNumModes, 2 * kNumModes), 40, seen); + BOOST_TEST(seen.sparse_terms == 0U); + BOOST_TEST(seen.fallback_terms == terms.size()); +} + +// --------------------------------------------------------------------------------------------------- +// The width-bound dense kernel against the width-agnostic one. +// --------------------------------------------------------------------------------------------------- + +namespace { + +// Which of the two cutoff paths a narrow-kernel run took, per term. Both have to occur across the +// cases below or one of them ships compared against nothing. +struct NarrowSeen { + size_t terms = 0; + size_t word_cutoff_terms = 0; // passes() answered off the words + size_t evaluator_terms = 0; // passes() went through the CutoffEvaluator + size_t passed = 0; + size_t failed = 0; + size_t paired_rescues = 0; // kept although longer than the cutoff, i.e. the fully-paired clause +}; + +// One term through both dense kernels, comparing every answer the scan reads plus the product itself. +// Unlike the sparse comparison the records must agree *byte for byte*: both push a dense monomial, so +// any difference here is a difference in the product. +template +auto check_narrow_term(DenseTermProducts &reference_kernel, + DenseTermProductsW &candidate_kernel, + const OperatorIndex &packed, + size_t i, + size_t gen_pop, + unsigned int cutoff, + NarrowSeen &seen) -> void { + const auto reference = reference_kernel.product(packed, i); + const auto candidate = candidate_kernel.product(packed, i); + BOOST_TEST(candidate.overlap == reference.overlap); + BOOST_TEST(candidate.phase_factor == reference.phase_factor); + BOOST_TEST((candidate_kernel.product_row() == reference_kernel.product_row())); + BOOST_TEST(candidate_kernel.record_words() == reference_kernel.record_words()); + + const size_t new_pop = packed.popcount(i) + gen_pop - (2 * reference.overlap); + const bool reference_passes = reference_kernel.passes(new_pop); + BOOST_TEST(candidate_kernel.passes(new_pop) == reference_passes); + + if (reference_passes) { + for (const size_t rank_count : {2U, 3U, 8U}) { + BOOST_TEST(candidate_kernel.owner(rank_count) == reference_kernel.owner(rank_count)); + } + + VecZ unused_escapes; + VecZ reference_record = query_buffer(); + reference_kernel.push(QueryOut{reference_record, unused_escapes}, -1); + VecZ candidate_record = query_buffer(); + candidate_kernel.push(QueryOut{candidate_record, unused_escapes}, -1); + BOOST_TEST(candidate_record == reference_record, boost::test_tools::per_element()); + BOOST_TEST(unused_escapes.empty()); // neither dense kernel has anywhere to escape to + seen.paired_rescues += new_pop > cutoff ? 1 : 0; + } + + ++seen.terms; + seen.word_cutoff_terms += candidate_kernel.uses_word_cutoff() ? 1 : 0; + seen.evaluator_terms += candidate_kernel.uses_word_cutoff() ? 0 : 1; + seen.passed += reference_passes ? 1 : 0; + seen.failed += reference_passes ? 0 : 1; +} + +// Every term of `terms` against every generator of `gens`, at the one width W the terms are built for. +template +auto sweep_narrow(const std::vector &terms, + const std::vector &gens, + const CutoffFn &cutoff_fn, + unsigned int cutoff, + NarrowSeen &seen) -> void { + BOOST_REQUIRE(!terms.empty()); + const size_t num_bits = terms.front().size(); + BOOST_REQUIRE(num_bits == W * Bitset::word_width); + OperatorIndex packed(num_bits); + for (const auto &mono : terms) { + packed.push_back(mono); + } + + const CutoffEvaluator cutoff_eval{cutoff_fn}; + for (const auto &gen : gens) { + DenseTermProducts reference_kernel(gen, cutoff_eval); + DenseTermProductsW candidate_kernel(gen, cutoff_eval); + const size_t gen_pop = gen.count(); + for (size_t i = 0; i < terms.size(); ++i) { + check_narrow_term(reference_kernel, candidate_kernel, packed, i, gen_pop, cutoff, seen); + } + } +} + +// The W the dispatch bound, read back out of the arm it selected -- the only thing about the seam that +// is observable, since every arm answers identically. +template +auto bound_kernel_width(size_t num_words) -> size_t { + return with_kernel_width(num_words, [](std::integral_constant) -> size_t { return W; }); +} + +// Terms wide enough to be rejected, narrow enough to survive, and some fully paired well above the +// cutoff -- which is the clause the word cutoff answers with its own fold rather than a popcount. +auto narrow_kernel_terms(std::mt19937_64 &rng, size_t num_modes) -> std::vector { + std::vector terms; + for (size_t t = 0; t < 60; ++t) { + terms.push_back(test_utils::random_monomial(rng, num_modes, 7)); + } + for (const size_t paired : {1U, 3U, 8U}) { + terms.push_back(paired_term(num_modes, paired)); + } + return terms; +} + +} // namespace + +// A length cutoff over the whole register: the one shape the word cutoff answers, so this is the case +// where the specialized path is actually under test. Asserted through uses_word_cutoff(), because a +// change that quietly stopped engaging it would leave every other assertion here still passing. +BOOST_AUTO_TEST_CASE(narrow_kernel_matches_dense_under_a_whole_register_length_cutoff) { + std::mt19937_64 rng(20260821U); + NarrowSeen seen; + test_utils::for_each_inline_width([&](std::integral_constant) { + const size_t num_modes = (W * Bitset::word_width) / 2; + const auto terms = narrow_kernel_terms(rng, num_modes); + const std::vector gens{test_utils::random_monomial(rng, num_modes, 2), + test_utils::random_monomial(rng, num_modes, 4), + paired_term(num_modes, 2)}; + for (const unsigned int cutoff : {4U, 8U}) { + const auto length = cutoff_function(CutoffType::Length, cutoff, num_modes, 2 * num_modes); + sweep_narrow(terms, gens, length, cutoff, seen); + // The Pauli algebra with a length cutoff: the kind and the basis are independent here even + // though the shipping models pair them, and the kernel's cutoff arm must not depend on the + // algebra it was instantiated with. + sweep_narrow(terms, gens, length, cutoff, seen); + } + }); + BOOST_TEST(seen.terms > 0U); + BOOST_TEST(seen.evaluator_terms == 0U); // every term took the word cutoff + BOOST_TEST(seen.word_cutoff_terms == seen.terms); + BOOST_TEST(seen.passed > 0U); + BOOST_TEST(seen.failed > 0U); + // Products kept although longer than the cutoff, i.e. the fully-paired fold answering yes. Without + // these the word cutoff would be tested as nothing but `new_pop <= cutoff`. + BOOST_TEST(seen.paired_rescues > 0U); +} + +// The two shapes the word cutoff declines: a support cutoff, and a length cutoff whose active window is +// narrower than the storage register -- which is what storage_modes_for() produces for any mode count +// that is not a whole 32-mode block. Both must fall through to the evaluator and still agree; the +// failure this guards against is answering a *different* cutoff, which changes which terms survive. +BOOST_AUTO_TEST_CASE(narrow_kernel_defers_to_the_evaluator_off_the_whole_register_length_cutoff) { + std::mt19937_64 rng(606U); + NarrowSeen seen; + test_utils::for_each_inline_width([&](std::integral_constant) { + const size_t num_modes = (W * Bitset::word_width) / 2; + const size_t logical_modes = num_modes - 5; // an inactive prefix of 5 modes + const auto terms = narrow_kernel_terms(rng, num_modes); + const std::vector gens{test_utils::random_monomial(rng, num_modes, 3), + test_utils::random_monomial(rng, num_modes, 2)}; + for (const unsigned int cutoff : {4U, 6U}) { + const auto support = cutoff_function(CutoffType::Support, cutoff, num_modes, 2 * num_modes); + const auto narrow = cutoff_function(CutoffType::Length, cutoff, logical_modes, 2 * num_modes); + sweep_narrow(terms, gens, support, cutoff, seen); + sweep_narrow(terms, gens, support, cutoff, seen); + sweep_narrow(terms, gens, narrow, cutoff, seen); + } + }); + BOOST_TEST(seen.terms > 0U); + BOOST_TEST(seen.word_cutoff_terms == 0U); + BOOST_TEST(seen.evaluator_terms == seen.terms); + BOOST_TEST(seen.passed > 0U); + BOOST_TEST(seen.failed > 0U); +} + +// A cutoff that is neither concrete functor, so CutoffEvaluator recovered nothing: the kernel keeps its +// word product and defers the whole predicate. Same case as the sparse kernel's, for the same reason. +BOOST_AUTO_TEST_CASE(narrow_kernel_defers_when_the_cutoff_has_no_concrete_functor) { + std::mt19937_64 rng(707U); + constexpr size_t kWords = 2; + constexpr size_t kNumModes = (kWords * Bitset::word_width) / 2; + const auto terms = narrow_kernel_terms(rng, kNumModes); + const std::vector gens{test_utils::random_monomial(rng, kNumModes, 3)}; + + MonomialList basis; + for (size_t b = 0; b < 2 * kNumModes; ++b) { + Bitset single(2 * kNumModes); + single.set(b); + basis.push_back(single); + } + const auto wrapped = cutoff_function_basis_change(CutoffType::Length, 4, basis, kNumModes); + BOOST_REQUIRE(CutoffEvaluator{wrapped}.length_cutoff() == nullptr); + + NarrowSeen seen; + sweep_narrow(terms, gens, wrapped, 4, seen); + BOOST_TEST(seen.terms == terms.size()); + BOOST_TEST(seen.word_cutoff_terms == 0U); + BOOST_TEST(seen.passed > 0U); + BOOST_TEST(seen.failed > 0U); +} + +// The seam itself: which W the scan binds for a given storage width and store. Pinned because it is the +// one thing above that no differential test can see -- every arm computes the same answers, so a +// dispatch that always chose 0 would leave the whole suite green and only the benchmark different. +BOOST_AUTO_TEST_CASE(with_kernel_width_binds_the_capped_storage_word_count) { + for (size_t words = 1; words <= Bitset::kInlineWords; ++words) { + BOOST_TEST(bound_kernel_width(words) == (words <= kNarrowKernelWords ? words : 0)); + // The sparse store is never specialized: its per-term work is O(slots), not O(storage words). + BOOST_TEST(bound_kernel_width(words) == 0U); + } + // Above the inline regime the words are on the heap, so there is no width to bind. + BOOST_TEST(bound_kernel_width(Bitset::kInlineWords + 1) == 0U); +} + +// with_kernel_width binds W from the store's row word count, independently of whatever gen a caller +// hands to the constructor it dispatches into -- so a mismatch is a real (if never-yet-observed) call +// site bug, not an internal-only invariant, and unlike the per-term word ops in Bitset.h (deliberately +// assert-only) this must not compile away under NDEBUG. +BOOST_AUTO_TEST_CASE(narrow_kernel_constructor_rejects_a_generator_of_the_wrong_width) { + constexpr size_t kWords = 2; + constexpr size_t kNumModes = (kWords * Bitset::word_width) / 2; + const Bitset narrower_gen(2 * (kNumModes - Bitset::word_width / 2)); + const Bitset wider_gen(2 * (kNumModes + Bitset::word_width / 2)); + + const CutoffEvaluator cutoff_eval{cutoff_function(CutoffType::Length, 4, kNumModes, 2 * kNumModes)}; + BOOST_CHECK_THROW((DenseTermProductsW(narrower_gen, cutoff_eval)), KernelWidthMismatch); + BOOST_CHECK_THROW((DenseTermProductsW(wider_gen, cutoff_eval)), KernelWidthMismatch); +} diff --git a/cpp/tests/unit_tests.cpp b/cpp/tests/unit_tests.cpp index 06ceb530..af248b8d 100644 --- a/cpp/tests/unit_tests.cpp +++ b/cpp/tests/unit_tests.cpp @@ -14,6 +14,7 @@ #define BOOST_TEST_MODULE "MonoProp Unit Tests" +#include #include #include @@ -27,6 +28,16 @@ static auto init() -> bool { auto main(int argc, char* argv[]) -> int { // overwrite=0, so an explicit environment override still wins; why it is off: tests/cpp/README.md. setenv("monoprop_PARTITIONS", "off", 0); + // Must precede mpi::init, and only the harness may do it -- changing a signal disposition is a + // process-wide act, so the library cannot. + // + // In an MPI build every one of these per-case processes runs a *singleton* MPI_Init: no launcher, so + // PMIx opens a socket to a daemon that is not there. Under enough concurrency a write to that dead + // socket lands, and SIGPIPE's default action kills the process mid-init -- which surfaced as + // load-dependent SIGPIPE exceptions in random cases under `ctest -j`, each passing when re-run alone. + // Ignoring it makes the write return EPIPE for the MPI layer to handle. Python callers never saw this + // because CPython already ignores SIGPIPE at startup. + std::signal(SIGPIPE, SIG_IGN); monoprop::mpi::init(&argc, &argv); int result = boost::unit_test::unit_test_main(&init, argc, argv); monoprop::mpi::finalize(); diff --git a/cpp/tests/update_initial_operator.cpp b/cpp/tests/update_initial_operator.cpp index 76dc74ac..c9c2be87 100644 --- a/cpp/tests/update_initial_operator.cpp +++ b/cpp/tests/update_initial_operator.cpp @@ -19,6 +19,7 @@ #include #include +#include "TestPropagator.h" #include "monoprop/MonomialPropagator.h" #include "monoprop/detail/mpi/MPICompat.h" @@ -32,15 +33,16 @@ BOOST_AUTO_TEST_CASE(update_initial_operator_updates_core_expval) { initial_ham[VecZ{}] = std::complex{1.0, 0.0}; VecZ initial_state{0, 1}; - MonomialPropagator simulator(initial_ham, - 2 * n_modes, - initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Support, - std::nullopt); + auto simulator = test_utils::make_propagator(n_modes, + initial_ham, + 2 * n_modes, + initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Support, + std::nullopt); const VecD empty_params; auto expval_fn = simulator.expectation_value_functional(std::nullopt); @@ -64,15 +66,16 @@ BOOST_AUTO_TEST_CASE(update_initial_operator_invalidates_gradient_functional) { initial_ham[VecZ{}] = std::complex{1.0, 0.0}; VecZ initial_state{0, 1}; - MonomialPropagator simulator(initial_ham, - 2 * n_modes, - initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Support, - std::nullopt); + MonomialPropagator simulator(initial_ham, + 6, + initial_state, + 2 * n_modes, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Support, + std::nullopt); const VecD empty_params; auto grad_fn = simulator.expectation_value_and_gradient_functional(std::nullopt); @@ -93,15 +96,16 @@ BOOST_AUTO_TEST_CASE(update_initial_operator_throws_for_unknown_term_in_heisenbe initial_ham[VecZ{0, 1}] = std::complex{0, 1.0}; VecZ initial_state{0, 1}; - MonomialPropagator simulator(initial_ham, - 2 * n_modes, - initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Support, - std::nullopt); + auto simulator = test_utils::make_propagator(n_modes, + initial_ham, + 2 * n_modes, + initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Support, + std::nullopt); const VecZ invalid_term{2, 3}; OperatorDict missing_term; @@ -118,15 +122,16 @@ BOOST_AUTO_TEST_CASE(update_initial_operator_accepts_new_terms_in_schrodinger) { VecZ initial_state{0, 1}; const unsigned int cutoff = static_cast(2 * n_modes); - MonomialPropagator simulator(initial_ham, - cutoff, - initial_state, - cutoff, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Support, - std::nullopt); + auto simulator = test_utils::make_propagator(n_modes, + initial_ham, + cutoff, + initial_state, + cutoff, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + CutoffType::Support, + std::nullopt); OperatorDict new_term; new_term[VecZ{2, 3}] = std::complex{0.0, 0.25}; diff --git a/cpp/tests/validation_tests.cpp b/cpp/tests/validation_tests.cpp index 173c3a2d..0f6d87ca 100644 --- a/cpp/tests/validation_tests.cpp +++ b/cpp/tests/validation_tests.cpp @@ -62,8 +62,8 @@ BOOST_AUTO_TEST_CASE(validation_expected_graph_layers) { } BOOST_AUTO_TEST_CASE(validation_only_rotate_len_k) { - BOOST_CHECK_NO_THROW(validate_only_rotate_len_k_(std::nullopt, 8)); - BOOST_CHECK_NO_THROW(validate_only_rotate_len_k_(8u, 8)); - BOOST_CHECK_THROW(validate_only_rotate_len_k_(0u, 8), std::runtime_error); - BOOST_CHECK_THROW(validate_only_rotate_len_k_(9u, 8), std::runtime_error); + BOOST_CHECK_NO_THROW(validate_only_rotate_len_k(std::nullopt, 8)); + BOOST_CHECK_NO_THROW(validate_only_rotate_len_k(8u, 8)); + BOOST_CHECK_THROW(validate_only_rotate_len_k(0u, 8), std::runtime_error); + BOOST_CHECK_THROW(validate_only_rotate_len_k(9u, 8), std::runtime_error); } diff --git a/cpp/tests/wide_system_tests.cpp b/cpp/tests/wide_system_tests.cpp new file mode 100644 index 00000000..79398523 --- /dev/null +++ b/cpp/tests/wide_system_tests.cpp @@ -0,0 +1,145 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The engine at a storage width that runs the support-form row store in a released wheel. +// +// Every checked-in fixture is 28 modes or fewer, so all of them store monomials in a single 32-mode +// block. That is below every sparse-row crossover, which leaves a gap the rest of the suite cannot +// close: monoprop_ROW_STORE=sparse can force the support-form store onto fixture-width rows and pass, +// while the width it was built for -- several words per monomial, an active window short of its +// storage, mode lanes spread over more than one codes word -- stays unexercised. +// +// So this case embeds a fixture instead of adding one: LiH's 12 modes relabelled into a 260-mode system +// (test_utils::ModeEmbedding), which stores at 288 -- nine words, so past Bitset's eight inline ones and +// into the regime where every by-value monomial spills to the heap. The embedding is also the oracle -- a +// monotone mode relabelling is a canonical transformation, so the fixture's exact energy still applies, +// and the truncated run still owes the narrow run's value. + +#include + +#include +#include +#include +#include + +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/EnvConfig.h" + +#include "TestData.h" +#include "TestPropagator.h" +#include "TestUtilities.h" + +using namespace monoprop; + +namespace { + +// The source fixture's own width. The embedding only relabels, so the physics never leaves these 12 +// modes: no evolved term can carry more than 24 Majorana indices or occupy more than 12 modes, which is +// what makes the cutoffs below untruncated -- and the reference energy exact -- at 260 modes. +constexpr size_t kSourceModes = 12; + +constexpr size_t kWideModes = 260; +constexpr size_t kWideStorageModes = 288; + +// The twin of tests/cases.py's WIDE_EMBEDDING, position for position; see there for why these twelve. +// In short: 0 and 259 are the ends of the active window; 3/4, 35/36 and 67/68 straddle the storage-word +// boundaries once the 28-mode window offset is applied; and 227/228 straddles physical mode 256, the +// inline-to-heap boundary. +auto wide_embedding() -> test_utils::ModeEmbedding { + return {.num_modes = kWideModes, .modes = {0, 3, 4, 35, 36, 67, 68, 227, 228, 257, 258, 259}}; +} + +auto wide_case() -> test_utils::CaseData { + return test_utils::embed_case(test_utils::load_case_data("lih_fermionic_spin_exact.msgpack"), wide_embedding()); +} + +// 260 logical modes, which storage_modes_for rounds to the 288 the two cases below assert on. +auto wide_propagator(const test_utils::CaseData &data, unsigned int cutoff, CutoffType cutoff_type) + -> MonomialPropagator { + return test_utils::make_propagator(kWideModes, + data.hamiltonian, + cutoff, + data.initial_state, + /*schrodinger_cutoff=*/std::nullopt, + MPI_COMM_SELF, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::nullopt, + cutoff_type, + /*basis_change=*/std::nullopt); +} + +// (cutoff_type, cutoff) pairs that truncate nothing for this problem, and the pair that does. +constexpr std::array, 2> kUntruncated{ + {{CutoffType::Length, 2 * kSourceModes}, {CutoffType::Support, kSourceModes}}}; + +} // namespace + +// The premise the two cases below rest on. 288 storage modes is at or above the crossover a released +// wheel is built with (monoprop_SPARSE_ROW_MIN_MODES is 256 unless -march=native moves it to 768), so on +// a wheel this width selects the support-form store by itself; a dev build with arch flags on gets there +// through monoprop_ROW_STORE=sparse, i.e. the sparse-rows ctest variant. Asserted so that variant cannot +// quietly have run dense rows a second time. Nine words also puts every monomial here on the heap. +BOOST_AUTO_TEST_CASE(wide_case_stores_at_two_hundred_eighty_eight_modes) { + const auto data = wide_case(); + const auto propagator = wide_propagator(data, 2 * kSourceModes, CutoffType::Length); + BOOST_TEST(data.num_modes == kWideModes); + BOOST_TEST(propagator.num_modes() == kWideModes); + BOOST_TEST(propagator.storage_num_modes() == kWideStorageModes); + BOOST_TEST(monoprop::detail::storage_modes_for(kWideModes) == kWideStorageModes); + if (config::get().row_store == config::RowStore::Sparse) { + BOOST_TEST(propagator.rows_are_sparse()); + } +} + +// The fixture's exact energy, reached at a width no fixture on disk has. Both cutoff kinds run: each +// has its own evaluator, and each has a codes-word form the sparse rows use instead. +BOOST_AUTO_TEST_CASE(wide_case_reaches_its_exact_expectation_value) { + const auto data = wide_case(); + for (const auto &[cutoff_type, cutoff] : kUntruncated) { + BOOST_TEST_CONTEXT("cutoff_type=" << static_cast(cutoff_type) << " cutoff=" << cutoff) { + auto propagator = wide_propagator(data, cutoff, cutoff_type); + const double expval = test_utils::evaluate_expval(propagator, data, /*pare=*/false); + test_utils::check_expval_close("wide embedding", expval, data.actual_expval); + } + } +} + +// With truncation on, which terms survive must not depend on the storage width: both cutoffs count +// something the relabelling preserves -- Majorana indices, or occupied modes. So the wide run owes the +// narrow run's energy, to rather more than the 1e-9 an exact-value check asks for. Compared as a value +// rather than term by term because that comparison is the Python suite's (tests/test_wide_system.py), +// which can hold the two term sets side by side. +BOOST_AUTO_TEST_CASE(truncated_wide_run_matches_the_narrow_run) { + const auto narrow_data = test_utils::load_case_data("lih_fermionic_spin_exact.msgpack"); + const auto wide_data = test_utils::embed_case(narrow_data, wide_embedding()); + constexpr unsigned int kTruncating = 4; + + for (const auto cutoff_type : {CutoffType::Length, CutoffType::Support}) { + BOOST_TEST_CONTEXT("cutoff_type=" << static_cast(cutoff_type)) { + auto narrow = test_utils::make_propagator(kSourceModes, + narrow_data.hamiltonian, + kTruncating, + narrow_data.initial_state, + std::nullopt, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + cutoff_type); + auto wide = wide_propagator(wide_data, kTruncating, cutoff_type); + const double narrow_expval = test_utils::evaluate_expval(narrow, narrow_data, /*pare=*/false); + const double wide_expval = test_utils::evaluate_expval(wide, wide_data, /*pare=*/false); + BOOST_TEST(test_utils::near(wide_expval, narrow_expval, /*atol=*/1e-12, /*rtol=*/1e-12)); + } + } +} diff --git a/cpp/tests/word_kernel_tests.cpp b/cpp/tests/word_kernel_tests.cpp new file mode 100644 index 00000000..a14a3a55 --- /dev/null +++ b/cpp/tests/word_kernel_tests.cpp @@ -0,0 +1,204 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The scan's bound-width word passes against the computations they restate: detail::WordKernel's +// four Bitset methods, and detail::fully_paired_words, whose oracle is cutoff_sums rather than a +// Bitset method (which is why it lives in AlgebraCommon.h and is tested here anyway -- one sweep over +// the inline regime, one set of word patterns). Every one of them is a second copy of an existing +// computation, so the only thing worth testing is that the two copies agree -- at every W in the +// inline regime, on the word patterns that distinguish a per-word fold from a whole-register one. +// +// splitmix carries the strongest obligation and gets the strictest test: that value is monomial_hash, +// which routes MPI ownership, so a divergence would move terms between ranks rather than merely run +// slower. It is asserted equal to SplitmixHash for every W, not merely well-distributed. + +#include + +#include +#include +#include +#include +#include +#include + +#include "monoprop/Bitset.h" +// AlgebraCommon.h before Monomial.h and without Utilities.h: Utilities.h is not self-contained (it is +// reached through TypeAliases.h, which pulls MPOperator.h in ahead of the free functions that header +// calls), so even_bits arrives transitively here the same way term_product_tests.cpp gets it. +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/core/Monomial.h" + +#include "InlineWidths.h" + +using monoprop::Bitset; +using monoprop::detail::WordKernel; +using test_utils::for_each_inline_width; + +namespace { + +// A bitset of exactly W words with the given words written straight in. Going through data() rather +// than set() because these tests are about the words: a pattern like "every odd bit of word 3" is a +// word, not a bit list. +template +auto from_words(const std::array &words) -> Bitset { + Bitset bs(W * Bitset::word_width); + BOOST_REQUIRE(bs.num_words() == W); + for (size_t w = 0; w < W; ++w) { + bs.data()[w] = words[w]; + } + return bs; +} + +// The patterns a per-word fold can get wrong where a whole-register one cannot: all-zero and all-ones +// (parity of an even count of set bits), the two single-mode halves (a fold that dropped a word would +// still see one of them), and the paired/unpaired even-odd patterns fully_paired keys on. Randomized +// words follow in every case; these are the ones worth naming. +template +auto interesting_words(std::mt19937_64 &rng) -> std::vector> { + std::vector> out; + const auto fill = [&](uint64_t v) { + std::array a{}; + a.fill(v); + out.push_back(a); + }; + fill(0); + fill(~uint64_t{0}); + fill(0x5555555555555555ULL); // every even bit: every occupied mode singly occupied + fill(0xAAAAAAAAAAAAAAAAULL); // every odd bit: likewise, the other Majorana + fill(0xFFFFFFFFFFFFFFFFULL); // every mode fully paired + // One word at a time set, so a fold that skipped word w fails on exactly one entry. + for (size_t w = 0; w < W; ++w) { + std::array a{}; + a[w] = 0x0123456789ABCDEFULL; + out.push_back(a); + std::array b{}; + b[w] = uint64_t{1} << (w % Bitset::word_width); + out.push_back(b); + } + for (size_t t = 0; t < 64; ++t) { + std::array a{}; + for (auto &word : a) { + word = rng(); + } + out.push_back(a); + // ...and the same words sparsified, since real monomials are sparse and a dense random word + // never exercises the "one set bit in the whole register" shapes. + std::array sparse{}; + for (size_t k = 0; k < 3; ++k) { + const size_t pos = rng() % (W * Bitset::word_width); + sparse[pos / Bitset::word_width] |= uint64_t{1} << (pos % Bitset::word_width); + } + out.push_back(sparse); + } + return out; +} + +} // namespace + +// The owner-routing value. Equality with SplitmixHash (and hence monomial_hash) at every W, including +// the W == 1 arm both sides special-case. +BOOST_AUTO_TEST_CASE(word_kernel_splitmix_is_the_owner_routing_hash) { + std::mt19937_64 rng(20260821U); + for_each_inline_width([&](std::integral_constant) { + for (const auto &words : interesting_words(rng)) { + const Bitset bs = from_words(words); + const size_t expected = monoprop::SplitmixHash{}(bs); + BOOST_TEST(WordKernel::splitmix(bs.data()) == expected); + // The name that marks the value as pinned, asserted separately from the hash functor so a + // future indirection between them cannot pass this test by tautology. + BOOST_TEST(WordKernel::splitmix(bs.data()) == monoprop::monomial_hash(bs)); + } + }); +} + +// The product and its two counts, in the same destination the scan writes. +BOOST_AUTO_TEST_CASE(word_kernel_fused_xor_into_matches_bitset) { + std::mt19937_64 rng(31337U); + for_each_inline_width([&](std::integral_constant) { + const auto lhs_words = interesting_words(rng); + const auto rhs_words = interesting_words(rng); + for (size_t i = 0; i < lhs_words.size(); ++i) { + const Bitset lhs = from_words(lhs_words[i]); + const Bitset rhs = from_words(rhs_words[i]); + + Bitset reference_out(W * Bitset::word_width); + const auto reference = lhs.fused_xor_into(rhs, reference_out); + + Bitset candidate_out(W * Bitset::word_width); + const auto candidate = WordKernel::fused_xor_into(lhs.data(), rhs.data(), candidate_out.data()); + + BOOST_TEST(candidate.overlap == reference.overlap); + BOOST_TEST(candidate.result_count == reference.result_count); + BOOST_TEST((candidate_out == reference_out)); + } + }); +} + +// The Majorana rotation sign's parity, which is of the whole AND and not of any per-word rounding -- +// so the all-ones patterns above matter: they make the per-word popcounts even and the total even too, +// where a wrong fold would still agree. +BOOST_AUTO_TEST_CASE(word_kernel_parity_and_matches_bitset) { + std::mt19937_64 rng(4242U); + for_each_inline_width([&](std::integral_constant) { + const auto lhs_words = interesting_words(rng); + const auto rhs_words = interesting_words(rng); + for (size_t i = 0; i < lhs_words.size(); ++i) { + const Bitset lhs = from_words(lhs_words[i]); + const Bitset rhs = from_words(rhs_words[i]); + BOOST_TEST(WordKernel::parity_and(lhs.data(), rhs.data()) == lhs.parity_and(rhs)); + } + }); +} + +// The cutoff's fully-paired clause. The oracle is cutoff_sums' xor_sum over the whole register, which +// is the only window the pass is allowed to answer for (a narrower one keeps going through the +// evaluator -- see DenseTermProductsW). It carries the even-bit pattern as a literal, so the mask built +// here is also the check that the literal is what even_bits would have produced. +BOOST_AUTO_TEST_CASE(fully_paired_words_matches_cutoff_sums) { + std::mt19937_64 rng(5150U); + for_each_inline_width([&](std::integral_constant) { + const size_t num_bits = W * Bitset::word_width; + const Bitset mask = monoprop::even_bits(num_bits); + for (size_t w = 0; w < W; ++w) { + BOOST_TEST(mask.word(w) == 0x5555555555555555ULL); + } + size_t paired = 0; + size_t unpaired = 0; + for (const auto &words : interesting_words(rng)) { + const Bitset bs = from_words(words); + const bool expected = monoprop::cutoff_sums(bs, num_bits / 2).xor_sum == 0; + BOOST_TEST(monoprop::detail::fully_paired_words(bs.data()) == expected); + paired += expected ? 1 : 0; + unpaired += expected ? 0 : 1; + } + // Both answers occur, so neither is passing by always returning the same one. + BOOST_TEST(paired > 0U); + BOOST_TEST(unpaired > 0U); + }); +} + +// clear() zeroes exactly W words. The word above is checked because the kernel's whole contract is +// "the caller bound the width": writing one word too many would corrupt an unrelated monomial's +// storage, and no other test reads that word. +BOOST_AUTO_TEST_CASE(word_kernel_clear_zeroes_exactly_its_width) { + for_each_inline_width([&](std::integral_constant) { + std::array buffer{}; + buffer.fill(~uint64_t{0}); + WordKernel::clear(buffer.data()); + for (size_t w = 0; w < W; ++w) { + BOOST_TEST(buffer[w] == 0U); + } + BOOST_TEST(buffer[W] == ~uint64_t{0}); + }); +} diff --git a/docs/content/docs/benchmarks.mdx b/docs/content/docs/benchmarks.mdx index 97bbf9be..55687663 100644 --- a/docs/content/docs/benchmarks.mdx +++ b/docs/content/docs/benchmarks.mdx @@ -278,10 +278,30 @@ The suite contains two benchmark groups: - `bench_models.py` measures two fixed models marked `slow`: a 120-qubit Fermi-Hubbard 29-step Trotter run (`test_model[hubbard]`) and a 127-qubit kicked-Ising run over 20 layers (`test_model[pauli]`). Override any config field with `---`, e.g. - `--pauli-num-layers 30`; `just bench --help` lists them all. + `--pauli-num-layers 30`; `just bench --help` lists them all. `--model-rounds` (1) sets the rounds + per model, each rebuilding the model first. Term counts, the operator accounting and `Baseline + RSS` are round count-independent; `Peak RSS` is not, because from the second round on the model + is rebuilt while pytest-benchmark still holds the previous round's, so the high-water mark covers + two of them. Pass `--model-rounds 5` or more whenever a timing *difference* is the point of the + run — a single sample on these models can sit well off the median, and comparing two one-sample + runs will read noise as signal. Each run writes pytest-benchmark timings to `results/time-