refactor!: ♻️ drop non-type template parameter for number of modes

.devcontainer/devcontainer.json

@@ -16,6 +16,9 @@
     "source=monoprop-docs-next,target=${containerWorkspaceFolder}/docs/.next,type=volume",
     "source=monoprop-docs-node-modules,target=${containerWorkspaceFolder}/docs/node_modules,type=volume"
   ],
+  "runArgs": [
+    "--shm-size=1g"
+  ],
   "remoteUser": "vscode",
   "customizations": {
     "vscode": {

.github/workflows/test.yml

@@ -132,37 +132,37 @@ jobs:
           using namespace monoprop;
 
           auto main() -> int {
-              constexpr size_t kModes = 2;
               OperatorDict ham;
               ham[VecZ{0, 1}] = std::complex<double>{0.0, 1.0};
 
               // Graph-building / Schrodinger path: detail/graph_encoding/MPGraphEncodingStorage.h.
-              MonomialPropagator<kModes> graph_sim(ham,
-                                                   2 * kModes,
-                                                   VecZ{0, 1},
-                                                   std::optional<unsigned int>{4U},
-                                                   MPI_COMM_SELF,
-                                                   std::nullopt,
-                                                   std::nullopt,
-                                                   CutoffType::Length,
-                                                   std::nullopt);
+              MonomialPropagator graph_sim(ham,
+                                           6,
+                                           VecZ{0, 1},
+                                           4,
+                                           std::optional{4U},
+                                           MPI_COMM_SELF,
+                                           std::nullopt,
+                                           std::nullopt,
+                                           CutoffType::Length,
+                                           std::nullopt);
               const std::vector<VecZ> monos{{0}, {1}, {2}};
               graph_sim.build_graph(monos, VecZ{0, 1, 2}, VecD{1.0, 1.0, 1.0});
               graph_sim.graph_memory_usage();
               graph_sim.expectation_value_and_gradient(VecD{0.1, 0.2, 0.3});
 
-              MonomialPropagator<kModes> partition_sim(ham,
-                                                       2 * kModes,
-                                                       VecZ{0, 1},
-                                                       std::nullopt,
-                                                       MPI_COMM_SELF,
-                                                       std::nullopt,
-                                                       std::nullopt,
-                                                       CutoffType::Length,
-                                                       std::nullopt,
-                                                       kModes,
-                                                       Basis::Majorana,
-                                                       2);
+              MonomialPropagator partition_sim(ham,
+                                               6,
+                                               VecZ{0, 1},
+                                               4,
+                                               std::nullopt,
+                                               MPI_COMM_SELF,
+                                               std::nullopt,
+                                               std::nullopt,
+                                               CutoffType::Length,
+                                               std::nullopt,
+                                               Basis::Majorana,
+                                               2);
               partition_sim.size();
 
               return 0;

.gitignore

@@ -161,7 +161,6 @@ Thumbs.db
 
 external/upstream/_srcs/
 tests/cpp/_srcs/
-_dispatch*.py
 _constants.py
 build*/
 Testing/
@@ -178,6 +177,11 @@ benches/results/**
 # devcontainer files
 .devcontainer/devcontainer-lock.json
 
+notes/**
+
+# `just capture-baseline` / `just diff-baseline` output (tools/capture-baseline.py)
+.baseline-capture/**
+
 # Useful when running in clusters
 logs/
 

CMakeLists.txt

@@ -42,12 +42,6 @@ if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "Release")
 endif()
 
-set(
-  monoprop_MAX_NUM_MODES
-  "250"
-  CACHE STRING
-  "Maximum number of simulable Fermionic modes with Python bindings"
-)
 option(monoprop_ENABLE_MPI "Enable MPI parallelization" OFF)
 option(
   monoprop_WIDE_TERM_INDEX
@@ -77,6 +71,23 @@ endif()
 include(${PROJECT_SOURCE_DIR}/cmake/compiler_flags/Sanitizers.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/compiler_flags/CXXFlags.cmake)
 
+# Sparse/dense crossover depends on vector popcount support.
+# Dense scales with storage-word passes, while sparse is mostly width-flat.
+# Without vector popcount, dense degrades earlier.
+#
+# This is intentionally not a cache variable: it should track the flags actually used
+# for compilation. monoprop_ROW_STORE=dense|sparse already lets users force a backend
+# at runtime without rebuilding.
+#
+# Thresholds are the first full 32-mode block where sparse is clearly faster than dense
+# beyond run-to-run noise, measured end-to-end in the propagator. Expect about +/-1 block
+# variation across machines due to cache and popcount throughput.
+if(ARCH_FLAG)
+  set(monoprop_SPARSE_ROW_MIN_MODES 768)
+else()
+  set(monoprop_SPARSE_ROW_MIN_MODES 256)
+endif()
+
 # report on compiler flags in use
 message(STATUS "Configuring a ${CMAKE_BUILD_TYPE} build")
 string(TOUPPER ${CMAKE_BUILD_TYPE} _cmake_build_type_upper)
@@ -101,8 +112,11 @@ message(STATUS "   Sanitizer profile      : ${monoprop_SANITIZER}")
 
 message(STATUS "   MPI parallelization    : ${monoprop_ENABLE_MPI}")
 message(STATUS "   Wide term index        : ${monoprop_WIDE_TERM_INDEX}")
-message(STATUS "   Max simulable modes    : ${monoprop_MAX_NUM_MODES}")
 message(STATUS "   C++ unit tests         : ${monoprop_ENABLE_CXX_UNIT_TESTS}")
+message(
+  STATUS
+  "   Sparse rows from       : ${monoprop_SPARSE_ROW_MIN_MODES} modes"
+)
 
 include(GNUInstallDirs)
 
@@ -112,8 +126,13 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
 add_library(monoprop-objs OBJECT "")
 add_library(monoprop SHARED $<TARGET_OBJECTS:monoprop-objs>)
 
-# must run before add_subdirectory(cpp): CTest's enabled-ness does not propagate
-# back up to a parent directory that has already been added as a subdirectory.
+# Testing is enabled from the *top-level* list file, and must run before add_subdirectory(cpp), on
+# purpose: CTest's root is wherever enable_testing() was called, so called from cpp/ it wrote no
+# top-level CTestTestfile.cmake and every documented entry point (the CMakePresets test presets,
+# `just test-mpi`, `just test-wide`) pointed ctest at a directory with no tests -- ctest reports "No
+# tests were found" and exits 0 for that, so those commands were silently running nothing. And
+# CTest's enabled-ness does not propagate back up to a parent directory that has already been added
+# as a subdirectory, so this must precede add_subdirectory(cpp) below rather than follow it.
 if(monoprop_ENABLE_CXX_UNIT_TESTS)
   enable_testing()
   include(CTest)

README.md

@@ -117,10 +117,13 @@ uv sync --all-groups --all-extras -v    # installs the workspace, incl. the benc
 uv run python -m pytest -m "not mpi"   # Python tests (serial)
 just test-mpi                          # Python + C++ tests under MPI
 just test-wide                         # Python + C++ unit tests with a 64-bit TermIndex
+just test-sparse-rows                  # Python tests with the support-form row backend forced
 ```
 
-See the [testing guide](https://docs.monoprop.algorithmiq.tech/testing)
-for the with/without-MPI details and the rank matrix.
+`ctest` runs every C++ case twice, once per row backend — the second pass carries
+the `sparse-rows` label. See the
+[testing guide](https://docs.algorithmiq.fi/monoprop/docs/testing) for that, the
+with/without-MPI details, and the rank matrix.
 
 ## Repository layout
 

benches/bench_models.py

@@ -51,6 +51,7 @@ def test_model(
     benchmark,
     bench_comm,
     model_configs,
+    model_rounds,
     model,
     record_model_config,
     record_model_stats,
@@ -64,7 +65,12 @@ def test_model(
     state: dict[str, Any] = {}
 
     def setup():
-        state["baseline_rss"] = resting_rss_bytes()
+        # First round only: only then does setup() run before any model is built, matching
+        # `Baseline RSS` (resting memory before construction). In later rounds,
+        # pytest-benchmark still holds the previous round's args during setup(), so the
+        # old propagator is still live and cannot be reclaimed. That would make the reading
+        # baseline + one full model and misstate the model's memory cost versus `Peak RSS`.
+        state.setdefault("baseline_rss", resting_rss_bytes())
         state["built"] = build_fn(config, comm=bench_comm)
         return (state["built"], steps), {}
 
@@ -74,10 +80,14 @@ def run(built, n_steps):
             propagator.propagate(circuit)
         return propagator.expectation_value()
 
+    # setup() runs before every round, so each round rebuilds the model and evolves a fresh
+    # propagator -- these simulations are in place, and replaying a mutated one would time the wrong
+    # thing. record_model_stats below then describes the last round, which is what any round would
+    # produce: the term counts and memory are deterministic.
     result = benchmark.pedantic(
         barriered(run, bench_comm),
         setup=barrier_setup(bench_comm, setup),
-        rounds=1,
+        rounds=model_rounds,
         iterations=1,
     )
     assert isinstance(result, float)

benches/conftest.py

@@ -178,6 +178,16 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         group.addoption(f"--{name}", type=int, default=default, help=help_text)
 
     models = parser.getgroup("monoprop-models", "monoprop fixed-model overrides")
+    # One round is enough for the memory and term-count stats, which are deterministic, but it yields
+    # no spread for the timing -- and these models are expensive enough that a single sample can sit
+    # well off the median. Raise this when a timing difference is the point of the run.
+    models.addoption(
+        "--model-rounds",
+        type=int,
+        default=1,
+        help="Rounds per fixed model; each rebuilds the model first. >1 gives a median and stddev "
+        "(default: 1).",
+    )
     for model, (config_cls, _builder, _steps) in MODELS.items():
         for field in fields(config_cls):
             models.addoption(
@@ -214,6 +224,7 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]:
         "nodes": nodes,
         "ranks_per_node": ranks_per_node,
         "monoprop_threads": os.environ.get("monoprop_NUM_THREADS", "default"),  # noqa: SIM112
+        "monoprop_row_store": os.environ.get("monoprop_ROW_STORE") or "auto",  # noqa: SIM112
         "cpu_count_logical": psutil.cpu_count(logical=True),
         "cpu_count_physical": psutil.cpu_count(logical=False),
         "hostname": socket.gethostname(),
@@ -224,7 +235,6 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]:
         "python_version": platform.python_version(),
         "nanobind_version": monoprop.__nanobind_version__,
         "nanobind_backend_version": nanobind_backend_version,
-        "monoprop_max_num_modes": monoprop.MAX_NUM_MODES,
         "malloc_arena_max": os.environ.get("MALLOC_ARENA_MAX", "default"),
         "omp_num_threads": os.environ.get("OMP_NUM_THREADS", "default"),
         # Filled by _record_placement: the threads exist only once a propagator does.
@@ -238,6 +248,27 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]:
     return meta
 
 
+def _record_row_store(propagator: Any) -> None:
+    """Fold one propagator's resolved row backend into this run's metadata.
+
+    ``monoprop_ROW_STORE`` says what was asked for, not what ran: unset lets the storage width pick,
+    and the crossover it picks against is a build-time constant. The two backends accumulate a term
+    sum in different orders and have different footprints, so a report has to name the one that ran.
+    Widths differ within a run, hence so can the backend: a disagreement records as ``"mixed"``
+    rather than letting the last propagator speak for the others.
+    """
+    if _rank() != 0:
+        return
+    # Read straight off the binding, with no getattr fallback: a benchmark whose whole job is to name
+    # the backend that ran must fail loudly against an extension that cannot say, not quietly record
+    # nothing.
+    resolved = "sparse" if propagator._simulator.rows_are_sparse else "dense"
+    seen = _RESULTS["meta"].get("row_store_effective")
+    _RESULTS["meta"]["row_store_effective"] = (
+        resolved if seen in (None, resolved) else "mixed"
+    )
+
+
 def _params(config: pytest.Config) -> dict[str, Any]:
     """Return the resolved random-problem hyperparameters (defaults included)."""
     return {
@@ -295,6 +326,12 @@ def bench_rounds(request: pytest.FixtureRequest) -> int:
     return int(request.config.getoption("--bench-rounds"))
 
 
+@pytest.fixture(scope="session")
+def model_rounds(request: pytest.FixtureRequest) -> int:
+    """Return the round count for the fixed-model benchmarks."""
+    return int(request.config.getoption("--model-rounds"))
+
+
 @pytest.fixture(scope="session")
 def model_configs(request: pytest.FixtureRequest) -> dict[str, Any]:
     """Return each fixed model's config, every field resolved from the CLI.
@@ -353,6 +390,7 @@ def _record_model_stats(
 ) -> None:
     """Record term count, operator memory breakdown and footprint under ``key``."""
     _record("opsize", key, {"terms": _reduce_sum(comm, propagator.size())})
+    _record_row_store(propagator)
 
     # Placement is only observable while the propagator's threads are alive.
     _record_placement(comm)
@@ -553,6 +591,7 @@ def built_graph(
 
     # Under MPI the operator is partitioned, so sum the partitions.
     _record("opsize", picture, {"terms": _reduce_sum(bench_comm, mp.size())})
+    _record_row_store(mp)
 
     # Settled RSS once the build's transients are released -- the persistent
     # footprint the per-operation peak cannot see.