diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 2e3979c..73436c3 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -25,6 +25,21 @@ endif() FetchContent_MakeAvailable(googlebenchmark) +# Kernel selection is compile-time (nsparse/utils/distance_simd.h) and the -m +# flags live on the per-level library targets, so linking the plain `nsparse` +# gets the scalar kernels no matter what NSPARSE_OPT_LEVEL says. Pick the library +# that matches the requested level (as nsparse/python/CMakeLists.txt does) and +# print which one, so a mismatch is visible instead of silently scalar. The +# foreach over an explicit list (not nsparse_${NSPARSE_OPT_LEVEL}) makes a typo'd +# level fall back to a target that exists rather than fail at link time. +set(NSPARSE_BENCH_LIB nsparse) +foreach(_level avx2 avx512 sve) + if(NSPARSE_OPT_LEVEL STREQUAL "${_level}") + set(NSPARSE_BENCH_LIB nsparse_${_level}) + endif() +endforeach() +message(STATUS "benchmarks: NSPARSE_OPT_LEVEL=${NSPARSE_OPT_LEVEL} -> linking ${NSPARSE_BENCH_LIB}") + if(NSPARSE_BENCHMARK_SRC) find_package(OpenMP REQUIRED) @@ -33,7 +48,7 @@ if(NSPARSE_BENCHMARK_SRC) target_include_directories(nsparse_benchmark PRIVATE ${PROJECT_SOURCE_DIR}) target_link_libraries(nsparse_benchmark PRIVATE - nsparse + ${NSPARSE_BENCH_LIB} OpenMP::OpenMP_CXX benchmark::benchmark benchmark::benchmark_main @@ -48,7 +63,7 @@ endif() add_executable(sq_residency_bench sq_residency_bench.cpp) target_include_directories(sq_residency_bench PRIVATE ${PROJECT_SOURCE_DIR}) target_link_libraries(sq_residency_bench PRIVATE - nsparse + ${NSPARSE_BENCH_LIB} OpenMP::OpenMP_CXX absl::flat_hash_set absl::flat_hash_map @@ -60,7 +75,7 @@ target_link_libraries(sq_residency_bench PRIVATE add_executable(nsparse_build_benchmark index_build_benchmark.cpp) target_include_directories(nsparse_build_benchmark PRIVATE ${PROJECT_SOURCE_DIR}) target_link_libraries(nsparse_build_benchmark PRIVATE - nsparse + ${NSPARSE_BENCH_LIB} OpenMP::OpenMP_CXX benchmark::benchmark absl::flat_hash_set diff --git a/benchmarks/DISK_SEISMIC_BENCH.md b/benchmarks/DISK_SEISMIC_BENCH.md new file mode 100644 index 0000000..c70a50d --- /dev/null +++ b/benchmarks/DISK_SEISMIC_BENCH.md @@ -0,0 +1,76 @@ +# DiskSeismic C++ benchmark (disk-vs-RAM residency) + +Measures whether the disk-resident DiskSeismic index holds up on latency/recall as +RAM shrinks below the index size — the core question before investing in the full +OpenSearch plugin integration. Pure C++; no OpenSearch, no plugin. + +## What it compares + +Three residency regimes over the same MS MARCO corpus, single-thread query: + +| # | Variant | Index / residency | Search | +|---|---------|-------------------|--------| +| V1 | in-RAM SEISMIC | `seismic`, in-memory (heap copy) | heap_factor traversal | +| V2 | scattered-disk SEISMIC | `seismic`, mmap (mapped forward index) | heap_factor traversal | +| V3 | **DiskSeismic** | `disk_seismic`, mmap (inline forward index) | **GroC top-k'** | + +The result is the **RAM-cap sweep**: below the index size, V1 (heap) is OOM-killed +while the mmap variants keep serving by paging from disk. How V2/V3 latency degrades +as RAM shrinks — and where V1 dies — is the finding. DiskSeismic keeps only cluster +summaries resident (`RssAnon`), paging the forward index on demand (`RssFile`). + +## Host prereqs (fresh g5.12xlarge) + +- **g5.12xlarge = AMD Zen2 (EPYC 7R32): no AVX-512.** Build uses `OPT_LEVEL=avx2` (default). `avx512` would not run. +- Toolchain: `gcc10` (C++20) + `cmake3`. Build with `OPT_LEVEL=avx2` (the default); `OPT_LEVEL=generic` uses the scalar kernels and costs ~1.11–1.21× on the search path. +- **Run as root** — the cgroup cap (`systemd-run --scope`) and dropping the page cache between runs both need it. As a normal user the cap runs fail with "Interactive authentication required". +- ~24 GB free disk for the corpus + ~15 GB per serialized index (two here). + +## Get the data + +```bash +export NSPARSE_DATA_DIR=/data +mkdir -p "$NSPARSE_DATA_DIR" && cd "$NSPARSE_DATA_DIR" +curl -O https://do0ia2psryw9c.cloudfront.net/base_full.csr.gz +curl -O https://do0ia2psryw9c.cloudfront.net/queries.dev.csr.gz +curl -O https://do0ia2psryw9c.cloudfront.net/iv_array.txt +gunzip base_full.csr.gz queries.dev.csr.gz +# verify: (8841823, 30109, 1121199371) +python3 -c "import struct;print(struct.unpack('<3q',open('base_full.csr','rb').read(24)))" +``` + +## Run + +```bash +sudo NSPARSE_DATA_DIR=/data \ + CAPS="unlimited 12G 8G 4G" \ + ./benchmarks/run_disk_seismic_bench.sh +``` + +It builds the binary, builds both indexes once (reused on re-runs), then sweeps +caps × variants and writes `summary.tsv` + per-run logs to +`$NSPARSE_DATA_DIR/disk_seismic_bench_out/`. + +Knobs (env): `OPT_LEVEL=avx2 LAMBDA=6000 BETA=400 ALPHA=0.4 CUT=3 KPRIME=50 K=10 REPS=5 CAPS="unlimited 12G 8G 4G"`. +Set caps around and below the index size (~15 GB float; DiskSeismic's inline file is +larger on disk but only its summaries stay resident) to expose the disk benefit. + +## Reading the output (`summary.tsv`) + +`variant cap status p50_ms p90_ms p99_ms qps recall RssAnon/RssFile_GiB` + +- `status=OOM/FAIL` at a cap = that variant could not run there. Expect V1 to hit this + once the cap drops below its ~15 GB heap; V2/V3 should keep `status=ok`. +- Compare V3 (DiskSeismic) latency and recall to V1's uncapped baseline: the bet is + that V3 stays acceptable at caps where V1 is dead. +- Memory split: V1 is almost all `RssAnon` (heap); V2/V3 are almost all `RssFile` + (mapped, reclaimable), V3 with a small `RssAnon` for summaries. + +## Caveats + +- **`base_small` is a smoke test, not a benchmark** — it fits in cache, so the + memory-bound behavior that is the whole point disappears. Only report `base_full`. +- Regenerate the `.dat` files whenever the source tree changes; the serialized format + carries no version and an old binary silently misreads a new file. +- Query threads are pinned to 1 (`OMP_NUM_THREADS=1`) — per-query cost is the metric + and the workload is bandwidth-bound. diff --git a/benchmarks/run_disk_seismic_bench.sh b/benchmarks/run_disk_seismic_bench.sh new file mode 100755 index 0000000..1f2fa8a --- /dev/null +++ b/benchmarks/run_disk_seismic_bench.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Self-contained DiskSeismic C++ benchmark: compares three sparse-index residency +# regimes on MS MARCO under a sweep of RAM caps, single-thread query. +# +# V1 in-RAM SEISMIC seismic .dat, inmem (heap copy, SEISMIC heap_factor) +# V2 scattered-disk SEISMIC seismic .dat, mmap (mapped forward index, SEISMIC heap_factor) +# V3 DiskSeismic disk_seismic .dat, mmap (inline forward index, GroC top-k') +# +# The point of the RAM-cap sweep: below the index size, V1 (heap) is OOM-killed +# while the mmap variants keep serving by paging from disk. That contrast — and +# how V2/V3 latency degrades as RAM shrinks — is the result. +# +# Prereqs on the host: gcc10 (C++20), cmake3, and MS MARCO CSR data. For the cap +# sweep, systemd (systemd-run --scope) and permission to set MemoryMax; dropping +# the page cache between runs needs root (falls back to sudo, else warm-cache). +# +# Usage: +# NSPARSE_DATA_DIR=/data ./benchmarks/run_disk_seismic_bench.sh +# Env knobs (defaults): OPT_LEVEL=avx2 LAMBDA=6000 BETA=400 ALPHA=0.4 +# CUT=3 KPRIME=50 K=10 REPS=5 CAPS="unlimited 8G 4G 2G" +set -euo pipefail + +: "${NSPARSE_DATA_DIR:?set NSPARSE_DATA_DIR to a dir with base_full.csr, queries.dev.csr, iv_array.txt}" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DATA="$NSPARSE_DATA_DIR" +CORPUS="$DATA/base_full.csr"; Q="$DATA/queries.dev.csr"; TRUTH="$DATA/iv_array.txt" +for f in "$CORPUS" "$Q" "$TRUTH"; do [ -f "$f" ] || { echo "missing $f"; exit 1; }; done + +# g5.12xlarge is AMD Zen2 (EPYC 7R32): NO AVX-512. avx2 is correct; avx512 would not run. +OPT_LEVEL="${OPT_LEVEL:-avx2}" +CMAKE="${CMAKE:-cmake3}"; CXX="${CXX:-/usr/bin/gcc10-g++}"; CC="${CC:-/usr/bin/gcc10-gcc}" +LAMBDA="${LAMBDA:-6000}"; BETA="${BETA:-400}"; ALPHA="${ALPHA:-0.4}" +CUT="${CUT:-3}"; KPRIME="${KPRIME:-50}"; K="${K:-10}"; REPS="${REPS:-5}" +CAPS="${CAPS:-unlimited 8G 4G 2G}" +BUILD_DIR="${BUILD_DIR:-$REPO/build-bench}" +OUT="${OUT:-$DATA/disk_seismic_bench_out}"; mkdir -p "$OUT" + +echo "=== 1. build (Release, $OPT_LEVEL) ===" +"$CMAKE" -S "$REPO" -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE=Release -DNSPARSE_OPT_LEVEL="$OPT_LEVEL" \ + -DNSPARSE_ENABLE_BENCHMARKS=ON -DCMAKE_CXX_COMPILER="$CXX" -DCMAKE_C_COMPILER="$CC" >"$OUT/configure.log" 2>&1 +"$CMAKE" --build "$BUILD_DIR" --target sq_residency_bench -j"$(nproc)" >"$OUT/build.log" 2>&1 +BIN="$BUILD_DIR/benchmarks/sq_residency_bench" +[ -x "$BIN" ] || { echo "build failed; see $OUT/build.log"; exit 1; } +grep -q "CMAKE_BUILD_TYPE:STRING=Release" "$BUILD_DIR/CMakeCache.txt" || { echo "NOT a Release build"; exit 1; } + +echo "=== 2. build indexes from base_full (all cores; reused if present) ===" +SEISMIC_DAT="$OUT/base_full.seismic.dat"; DISK_DAT="$OUT/base_full.disk_seismic.dat" +[ -f "$SEISMIC_DAT" ] || "$BIN" build "$CORPUS" "seismic,lambda=$LAMBDA|beta=$BETA|alpha=$ALPHA" "$SEISMIC_DAT" +[ -f "$DISK_DAT" ] || "$BIN" build "$CORPUS" "disk_seismic,lambda=$LAMBDA|beta=$BETA|alpha=$ALPHA" "$DISK_DAT" + +drop_caches() { + sync + if [ -w /proc/sys/vm/drop_caches ]; then echo 3 >/proc/sys/vm/drop_caches + else sudo sh -c 'echo 3 >/proc/sys/vm/drop_caches' 2>/dev/null || echo " (warn: could not drop page cache; run as root for cold-cache numbers)"; fi +} + +# systemd renamed the cgroup memory knob at v231 (MemoryMax) from v219 (MemoryLimit); +# MemorySwapMax also only exists on the newer line. EC2 hosts have no swap by default, +# so the limit is hard either way. Running as root avoids cgroup-delegation issues. +SYSTEMD_VER="$(systemctl --version 2>/dev/null | head -1 | awk '{print $2}')" +mem_props() { + if [ "${SYSTEMD_VER:-0}" -ge 231 ] 2>/dev/null; then echo "-p MemoryMax=$1 -p MemorySwapMax=0" + else echo "-p MemoryLimit=$1"; fi +} + +# run one search, optionally inside a memory-capped transient scope +run_capped() { + local cap="$1" logf="$2"; shift 2 + drop_caches + if [ "$cap" = "unlimited" ]; then + OMP_NUM_THREADS=1 "$@" >"$logf" 2>&1 || echo "RUN_FAILED rc=$?" >>"$logf" + else + # shellcheck disable=SC2046 + systemd-run --scope -q $(mem_props "$cap") \ + env OMP_NUM_THREADS=1 "$@" >"$logf" 2>&1 || echo "OOM_OR_FAIL rc=$? cap=$cap" >>"$logf" + fi +} + +echo "=== 3. sweep caps x variants (single-thread) ===" +for cap in $CAPS; do + echo " cap=$cap ..." + run_capped "$cap" "$OUT/v1_inram.$cap.log" "$BIN" search "$SEISMIC_DAT" "$Q" "$K" "$REPS" inmem "$TRUTH" "$CUT" + run_capped "$cap" "$OUT/v2_scattered.$cap.log" "$BIN" search "$SEISMIC_DAT" "$Q" "$K" "$REPS" mmap "$TRUTH" "$CUT" + run_capped "$cap" "$OUT/v3_diskseismic.$cap.log" "$BIN" search "$DISK_DAT" "$Q" "$K" "$REPS" mmap "$TRUTH" "$CUT" "$KPRIME" +done + +echo "=== 4. summary (also written to $OUT/summary.tsv) ===" +python3 - "$OUT" <<'PY' | tee "$OUT/summary.tsv" +import os, re, sys, glob +outdir = sys.argv[1] +def grab(txt, pat): + m = re.search(pat, txt); return m.group(1) if m else "" +rows = [] +for f in sorted(glob.glob(os.path.join(outdir, "v*.log"))): + base = os.path.basename(f)[:-4] # v1_inram.8G + variant, cap = base.split(".", 1) + txt = open(f).read() + if "OOM_OR_FAIL" in txt or "RUN_FAILED" in txt or "error:" in txt: + rows.append((variant, cap, "OOM/FAIL", "", "", "", "", "", "")); continue + p50 = grab(txt, r"p50 ([\d.]+)"); p90 = grab(txt, r"p90 ([\d.]+)"); p99 = grab(txt, r"p99 ([\d.]+)") + qps = grab(txt, r"qps ([\d.]+)") # last batch qps + rec = grab(txt, r"recall@\d+ ([\d.]+)") + # memory from the after_search line + aft = txt.split("mem[after_search]")[-1] if "after_search" in txt else "" + anon = grab(aft, r"RssAnon: ([\d.]+)"); rfile = grab(aft, r"RssFile: ([\d.]+)") + rows.append((variant, cap, "ok", p50, p90, p99, qps, rec, f"{anon}/{rfile}")) +print("variant\tcap\tstatus\tp50_ms\tp90_ms\tp99_ms\tqps\trecall\tRssAnon/File_GiB") +for r in rows: print("\t".join(map(str, r))) +PY +echo "done. logs + summary in $OUT" diff --git a/benchmarks/sq_residency_bench.cpp b/benchmarks/sq_residency_bench.cpp index 2278479..9d4797a 100644 --- a/benchmarks/sq_residency_bench.cpp +++ b/benchmarks/sq_residency_bench.cpp @@ -35,6 +35,7 @@ #include #include +#include "nsparse/disk_seismic_index.h" #include "nsparse/index.h" #include "nsparse/index_factory.h" #include "nsparse/io/index_io.h" @@ -185,7 +186,7 @@ int do_build(int argc, char** argv) { int do_search(int argc, char** argv) { if (argc < 7) { std::cerr << "search " - "[truth.txt]\n"; + "[truth.txt|-] [cut] [k_prime] [heap_factor]\n"; return 2; } std::string dat_path = argv[2]; @@ -193,7 +194,16 @@ int do_search(int argc, char** argv) { const int k = std::atoi(argv[4]); const int reps = std::atoi(argv[5]); const std::string residency = argv[6]; - const std::string truth_path = argc > 7 ? argv[7] : ""; + const std::string truth_path = + (argc > 7 && std::strcmp(argv[7], "-") != 0) ? argv[7] : ""; + const int cut = argc > 8 ? std::atoi(argv[8]) : 3; + // k_prime > 0 selects the DiskSeismic GroC path (score summaries, take the + // global top-k' blocks); 0 uses the plain SEISMIC heap_factor traversal. + const int k_prime = argc > 9 ? std::atoi(argv[9]) : 0; + // SEISMIC pruning slack (used only when k_prime==0). Skip a cluster when + // cluster_score*heap_factor < heap.peek, so LARGER = prune fewer = higher + // recall + slower. Sweep this to raise SEISMIC recall for iso-recall points. + const float heap_factor = argc > 10 ? static_cast(std::atof(argv[10])) : 1.0F; const int io_flags = residency == "mmap" ? nsparse::IndexIoFlag::kUseMmap : 0; @@ -220,7 +230,14 @@ int do_search(int argc, char** argv) { << (vectors == nullptr ? 0 : vectors->get_element_size()) << "\n"; print_memory("after_load"); - nsparse::SeismicSearchParameters params(/*cut=*/3, /*heap_factor=*/1.0F); + std::unique_ptr params; + if (k_prime > 0) { + params = std::make_unique(cut, k_prime); + std::cout << "search_params cut " << cut << " k_prime " << k_prime << "\n"; + } else { + params = std::make_unique(cut, heap_factor); + std::cout << "search_params cut " << cut << " heap_factor " << heap_factor << "\n"; + } std::vector distances(static_cast(n_queries) * k); std::vector labels(static_cast(n_queries) * k); @@ -231,7 +248,7 @@ int do_search(int argc, char** argv) { index->search(static_cast(n_queries), query.indptr.data(), query.indices.data(), query.data.data(), k, distances.data(), labels.data(), - ¶ms); + params.get()); const auto t1 = std::chrono::steady_clock::now(); const double ms = std::chrono::duration(t1 - t0).count(); @@ -255,7 +272,7 @@ int do_search(int argc, char** argv) { const auto t0 = std::chrono::steady_clock::now(); index->search(1, q_indptr.data(), query.indices.data() + start, query.data.data() + start, k, distances.data(), - labels.data(), ¶ms); + labels.data(), params.get()); const auto t1 = std::chrono::steady_clock::now(); per_query_ms.push_back( std::chrono::duration(t1 - t0).count()); @@ -271,7 +288,7 @@ int do_search(int argc, char** argv) { // Recall from a final batched pass, so `labels` holds every query again. index->search(static_cast(n_queries), query.indptr.data(), query.indices.data(), query.data.data(), k, distances.data(), - labels.data(), ¶ms); + labels.data(), params.get()); if (!truth_path.empty()) { std::cout << "recall@" << k << " " << recall_at_k(truth_path, labels, k, n_queries) << "\n";