From 1b8f069722401329d66583170c31321af0e6cd5b Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 19 Mar 2026 15:35:21 +0800 Subject: [PATCH 1/4] Fix IDMapIndex read/write segment fault & fix ExactMatch score no scaling back Signed-off-by: Liyun Xiu --- nsparse/id_map_index.cpp | 4 +- nsparse/io/index_io.cpp | 33 ++-- nsparse/io/index_io.h | 7 +- nsparse/seismic_scalar_quantized_index.cpp | 9 +- tests/index_io_test.cpp | 152 ++++++++++++++++++ tests/seismic_scalar_quantized_index_test.cpp | 58 +++++++ 6 files changed, 250 insertions(+), 13 deletions(-) diff --git a/nsparse/id_map_index.cpp b/nsparse/id_map_index.cpp index 28bbe26..e821423 100644 --- a/nsparse/id_map_index.cpp +++ b/nsparse/id_map_index.cpp @@ -75,7 +75,7 @@ void IDMapIndex::add_with_ids(idx_t n, const idx_t* indptr, } } void IDMapIndex::write_index(IOWriter* io_writer) { - nsparse::write_index(delegate_, io_writer); + nsparse::detail::write_index(delegate_, io_writer, true); // Write internal_to_external_ vector size_t map_size = internal_to_external_.size(); @@ -86,7 +86,7 @@ void IDMapIndex::write_index(IOWriter* io_writer) { } void IDMapIndex::read_index(IOReader* io_reader) { - delegate_ = nsparse::read_index(io_reader); + delegate_ = nsparse::detail::read_index(io_reader, true); // Read internal_to_external_ vector size_t map_size = 0; diff --git a/nsparse/io/index_io.cpp b/nsparse/io/index_io.cpp index f9d9c42..75b4776 100644 --- a/nsparse/io/index_io.cpp +++ b/nsparse/io/index_io.cpp @@ -57,7 +57,9 @@ Index* read_header(IOReader* io_reader) { } } } // namespace -void write_index(Index* index, IOWriter* io_writer) { + +namespace detail { +void write_index(Index* index, IOWriter* io_writer, bool keep_open) { auto* index_io = dynamic_cast(index); if (index_io == nullptr) { throw std::runtime_error("Index does not support serialization"); @@ -66,24 +68,37 @@ void write_index(Index* index, IOWriter* io_writer) { write_header(index, io_writer); // write index customized payload index_io->write_index(io_writer); - io_writer->close(); -} - -void write_index(Index* index, char* filename) { - FileIOWriter writer(filename); - write_index(index, &writer); + if (!keep_open) { + io_writer->close(); + } } -Index* read_index(IOReader* io_reader) { +Index* read_index(IOReader* io_reader, bool keep_open) { Index* index = read_header(io_reader); auto* index_io = dynamic_cast(index); if (index_io == nullptr) { throw std::runtime_error("Index does not support serialization"); } index_io->read_index(io_reader); - io_reader->close(); + if (!keep_open) { + io_reader->close(); + } return index; } +} // namespace detail + +void write_index(Index* index, IOWriter* io_writer) { + detail::write_index(index, io_writer, false); +} + +void write_index(Index* index, char* filename) { + FileIOWriter writer(filename); + write_index(index, &writer); +} + +Index* read_index(IOReader* io_reader) { + return detail::read_index(io_reader, false); +} Index* read_index(char* filename) { FileIOReader reader(filename); diff --git a/nsparse/io/index_io.h b/nsparse/io/index_io.h index ea96bbe..e08a43d 100644 --- a/nsparse/io/index_io.h +++ b/nsparse/io/index_io.h @@ -14,9 +14,14 @@ #include "nsparse/io/io.h" namespace nsparse { +namespace detail { +void write_index(Index* index, IOWriter* io_writer, bool keep_open); +Index* read_index(IOReader* io_reader, bool keep_open); +} // namespace detail + void write_index(Index* index, char* filename); -Index* read_index(char* filename); void write_index(Index* index, IOWriter* io_writer); +Index* read_index(char* filename); Index* read_index(IOReader* io_reader); } // namespace nsparse diff --git a/nsparse/seismic_scalar_quantized_index.cpp b/nsparse/seismic_scalar_quantized_index.cpp index 73cdd2e..41b1111 100644 --- a/nsparse/seismic_scalar_quantized_index.cpp +++ b/nsparse/seismic_scalar_quantized_index.cpp @@ -208,11 +208,18 @@ auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, // if filter ids size is <= k, just run exact match if (detail::should_run_exact_match(search_parameters->get_id_selector(), k, &query_vectors)) { - return detail::ExactMatcher::search( + auto [distances, labels] = detail::ExactMatcher::search( vectors_.get(), dynamic_cast( search_parameters->get_id_selector()), &query_vectors, element_size, k); + // Decode quantized dot product scores + for (auto& query_distances : distances) { + for (auto& dist : query_distances) { + dist = sq_.decode_dot_product(dist, query_sq); + } + } + return {distances, labels}; } std::vector> result_distances(n); diff --git a/tests/index_io_test.cpp b/tests/index_io_test.cpp index 840058e..fa70be1 100644 --- a/tests/index_io_test.cpp +++ b/tests/index_io_test.cpp @@ -19,6 +19,7 @@ #include "nsparse/brutal_index.h" #include "nsparse/id_map_index.h" #include "nsparse/index.h" +#include "nsparse/inverted_index.h" #include "nsparse/io/buffered_io.h" #include "nsparse/io/io.h" #include "nsparse/seismic_index.h" @@ -28,6 +29,49 @@ namespace { +// IOWriter that throws if write() is called after close(). +// Simulates real file I/O where writing to a closed stream is invalid. +class StrictBufferedIOWriter : public nsparse::IOWriter { +public: + void write(void* ptr, size_t size, size_t nitems) override { + if (closed_) { + throw std::runtime_error( + "Write after close: stream already closed"); + } + delegate_.write(ptr, size, nitems); + } + + void close() override { closed_ = true; } + + const std::vector& data() const { return delegate_.data(); } + size_t size() const { return delegate_.size(); } + +private: + nsparse::BufferedIOWriter delegate_; + bool closed_ = false; +}; + +// IOReader that throws if read() is called after close(). +// Simulates real file I/O where reading from a closed stream is invalid. +class StrictBufferedIOReader : public nsparse::IOReader { +public: + explicit StrictBufferedIOReader(const std::vector& data) + : delegate_(data) {} + + size_t read(void* ptr, size_t size, size_t nitems) override { + if (closed_) { + throw std::runtime_error("Read after close: stream already closed"); + } + return delegate_.read(ptr, size, nitems); + } + + void close() override { closed_ = true; } + +private: + nsparse::BufferedIOReader delegate_; + bool closed_ = false; +}; + // Mock Index that implements both Index and IndexIO for testing class MockIndex : public nsparse::Index, public nsparse::IndexIO { public: @@ -302,3 +346,111 @@ TEST(IndexIO, RoundtripIDMapIndexWithData) { delete original; delete loaded; } + +// Verify StrictBufferedIOWriter throws on write after close +TEST(IndexIO, StrictWriterThrowsAfterClose) { + StrictBufferedIOWriter writer; + int val = 42; + writer.write(&val, sizeof(int), 1); + writer.close(); + ASSERT_THROW(writer.write(&val, sizeof(int), 1), std::runtime_error); +} + +// Verify StrictBufferedIOReader throws on read after close +TEST(IndexIO, StrictReaderThrowsAfterClose) { + std::vector buf(sizeof(int), 0); + StrictBufferedIOReader reader(buf); + int val = 0; + reader.read(&val, sizeof(int), 1); + reader.close(); + ASSERT_THROW(reader.read(&val, sizeof(int), 1), std::runtime_error); +} + +// IDMapIndex wrapping SeismicIndex: write then read with strict IO. +// Before the keep_open fix, write_index closed the stream before +// IDMapIndex could write its id map, causing a write-after-close crash. +TEST(IndexIO, StrictIO_RoundtripIDMapSeismicIndex) { + auto* seismic = new nsparse::SeismicIndex(128); + auto* original = new nsparse::IDMapIndex(seismic); + + std::vector indptr = {0, 2, 4}; + std::vector indices = {0, 1, 2, 3}; + std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; + std::vector ids = {100, 200}; + original->add_with_ids(2, indptr.data(), indices.data(), values.data(), + ids.data()); + + StrictBufferedIOWriter writer; + ASSERT_NO_THROW(nsparse::write_index(original, &writer)); + + StrictBufferedIOReader reader(writer.data()); + nsparse::Index* loaded = nullptr; + ASSERT_NO_THROW(loaded = nsparse::read_index(&reader)); + + ASSERT_NE(loaded, nullptr); + ASSERT_EQ(loaded->id(), original->id()); + ASSERT_EQ(loaded->num_vectors(), 2); + + delete original; + delete loaded; +} + +// IDMapIndex wrapping InvertedIndex: the original segfault scenario. +// InvertedIndex has a non-trivial write_index/read_index, so the +// stream must stay open for IDMapIndex to write/read its id map after +// the delegate is serialized. +// Note: InvertedIndex::get_vectors() returns nullptr after build() +// (vectors_ is consumed to create inverted_lists_), so num_vectors() +// returns 0. We verify the roundtrip by checking the index type instead. +TEST(IndexIO, StrictIO_RoundtripIDMapInvertedIndex) { + auto* inverted = new nsparse::InvertedIndex(128); + auto* original = new nsparse::IDMapIndex(inverted); + + std::vector indptr = {0, 2, 4}; + std::vector indices = {0, 1, 2, 3}; + std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; + std::vector ids = {100, 200}; + original->add_with_ids(2, indptr.data(), indices.data(), values.data(), + ids.data()); + original->build(); + + StrictBufferedIOWriter writer; + ASSERT_NO_THROW(nsparse::write_index(original, &writer)); + + StrictBufferedIOReader reader(writer.data()); + nsparse::Index* loaded = nullptr; + ASSERT_NO_THROW(loaded = nsparse::read_index(&reader)); + + ASSERT_NE(loaded, nullptr); + ASSERT_EQ(loaded->id(), original->id()); + + delete original; + delete loaded; +} + +// IDMapIndex wrapping SeismicScalarQuantizedIndex with strict IO +TEST(IndexIO, StrictIO_RoundtripIDMapSeismicSQIndex) { + auto* sq_index = new nsparse::SeismicScalarQuantizedIndex(128); + auto* original = new nsparse::IDMapIndex(sq_index); + + std::vector indptr = {0, 2, 4}; + std::vector indices = {0, 1, 2, 3}; + std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; + std::vector ids = {100, 200}; + original->add_with_ids(2, indptr.data(), indices.data(), values.data(), + ids.data()); + + StrictBufferedIOWriter writer; + ASSERT_NO_THROW(nsparse::write_index(original, &writer)); + + StrictBufferedIOReader reader(writer.data()); + nsparse::Index* loaded = nullptr; + ASSERT_NO_THROW(loaded = nsparse::read_index(&reader)); + + ASSERT_NE(loaded, nullptr); + ASSERT_EQ(loaded->id(), original->id()); + ASSERT_EQ(loaded->num_vectors(), 2); + + delete original; + delete loaded; +} diff --git a/tests/seismic_scalar_quantized_index_test.cpp b/tests/seismic_scalar_quantized_index_test.cpp index 86fdc28..acd70d7 100644 --- a/tests/seismic_scalar_quantized_index_test.cpp +++ b/tests/seismic_scalar_quantized_index_test.cpp @@ -727,6 +727,64 @@ TEST(SeismicSQIndexSearch, search_exact_match_with_small_selector) { EXPECT_GT(distances[0], distances[1]); } +TEST(SeismicSQIndexSearch, search_exact_match_scores_match_normal_path_scores) { + TestableSeismicSQIndex index(QuantizerType::QT_8bit, 0.0F, 1.0F, 10, 3, + 0.5F, 3); + Index* idx = &index; + + // doc0: term0=1.0, doc1: term0=0.5, doc2: term0=0.8 + index.add_docs({{{0, 1.0F}}, {{0, 0.5F}}, {{0, 0.8F}}}); + index.build(); + + std::vector query_indptr = {0, 1}; + std::vector query_indices = {0}; + std::vector query_values = {1.0F}; + + // Normal path (no selector): k=3, all 3 docs returned + std::vector labels_normal(3, -1); + std::vector distances_normal(3, -1.0F); + SeismicSearchParameters params_normal(5, 1000.0F); + idx->search(1, query_indptr.data(), query_indices.data(), + query_values.data(), 3, distances_normal.data(), + labels_normal.data(), ¶ms_normal); + + // Exact match path: selector size (2) <= k (2) + std::vector allowed_ids = {0, 2}; + ArrayIDSelector selector(allowed_ids.size(), allowed_ids.data()); + SeismicSearchParameters params_filtered(5, 1000.0F); + params_filtered.set_id_selector(&selector); + + std::vector labels_filtered(2, -1); + std::vector distances_filtered(2, -1.0F); + idx->search(1, query_indptr.data(), query_indices.data(), + query_values.data(), 2, distances_filtered.data(), + labels_filtered.data(), ¶ms_filtered); + + // Find doc0's score from the normal path + float doc0_score_normal = -1.0F; + for (int i = 0; i < 3; ++i) { + if (labels_normal[i] == 0) { + doc0_score_normal = distances_normal[i]; + break; + } + } + // Find doc0's score from the exact match path + float doc0_score_filtered = -1.0F; + for (int i = 0; i < 2; ++i) { + if (labels_filtered[i] == 0) { + doc0_score_filtered = distances_filtered[i]; + break; + } + } + + ASSERT_GE(doc0_score_normal, 0.0F); + ASSERT_GE(doc0_score_filtered, 0.0F); + // Scores must match — both paths should decode quantized dot products + EXPECT_FLOAT_EQ(doc0_score_normal, doc0_score_filtered); + // Sanity: decoded score should be in a reasonable range (not raw quantized) + EXPECT_LT(doc0_score_filtered, 10.0F); +} + TEST(SeismicSQIndexSearch, search_with_id_selector_filters_results) { TestableSeismicSQIndex index(QuantizerType::QT_8bit, 0.0F, 1.0F, 10, 3, 0.5F, 3); From a64653cbf13810f39de43ad2e0ed82aaf0c64f20 Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 2 Jul 2026 14:43:22 +0800 Subject: [PATCH 2/4] Fix a bug Signed-off-by: Liyun Xiu --- nsparse/inverted_index.cpp | 4 + tests/CMakeLists.txt | 1 + tests/inverted_index_recall_test.cpp | 266 +++++++++++++++++++++++++++ tests/inverted_index_test.cpp | 53 ++++++ 4 files changed, 324 insertions(+) create mode 100644 tests/inverted_index_recall_test.cpp diff --git a/nsparse/inverted_index.cpp b/nsparse/inverted_index.cpp index 7813bca..806e837 100644 --- a/nsparse/inverted_index.cpp +++ b/nsparse/inverted_index.cpp @@ -135,6 +135,10 @@ void score_essential_terms(std::vector& scorers, if (doc >= window_end) { break; } + if (doc < window_base) { + ++scorer.current_index; + continue; + } int slot = doc - window_base; bitmap[slot >> 6] |= (1ULL << (slot & 63)); window_scores[slot] += diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2f58059..2d59fa7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,7 @@ set(NSPARSE_TEST_SRC index_factory_test.cpp index_io_test.cpp inverted_index_test.cpp + inverted_index_recall_test.cpp inverted_list_clusters_test.cpp inverted_lists_test.cpp kmeans_utils_test.cpp diff --git a/tests/inverted_index_recall_test.cpp b/tests/inverted_index_recall_test.cpp new file mode 100644 index 0000000..80f0529 --- /dev/null +++ b/tests/inverted_index_recall_test.cpp @@ -0,0 +1,266 @@ +/** + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "nsparse/inverted_index.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "nsparse/types.h" + +namespace nsparse { +namespace { + +// Brute-force top-k: compute dot product for all docs and return top-k doc ids. +std::vector brute_force_top_k( + const std::vector>>& docs, + const std::vector>& query, int k) { + std::vector> scores; + for (idx_t d = 0; d < static_cast(docs.size()); ++d) { + float score = 0.0F; + // Build a quick lookup for doc terms. + std::map doc_map; + for (auto& [t, v] : docs[d]) { + doc_map[t] = v; + } + for (auto& [qt, qv] : query) { + auto it = doc_map.find(qt); + if (it != doc_map.end()) { + score += qv * it->second; + } + } + if (score > 0.0F) { + scores.emplace_back(score, d); + } + } + // Sort descending by score, then ascending by doc_id for tie-breaking. + std::sort(scores.begin(), scores.end(), + [](const auto& a, const auto& b) { + if (a.first != b.first) return a.first > b.first; + return a.second < b.second; + }); + std::vector result; + for (int i = 0; i < k && i < static_cast(scores.size()); ++i) { + result.push_back(scores[i].second); + } + return result; +} + +// Generate random sparse documents. +struct RandomSparseData { + int dim; + int n_docs; + int n_queries; + int avg_nnz; + std::vector>> docs; + std::vector>> queries; +}; + +RandomSparseData generate_random_data(int dim, int n_docs, int n_queries, + int avg_nnz, unsigned seed = 42) { + std::mt19937 rng(seed); + std::uniform_int_distribution nnz_dist( + std::max(1, avg_nnz / 2), avg_nnz * 2); + std::uniform_int_distribution term_dist(0, dim - 1); + std::uniform_real_distribution val_dist(0.01F, 2.0F); + + RandomSparseData data; + data.dim = dim; + data.n_docs = n_docs; + data.n_queries = n_queries; + data.avg_nnz = avg_nnz; + + // Generate documents. + for (int d = 0; d < n_docs; ++d) { + int nnz = nnz_dist(rng); + std::set used_terms; + std::vector> doc; + for (int j = 0; j < nnz; ++j) { + term_t t = static_cast(term_dist(rng)); + if (used_terms.insert(t).second) { + doc.emplace_back(t, val_dist(rng)); + } + } + // Sort by term for consistent ordering. + std::sort(doc.begin(), doc.end()); + data.docs.push_back(std::move(doc)); + } + + // Generate queries. + for (int q = 0; q < n_queries; ++q) { + int nnz = nnz_dist(rng); + std::set used_terms; + std::vector> query; + for (int j = 0; j < nnz; ++j) { + term_t t = static_cast(term_dist(rng)); + if (used_terms.insert(t).second) { + query.emplace_back(t, val_dist(rng)); + } + } + std::sort(query.begin(), query.end()); + data.queries.push_back(std::move(query)); + } + + return data; +} + +void add_docs_from_data( + InvertedIndex& index, + const std::vector>>& docs) { + std::vector indptr; + std::vector indices; + std::vector values; + + indptr.push_back(0); + for (const auto& doc : docs) { + for (const auto& [term, value] : doc) { + indices.push_back(term); + values.push_back(value); + } + indptr.push_back(static_cast(indices.size())); + } + + index.add(static_cast(docs.size()), indptr.data(), indices.data(), + values.data()); +} + +float compute_recall(const std::vector& predicted, + const std::vector& ground_truth) { + std::set gt_set(ground_truth.begin(), ground_truth.end()); + int hits = 0; + for (auto id : predicted) { + if (id != INVALID_IDX && gt_set.count(id)) { + ++hits; + } + } + return ground_truth.empty() + ? 1.0F + : static_cast(hits) / static_cast(gt_set.size()); +} + +// Test recall with various dataset sizes and k values. +class InvertedIndexRecallTest + : public ::testing::TestWithParam> {}; + +TEST_P(InvertedIndexRecallTest, recall_should_be_perfect) { + auto [n_docs, dim, avg_nnz, k] = GetParam(); + int n_queries = 50; + + auto data = generate_random_data(dim, n_docs, n_queries, avg_nnz); + + InvertedIndex index(dim); + add_docs_from_data(index, data.docs); + index.build(); + Index* idx = &index; + + float total_recall = 0.0F; + int n_valid_queries = 0; + + for (int q = 0; q < n_queries; ++q) { + const auto& query = data.queries[q]; + if (query.empty()) continue; + + // Build query CSR. + std::vector q_indptr = {0, + static_cast(query.size())}; + std::vector q_indices; + std::vector q_values; + for (auto& [t, v] : query) { + q_indices.push_back(t); + q_values.push_back(v); + } + + std::vector labels(k, INVALID_IDX); + std::vector distances(k, -1.0F); + + idx->search(1, q_indptr.data(), q_indices.data(), q_values.data(), k, + distances.data(), labels.data()); + + // Brute-force ground truth. + auto gt = brute_force_top_k(data.docs, query, k); + + float recall = compute_recall( + std::vector(labels.begin(), labels.end()), gt); + + total_recall += recall; + ++n_valid_queries; + + // Print failing queries for debugging. + if (recall < 1.0F) { + std::cout << "Query " << q << ": recall=" << recall + << " predicted=["; + for (int i = 0; i < k; ++i) { + if (i > 0) std::cout << ","; + std::cout << labels[i]; + } + std::cout << "] gt=["; + for (size_t i = 0; i < gt.size(); ++i) { + if (i > 0) std::cout << ","; + std::cout << gt[i]; + } + std::cout << "]" << std::endl; + + // Print scores for debugging. + std::cout << " predicted_scores=["; + for (int i = 0; i < k; ++i) { + if (i > 0) std::cout << ","; + std::cout << distances[i]; + } + std::cout << "]" << std::endl; + + // Print ground truth scores. + std::cout << " gt_scores=["; + for (size_t i = 0; i < gt.size(); ++i) { + if (i > 0) std::cout << ","; + // Compute score for gt[i]. + float score = 0.0F; + std::map doc_map; + for (auto& [t, v] : data.docs[gt[i]]) doc_map[t] = v; + for (auto& [qt, qv] : query) { + auto it = doc_map.find(qt); + if (it != doc_map.end()) score += qv * it->second; + } + std::cout << score; + } + std::cout << "]" << std::endl; + } + } + + float avg_recall = + n_valid_queries > 0 ? total_recall / n_valid_queries : 1.0F; + std::cout << "Average recall: " << avg_recall << " (n_docs=" << n_docs + << ", dim=" << dim << ", avg_nnz=" << avg_nnz << ", k=" << k + << ")" << std::endl; + + // InvertedIndex should have PERFECT recall (it's an exact algorithm). + EXPECT_FLOAT_EQ(avg_recall, 1.0F) + << "InvertedIndex recall should be 1.0 but got " << avg_recall; +} + +INSTANTIATE_TEST_SUITE_P( + RecallTests, InvertedIndexRecallTest, + ::testing::Values( + // (n_docs, dim, avg_nnz, k) + // Small: single window, no pruning. + std::make_tuple(100, 500, 10, 10), + // Medium: multiple windows, pruning active. + std::make_tuple(5000, 1000, 20, 10), + // Large: many windows, aggressive pruning. + std::make_tuple(10000, 2000, 30, 10), + // Large with higher k. + std::make_tuple(10000, 2000, 30, 50), + // Dense queries, many terms. + std::make_tuple(5000, 500, 50, 10), + // Very sparse queries and docs. + std::make_tuple(10000, 5000, 5, 10))); + +} // namespace +} // namespace nsparse diff --git a/tests/inverted_index_test.cpp b/tests/inverted_index_test.cpp index 427d9d2..95f379d 100644 --- a/tests/inverted_index_test.cpp +++ b/tests/inverted_index_test.cpp @@ -411,5 +411,58 @@ TEST(InvertedIndexIO, roundtrip_preserves_fourcc) { delete loaded; } +// Regression test: score_essential_terms must skip docs before window_base. +// When scorers span multiple windows (>4096 docs) and re-partitioning changes +// first_essential, a newly-essential scorer may have its cursor pointing at +// docs before the current window. Without the bounds check, this produces a +// negative slot index causing SIGBUS / out-of-bounds write. +TEST(InvertedIndexSearch, search_multi_window_no_crash) { + // Use 2 terms. term0 has docs in early range, term1 has docs spread across + // multiple windows (>4096). This forces the BMW algorithm to process + // multiple scoring windows and potentially re-partition essential terms. + constexpr int kDim = 2; + constexpr int kNumDocs = 6000; // > kScoreWindowSize (4096) + InvertedIndex index(kDim); + + // Build docs: every doc has term0, every 3rd doc also has term1. + // This creates different posting list lengths that affect max_score + // partitioning. + std::vector> docs; + docs.reserve(kNumDocs); + for (int i = 0; i < kNumDocs; ++i) { + std::map doc; + doc[0] = 0.1F + static_cast(i % 10) * 0.01F; + if (i % 3 == 0) { + doc[1] = 0.5F + static_cast(i % 7) * 0.05F; + } + docs.push_back(doc); + } + add_docs(index, docs); + index.build(); + + // Query both terms — forces multi-term scoring across multiple windows. + std::vector query_indptr = {0, 2}; + std::vector query_indices = {0, 1}; + std::vector query_values = {1.0F, 1.0F}; + + constexpr int kTopK = 10; + std::vector labels(kTopK, -1); + std::vector distances(kTopK, -1.0F); + + Index* idx = &index; + // The main assertion is that this does not crash (SIGBUS). + idx->search(1, query_indptr.data(), query_indices.data(), + query_values.data(), kTopK, distances.data(), labels.data()); + + // Verify we got valid results. + EXPECT_GE(labels[0], 0); + EXPECT_GT(distances[0], 0.0F); + + // Results should be sorted by score descending. + for (int i = 1; i < kTopK; ++i) { + EXPECT_GE(distances[i - 1], distances[i]); + } +} + } // namespace } // namespace nsparse From 3d5569c2cb274c24e62ddef9d8c6fb9db686a473 Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 2 Jul 2026 15:02:37 +0800 Subject: [PATCH 3/4] Pin tj-actions/changed-files to commit SHA in check-files action Signed-off-by: Liyun Xiu --- .github/actions/check-files/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/check-files/action.yml b/.github/actions/check-files/action.yml index a4f1e61..303191a 100644 --- a/.github/actions/check-files/action.yml +++ b/.github/actions/check-files/action.yml @@ -24,7 +24,7 @@ runs: echo "combined_files=$COMBINED_FILES" >> $GITHUB_OUTPUT - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v47.0.0 + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: files: ${{ steps.combine-files.outputs.combined_files }} files_separator: "," From 5bc8d61afdce1431b16af8a2ca9e2a13e8eb962b Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 2 Jul 2026 15:10:24 +0800 Subject: [PATCH 4/4] Pin transitive GitHub Actions to commit SHAs to satisfy org policy Signed-off-by: Liyun Xiu --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7cba0d5..db13712 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -26,7 +26,7 @@ jobs: Get-CI-Image-Tag: needs: check-files if: needs.check-files.outputs.RUN_BUILD_AND_TEST == 'true' - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@6631dc102f1e64c182eccfc48a319a87bc9079f6 # main with: product: opensearch