diff --git a/benchmarks/batched_build_mem_bench.cpp b/benchmarks/batched_build_mem_bench.cpp index efc7184..b35adba 100644 --- a/benchmarks/batched_build_mem_bench.cpp +++ b/benchmarks/batched_build_mem_bench.cpp @@ -216,10 +216,10 @@ void streaming_add(nsparse::Index* index, const std::string& path) { const int64_t bnnz = indptr64[row_end] - indptr64[row_start]; const int64_t boff = indptr64[row_start]; - std::vector bindptr(brows + 1); + std::vector bindptr(brows + 1); for (int64_t i = 0; i <= brows; ++i) { bindptr[i] = - static_cast(indptr64[row_start + i] - boff); + static_cast(indptr64[row_start + i] - boff); } std::vector bindices(bnnz); { diff --git a/benchmarks/index_build_benchmark.cpp b/benchmarks/index_build_benchmark.cpp index d0aee4f..7516886 100644 --- a/benchmarks/index_build_benchmark.cpp +++ b/benchmarks/index_build_benchmark.cpp @@ -48,7 +48,7 @@ struct CSRMatrix { int64_t nrow; int64_t ncol; int64_t nnz; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector data; }; @@ -71,7 +71,7 @@ CSRMatrix read_csr(const std::string& path) { static_cast((m.nrow + 1) * sizeof(int64_t))); m.indptr.resize(m.nrow + 1); for (int64_t i = 0; i <= m.nrow; ++i) { - m.indptr[i] = static_cast(indptr64[i]); + m.indptr[i] = static_cast(indptr64[i]); } std::vector indices32(m.nnz); @@ -112,7 +112,7 @@ const CSRMatrix& shared_data() { } // Seismic cluster parameters comparable to the search benchmark's index. -constexpr nsparse::SeismicClusterParameters kParams = { +const nsparse::SeismicClusterParameters kParams = { .lambda = 6000, .beta = 400, .alpha = 0.4F}; // Builds a fresh SeismicIndex from the shared corpus and times only build(). diff --git a/benchmarks/index_search_benchmark.cpp b/benchmarks/index_search_benchmark.cpp index 5f6bda6..9cd35cf 100644 --- a/benchmarks/index_search_benchmark.cpp +++ b/benchmarks/index_search_benchmark.cpp @@ -29,7 +29,7 @@ struct CSRMatrix { int64_t nrow; int64_t ncol; int64_t nnz; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector data; }; @@ -52,7 +52,7 @@ CSRMatrix read_csr(const std::string& path) { static_cast((m.nrow + 1) * sizeof(int64_t))); m.indptr.resize(m.nrow + 1); for (int64_t i = 0; i <= m.nrow; ++i) { - m.indptr[i] = static_cast(indptr64[i]); + m.indptr[i] = static_cast(indptr64[i]); } std::vector indices32(m.nnz); diff --git a/benchmarks/sq_residency_bench.cpp b/benchmarks/sq_residency_bench.cpp index 9d4797a..5fd36d5 100644 --- a/benchmarks/sq_residency_bench.cpp +++ b/benchmarks/sq_residency_bench.cpp @@ -48,7 +48,7 @@ struct CSRMatrix { int64_t nrow = 0; int64_t ncol = 0; int64_t nnz = 0; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector data; }; @@ -70,7 +70,7 @@ CSRMatrix read_csr(const std::string& path) { static_cast((m.nrow + 1) * sizeof(int64_t))); m.indptr.resize(m.nrow + 1); for (int64_t i = 0; i <= m.nrow; ++i) { - m.indptr[i] = static_cast(indptr64[i]); + m.indptr[i] = static_cast(indptr64[i]); } std::vector indices32(m.nnz); @@ -266,9 +266,9 @@ int do_search(int argc, char** argv) { std::vector per_query_ms; per_query_ms.reserve(static_cast(n_queries)); for (int qi = 0; qi < n_queries; ++qi) { - const nsparse::idx_t start = query.indptr[qi]; - const nsparse::idx_t end = query.indptr[qi + 1]; - std::vector q_indptr = {0, end - start}; + const nsparse::offset_t start = query.indptr[qi]; + const nsparse::offset_t end = query.indptr[qi + 1]; + std::vector q_indptr = {0, end - start}; const auto t0 = std::chrono::steady_clock::now(); index->search(1, q_indptr.data(), query.indices.data() + start, query.data.data() + start, k, distances.data(), diff --git a/nsparse/brutal_index.cpp b/nsparse/brutal_index.cpp index 6f2a05f..8a7e3a5 100644 --- a/nsparse/brutal_index.cpp +++ b/nsparse/brutal_index.cpp @@ -24,7 +24,7 @@ namespace nsparse { BrutalIndex::BrutalIndex(int dim) : Index(dim) {} -void BrutalIndex::add(idx_t n, const idx_t* indptr, const term_t* indices, +void BrutalIndex::add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) { throw_if_not_positive(n); throw_if_any_null(indptr, indices, values); @@ -42,7 +42,7 @@ void BrutalIndex::add(idx_t n, const idx_t* indptr, const term_t* indices, nnz * element_size); } -auto BrutalIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, +auto BrutalIndex::search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters) -> pair_of_score_id_vectors_t { @@ -84,7 +84,7 @@ auto BrutalIndex::single_query(const std::vector& dense, int k) const auto& [indptr, indices, values] = vectors_->get_all_data(); for (size_t i = 0; i < num_docs; ++i) { - const idx_t start = indptr[i]; + const offset_t start = indptr[i]; const size_t len = indptr[i + 1] - start; float score = detail::dot_product_float_dense( indices + start, values + start, len, dense.data()); diff --git a/nsparse/brutal_index.h b/nsparse/brutal_index.h index 4c18bee..a66e048 100644 --- a/nsparse/brutal_index.h +++ b/nsparse/brutal_index.h @@ -25,14 +25,14 @@ class BrutalIndex : public Index { BrutalIndex(const BrutalIndex&) = delete; BrutalIndex& operator=(const BrutalIndex&) = delete; - void add(idx_t n, const idx_t* indptr, const term_t* indices, + void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) override; std::array id() const override { return name; } static constexpr std::array name = {'B', 'R', 'U', 'T'}; protected: - auto search(idx_t n, const idx_t* indptr, const term_t* indices, + auto search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters = nullptr) -> pair_of_score_id_vectors_t override; diff --git a/nsparse/cluster/inverted_list_clusters.cpp b/nsparse/cluster/inverted_list_clusters.cpp index 1bb2e68..c8f637a 100644 --- a/nsparse/cluster/inverted_list_clusters.cpp +++ b/nsparse/cluster/inverted_list_clusters.cpp @@ -93,9 +93,9 @@ SparseVectors summarize_with_cpu_(const SparseVectors* vectors, auto doc_ids = std::span( group_of_doc_ids.data() + offsets[i], offsets[i + 1] - offsets[i]); for (const auto& doc_id : doc_ids) { - int start = indptr_data[doc_id]; - int end = indptr_data[doc_id + 1]; - for (size_t j = start; j < end; ++j) { + offset_t start = indptr_data[doc_id]; + offset_t end = indptr_data[doc_id + 1]; + for (offset_t j = start; j < end; ++j) { const term_t term = indices_data[j]; // j is element index, need byte offset for T access const T v = @@ -280,9 +280,9 @@ void InvertedListClusters::build_transpose(const SparseVectors& summaries) { std::vector csc_value(nnz * esz); std::vector cursor(term_ptr.begin(), term_ptr.end() - 1); for (size_t cluster = 0; cluster < n_clusters_; ++cluster) { - const idx_t start = indptr[cluster]; - const idx_t end = indptr[cluster + 1]; - for (idx_t j = start; j < end; ++j) { + const offset_t start = indptr[cluster]; + const offset_t end = indptr[cluster + 1]; + for (offset_t j = start; j < end; ++j) { const size_t col = term_column(indices[j]); const idx_t pos = cursor[col]++; csc_cluster[pos] = static_cast(cluster); diff --git a/nsparse/cluster/kmeans_utils.cpp b/nsparse/cluster/kmeans_utils.cpp index 5247182..521ec56 100644 --- a/nsparse/cluster/kmeans_utils.cpp +++ b/nsparse/cluster/kmeans_utils.cpp @@ -52,7 +52,7 @@ template CentroidIndex build_centroid_index( const SparseVectors* vectors, const std::vector>& clusters) { - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const T* values = vectors->typed_values_data(); const size_t n_clusters = clusters.size(); @@ -63,7 +63,7 @@ CentroidIndex build_centroid_index( size_t nnz = 0; for (const auto& cluster : clusters) { const idx_t centroid = cluster.at(0); - for (idx_t j = indptr[centroid]; j < indptr[centroid + 1]; ++j) { + for (offset_t j = indptr[centroid]; j < indptr[centroid + 1]; ++j) { max_term = std::max(max_term, indices[j]); ++nnz; } @@ -74,7 +74,7 @@ CentroidIndex build_centroid_index( index.term_ptr.assign(index.n_cols + 1, 0); for (const auto& cluster : clusters) { const idx_t centroid = cluster.at(0); - for (idx_t j = indptr[centroid]; j < indptr[centroid + 1]; ++j) { + for (offset_t j = indptr[centroid]; j < indptr[centroid + 1]; ++j) { index.term_ptr[indices[j] + 1]++; } } @@ -87,7 +87,7 @@ CentroidIndex build_centroid_index( std::vector cursor(index.term_ptr.begin(), index.term_ptr.end() - 1); for (size_t c = 0; c < n_clusters; ++c) { const idx_t centroid = clusters[c].at(0); - for (idx_t j = indptr[centroid]; j < indptr[centroid + 1]; ++j) { + for (offset_t j = indptr[centroid]; j < indptr[centroid + 1]; ++j) { const idx_t pos = cursor[indices[j]]++; index.cluster[pos] = static_cast(c); index.weight[pos] = values[j]; @@ -108,7 +108,7 @@ template void map_docs_to_clusters_typed(const SparseVectors* vectors, const std::vector& docs, std::vector>& clusters) { - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const T* values = vectors->typed_values_data(); const size_t n_clusters = clusters.size(); @@ -131,7 +131,7 @@ void map_docs_to_clusters_typed(const SparseVectors* vectors, continue; } std::ranges::fill(similarities, acc_t(0)); - for (idx_t j = indptr[doc_id]; j < indptr[doc_id + 1]; ++j) { + for (offset_t j = indptr[doc_id]; j < indptr[doc_id + 1]; ++j) { const size_t term = indices[j]; if (term >= index.n_cols) { continue; // no centroid carries this term diff --git a/nsparse/disk_seismic_index_base.cpp b/nsparse/disk_seismic_index_base.cpp index 8e70603..256b931 100644 --- a/nsparse/disk_seismic_index_base.cpp +++ b/nsparse/disk_seismic_index_base.cpp @@ -42,7 +42,7 @@ DiskSeismicIndexBase::DiskSeismicIndexBase(int dim, SeismicClusterParameters parameter) : MmapIndex(dim), cluster_parameter_(parameter) {} -void DiskSeismicIndexBase::add(idx_t n, const idx_t* indptr, +void DiskSeismicIndexBase::add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) { throw_if_not_positive(n); throw_if_any_null(indptr, indices, values); @@ -70,7 +70,7 @@ void DiskSeismicIndexBase::build() { &batch_spill_); } -auto DiskSeismicIndexBase::search(idx_t n, const idx_t* indptr, +auto DiskSeismicIndexBase::search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters) -> pair_of_score_id_vectors_t { @@ -174,7 +174,7 @@ auto DiskSeismicIndexBase::search(idx_t n, const idx_t* indptr, #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { - const idx_t start = indptr[query_idx]; + const offset_t start = indptr[query_idx]; const size_t len = indptr[query_idx + 1] - start; const term_t* query_indices = indices + start; const uint8_t* query_codes = @@ -251,7 +251,7 @@ void DiskSeismicIndexBase::write_doc_directory( SparseVectors remainder( {.element_size = element_size, .dimension = static_cast(get_dimension())}); - const idx_t* indptr = vectors.indptr_data(); + const offset_t* indptr = vectors.indptr_data(); const term_t* indices = vectors.indices_data(); const uint8_t* values = vectors.values_data(); uint32_t remainder_row = 0; @@ -259,9 +259,9 @@ void DiskSeismicIndexBase::write_doc_directory( if (covered[doc_id]) { continue; } - const idx_t start = indptr[doc_id]; + const offset_t start = indptr[doc_id]; const size_t nnz = static_cast(indptr[doc_id + 1] - start); - const idx_t row_indptr[2] = {0, static_cast(nnz)}; + const offset_t row_indptr[2] = {0, static_cast(nnz)}; remainder.add_vectors( row_indptr, 2, indices + start, nnz, values + static_cast(start) * element_size, @@ -334,8 +334,8 @@ auto DiskSeismicIndexBase::get_doc(idx_t doc_id, size_t element_size) const throw std::runtime_error( "DiskSeismic exact match: remainder locator out of range"); } - const idx_t* r_indptr = remainder_.indptr_data(); - const idx_t r_start = r_indptr[loc.block]; + const offset_t* r_indptr = remainder_.indptr_data(); + const offset_t r_start = r_indptr[loc.block]; return {remainder_.indices_data() + r_start, remainder_.values_data() + static_cast(r_start) * element_size, @@ -352,7 +352,7 @@ auto DiskSeismicIndexBase::get_doc(idx_t doc_id, size_t element_size) const } auto DiskSeismicIndexBase::exact_match_directory( - idx_t n, const idx_t* indptr, const term_t* indices, const float* values, + idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, const IDSelectorEnumerable& selector, const SearchParameters* search_parameters) const -> pair_of_score_id_vectors_t { @@ -374,7 +374,7 @@ auto DiskSeismicIndexBase::exact_match_directory( std::vector dense(dense_bytes, 0); #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { - const idx_t start = indptr[query_idx]; + const offset_t start = indptr[query_idx]; const size_t len = static_cast(indptr[query_idx + 1] - start); const term_t* q_indices = indices + start; @@ -396,8 +396,8 @@ auto DiskSeismicIndexBase::exact_match_directory( const DocSlice doc = get_doc(doc_id, element_size); // Dot the doc's slice against the dense query via a 2-entry // indptr. - const idx_t slice_indptr[2] = {0, - static_cast(doc.nnz)}; + const offset_t slice_indptr[2] = { + 0, static_cast(doc.nnz)}; const float score = detail::compute_similarity( 0, slice_indptr, doc.comps, doc.vals, dense.data(), element_size); diff --git a/nsparse/disk_seismic_index_base.h b/nsparse/disk_seismic_index_base.h index 74353a7..6a44693 100644 --- a/nsparse/disk_seismic_index_base.h +++ b/nsparse/disk_seismic_index_base.h @@ -56,7 +56,7 @@ class DiskSeismicIndexBase : public MmapIndex, public IndexIO { // Persisted, since a mapped index has no in-RAM vectors_ to derive it from. size_t num_vectors() const override { return num_vectors_; } - void add(idx_t n, const idx_t* indptr, const term_t* indices, + void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) override; void build() override; @@ -123,7 +123,7 @@ class DiskSeismicIndexBase : public MmapIndex, public IndexIO { detail::InlineForwardIndex fwd_; private: - auto search(idx_t n, const idx_t* indptr, const term_t* indices, + auto search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters = nullptr) -> pair_of_score_id_vectors_t override; @@ -152,7 +152,7 @@ class DiskSeismicIndexBase : public MmapIndex, public IndexIO { // Scores every selected doc directly through the doc-locator directory, for // a mapped index. Requires doc_locators_ populated. [[nodiscard]] auto exact_match_directory( - idx_t n, const idx_t* indptr, const term_t* indices, + idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, const IDSelectorEnumerable& selector, const SearchParameters* search_parameters) const -> pair_of_score_id_vectors_t; diff --git a/nsparse/disk_seismic_search.cpp b/nsparse/disk_seismic_search.cpp index aeffad9..c0807c1 100644 --- a/nsparse/disk_seismic_search.cpp +++ b/nsparse/disk_seismic_search.cpp @@ -77,11 +77,11 @@ void score_block(const InlineForwardIndex* fwd, const SparseVectors* vectors, element_size, id_selector, heap, visited); } } else if (vectors != nullptr) { - const idx_t* const indptr = vectors->indptr_data(); + const offset_t* const indptr = vectors->indptr_data(); const term_t* const indices = vectors->indices_data(); const uint8_t* const values = vectors->values_data(); for (const idx_t doc_id : clusters[pl].get_docs(cid)) { - const idx_t start = indptr[doc_id]; + const offset_t start = indptr[doc_id]; const size_t len = indptr[doc_id + 1] - start; score_doc(doc_id, indices + start, values + static_cast(start) * element_size, len, diff --git a/nsparse/gpu/gpu_cluster_assigner.cu b/nsparse/gpu/gpu_cluster_assigner.cu index bf02a8a..8a79f13 100644 --- a/nsparse/gpu/gpu_cluster_assigner.cu +++ b/nsparse/gpu/gpu_cluster_assigner.cu @@ -183,7 +183,7 @@ void GpuClusterAssigner::assign(const SparseVectors* vectors, return; } - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const size_t dim = vectors->get_dimension(); // Centroids are clusters[j].front(); collect them and record which input diff --git a/nsparse/gpu/gpu_common.cuh b/nsparse/gpu/gpu_common.cuh index 9fcfe5e..b736390 100644 --- a/nsparse/gpu/gpu_common.cuh +++ b/nsparse/gpu/gpu_common.cuh @@ -89,17 +89,29 @@ public: // are a cheap identity check. const DeviceCorpus& ensure_resident(const SparseVectors* vectors) { const size_t n_vectors = vectors->num_vectors(); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const float* values = vectors->values_data_float(); const int64_t nnz = indptr[n_vectors]; + // The device stores CSR offsets as int32 (and cuSPARSE's CSR API is + // 32-bit), so this path cannot represent nnz > INT32_MAX; rebuild such a + // corpus with the CPU path. + if (nnz > INT32_MAX) { + throw std::runtime_error( + "GPU build path requires nnz <= INT32_MAX; use the CPU path"); + } + std::lock_guard lock(mutex_); if (corpus_.matches(vectors, n_vectors, nnz)) { return corpus_; } free_locked(); + std::vector indptr32(n_vectors + 1); + for (size_t i = 0; i <= n_vectors; ++i) { + indptr32[i] = static_cast(indptr[i]); + } std::vector indices32(static_cast(nnz)); for (int64_t i = 0; i < nnz; ++i) { indices32[i] = static_cast(indices[i]); @@ -110,7 +122,7 @@ public: "cudaMalloc(corpus.indices)"); check_cuda(cudaMalloc(&corpus_.values, nnz * sizeof(float)), "cudaMalloc(corpus.values)"); - check_cuda(cudaMemcpy(corpus_.indptr, indptr, + check_cuda(cudaMemcpy(corpus_.indptr, indptr32.data(), (n_vectors + 1) * sizeof(int32_t), cudaMemcpyHostToDevice), "cudaMemcpy(corpus.indptr)"); diff --git a/nsparse/gpu/gpu_summarizer.cu b/nsparse/gpu/gpu_summarizer.cu index 3bffdda..f89fe53 100644 --- a/nsparse/gpu/gpu_summarizer.cu +++ b/nsparse/gpu/gpu_summarizer.cu @@ -165,7 +165,7 @@ bool summarize_list_impl(const SparseVectors* vectors, const idx_t* docs, const idx_t* offsets, size_t n_clusters, std::vector& out) { const size_t dim = vectors->get_dimension(); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const size_t n_docs = static_cast(offsets[n_clusters] - offsets[0]); if (n_docs == 0) { diff --git a/nsparse/id_map_index.cpp b/nsparse/id_map_index.cpp index 509f17f..3cf6c31 100644 --- a/nsparse/id_map_index.cpp +++ b/nsparse/id_map_index.cpp @@ -27,14 +27,14 @@ namespace nsparse { IDMapIndex::IDMapIndex(Index* index) : delegate_(index) {} -void IDMapIndex::add(idx_t n, const idx_t* indptr, const term_t* indices, +void IDMapIndex::add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) { delegate_->add(n, indptr, indices, values); } void IDMapIndex::build() { delegate_->build(); } -void IDMapIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, +void IDMapIndex::search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, float* distances, idx_t* labels, SearchParameters* search_parameters) { std::unique_ptr id_selector_idmap = nullptr; @@ -75,7 +75,7 @@ size_t IDMapIndex::num_vectors() const { return delegate_ == nullptr ? 0 : delegate_->num_vectors(); } -void IDMapIndex::add_with_ids(idx_t n, const idx_t* indptr, +void IDMapIndex::add_with_ids(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, const idx_t* ids) { size_t old_size = delegate_->num_vectors(); diff --git a/nsparse/id_map_index.h b/nsparse/id_map_index.h index d448c42..888ecb6 100644 --- a/nsparse/id_map_index.h +++ b/nsparse/id_map_index.h @@ -88,10 +88,10 @@ class IDMapIndex : public Index, public IndexIO { explicit IDMapIndex(Index*); std::array id() const override { return name; } - void add(idx_t n, const idx_t* indptr, const term_t* indices, + void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) override; void build() override; - void search(idx_t n, const idx_t* indptr, const term_t* indices, + void search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, float* distances, idx_t* labels, SearchParameters* search_parameters = nullptr) override; const SparseVectors* get_vectors() const override; @@ -99,7 +99,7 @@ class IDMapIndex : public Index, public IndexIO { // exposing its vectors. size_t num_vectors() const override; - void add_with_ids(idx_t n, const idx_t* indptr, const term_t* indices, + void add_with_ids(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, const idx_t* ids) override; void read_csr_and_ids(const char* csr_path, const char* id_path, diff --git a/nsparse/index.cpp b/nsparse/index.cpp index c01f063..9c8fd62 100644 --- a/nsparse/index.cpp +++ b/nsparse/index.cpp @@ -29,7 +29,7 @@ Index::Index(int dim) : dimension_(dim) {} void Index::build() { throw_not_implemented(); } -void Index::search(idx_t n, const idx_t* indptr, const term_t* indices, +void Index::search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, float* distances, idx_t* labels, SearchParameters* search_parameters) { throw_if_not_positive(n); @@ -48,14 +48,14 @@ void Index::search(idx_t n, const idx_t* indptr, const term_t* indices, } } -auto Index::search(idx_t n, const idx_t* indptr, const term_t* indices, +auto Index::search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters) -> pair_of_score_id_vectors_t { throw_not_implemented("search not implementted in Index"); } -void Index::add_with_ids(idx_t n, const idx_t* indptr, const term_t* indices, +void Index::add_with_ids(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, const idx_t* ids) { throw_not_implemented("add_with_ids not implemented in Index"); } @@ -92,10 +92,10 @@ void Index::read_csr(const char* file_path, Residency residency) { throw std::invalid_argument(std::string("Invalid CSR header in: ") + file_path); } - if (num_rows > std::numeric_limits::max() || - nnz > std::numeric_limits::max()) { - throw std::invalid_argument(std::string("CSR file too large for ") + - "32-bit offsets: " + file_path); + if (num_rows > std::numeric_limits::max()) { + throw std::invalid_argument( + std::string("CSR row count exceeds 32-bit doc-id range: ") + + file_path); } if (num_cols > dimension_) { throw std::invalid_argument( @@ -119,7 +119,6 @@ void Index::read_csr(const char* file_path, Residency residency) { throw std::invalid_argument( std::string("Inconsistent CSR indptr in: ") + file_path); } - std::vector indptr(file_indptr.begin(), file_indptr.end()); std::vector file_indices(nnz_size); read_or_throw(file_indices.data(), file_indices.size() * sizeof(int32_t)); @@ -137,7 +136,7 @@ void Index::read_csr(const char* file_path, Residency residency) { std::vector values(nnz_size); read_or_throw(values.data(), values.size() * sizeof(float)); - add(static_cast(num_rows), indptr.data(), indices.data(), + add(static_cast(num_rows), file_indptr.data(), indices.data(), values.data()); } } // namespace nsparse diff --git a/nsparse/index.h b/nsparse/index.h index 83f194c..fb7153e 100644 --- a/nsparse/index.h +++ b/nsparse/index.h @@ -32,11 +32,11 @@ class Index { explicit Index(int dim = 0); virtual ~Index() = default; virtual std::array id() const = 0; - virtual void add(idx_t n, const idx_t* indptr, const term_t* indices, + virtual void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) = 0; virtual void build(); virtual void search( - idx_t n, const idx_t* indptr, const term_t* indices, + idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, float* distances, idx_t* labels, SearchParameters* search_parameters = nullptr); // Pre-allocated: n * k @@ -50,7 +50,7 @@ class Index { const auto* vectors = get_vectors(); return vectors == nullptr ? 0 : vectors->num_vectors(); } - virtual void add_with_ids(idx_t n, const idx_t* indptr, + virtual void add_with_ids(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, const idx_t* ids); @@ -59,7 +59,7 @@ class Index { virtual void read_csr(const char* file_path, Residency residency = Residency::kInMemory); protected: - virtual auto search(idx_t n, const idx_t* indptr, const term_t* indices, + virtual auto search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters = nullptr) -> pair_of_score_id_vectors_t; diff --git a/nsparse/inverted_index.cpp b/nsparse/inverted_index.cpp index 214b27f..abd84f0 100644 --- a/nsparse/inverted_index.cpp +++ b/nsparse/inverted_index.cpp @@ -245,7 +245,7 @@ void evaluate_window_candidates(std::vector& scorers, InvertedIndex::InvertedIndex(int dim) : MmapIndex(dim) {} -void InvertedIndex::add(idx_t n, const idx_t* indptr, const term_t* indices, +void InvertedIndex::add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) { throw_if_not_positive(n); throw_if_any_null(indptr, indices, values); @@ -294,8 +294,8 @@ void InvertedIndex::build() { max_term_scores_ = Buf::own(std::move(max_term_scores)); } -auto InvertedIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, - const float* values, int k, +auto InvertedIndex::search(idx_t n, const offset_t* indptr, + const term_t* indices, const float* values, int k, SearchParameters* search_parameters) -> pair_of_score_id_vectors_t { if (inverted_lists_ == nullptr || n == 0) { @@ -323,7 +323,7 @@ auto InvertedIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, #pragma omp parallel for for (idx_t query_idx = 0; query_idx < n; ++query_idx) { - const idx_t start = query_indptr[query_idx]; + const offset_t start = query_indptr[query_idx]; const size_t len = query_indptr[query_idx + 1] - start; auto [distances, labels] = single_query( diff --git a/nsparse/inverted_index.h b/nsparse/inverted_index.h index 10d08ad..1822abc 100644 --- a/nsparse/inverted_index.h +++ b/nsparse/inverted_index.h @@ -30,7 +30,7 @@ class InvertedIndex : public MmapIndex, public IndexIO { InvertedIndex(const InvertedIndex&) = delete; InvertedIndex& operator=(const InvertedIndex&) = delete; - void add(idx_t n, const idx_t* indptr, const term_t* indices, + void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) override; void build() override; size_t num_vectors() const override { return num_vectors_; } @@ -63,7 +63,7 @@ class InvertedIndex : public MmapIndex, public IndexIO { const char* index_file, size_t pos); protected: - auto search(idx_t n, const idx_t* indptr, const term_t* indices, + auto search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters = nullptr) -> pair_of_score_id_vectors_t override; diff --git a/nsparse/invlists/inverted_lists.cpp b/nsparse/invlists/inverted_lists.cpp index 7792e07..0de9f7b 100644 --- a/nsparse/invlists/inverted_lists.cpp +++ b/nsparse/invlists/inverted_lists.cpp @@ -233,7 +233,7 @@ std::unique_ptr ArrayInvertedLists::build_inverted_lists( // every entry, and leave the geometric growth's slack behind. std::vector counts(n_term, 0); for (size_t i = 0; i < n_docs; ++i) { - for (idx_t j = indptr_data[i]; j < indptr_data[i + 1]; ++j) { + for (offset_t j = indptr_data[i]; j < indptr_data[i + 1]; ++j) { const term_t term_id = indices_data[j]; if (term_id >= n_term) { throw std::invalid_argument("term_id out of range"); @@ -251,7 +251,7 @@ std::unique_ptr ArrayInvertedLists::build_inverted_lists( // Documents in ascending order, so every posting list comes out sorted by // doc id -- which the search path relies on. for (size_t i = 0; i < n_docs; ++i) { - for (idx_t j = indptr_data[i]; j < indptr_data[i + 1]; ++j) { + for (offset_t j = indptr_data[i]; j < indptr_data[i + 1]; ++j) { const term_t term_id = indices_data[j]; doc_ids[term_id].push_back(static_cast(i)); const uint8_t* code = values_data + (j * element_size); diff --git a/nsparse/io/align.h b/nsparse/io/align.h index ac70cdb..2a03a1c 100644 --- a/nsparse/io/align.h +++ b/nsparse/io/align.h @@ -16,6 +16,7 @@ #include #include "nsparse/io/io.h" +#include "nsparse/types.h" #include "nsparse/utils/buf.h" #include "nsparse/utils/mmap_cursor.h" @@ -36,6 +37,8 @@ namespace nsparse::io_align { // Largest alignment any serialized element needs, and so the most padding a // single call can insert. constexpr size_t kMaxAlignment = 8; +static_assert(alignof(offset_t) <= kMaxAlignment, + "offset_t alignment must fit the array-padding budget"); // Bytes to insert at `pos` to reach an `alignment` boundary. Zero is reachable // input, not a caller bug: the alignment can be an element width read from the diff --git a/nsparse/io/inline_forward_index_io.cpp b/nsparse/io/inline_forward_index_io.cpp index 393ee8c..80f99ca 100644 --- a/nsparse/io/inline_forward_index_io.cpp +++ b/nsparse/io/inline_forward_index_io.cpp @@ -97,7 +97,7 @@ InlineForwardIndex& InlineForwardIndex::operator=( } InlineForwardIndex::BlockCounts InlineForwardIndex::count_block( - std::span docs, const idx_t* indptr, size_t num_vectors) { + std::span docs, const offset_t* indptr, size_t num_vectors) { // n_docs and the within-block offsets are u32 on the wire. if (docs.size() > UINT32_MAX) { throw std::length_error( @@ -133,7 +133,7 @@ uint64_t InlineForwardIndex::section_length() const { "InlineForwardIndex: element_size must be 1, 2, or 4, got " + std::to_string(element_size)); } - const idx_t* indptr = vectors.indptr_data(); + const offset_t* indptr = vectors.indptr_data(); const size_t num_vectors = vectors.num_vectors(); const uint64_t align = write_alignment(); @@ -200,7 +200,7 @@ void InlineForwardIndex::write_body(IOWriter* writer) const { "InlineForwardIndex: element_size must be 1, 2, or 4, got " + std::to_string(element_size)); } - const idx_t* indptr = vectors.indptr_data(); + const offset_t* indptr = vectors.indptr_data(); const term_t* indices = vectors.indices_data(); const uint8_t* values = vectors.values_data(); const size_t num_vectors = vectors.num_vectors(); @@ -269,7 +269,7 @@ void InlineForwardIndex::write_body(IOWriter* writer) const { // comps[] then (pad to element_size) then vals[], each doc's slice // concatenated in block order. for (const idx_t doc_id : docs) { - const idx_t start = indptr[doc_id]; + const offset_t start = indptr[doc_id]; const size_t nnz = indptr[doc_id + 1] - start; if (nnz > 0) { writer->write(const_cast(indices + start), @@ -284,7 +284,7 @@ void InlineForwardIndex::write_body(IOWriter* writer) const { vals_pad); } for (const idx_t doc_id : docs) { - const idx_t start = indptr[doc_id]; + const offset_t start = indptr[doc_id]; const size_t nnz = indptr[doc_id + 1] - start; if (nnz > 0) { writer->write( diff --git a/nsparse/io/inline_forward_index_io.h b/nsparse/io/inline_forward_index_io.h index a356ad5..0d955af 100644 --- a/nsparse/io/inline_forward_index_io.h +++ b/nsparse/io/inline_forward_index_io.h @@ -130,7 +130,7 @@ class InlineForwardIndex : public MmapSerializable { // INT32_MAX so off[] stays idx_t-convertible. Shared by section_length() // (sizing) and write_body() (emitting) so their per-block math can't drift. static BlockCounts count_block(std::span docs, - const idx_t* indptr, size_t num_vectors); + const offset_t* indptr, size_t num_vectors); // Byte length of the section body, from a payload-free sizing pass over // lists + indptr. Lets serialize() write the length prefix without first // rendering the body into a buffer. diff --git a/nsparse/mmap_index.h b/nsparse/mmap_index.h index 42cb109..d09c4ce 100644 --- a/nsparse/mmap_index.h +++ b/nsparse/mmap_index.h @@ -111,10 +111,10 @@ class MmapIndex : public Index { throw std::invalid_argument(std::string("Invalid CSR header in: ") + file_path); } - if (num_rows > std::numeric_limits::max() || - nnz > std::numeric_limits::max()) { - throw std::invalid_argument(std::string("CSR file too large for ") + - "32-bit offsets: " + file_path); + if (num_rows > std::numeric_limits::max()) { + throw std::invalid_argument( + std::string("CSR row count exceeds 32-bit doc-id range: ") + + file_path); } if (num_cols > dimension_) { throw std::invalid_argument( @@ -139,7 +139,7 @@ class MmapIndex : public Index { file_path); } - const auto* indptr = cursor.read_array(indptr_size); + const auto* indptr = cursor.read_array(indptr_size); const auto* indices = cursor.read_array(nnz_size); cursor.skip(csr_layout::native_values_offset(indptr_size, nnz_size) - cursor.pos()); diff --git a/nsparse/python/class_wrappers.py b/nsparse/python/class_wrappers.py index a0be135..6029ea1 100644 --- a/nsparse/python/class_wrappers.py +++ b/nsparse/python/class_wrappers.py @@ -27,7 +27,7 @@ def replacement_search(self, n, indptr, indices, values, k, params=None): n : int Number of query vectors indptr : array_like - CSR indptr array (int32) + CSR indptr array (int32 or int64) indices : array_like CSR indices array (uint16) values : array_like @@ -45,7 +45,7 @@ def replacement_search(self, n, indptr, indices, values, k, params=None): Array of shape (n, k) with neighbor indices """ # Ensure arrays are contiguous with correct dtypes for SWIG typemaps - indptr = np.ascontiguousarray(indptr, dtype=np.int32) + indptr = np.ascontiguousarray(indptr, dtype=np.int64) indices = np.ascontiguousarray(indices, dtype=np.uint16) values = np.ascontiguousarray(values, dtype=np.float32) @@ -69,7 +69,7 @@ def replacement_add_with_ids(self, n, indptr, indices, values, ids): n : int Number of vectors to add indptr : array_like - CSR indptr array (int32) + CSR indptr array (int32 or int64) indices : array_like CSR indices array (uint16) values : array_like @@ -77,7 +77,7 @@ def replacement_add_with_ids(self, n, indptr, indices, values, ids): ids : array_like Custom IDs for the vectors (int32) """ - indptr = np.ascontiguousarray(indptr, dtype=np.int32) + indptr = np.ascontiguousarray(indptr, dtype=np.int64) indices = np.ascontiguousarray(indices, dtype=np.uint16) values = np.ascontiguousarray(values, dtype=np.float32) ids = np.ascontiguousarray(ids, dtype=np.int32) diff --git a/nsparse/python/nsparse_typemaps.i b/nsparse/python/nsparse_typemaps.i index 38176e6..5036009 100644 --- a/nsparse/python/nsparse_typemaps.i +++ b/nsparse/python/nsparse_typemaps.i @@ -25,24 +25,44 @@ } } -%typemap(in)(const nsparse::idx_t* indptr)(Py_buffer view) { +%typemap(in)(const nsparse::offset_t* indptr)(Py_buffer view = {}, + nsparse::offset_t* tmp = 0) { if (PyObject_GetBuffer($input, &view, PyBUF_FORMAT | PyBUF_C_CONTIGUOUS) == -1) { SWIG_fail; } - if (strcmp(view.format, "i") != 0) { + // Accept a 32- or 64-bit signed-int CSR indptr and widen it into the + // offset_t (int64) the C++ API takes; the freearg frees this copy. Require + // an explicit int format ('i'/'l'/'q'): a null or float format is rejected + // rather than reinterpreted by width. + const char* fmt = view.format; + const bool is_i32 = fmt != nullptr && strcmp(fmt, "i") == 0; + const bool is_i64 = fmt != nullptr && + (strcmp(fmt, "l") == 0 || strcmp(fmt, "q") == 0); + if (!is_i32 && !is_i64) { PyBuffer_Release(&view); - PyErr_SetString(PyExc_TypeError, "Expected int32 array for indptr"); + PyErr_SetString(PyExc_TypeError, + "Expected a 32- or 64-bit int array for indptr"); SWIG_fail; } - $1 = (nsparse::idx_t*)view.buf; + const size_t count = view.len / view.itemsize; + tmp = (nsparse::offset_t*)malloc(count * sizeof(nsparse::offset_t)); + if (is_i64) { + const int64_t* src = (const int64_t*)view.buf; + for (size_t i = 0; i < count; ++i) tmp[i] = src[i]; + } else { + const int32_t* src = (const int32_t*)view.buf; + for (size_t i = 0; i < count; ++i) tmp[i] = src[i]; + } + $1 = tmp; } -%typemap(freearg)(const nsparse::idx_t* indptr) { +%typemap(freearg)(const nsparse::offset_t* indptr) { + free(tmp$argnum); PyBuffer_Release(&view$argnum); } -%typemap(in)(const nsparse::idx_t* ids)(Py_buffer view) { +%typemap(in)(const nsparse::idx_t* ids)(Py_buffer view = {}) { if (PyObject_GetBuffer($input, &view, PyBUF_FORMAT | PyBUF_C_CONTIGUOUS) == -1) { SWIG_fail; @@ -59,7 +79,7 @@ PyBuffer_Release(&view$argnum); } -%typemap(in)(const nsparse::term_t* indices)(Py_buffer view) { +%typemap(in)(const nsparse::term_t* indices)(Py_buffer view = {}) { if (PyObject_GetBuffer($input, &view, PyBUF_FORMAT | PyBUF_C_CONTIGUOUS) == -1) { SWIG_fail; @@ -76,7 +96,7 @@ PyBuffer_Release(&view$argnum); } -%typemap(in)(const float* values)(Py_buffer view) { +%typemap(in)(const float* values)(Py_buffer view = {}) { if (PyObject_GetBuffer($input, &view, PyBUF_FORMAT | PyBUF_C_CONTIGUOUS) == -1) { SWIG_fail; @@ -93,7 +113,7 @@ // Multi-argument typemap for the add method signature to ensure proper // validation -%typemap(check)(nsparse::idx_t n, const nsparse::idx_t* indptr, +%typemap(check)(nsparse::idx_t n, const nsparse::offset_t* indptr, const nsparse::term_t* indices, const float* values) { // Validation is handled in C++ code, this just ensures the signature is // recognized @@ -118,7 +138,7 @@ // Use a multi-argument typemap to capture n, k, distances, and labels together // For member functions, self is arg 1, so distances is arg 7, labels is arg 8 -%typemap(check)(nsparse::idx_t n, const nsparse::idx_t* indptr, +%typemap(check)(nsparse::idx_t n, const nsparse::offset_t* indptr, const nsparse::term_t* indices, const float* values, int k, float* distances, nsparse::idx_t* labels) { // Store n and k in the local variables from the labels typemap @@ -235,7 +255,7 @@ // Multi-argument typemap for SeismicIndex::search with cut and heap_factor (8 // args, labels is arg 9 including self) -%typemap(check)(nsparse::idx_t n, const nsparse::idx_t* indptr, +%typemap(check)(nsparse::idx_t n, const nsparse::offset_t* indptr, const nsparse::term_t* indices, const float* values, int k, int cut, float heap_factor, nsparse::idx_t* labels) { n_store9 = $1; // n @@ -251,7 +271,7 @@ // Multi-argument typemap for search with distances, labels, followed by // SearchParameters For member functions, self is arg 1, so distances is arg 7, // labels is arg 8, search_parameters is arg 9 -%typemap(check)(nsparse::idx_t n, const nsparse::idx_t* indptr, +%typemap(check)(nsparse::idx_t n, const nsparse::offset_t* indptr, const nsparse::term_t* indices, const float* values, int k, float* distances, nsparse::idx_t* labels, nsparse::SearchParameters* search_parameters) { diff --git a/nsparse/python/swignsparse.swig b/nsparse/python/swignsparse.swig index 109a8d6..2fa3015 100644 --- a/nsparse/python/swignsparse.swig +++ b/nsparse/python/swignsparse.swig @@ -168,13 +168,13 @@ import_array(); // (labels and distances have numinputs=0, so both would appear as 5-arg functions to Python) %extend nsparse::Index { // Version without SearchParameters - void search_c(idx_t n, const idx_t* indptr, const term_t* indices, + void search_c(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, float* distances, idx_t* labels) { $self->search(n, indptr, indices, values, k, distances, labels, nullptr); } // Version with SearchParameters - different name to avoid overload ambiguity - void search_with_params(idx_t n, const idx_t* indptr, const term_t* indices, + void search_with_params(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, float* distances, idx_t* labels, nsparse::SearchParameters* search_parameters) { diff --git a/nsparse/seismic_common.cpp b/nsparse/seismic_common.cpp index ca024fe..3e83271 100644 --- a/nsparse/seismic_common.cpp +++ b/nsparse/seismic_common.cpp @@ -114,10 +114,10 @@ std::vector make_windows(const std::vector& term_counts, std::vector count_postings_per_term(const SparseVectors& vectors, size_t dim) { std::vector counts(dim, 0); - const idx_t* indptr = vectors.indptr_data(); + const offset_t* indptr = vectors.indptr_data(); const term_t* indices = vectors.indices_data(); - const idx_t nnz = indptr[vectors.num_vectors()]; - for (idx_t j = 0; j < nnz; ++j) { + const offset_t nnz = indptr[vectors.num_vectors()]; + for (offset_t j = 0; j < nnz; ++j) { const size_t term = indices[j]; if (term >= dim) { throw std::invalid_argument( @@ -198,12 +198,12 @@ class WindowLists { // cost the ordering and buy nothing. void fill_from_corpus(const SparseVectors& vectors, const TermWindow& window, size_t element_size, WindowLists* lists) { - const idx_t* indptr = vectors.indptr_data(); + const offset_t* indptr = vectors.indptr_data(); const term_t* indices = vectors.indices_data(); const uint8_t* codes = vectors.values_data(); const auto n_docs = static_cast(vectors.num_vectors()); for (idx_t doc = 0; doc < n_docs; ++doc) { - for (idx_t j = indptr[doc]; j < indptr[doc + 1]; ++j) { + for (offset_t j = indptr[doc]; j < indptr[doc + 1]; ++j) { const size_t term = indices[j]; if (term < window.begin || term >= window.end) { continue; diff --git a/nsparse/seismic_common.h b/nsparse/seismic_common.h index 4cd07f4..d397f5a 100644 --- a/nsparse/seismic_common.h +++ b/nsparse/seismic_common.h @@ -99,10 +99,10 @@ inline std::vector calculate_summary_scores( return summary_scores; } -inline float compute_similarity(idx_t doc_id, const idx_t* indptr, +inline float compute_similarity(idx_t doc_id, const offset_t* indptr, const term_t* indices, const uint8_t* values, const uint8_t* dense, size_t element_size) { - const idx_t start = indptr[doc_id]; + const offset_t start = indptr[doc_id]; const size_t len = indptr[doc_id + 1] - start; float score = 0.0F; if (element_size == U32) { diff --git a/nsparse/seismic_index.cpp b/nsparse/seismic_index.cpp index df3fe40..0104f8e 100644 --- a/nsparse/seismic_index.cpp +++ b/nsparse/seismic_index.cpp @@ -95,7 +95,7 @@ void query_single_inverted_list( const auto& doc_id = docs[i]; if (i + kPrefetchDist < n_docs) { const idx_t next_doc = docs[i + kPrefetchDist]; - const idx_t next_start = indptr[next_doc]; + const offset_t next_start = indptr[next_doc]; const size_t next_len = indptr[next_doc + 1] - next_start; detail::prefetch_vector_head(indices + next_start, values + next_start, next_len, @@ -108,7 +108,7 @@ void query_single_inverted_list( if (id_selector != nullptr && !id_selector->is_member(doc_id)) { continue; } - const idx_t start = indptr[doc_id]; + const offset_t start = indptr[doc_id]; const size_t len = indptr[doc_id + 1] - start; auto score = detail::dot_product_float_dense( indices + start, values + start, len, dense.data()); @@ -124,7 +124,7 @@ SeismicIndex::SeismicIndex(int dim) SeismicIndex::SeismicIndex(int dim, SeismicClusterParameters parameter) : MmapIndex(dim), cluster_parameter_(parameter) {} -void SeismicIndex::add(idx_t n, const idx_t* indptr, const term_t* indices, +void SeismicIndex::add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) { throw_if_not_positive(n); throw_if_any_null(indptr, indices, values); @@ -147,8 +147,8 @@ void SeismicIndex::build() { &batch_spill_); } -auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, - const float* values, int k, +auto SeismicIndex::search(idx_t n, const offset_t* indptr, + const term_t* indices, const float* values, int k, SearchParameters* search_parameters) -> pair_of_score_id_vectors_t { if (vectors_ == nullptr || n == 0) { @@ -200,7 +200,7 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { - const idx_t start = indptr[query_idx]; + const offset_t start = indptr[query_idx]; const size_t len = indptr[query_idx + 1] - start; const term_t* q_indices = indices + start; const float* q_values = values + start; diff --git a/nsparse/seismic_index.h b/nsparse/seismic_index.h index df3e804..64712c9 100644 --- a/nsparse/seismic_index.h +++ b/nsparse/seismic_index.h @@ -48,7 +48,7 @@ class SeismicIndex : public MmapIndex, public IndexIO { void build() override; - void add(idx_t n, const idx_t* indptr, const term_t* indices, + void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) override; static SeismicIndex* mmap_index(const IndexHeader& header, @@ -66,7 +66,7 @@ class SeismicIndex : public MmapIndex, public IndexIO { void read_index(IOReader* io_reader, const IndexHeader& header, int io_flags = 0) override; - auto search(idx_t n, const idx_t* indptr, const term_t* indices, + auto search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters = nullptr) -> pair_of_score_id_vectors_t override; diff --git a/nsparse/seismic_scalar_quantized_index.cpp b/nsparse/seismic_scalar_quantized_index.cpp index e46d737..1ce2259 100644 --- a/nsparse/seismic_scalar_quantized_index.cpp +++ b/nsparse/seismic_scalar_quantized_index.cpp @@ -114,7 +114,7 @@ void query_single_inverted_list(const SparseVectors* vectors, const auto& doc_id = docs[i]; if (i + kPrefetchDist < n_docs) { const idx_t next_doc = docs[i + kPrefetchDist]; - const idx_t next_start = indptr[next_doc]; + const offset_t next_start = indptr[next_doc]; const size_t next_len = indptr[next_doc + 1] - next_start; detail::prefetch_vector_head(indices + next_start, values + next_start, next_len, @@ -146,7 +146,7 @@ SeismicScalarQuantizedIndex::SeismicScalarQuantizedIndex( sq_(quantizer_type, vmin, vmax), cluster_parameter_(parameter) {} -void SeismicScalarQuantizedIndex::add(idx_t n, const idx_t* indptr, +void SeismicScalarQuantizedIndex::add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) { throw_if_not_positive(n); @@ -188,7 +188,7 @@ void SeismicScalarQuantizedIndex::build() { &batch_spill_); } -auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, +auto SeismicScalarQuantizedIndex::search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters) @@ -268,7 +268,7 @@ auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { - const idx_t start = indptr[query_idx]; + const offset_t start = indptr[query_idx]; const size_t len = indptr[query_idx + 1] - start; const term_t* q_indices = indices + start; const uint8_t* q_val_bytes = query_values + start * element_size; diff --git a/nsparse/seismic_scalar_quantized_index.h b/nsparse/seismic_scalar_quantized_index.h index f760e15..3270fc8 100644 --- a/nsparse/seismic_scalar_quantized_index.h +++ b/nsparse/seismic_scalar_quantized_index.h @@ -50,7 +50,7 @@ class SeismicScalarQuantizedIndex : public MmapIndex, public IndexIO { SeismicScalarQuantizedIndex& operator=(const SeismicScalarQuantizedIndex&) = delete; std::array id() const override { return name; } - void add(idx_t n, const idx_t* indptr, const term_t* indices, + void add(idx_t n, const offset_t* indptr, const term_t* indices, const float* values) override; void build() override; @@ -86,7 +86,7 @@ class SeismicScalarQuantizedIndex : public MmapIndex, public IndexIO { // SeismicIndex and as the base signature's default argument implies. This // index used to reject it, which made that default argument -- and the // bindings' params=None -- unusable here alone. - auto search(idx_t n, const idx_t* indptr, const term_t* indices, + auto search(idx_t n, const offset_t* indptr, const term_t* indices, const float* values, int k, SearchParameters* search_parameters = nullptr) -> pair_of_score_id_vectors_t override; diff --git a/nsparse/sparse_vectors.cpp b/nsparse/sparse_vectors.cpp index ca31492..1680264 100644 --- a/nsparse/sparse_vectors.cpp +++ b/nsparse/sparse_vectors.cpp @@ -26,7 +26,7 @@ SparseVectors::SparseVectors(SparseVectorsConfig config) : config_(config) { } SparseVectors SparseVectors::map_vectors(SparseVectorsConfig config, - const idx_t* indptr, + const offset_t* indptr, size_t indptr_size, const term_t* indices, size_t indices_size, @@ -76,20 +76,20 @@ SparseVectors SparseVectors::map_vectors(SparseVectorsConfig config, "Mapped indptr does not end at the index count"); } - vectors.indptr_ = Buf::borrow(indptr, indptr_size); + vectors.indptr_ = Buf::borrow(indptr, indptr_size); vectors.indices_ = Buf::borrow(indices, indices_size); vectors.values_ = Buf::borrow(values, values_size); return vectors; } -void SparseVectors::add_vectors(const std::vector& indptr, +void SparseVectors::add_vectors(const std::vector& indptr, const std::vector& indices, const std::vector& weights) { add_vectors(indptr.data(), indptr.size(), indices.data(), indices.size(), weights.data(), weights.size()); } -void SparseVectors::add_vectors(const idx_t* indptr, size_t indptr_size, +void SparseVectors::add_vectors(const offset_t* indptr, size_t indptr_size, const term_t* indices, size_t indices_size, const uint8_t* weights, size_t weights_size) { if (indices_size * config_.element_size != weights_size) { @@ -102,7 +102,7 @@ void SparseVectors::add_vectors(const idx_t* indptr, size_t indptr_size, // Always copies: the arguments are often a caller-local buffer (freshly // quantized codes, say), and the incoming offsets are rebased onto what is // already stored. Borrowing is map_vectors' job. - std::vector indptr_vec = indptr_.take_vector(); + std::vector indptr_vec = indptr_.take_vector(); std::vector indices_vec = indices_.take_vector(); std::vector values_vec = values_.take_vector(); @@ -115,12 +115,12 @@ void SparseVectors::add_vectors(const idx_t* indptr, size_t indptr_size, // Append weights directly (already in uint8_t format) values_vec.insert(values_vec.end(), weights, weights + weights_size); - idx_t offset = indptr_vec.back(); + offset_t offset = indptr_vec.back(); for (size_t i = 1; i < indptr_size; ++i) { indptr_vec.push_back(indptr[i] + offset); } - indptr_ = Buf::own(std::move(indptr_vec)); + indptr_ = Buf::own(std::move(indptr_vec)); indices_ = Buf::own(std::move(indices_vec)); values_ = Buf::own(std::move(values_vec)); } @@ -133,12 +133,12 @@ void SparseVectors::add_vector(const std::vector& indices, void SparseVectors::add_vector(const term_t* indices, size_t indices_size, const uint8_t* weights, size_t weights_size) { // Copies for the same reason add_vectors does; see the note there. - std::vector indptr_vec = indptr_.take_vector(); + std::vector indptr_vec = indptr_.take_vector(); std::vector indices_vec = indices_.take_vector(); std::vector values_vec = values_.take_vector(); // Get the current offset (where the new vector starts) - idx_t offset = indptr_vec.empty() ? 0 : indptr_vec.back(); + offset_t offset = indptr_vec.empty() ? 0 : indptr_vec.back(); // If this is the first vector, initialize indptr with 0 if (indptr_vec.empty()) { @@ -147,9 +147,9 @@ void SparseVectors::add_vector(const term_t* indices, size_t indices_size, indices_vec.insert(indices_vec.end(), indices, indices + indices_size); values_vec.insert(values_vec.end(), weights, weights + weights_size); - indptr_vec.push_back(offset + static_cast(indices_size)); + indptr_vec.push_back(offset + static_cast(indices_size)); - indptr_ = Buf::own(std::move(indptr_vec)); + indptr_ = Buf::own(std::move(indptr_vec)); indices_ = Buf::own(std::move(indices_vec)); values_ = Buf::own(std::move(values_vec)); } @@ -160,12 +160,12 @@ std::vector SparseVectors::get_dense_vector_float( throw std::out_of_range("Vector index out of range"); } - idx_t start = indptr_[vector_idx]; - idx_t end = indptr_[vector_idx + 1]; + offset_t start = indptr_[vector_idx]; + offset_t end = indptr_[vector_idx + 1]; std::vector dense_vector( config_.dimension > 0 ? config_.dimension : indices_[end - 1] + 1, 0.0F); - for (idx_t i = start; i < end; ++i) { + for (offset_t i = start; i < end; ++i) { const uint8_t* value_ptr = values_.data() + (i * config_.element_size); if (config_.element_size == U32) { dense_vector[indices_[i]] = @@ -184,12 +184,12 @@ std::vector SparseVectors::get_dense_vector(idx_t vector_idx) const { if (vector_idx < 0 || vector_idx > static_cast(indptr_.size()) - 2) { throw std::out_of_range("Vector index out of range"); } - idx_t start = indptr_[vector_idx]; - idx_t end = indptr_[vector_idx + 1]; + offset_t start = indptr_[vector_idx]; + offset_t end = indptr_[vector_idx + 1]; size_t size = end - start; std::vector dense_vector(config_.dimension * config_.element_size, 0.0F); - for (idx_t i = start; i < end; ++i) { + for (offset_t i = start; i < end; ++i) { for (idx_t j = 0; j < config_.element_size; ++j) { dense_vector[indices_[i] * config_.element_size + j] = values_[i * config_.element_size + j]; @@ -242,7 +242,7 @@ void SparseVectors::deserialize(IOReader* io_reader) { // Skips the padding serialize() wrote before each array. size_t indptr_size = vector_count + 1; - indptr_ = io_align::read_padded(io_reader, indptr_size); + indptr_ = io_align::read_padded(io_reader, indptr_size); size_t indices_size = indptr_[vector_count]; indices_ = io_align::read_padded(io_reader, indices_size); @@ -271,8 +271,8 @@ void SparseVectors::mmap_deserialize(MmapCursor* cursor) { if (indptr_size == 0) { throw std::runtime_error("mmap: implausible vector count in index file"); } - io_align::skip_padding(cursor, alignof(idx_t)); - const idx_t* indptr = cursor->read_array(indptr_size); + io_align::skip_padding(cursor, alignof(offset_t)); + const offset_t* indptr = cursor->read_array(indptr_size); const auto indices_size = static_cast(indptr[vector_count]); io_align::skip_padding(cursor, alignof(term_t)); diff --git a/nsparse/sparse_vectors.h b/nsparse/sparse_vectors.h index af04ffb..6db2f96 100644 --- a/nsparse/sparse_vectors.h +++ b/nsparse/sparse_vectors.h @@ -28,7 +28,7 @@ struct SparseVectorsConfig { }; struct SparseVectorsData { - const idx_t* indptr_data; + const offset_t* indptr_data; const term_t* indices_data; const float* values_data; }; @@ -46,11 +46,11 @@ class SparseVectors : public MmapSerializable { SparseVectors(SparseVectors&& other) noexcept = default; SparseVectors& operator=(SparseVectors&& other) noexcept = default; - void add_vectors(const std::vector& indptr, + void add_vectors(const std::vector& indptr, const std::vector& indices, const std::vector& weights); - void add_vectors(const idx_t* indptr, size_t indptr_size, + void add_vectors(const offset_t* indptr, size_t indptr_size, const term_t* indices, size_t indices_size, const uint8_t* weights, size_t weights_size); @@ -68,7 +68,7 @@ class SparseVectors : public MmapSerializable { // file fails here rather than mid-search, and `values` must be aligned for // `element_size` because it is reinterpreted in place. static SparseVectors map_vectors(SparseVectorsConfig config, - const idx_t* indptr, size_t indptr_size, + const offset_t* indptr, size_t indptr_size, const term_t* indices, size_t indices_size, const uint8_t* values, @@ -80,7 +80,7 @@ class SparseVectors : public MmapSerializable { std::vector get_dense_vector_float(idx_t vector_idx) const; std::vector get_dense_vector(idx_t vector_idx) const; - const idx_t* indptr_data() const { return indptr_.data(); } + const offset_t* indptr_data() const { return indptr_.data(); } const term_t* indices_data() const { return indices_.data(); } const float* values_data_float() const { return reinterpret_cast(values_.data()); @@ -104,7 +104,7 @@ class SparseVectors : public MmapSerializable { void mmap_deserialize(MmapCursor* cursor) override; private: - Buf indptr_; + Buf indptr_; Buf indices_; Buf values_; SparseVectorsConfig config_; diff --git a/nsparse/types.h b/nsparse/types.h index 9582c51..dda3a9b 100644 --- a/nsparse/types.h +++ b/nsparse/types.h @@ -16,6 +16,8 @@ namespace nsparse { using idx_t = int32_t; +using offset_t = int64_t; // CSR nnz offsets (indptr); distinct from idx_t so + // doc-ids / term-ids / counts stay 32-bit. using term_t = uint16_t; using weight_t = float; diff --git a/nsparse/utils/csr_layout.cpp b/nsparse/utils/csr_layout.cpp index 676a262..62241ea 100644 --- a/nsparse/utils/csr_layout.cpp +++ b/nsparse/utils/csr_layout.cpp @@ -110,9 +110,17 @@ void convert(const std::string& interchange_path, throw std::invalid_argument("Inconsistent CSR indptr in: " + interchange_path); } - const auto indptr = narrow(wide_indptr, "indptr", interchange_path); - write_or_throw(out, indptr.data(), indptr.size() * sizeof(idx_t), - native_path); + // The interchange indptr is already int64 (== offset_t), so it is written + // straight through rather than copied. A negative entry is still rejected + // (map_vectors re-validates monotonicity when the native file is loaded). + for (const int64_t off : wide_indptr) { + if (off < 0) { + throw std::invalid_argument( + "CSR indptr has a negative offset in: " + interchange_path); + } + } + write_or_throw(out, wide_indptr.data(), + wide_indptr.size() * sizeof(offset_t), native_path); // The large array; stream it. for (size_t done = 0; done < nnz_size;) { @@ -132,8 +140,9 @@ void convert(const std::string& interchange_path, done += count; } - const size_t values_pos = - kHeaderBytes + indptr_size * sizeof(idx_t) + nnz_size * sizeof(term_t); + const size_t values_pos = kHeaderBytes + + indptr_size * sizeof(offset_t) + + nnz_size * sizeof(term_t); const std::array pad{}; if (const size_t pad_bytes = padding(values_pos); pad_bytes > 0) { write_or_throw(out, pad.data(), pad_bytes, native_path); diff --git a/nsparse/utils/csr_layout.h b/nsparse/utils/csr_layout.h index baf18da..0b448c2 100644 --- a/nsparse/utils/csr_layout.h +++ b/nsparse/utils/csr_layout.h @@ -19,8 +19,8 @@ // Two on-disk CSR layouts, both after an int64 header of (rows, cols, nnz): // // interchange native -// int64 indptr[r + 1] idx_t indptr[r + 1] -// int32 indices[nnz] term_t indices[nnz] +// int64 indptr[r + 1] offset_t indptr[r + 1] +// int32 indices[nnz] term_t indices[nnz] // // float values[nnz] value values[nnz] // @@ -53,7 +53,7 @@ constexpr size_t interchange_file_size(size_t indptr_size, size_t nnz) { constexpr size_t native_values_offset(size_t indptr_size, size_t nnz) { const size_t unaligned = - kHeaderBytes + indptr_size * sizeof(idx_t) + nnz * sizeof(term_t); + kHeaderBytes + indptr_size * sizeof(offset_t) + nnz * sizeof(term_t); return unaligned + padding(unaligned); } diff --git a/nsparse/utils/distance.h b/nsparse/utils/distance.h index d5bbf66..8fcdeec 100644 --- a/nsparse/utils/distance.h +++ b/nsparse/utils/distance.h @@ -69,8 +69,8 @@ inline auto dot_product_vectors_dense(const SparseVectors* vectors, const auto* values = vectors->values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; const term_t* idx_ptr = indices + start; // Cast to T* at the correct byte offset diff --git a/nsparse/utils/distance_avx2.h b/nsparse/utils/distance_avx2.h index 5c7f0cb..b70f23d 100644 --- a/nsparse/utils/distance_avx2.h +++ b/nsparse/utils/distance_avx2.h @@ -432,12 +432,12 @@ inline auto dot_product_float_vectors_dense(const SparseVectors* vectors, const auto& [indptr, indices, values] = vectors->get_all_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -454,17 +454,17 @@ inline auto dot_product_uint8_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint8_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -481,17 +481,17 @@ inline auto dot_product_uint16_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint16_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); diff --git a/nsparse/utils/distance_avx512.h b/nsparse/utils/distance_avx512.h index 1969e23..424f4df 100644 --- a/nsparse/utils/distance_avx512.h +++ b/nsparse/utils/distance_avx512.h @@ -445,12 +445,12 @@ inline auto dot_product_float_vectors_dense(const SparseVectors* vectors, const auto& [indptr, indices, values] = vectors->get_all_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -467,17 +467,17 @@ inline auto dot_product_uint8_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint8_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -494,17 +494,17 @@ inline auto dot_product_uint16_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint16_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); diff --git a/nsparse/utils/distance_neon.h b/nsparse/utils/distance_neon.h index 9674ff5..50a9d9b 100644 --- a/nsparse/utils/distance_neon.h +++ b/nsparse/utils/distance_neon.h @@ -457,12 +457,12 @@ inline auto dot_product_float_vectors_dense(const SparseVectors* vectors, const auto& [indptr, indices, values] = vectors->get_all_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -479,17 +479,17 @@ inline auto dot_product_uint8_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint8_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -506,17 +506,17 @@ inline auto dot_product_uint16_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint16_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); diff --git a/nsparse/utils/distance_sve.h b/nsparse/utils/distance_sve.h index c9a9428..8d6c52a 100644 --- a/nsparse/utils/distance_sve.h +++ b/nsparse/utils/distance_sve.h @@ -371,12 +371,12 @@ inline auto dot_product_float_vectors_dense(const SparseVectors* vectors, const auto& [indptr, indices, values] = vectors->get_all_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -393,17 +393,17 @@ inline auto dot_product_uint8_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint8_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); @@ -420,17 +420,17 @@ inline auto dot_product_uint16_vectors_dense(const SparseVectors* vectors, size_t n_vectors = vectors->num_vectors(); std::vector results(n_vectors, 0); - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const uint16_t* values = vectors->typed_values_data(); for (size_t i = 0; i < n_vectors; ++i) { - const idx_t start = indptr[i]; - const idx_t end = indptr[i + 1]; + const offset_t start = indptr[i]; + const offset_t end = indptr[i + 1]; const size_t len = end - start; if (i + 1 < n_vectors) { - const idx_t next_start = indptr[i + 1]; + const offset_t next_start = indptr[i + 1]; const size_t next_len = indptr[i + 2] - next_start; prefetch_vector(indices + next_start, values + next_start, next_len); diff --git a/nsparse/utils/prefetch.h b/nsparse/utils/prefetch.h index 9b1733d..94e876b 100644 --- a/nsparse/utils/prefetch.h +++ b/nsparse/utils/prefetch.h @@ -70,7 +70,7 @@ inline void prefetch_vector_head(const term_t* indices, const T* values, } } -inline void prefetch_indptr(const idx_t* indptr, idx_t doc_id) { +inline void prefetch_indptr(const offset_t* indptr, idx_t doc_id) { NSPARSE_PREFETCH(&indptr[doc_id], 0, 0); } diff --git a/python_tests/_abort_probe.py b/python_tests/_abort_probe.py deleted file mode 100644 index 67bcee1..0000000 --- a/python_tests/_abort_probe.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Performs one crashing operation, for test_error_paths.py. - -Only the cases that take the interpreter down live here; everything that raises -is asserted in-process. See test_error_paths.py for why a wrong dtype on a -non-final buffer argument segfaults. -""" - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) - -import numpy as np - -import nsparse -from support import make_corpus - -DIM = 512 - -CASES = {} - - -def case(fn): - CASES[fn.__name__] = fn - return fn - - -def corpus(): - return make_corpus(200, DIM, 20, 0x11) - - -@case -def bad_indices_dtype(): - c = corpus() - index = nsparse.index_factory(DIM, "inverted") - index.add(c.n, c.indptr, c.indices.astype(np.int32), c.values) - - -@case -def bad_indptr_dtype(): - c = corpus() - index = nsparse.index_factory(DIM, "inverted") - index.add(c.n, c.indptr.astype(np.int64), c.indices, c.values) - - -if __name__ == "__main__": - CASES[sys.argv[1]]() - print("COMPLETED-WITHOUT-ERROR") diff --git a/python_tests/test_error_paths.py b/python_tests/test_error_paths.py index 86bb971..e303696 100644 --- a/python_tests/test_error_paths.py +++ b/python_tests/test_error_paths.py @@ -11,28 +11,18 @@ std::invalid_argument arrives as ValueError, everything else derived from std::exception as RuntimeError. -The dtype cases at the bottom still need a subprocess. A wrong dtype for indptr -or indices does not raise -- it segfaults. The typemaps do validate the buffer -format, but they call SWIG_fail after releasing their own Py_buffer, and the -fail: label then runs the freearg typemaps for the remaining arguments, whose -Py_buffer views were never initialised. So PyBuffer_Release() is handed -uninitialised stack memory. `values` is the last argument, so nothing follows it -and it fails cleanly -- which is exactly why only the other two crash. Fixable -by zero-initialising the views or guarding freearg on view.obj. +A wrong dtype on any buffer argument raises TypeError. The buffer typemaps +zero-initialise their Py_buffer view, so when one argument fails validation and +runs SWIG_fail, the freearg typemaps for the remaining arguments release a +zeroed view (a no-op) rather than uninitialised stack memory. """ -from pathlib import Path - import numpy as np import pytest import nsparse -from conftest import run_isolated from support import make_corpus, make_index, search -SIGSEGV = -11 -PROBE = str(Path(__file__).parent / "_abort_probe.py") - DIM = 512 @@ -101,15 +91,18 @@ def test_wrong_values_dtype_raises(small_corpus): ) -@pytest.mark.parametrize("case", ["bad_indptr_dtype", "bad_indices_dtype"]) -def test_wrong_dtype_segfaults(case): - """Pins the crash described in the module docstring. - - Rewrite as pytest.raises(TypeError) once the typemaps zero-initialise their - Py_buffer views -- see test_wrong_values_dtype_raises for the target shape. - """ - result = run_isolated([PROBE, case]) - assert "COMPLETED-WITHOUT-ERROR" not in result.stdout - assert result.returncode == SIGSEGV, ( - f"{case}: expected SIGSEGV, got {result.returncode}\n{result.stderr}" - ) +@pytest.mark.parametrize( + "mutate,match", + [ + (lambda c: (c.indptr.astype(np.float64), c.indices, c.values), "indptr"), + (lambda c: (c.indptr, c.indices.astype(np.int32), c.values), "indices"), + ], + ids=["bad_indptr_dtype", "bad_indices_dtype"], +) +def test_wrong_dtype_on_non_final_buffer_raises(small_corpus, mutate, match): + """A wrong dtype on indptr or indices raises TypeError, not a segfault: the + zero-initialised Py_buffer views make every argument fail gracefully.""" + index = nsparse.index_factory(DIM, "inverted") + indptr, indices, values = mutate(small_corpus) + with pytest.raises(TypeError, match=match): + index.add(small_corpus.n, indptr, indices, values) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 75da0a4..8fc3cd8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,6 +25,7 @@ set(NSPARSE_TEST_SRC kmeans_utils_test.cpp mmap_cursor_test.cpp mmap_index_test.cpp + offset_width_test.cpp prefetch_test.cpp random_kmeans_test.cpp ranker_test.cpp diff --git a/tests/csr_interchange_test_util.h b/tests/csr_interchange_test_util.h index 52692c3..92a68ad 100644 --- a/tests/csr_interchange_test_util.h +++ b/tests/csr_interchange_test_util.h @@ -53,13 +53,13 @@ void write_interchange_csr(const std::string& path, const Corpus& c, // Writes a corpus of pre-quantized codes as a NATIVE CSR -- the layout read_mcsr // borrows when a quantizing index maps it: int64 header {rows, num_cols, nnz}, -// idx_t indptr[rows+1], term_t indices[nnz], pad to alignof(float), then +// offset_t indptr[rows+1], term_t indices[nnz], pad to alignof(float), then // `element_size`-byte codes[nnz]. `codes` holds nnz*element_size bytes, // row-aligned with `indices`. This is the code-width analog of // csr_layout::convert's output, written directly since the codes path has no // interchange form. inline void write_native_codes_csr(const std::string& path, - const std::vector& indptr, + const std::vector& indptr, const std::vector& indices, const std::vector& codes, int64_t num_cols, size_t element_size) { @@ -77,11 +77,11 @@ inline void write_native_codes_csr(const std::string& path, out.write(reinterpret_cast(header.data()), header.size() * sizeof(int64_t)); out.write(reinterpret_cast(indptr.data()), - static_cast(indptr.size() * sizeof(idx_t))); + static_cast(indptr.size() * sizeof(offset_t))); out.write(reinterpret_cast(indices.data()), static_cast(indices.size() * sizeof(term_t))); const size_t values_pos = csr_layout::kHeaderBytes + - indptr.size() * sizeof(idx_t) + + indptr.size() * sizeof(offset_t) + indices.size() * sizeof(term_t); const std::array pad{}; if (const size_t pad_bytes = csr_layout::padding(values_pos); diff --git a/tests/csr_layout_test.cpp b/tests/csr_layout_test.cpp index fc57b56..e92d93b 100644 --- a/tests/csr_layout_test.cpp +++ b/tests/csr_layout_test.cpp @@ -24,6 +24,7 @@ namespace { using nsparse::idx_t; +using nsparse::offset_t; using nsparse::term_t; namespace layout = nsparse::csr_layout; @@ -48,7 +49,7 @@ void write_interchange(const std::filesystem::path& path, int64_t num_rows, struct NativeFile { std::array header{}; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; std::vector pad; @@ -68,11 +69,11 @@ NativeFile read_native(const std::filesystem::path& path) { result.indices.resize(nnz); result.values.resize(nnz); result.pad.resize(layout::padding(layout::kHeaderBytes + - indptr_size * sizeof(idx_t) + + indptr_size * sizeof(offset_t) + nnz * sizeof(term_t))); file.read(reinterpret_cast(result.indptr.data()), - static_cast(sizeof(idx_t) * indptr_size)); + static_cast(sizeof(offset_t) * indptr_size)); file.read(reinterpret_cast(result.indices.data()), static_cast(sizeof(term_t) * nnz)); file.read(reinterpret_cast(result.pad.data()), @@ -147,7 +148,7 @@ TEST_F(CsrLayoutConvert, rewrites_arrays_at_native_widths) { ASSERT_EQ(native.header[0], 3); ASSERT_EQ(native.header[1], 4); ASSERT_EQ(native.header[2], 3); - ASSERT_EQ(native.indptr, std::vector({0, 2, 2, 3})); + ASSERT_EQ(native.indptr, std::vector({0, 2, 2, 3})); ASSERT_EQ(native.indices, std::vector({0, 2, 1})); ASSERT_EQ(native.values, std::vector({1.5F, 2.5F, 3.5F})); } @@ -179,7 +180,7 @@ TEST_F(CsrLayoutConvert, handles_an_empty_matrix) { ASSERT_EQ(std::filesystem::file_size(out_path()), layout::native_file_size(3, 0)); const auto native = read_native(out_path()); - ASSERT_EQ(native.indptr, std::vector({0, 0, 0})); + ASSERT_EQ(native.indptr, std::vector({0, 0, 0})); ASSERT_TRUE(native.indices.empty()); } diff --git a/tests/disk_seismic_test_util.h b/tests/disk_seismic_test_util.h index f34cde6..7eb2d34 100644 --- a/tests/disk_seismic_test_util.h +++ b/tests/disk_seismic_test_util.h @@ -48,7 +48,7 @@ inline SeismicClusterParameters cluster_params() { } struct CSR { - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; idx_t n = 0; @@ -75,7 +75,7 @@ inline CSR make_corpus(idx_t rows, unsigned seed) { c.indices.push_back(static_cast(t)); c.values.push_back(val_dist(rng)); } - c.indptr.push_back(static_cast(c.indices.size())); + c.indptr.push_back(static_cast(c.indices.size())); } return c; } @@ -95,12 +95,12 @@ inline CSR make_corpus_with_remainder(int n_fillers, int n_victims) { c.values.push_back(1.0F); c.indices.push_back(static_cast(r + 1)); c.values.push_back(1.0F); - c.indptr.push_back(static_cast(c.indices.size())); + c.indptr.push_back(static_cast(c.indices.size())); } for (int v = 0; v < n_victims; ++v) { c.indices.push_back(0); c.values.push_back(0.01F); - c.indptr.push_back(static_cast(c.indices.size())); + c.indptr.push_back(static_cast(c.indices.size())); } return c; } diff --git a/tests/gpu_cluster_assigner_test.cpp b/tests/gpu_cluster_assigner_test.cpp index c4e9ab6..8d88a98 100644 --- a/tests/gpu_cluster_assigner_test.cpp +++ b/tests/gpu_cluster_assigner_test.cpp @@ -31,7 +31,7 @@ namespace { std::vector> cpu_reference_assign( const SparseVectors* vectors, const std::vector& docs, std::vector> clusters) { - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const float* values = vectors->values_data_float(); const size_t n_clusters = clusters.size(); @@ -46,7 +46,7 @@ std::vector> cpu_reference_assign( is_centroid = true; break; } - const idx_t start = indptr[c]; + const offset_t start = indptr[c]; const size_t len = indptr[c + 1] - start; float score = 0.0F; for (size_t t = 0; t < len; ++t) { @@ -164,12 +164,12 @@ void cpu_reference_maxpool(const SparseVectors* vectors, const std::vector& doc_ids, std::vector& terms, std::vector& values, float& sum) { - const idx_t* indptr = vectors->indptr_data(); + const offset_t* indptr = vectors->indptr_data(); const term_t* indices = vectors->indices_data(); const float* vals = vectors->values_data_float(); std::map m; for (idx_t d : doc_ids) { - for (idx_t j = indptr[d]; j < indptr[d + 1]; ++j) { + for (offset_t j = indptr[d]; j < indptr[d + 1]; ++j) { auto& v = m[indices[j]]; v = std::max(v, vals[j]); } diff --git a/tests/id_map_index_test.cpp b/tests/id_map_index_test.cpp index 2ab3930..a4a9813 100644 --- a/tests/id_map_index_test.cpp +++ b/tests/id_map_index_test.cpp @@ -62,7 +62,7 @@ TEST_F(IDMapIndexTest, num_vectors_empty) { } TEST_F(IDMapIndexTest, add_with_ids) { - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -74,7 +74,7 @@ TEST_F(IDMapIndexTest, add_with_ids) { } TEST_F(IDMapIndexTest, add_with_ids_multiple_batches) { - std::vector indptr1 = {0, 2}; + std::vector indptr1 = {0, 2}; std::vector indices1 = {0, 1}; std::vector values1 = {1.0F, 0.5F}; std::vector ids1 = {100}; @@ -83,7 +83,7 @@ TEST_F(IDMapIndexTest, add_with_ids_multiple_batches) { ids1.data()); EXPECT_EQ(idmap_->num_vectors(), 1); - std::vector indptr2 = {0, 2}; + std::vector indptr2 = {0, 2}; std::vector indices2 = {2, 3}; std::vector values2 = {0.8F, 0.3F}; std::vector ids2 = {200}; @@ -95,7 +95,7 @@ TEST_F(IDMapIndexTest, add_with_ids_multiple_batches) { TEST_F(IDMapIndexTest, search_returns_external_ids) { // Add vectors with custom external IDs - std::vector indptr = {0, 2, 4, 6}; + std::vector indptr = {0, 2, 4, 6}; std::vector indices = {0, 1, 0, 1, 0, 1}; std::vector values = {1.0F, 0.5F, 0.3F, 0.2F, 0.8F, 0.4F}; std::vector ids = {1000, 2000, 3000}; @@ -105,7 +105,7 @@ TEST_F(IDMapIndexTest, search_returns_external_ids) { idmap_->build(); // Query - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(3, -1); @@ -124,7 +124,7 @@ TEST_F(IDMapIndexTest, search_returns_external_ids) { TEST_F(IDMapIndexTest, search_preserves_negative_ids) { // Add one vector - std::vector indptr = {0, 2}; + std::vector indptr = {0, 2}; std::vector indices = {0, 1}; std::vector values = {1.0F, 0.5F}; std::vector ids = {1000}; @@ -134,7 +134,7 @@ TEST_F(IDMapIndexTest, search_preserves_negative_ids) { idmap_->build(); // Query for k=3 but only 1 result exists - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(3, -1); @@ -151,7 +151,7 @@ TEST_F(IDMapIndexTest, search_preserves_negative_ids) { } TEST_F(IDMapIndexTest, get_vectors_after_add) { - std::vector indptr = {0, 2}; + std::vector indptr = {0, 2}; std::vector indices = {0, 1}; std::vector values = {1.0F, 0.5F}; std::vector ids = {100}; @@ -170,7 +170,7 @@ TEST(IDMapIndex, default_constructor) { } TEST_F(IDMapIndexTest, search_with_null_search_parameters) { - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 0, 1}; std::vector values = {1.0F, 0.5F, 0.3F, 0.2F}; std::vector ids = {100, 200}; @@ -179,7 +179,7 @@ TEST_F(IDMapIndexTest, search_with_null_search_parameters) { ids.data()); idmap_->build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -196,7 +196,7 @@ TEST_F(IDMapIndexTest, search_with_null_search_parameters) { } TEST_F(IDMapIndexTest, search_with_search_parameters_no_id_selector) { - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 0, 1}; std::vector values = {1.0F, 0.5F, 0.3F, 0.2F}; std::vector ids = {100, 200}; @@ -205,7 +205,7 @@ TEST_F(IDMapIndexTest, search_with_search_parameters_no_id_selector) { ids.data()); idmap_->build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -224,7 +224,7 @@ TEST_F(IDMapIndexTest, search_with_search_parameters_no_id_selector) { } TEST_F(IDMapIndexTest, search_with_id_selector_filters_by_external_id) { - std::vector indptr = {0, 2, 4, 6}; + std::vector indptr = {0, 2, 4, 6}; std::vector indices = {0, 1, 0, 1, 0, 1}; std::vector values = {1.0F, 0.5F, 0.3F, 0.2F, 0.8F, 0.4F}; std::vector ids = {100, 200, 300}; @@ -233,7 +233,7 @@ TEST_F(IDMapIndexTest, search_with_id_selector_filters_by_external_id) { ids.data()); idmap_->build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(3, -1); @@ -257,7 +257,7 @@ TEST_F(IDMapIndexTest, search_with_id_selector_filters_by_external_id) { } TEST_F(IDMapIndexTest, search_with_id_selector_excludes_all) { - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 0, 1}; std::vector values = {1.0F, 0.5F, 0.3F, 0.2F}; std::vector ids = {100, 200}; @@ -266,7 +266,7 @@ TEST_F(IDMapIndexTest, search_with_id_selector_excludes_all) { ids.data()); idmap_->build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -299,7 +299,7 @@ class DestructionTrackingIndex : public nsparse::Index { ~DestructionTrackingIndex() override { *destroyed_ = true; } std::array id() const override { return {'T', 'E', 'S', 'T'}; } - void add(nsparse::idx_t, const nsparse::idx_t*, const nsparse::term_t*, + void add(nsparse::idx_t, const nsparse::offset_t*, const nsparse::term_t*, const float*) override {} private: @@ -325,7 +325,7 @@ TEST(IDMapIndexOwnership, DeletesDelegateOnDestruction) { TEST(IDMapIndexOwnership, DeletesDelegateAcquiredViaReadIndex) { // Build and serialize an idmap-wrapped inverted index. auto* original = new nsparse::IDMapIndex(new nsparse::InvertedIndex(16)); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -347,7 +347,7 @@ TEST(IDMapIndexOwnership, DeletesDelegateAcquiredViaReadIndex) { } TEST_F(IDMapIndexTest, search_with_not_id_selector) { - std::vector indptr = {0, 2, 4, 6}; + std::vector indptr = {0, 2, 4, 6}; std::vector indices = {0, 1, 0, 1, 0, 1}; std::vector values = {1.0F, 0.5F, 0.3F, 0.2F, 0.8F, 0.4F}; std::vector ids = {100, 200, 300}; @@ -356,7 +356,7 @@ TEST_F(IDMapIndexTest, search_with_not_id_selector) { ids.data()); idmap_->build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(3, -1); @@ -384,13 +384,14 @@ TEST_F(IDMapIndexTest, search_with_not_id_selector) { namespace { using nsparse::idx_t; +using nsparse::offset_t; using nsparse::term_t; // A CSR corpus exposing the fields csr_test::write_interchange_csr needs // (.n / .indptr / .indices / .values). struct Corpus { idx_t n = 0; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; }; @@ -415,7 +416,7 @@ Corpus make_corpus(idx_t rows, int dim, unsigned seed) { c.indices.push_back(static_cast(t)); c.values.push_back(val_dist(rng)); } - c.indptr.push_back(static_cast(c.indices.size())); + c.indptr.push_back(static_cast(c.indices.size())); } return c; } diff --git a/tests/index_io_test.cpp b/tests/index_io_test.cpp index e7068a5..0bf71f0 100644 --- a/tests/index_io_test.cpp +++ b/tests/index_io_test.cpp @@ -92,7 +92,7 @@ class MockIndex : public nsparse::Index, public nsparse::IndexIO { std::array id() const override { return name; } uint32_t format_version() const override { return kFormatVersion; } - void add(nsparse::idx_t /*n*/, const nsparse::idx_t* /*indptr*/, + void add(nsparse::idx_t /*n*/, const nsparse::offset_t* /*indptr*/, const nsparse::term_t* /*indices*/, const float* /*values*/) override {} @@ -171,8 +171,8 @@ TEST(IndexIO, WriteIndexThrowsForNonIndexIO) { public: NonSerializableIndex() : Index(10) {} std::array id() const override { return {'N', 'O', 'I', 'O'}; } - void add(nsparse::idx_t, const nsparse::idx_t*, const nsparse::term_t*, - const float*) override {} + void add(nsparse::idx_t, const nsparse::offset_t*, + const nsparse::term_t*, const float*) override {} const nsparse::SparseVectors* get_vectors() const override { return nullptr; } @@ -350,7 +350,7 @@ TEST(IndexIO, RoundtripIDMapIndexWithData) { auto* original = new nsparse::IDMapIndex(seismic); // Add some vectors with custom IDs - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -436,7 +436,7 @@ TEST(IndexIO, StrictIO_RoundtripIDMapSeismicIndex) { auto* seismic = new nsparse::SeismicIndex(128); auto* original = new nsparse::IDMapIndex(seismic); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -465,7 +465,7 @@ TEST(IndexIO, StrictIO_RoundtripIDMapInvertedIndex) { auto* inverted = new nsparse::InvertedIndex(128); auto* original = new nsparse::IDMapIndex(inverted); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -495,7 +495,7 @@ TEST(IndexIO, StrictIO_RoundtripIDMapInvertedIndex) { TEST(IndexIO, RoundtripInvertedIndexCountsATrailingEmptyDocument) { nsparse::InvertedIndex original(128); - std::vector indptr = {0, 2, 2}; + std::vector indptr = {0, 2, 2}; std::vector indices = {0, 1}; std::vector values = {1.0F, 0.5F}; original.add(2, indptr.data(), indices.data(), values.data()); @@ -519,7 +519,7 @@ TEST(IndexIO, RoundtripInvertedIndexCountsATrailingEmptyDocument) { TEST(IndexIO, ReadIndexRejectsAnInvertedIndexFileWithoutTheDocumentCount) { nsparse::InvertedIndex original(128); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; original.add(2, indptr.data(), indices.data(), values.data()); @@ -543,7 +543,7 @@ TEST(IndexIO, StrictIO_RoundtripIDMapSeismicSQIndex) { auto* sq_index = new nsparse::SeismicScalarQuantizedIndex(128); auto* original = new nsparse::IDMapIndex(sq_index); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -595,7 +595,7 @@ TEST(IndexIO, UseMmapFlagMapsTheInvertedIndexDelegateOfAnIDMap) { auto* inverted = new nsparse::InvertedIndex(128); nsparse::IDMapIndex original(inverted); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -612,7 +612,7 @@ TEST(IndexIO, UseMmapFlagMapsTheInvertedIndexDelegateOfAnIDMap) { ASSERT_EQ(loaded->num_vectors(), 2); // The id map still round-trips: search returns external ids, not internal. - std::vector q_indptr = {0, 1}; + std::vector q_indptr = {0, 1}; std::vector q_indices = {0}; std::vector q_values = {1.0F}; std::vector labels(1, nsparse::detail::INVALID_IDX); @@ -632,7 +632,7 @@ TEST(IndexIO, UseMmapFlagReachesTheIDMapDelegate) { new nsparse::SeismicIndex(5, {.lambda = 10, .beta = 2, .alpha = 0.5F}); nsparse::IDMapIndex original(seismic); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -658,10 +658,10 @@ TEST(IndexIO, UseMmapFlagReachesTheIDMapDelegate) { reinterpret_cast(vectors->indices_data()); EXPECT_EQ(indices_bytes - indptr_bytes, static_cast((vectors->num_vectors() + 1) * - sizeof(nsparse::idx_t))); + sizeof(nsparse::offset_t))); // The id map still round-trips: search returns external ids, not internal. - std::vector q_indptr = {0, 1}; + std::vector q_indptr = {0, 1}; std::vector q_indices = {0}; std::vector q_values = {1.0F}; std::vector labels(1, nsparse::detail::INVALID_IDX); @@ -681,7 +681,7 @@ TEST(IndexIO, UseMmapFlagReachesTheIDMapQuantizedDelegate) { {.lambda = 10, .beta = 2, .alpha = 0.5F}, 5); nsparse::IDMapIndex original(seismic_sq); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -706,12 +706,12 @@ TEST(IndexIO, UseMmapFlagReachesTheIDMapQuantizedDelegate) { reinterpret_cast(vectors->indices_data()); EXPECT_EQ(indices_bytes - indptr_bytes, static_cast((vectors->num_vectors() + 1) * - sizeof(nsparse::idx_t))); + sizeof(nsparse::offset_t))); // The quantizer header survived the mapped read, so the codes are 1 byte // wide rather than being strided as floats. EXPECT_EQ(vectors->get_element_size(), 1); - std::vector q_indptr = {0, 1}; + std::vector q_indptr = {0, 1}; std::vector q_indices = {0}; std::vector q_values = {1.0F}; std::vector labels(1, nsparse::detail::INVALID_IDX); @@ -854,7 +854,7 @@ TEST(IndexIOVersion, ReadIndexAcceptsTheVersionItWrites) { // SEIS, and vice versa. TEST(IndexIOVersion, ANestedDelegateCarriesItsOwnVersion) { nsparse::IDMapIndex original(new nsparse::SeismicIndex(128)); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; std::vector ids = {100, 200}; @@ -893,7 +893,7 @@ TEST(IndexIOVersion, ANestedDelegateCarriesItsOwnVersion) { TEST(IndexIOVersion, MappedReadAlsoRejectsAVersionFromTheFuture) { TempIndexFile file("nsparse_index_io_future_version_mmap.idx"); nsparse::InvertedIndex original(128); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; original.add(2, indptr.data(), indices.data(), values.data()); @@ -931,7 +931,7 @@ TEST(IndexIOVersion, MappedReadRejectsAFutureDiskSeismicVersion) { TempIndexFile file("nsparse_index_io_dsei_future_version.idx"); nsparse::DiskSeismicIndex original( 5, {.lambda = 10, .beta = 2, .alpha = 0.5F}); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {1.0F, 0.5F, 0.8F, 0.3F}; original.add(2, indptr.data(), indices.data(), values.data()); diff --git a/tests/index_test.cpp b/tests/index_test.cpp index 59e7c5e..92253e0 100644 --- a/tests/index_test.cpp +++ b/tests/index_test.cpp @@ -31,7 +31,7 @@ class RecordingIndex : public nsparse::Index { std::array id() const override { return {'T', 'E', 'S', 'T'}; } - void add(nsparse::idx_t n, const nsparse::idx_t* indptr, + void add(nsparse::idx_t n, const nsparse::offset_t* indptr, const nsparse::term_t* indices, const float* values) override { num_added = n; this->indptr.assign(indptr, indptr + n + 1); @@ -42,7 +42,7 @@ class RecordingIndex : public nsparse::Index { int add_calls = 0; nsparse::idx_t num_added = 0; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; }; @@ -116,7 +116,7 @@ TEST(IndexReadCSR, adds_vectors_from_file) { ASSERT_EQ(index.add_calls, 1); ASSERT_EQ(index.num_added, 3); - ASSERT_EQ(index.indptr, (std::vector{0, 2, 2, 3})); + ASSERT_EQ(index.indptr, (std::vector{0, 2, 2, 3})); ASSERT_EQ(index.indices, (std::vector{0, 2, 1})); ASSERT_EQ(index.values, (std::vector{1.5F, 2.5F, 3.5F})); } diff --git a/tests/inverted_index_recall_test.cpp b/tests/inverted_index_recall_test.cpp index a3461c6..0df307b 100644 --- a/tests/inverted_index_recall_test.cpp +++ b/tests/inverted_index_recall_test.cpp @@ -115,7 +115,7 @@ RandomSparseData generate_random_data(int dim, int n_docs, int n_queries, void add_docs_from_data( InvertedIndex& index, const std::vector>>& docs) { - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; @@ -125,7 +125,7 @@ void add_docs_from_data( indices.push_back(term); values.push_back(value); } - indptr.push_back(static_cast(indices.size())); + indptr.push_back(static_cast(indices.size())); } index.add(static_cast(docs.size()), indptr.data(), indices.data(), @@ -169,8 +169,8 @@ TEST_P(InvertedIndexRecallTest, recall_should_be_perfect) { if (query.empty()) continue; // Build query CSR. - std::vector q_indptr = {0, - static_cast(query.size())}; + std::vector q_indptr = {0, + static_cast(query.size())}; std::vector q_indices; std::vector q_values; for (auto& [t, v] : query) { diff --git a/tests/inverted_index_test.cpp b/tests/inverted_index_test.cpp index 40acdd9..bde26a8 100644 --- a/tests/inverted_index_test.cpp +++ b/tests/inverted_index_test.cpp @@ -34,7 +34,7 @@ namespace { // Helper to add docs from map format: {{term: value, ...}, ...} void add_docs(InvertedIndex& index, const std::vector>& docs) { - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; @@ -44,7 +44,7 @@ void add_docs(InvertedIndex& index, indices.push_back(static_cast(term)); values.push_back(value); } - indptr.push_back(static_cast(indices.size())); + indptr.push_back(static_cast(indices.size())); } index.add(static_cast(docs.size()), indptr.data(), indices.data(), @@ -107,7 +107,7 @@ TEST(InvertedIndexSearch, search_returns_empty_when_not_built) { InvertedIndex index(5); Index* idx = &index; - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.5F}; std::vector labels(5, -1); @@ -128,7 +128,7 @@ TEST(InvertedIndexSearch, search_finds_matching_doc) { add_docs(index, {{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -148,7 +148,7 @@ TEST(InvertedIndexSearch, search_multiple_queries) { add_docs(index, {{{0, 1.0F}, {1, 0.5F}}, {{2, 0.8F}, {3, 0.6F}}}); index.build(); - std::vector query_indptr = {0, 1, 2}; + std::vector query_indptr = {0, 1, 2}; std::vector query_indices = {0, 2}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -168,7 +168,7 @@ TEST(InvertedIndexSearch, search_respects_k_limit) { add_docs(index, {{{0, 1.0F}}, {{0, 0.9F}}, {{0, 0.8F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(2, -1); @@ -190,7 +190,7 @@ TEST(InvertedIndexSearch, search_returns_results_sorted_by_score) { add_docs(index, {{{0, 0.3F}}, {{0, 1.0F}}, {{0, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -213,7 +213,7 @@ TEST(InvertedIndexSearch, search_with_no_matching_term) { index.build(); // Query with term 3 which no doc has - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {3}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -241,7 +241,7 @@ TEST(InvertedIndexSearch, search_multi_term_dot_product) { // Scores: doc0 = 1.0*1.0 + 0.5*0.8 = 1.4 // doc1 = 0.8*1.0 = 0.8 // doc2 = 0.9*0.8 = 0.72 - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.8F}; std::vector labels(3, -1); @@ -265,7 +265,7 @@ TEST(InvertedIndexSearch, search_k_larger_than_num_docs) { add_docs(index, {{{0, 1.0F}}, {{0, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(5, -1); @@ -330,7 +330,7 @@ TEST(InvertedIndexIO, write_and_read_search_produces_same_results) { original.build(); // Search on original - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.8F}; std::vector labels_original(3, -1); @@ -381,7 +381,7 @@ TEST(InvertedIndexIO, write_and_read_with_empty_posting_lists) { EXPECT_EQ(loaded->get_dimension(), 5); // Verify search still works — query term 0 - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -409,7 +409,7 @@ TEST(InvertedIndexIO, write_and_read_multiple_docs_per_term) { Index* loaded = read_index(&reader); // Search for all 4 docs - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(4, -1); @@ -471,7 +471,7 @@ TEST(InvertedIndexSearch, search_multi_window_no_crash) { index.build(); // Query both terms — forces multi-term scoring across multiple windows. - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; @@ -510,7 +510,7 @@ TEST(InvertedIndexSearch, search_with_id_selector_filters_results) { SearchParameters params; params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -545,7 +545,7 @@ TEST(InvertedIndexSearch, DerivedSearchParameters params; params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -573,7 +573,7 @@ TEST(InvertedIndexSearch, search_with_id_selector_matching_nothing) { SearchParameters params; params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(2, -1); @@ -601,7 +601,7 @@ TEST(InvertedIndexSearch, search_with_non_enumerable_id_selector) { SearchParameters params; params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -632,7 +632,7 @@ TEST(InvertedIndexSearch, search_with_id_selector_multi_term) { SearchParameters params; params.set_id_selector(&selector); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.8F}; std::vector labels(3, -1); @@ -673,7 +673,7 @@ TEST(InvertedIndexSearch, search_with_id_selector_multi_window) { SearchParameters params; params.set_id_selector(&selector); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 1.0F}; @@ -707,7 +707,7 @@ TEST(InvertedIndexSearch, search_ignores_query_terms_beyond_the_dimension) { index.build(); // Term 0 exists; term 9 is past the dimension entirely. - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 9}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -768,7 +768,7 @@ std::unique_ptr built_index() { pair_of_score_id_vector_t search_one(Index* index, const std::vector& terms, const std::vector& weights, int k) { - std::vector indptr = {0, static_cast(terms.size())}; + std::vector indptr = {0, static_cast(terms.size())}; std::vector distances(k, -1.0F); std::vector labels(k, detail::INVALID_IDX); index->search(1, indptr.data(), terms.data(), weights.data(), k, diff --git a/tests/inverted_lists_test.cpp b/tests/inverted_lists_test.cpp index 3a6f2bf..8ce3072 100644 --- a/tests/inverted_lists_test.cpp +++ b/tests/inverted_lists_test.cpp @@ -643,7 +643,7 @@ std::unique_ptr lists_for_io() { .dimension = 3}; nsparse::SparseVectors vectors(config); // doc0: {0: 1.5, 2: 2.5}, doc1: {0: 0.5}, doc2: {0: 3.5} - std::vector indptr = {0, 2, 3, 4}; + std::vector indptr = {0, 2, 3, 4}; std::vector indices = {0, 2, 0, 0}; std::vector values = {1.5F, 2.5F, 0.5F, 3.5F}; vectors.add_vectors(indptr.data(), indptr.size(), indices.data(), diff --git a/tests/mmap_index_test.cpp b/tests/mmap_index_test.cpp index 2317096..59532db 100644 --- a/tests/mmap_index_test.cpp +++ b/tests/mmap_index_test.cpp @@ -30,7 +30,7 @@ class TestMmapIndex : public nsparse::MmapIndex { std::array id() const override { return {'M', 'M', 'A', 'P'}; } - void add(nsparse::idx_t n, const nsparse::idx_t* indptr, + void add(nsparse::idx_t n, const nsparse::offset_t* indptr, const nsparse::term_t* indices, const float* values) override { num_added = n; ++add_calls; @@ -58,17 +58,18 @@ class TempNativeCSRFile { TempNativeCSRFile& operator=(const TempNativeCSRFile&) = delete; void write(int64_t num_rows, int64_t num_cols, int64_t nnz, - const std::vector& indptr, + const std::vector& indptr, const std::vector& indices, const std::vector& values) { std::ofstream file(path_, std::ios::binary); const std::array header = {num_rows, num_cols, nnz}; write_all(file, header.data(), header.size() * sizeof(int64_t)); - write_all(file, indptr.data(), indptr.size() * sizeof(nsparse::idx_t)); + write_all(file, indptr.data(), + indptr.size() * sizeof(nsparse::offset_t)); write_all(file, indices.data(), indices.size() * sizeof(nsparse::term_t)); const size_t unaligned = header.size() * sizeof(int64_t) + - indptr.size() * sizeof(nsparse::idx_t) + + indptr.size() * sizeof(nsparse::offset_t) + indices.size() * sizeof(nsparse::term_t); const std::array padding{}; write_all(file, padding.data(), unaligned % alignof(float) == 0 @@ -150,7 +151,7 @@ TEST(MmapIndexReadCSR, values_point_into_the_mapping) { reinterpret_cast(vectors->indptr_data()); const auto* indices_bytes = reinterpret_cast(vectors->indices_data()); - ASSERT_EQ(indices_bytes - indptr_bytes, 2 * sizeof(nsparse::idx_t)); + ASSERT_EQ(indices_bytes - indptr_bytes, 2 * sizeof(nsparse::offset_t)); } TEST(MmapIndexReadCSR, falls_back_to_index_read_csr_when_disabled) { diff --git a/tests/offset_width_test.cpp b/tests/offset_width_test.cpp new file mode 100644 index 0000000..7e57c83 --- /dev/null +++ b/tests/offset_width_test.cpp @@ -0,0 +1,139 @@ +/** + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +#include + +#include +#include +#include +#include +#include + +#include "nsparse/index.h" +#include "nsparse/index_factory.h" +#include "nsparse/io/buffered_io.h" +#include "nsparse/sparse_vectors.h" +#include "nsparse/types.h" +#include "nsparse/utils/csr_layout.h" +#include "nsparse/utils/mmap_cursor.h" + +namespace nsparse { +namespace { + +// offset_t is the widened CSR nnz-offset type: a signed 64-bit integer, distinct +// from the 32-bit idx_t used for doc-ids/labels. +static_assert(std::is_same_v); +static_assert(sizeof(offset_t) == 8); +static_assert(std::is_signed_v); +static_assert(sizeof(idx_t) == 4); + +// The storage root hands out 64-bit offsets, not 32-bit doc indices. +static_assert(std::is_same_v() + .indptr_data()), + const offset_t*>); + +int64_t aligned_values_offset(int64_t indptr_size, int64_t nnz) { + const int64_t unaligned = 3 * 8 + indptr_size * 8 + nnz * 2; + return unaligned + (4 - unaligned % 4) % 4; +} + +// The native .mcsr layout stores indptr at 8 bytes/entry (offset_t), so the +// values section sits 4 bytes further out per entry than the old int32 layout. +TEST(OffsetWidth, NativeLayoutUsesEightByteIndptr) { + EXPECT_EQ(csr_layout::native_values_offset(3, 0), 24 + 24); + EXPECT_EQ(csr_layout::native_values_offset(6, 7), + aligned_values_offset(6, 7)); + EXPECT_EQ(csr_layout::native_values_offset(101, 50), + aligned_values_offset(101, 50)); + EXPECT_EQ(csr_layout::native_file_size(6, 7, sizeof(float)), + csr_layout::native_values_offset(6, 7) + 7 * sizeof(float)); +} + +// Round-trips a small corpus through both serialize/deserialize and +// serialize/mmap_deserialize, asserting indptr comes back identical as 64-bit +// words -- exercising the widened read_padded / read_array +// format path. The terminal offset fits int32 here; the >2^31 gate is the +// DISABLED_ test below. +TEST(OffsetWidth, IndptrRoundTripsAsSixtyFourBit) { + SparseVectors original({.element_size = U32, .dimension = 8}); + const std::vector indptr = {0, 2, 3, 6}; + const std::vector indices = {0, 1, 2, 3, 4, 5}; + const std::vector values(6 * U32, 0); + original.add_vectors(indptr, indices, values); + + BufferedIOWriter writer; + original.serialize(&writer); + const std::vector bytes = writer.data(); + + BufferedIOReader reader(bytes); + SparseVectors deserialized; + deserialized.deserialize(&reader); + + MmapCursor cursor(bytes.data(), bytes.size()); + SparseVectors mapped; + mapped.mmap_deserialize(&cursor); + + for (const SparseVectors* v : {&original, &deserialized, &mapped}) { + ASSERT_EQ(v->num_vectors(), 3U); + for (size_t i = 0; i < indptr.size(); ++i) { + EXPECT_EQ(v->indptr_data()[i], indptr[i]); + } + } +} + +// The real acceptance gate: build a corpus whose cumulative nnz exceeds +// INT32_MAX and search it end to end, so the widened offset_t flows through +// add -> build (clustering, inverted lists, forward index) -> search instead +// of wrapping negative past the ~2.1-billionth nnz. Disabled by default: the +// corpus needs tens of GB. Run on a large host with +// --gtest_also_run_disabled_tests. +TEST(OffsetWidth, DISABLED_CrossesInt32BoundaryEndToEnd) { + const idx_t n_docs = 36'000; + const int dim = 60'000; // < term_t max; every doc carries all `dim` terms + const int64_t per_doc = dim; + const int64_t total_nnz = static_cast(n_docs) * per_doc; + ASSERT_GT(total_nnz, + static_cast(std::numeric_limits::max())); + + std::vector indptr(n_docs + 1); + for (idx_t d = 0; d <= n_docs; ++d) indptr[d] = static_cast(d) * + per_doc; + std::vector indices(static_cast(total_nnz)); + for (idx_t d = 0; d < n_docs; ++d) { + for (int t = 0; t < dim; ++t) { + indices[static_cast(indptr[d]) + t] = + static_cast(t); + } + } + std::vector values(static_cast(total_nnz), 1.0F); + ASSERT_GT(indptr[n_docs], + static_cast(std::numeric_limits::max())); + + std::unique_ptr index(index_factory(dim, "seismic")); + index->add(n_docs, indptr.data(), indices.data(), values.data()); + index->build(); + ASSERT_EQ(index->num_vectors(), static_cast(n_docs)); + + std::vector q_indices(dim); + for (int t = 0; t < dim; ++t) q_indices[t] = static_cast(t); + const std::vector q_values(dim, 1.0F); + const offset_t q_indptr[2] = {0, dim}; + const int k = 10; + std::vector distances(k); + std::vector labels(k); + index->search(1, q_indptr, q_indices.data(), q_values.data(), k, + distances.data(), labels.data()); + + EXPECT_GE(labels[0], 0); + EXPECT_LT(labels[0], n_docs); + EXPECT_GT(distances[0], 0.0F); +} + +} // namespace +} // namespace nsparse diff --git a/tests/seismic_batched_build_test.cpp b/tests/seismic_batched_build_test.cpp index d83f191..12404ab 100644 --- a/tests/seismic_batched_build_test.cpp +++ b/tests/seismic_batched_build_test.cpp @@ -50,7 +50,7 @@ constexpr float kAlpha = 0.4F; struct Corpus { int dim; idx_t n = 0; - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; }; @@ -77,7 +77,7 @@ Corpus make_corpus(int n_docs, int dim, unsigned seed) { corpus.indices.push_back(static_cast(term)); corpus.values.push_back(val_dist(gen)); } - corpus.indptr.push_back(static_cast(corpus.indices.size())); + corpus.indptr.push_back(static_cast(corpus.indices.size())); } return corpus; } @@ -389,7 +389,7 @@ TEST(SeismicBatchedBuild, HandlesATermHeavierThanAWholeWindow) { skewed.indptr.push_back(0); for (idx_t doc = 0; doc < corpus.n; ++doc) { std::vector> row; - for (idx_t j = corpus.indptr[doc]; j < corpus.indptr[doc + 1]; ++j) { + for (offset_t j = corpus.indptr[doc]; j < corpus.indptr[doc + 1]; ++j) { if (corpus.indices[j] != 7) { row.emplace_back(corpus.indices[j], corpus.values[j]); } @@ -400,7 +400,7 @@ TEST(SeismicBatchedBuild, HandlesATermHeavierThanAWholeWindow) { skewed.indices.push_back(term); skewed.values.push_back(value); } - skewed.indptr.push_back(static_cast(skewed.indices.size())); + skewed.indptr.push_back(static_cast(skewed.indices.size())); } TempDir dir("skewed"); diff --git a/tests/seismic_common_test.cpp b/tests/seismic_common_test.cpp index 3b0162d..736872f 100644 --- a/tests/seismic_common_test.cpp +++ b/tests/seismic_common_test.cpp @@ -117,7 +117,7 @@ TEST(CalculateSummaryScores, multiple_vectors) { TEST(ComputeSimilarity, float_element_size) { // 2 docs: doc0 has indices {1}, values {2.0f}; doc1 has indices {0,2}, // values {1.0f, 3.0f} - std::vector indptr = {0, 1, 3}; + std::vector indptr = {0, 1, 3}; std::vector indices = {1, 0, 2}; // Values stored as raw bytes @@ -142,7 +142,7 @@ TEST(ComputeSimilarity, float_element_size) { } TEST(ComputeSimilarity, uint16_element_size) { - std::vector indptr = {0, 2}; + std::vector indptr = {0, 2}; std::vector indices = {0, 1}; std::vector values(2 * sizeof(uint16_t)); @@ -160,7 +160,7 @@ TEST(ComputeSimilarity, uint16_element_size) { } TEST(ComputeSimilarity, uint8_element_size) { - std::vector indptr = {0, 3}; + std::vector indptr = {0, 3}; std::vector indices = {0, 1, 2}; std::vector values = {2, 3, 4}; std::vector dense = {10, 20, 30}; @@ -172,7 +172,7 @@ TEST(ComputeSimilarity, uint8_element_size) { } TEST(ComputeSimilarity, empty_doc) { - std::vector indptr = {0, 0}; + std::vector indptr = {0, 0}; std::vector indices = {}; std::vector values = {}; std::vector dense = {1, 2, 3}; diff --git a/tests/seismic_index_test.cpp b/tests/seismic_index_test.cpp index 35a9fd9..c515e94 100644 --- a/tests/seismic_index_test.cpp +++ b/tests/seismic_index_test.cpp @@ -50,7 +50,7 @@ class TestableSeismicIndex : public SeismicIndex { // Helper to add docs from map format: {{term: value, ...}, ...} void add_docs(const std::vector>& docs) { - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; @@ -60,7 +60,7 @@ class TestableSeismicIndex : public SeismicIndex { indices.push_back(static_cast(term)); values.push_back(value); } - indptr.push_back(static_cast(indices.size())); + indptr.push_back(static_cast(indices.size())); } SeismicIndex::add(static_cast(docs.size()), indptr.data(), @@ -233,7 +233,7 @@ TEST(SeismicIndexSearch, search_returns_empty_when_no_vectors) { SeismicIndex index(5); Index* idx = &index; - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.5F}; std::vector labels(5, -1); @@ -254,7 +254,7 @@ TEST(SeismicIndexSearch, search_finds_matching_doc) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -275,7 +275,7 @@ TEST(SeismicIndexSearch, search_multiple_queries) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}, {{2, 0.8F}, {3, 0.6F}}}); index.build(); - std::vector query_indptr = {0, 1, 2}; + std::vector query_indptr = {0, 1, 2}; std::vector query_indices = {0, 2}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -297,7 +297,7 @@ TEST(SeismicIndexSearch, search_respects_k_limit) { index.add_docs({{{0, 1.0F}}, {{0, 0.9F}}, {{0, 0.8F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(2, -1); @@ -322,7 +322,7 @@ TEST(SeismicIndexSearch, search_with_no_matching_term) { // Query with term {3} which doesn't exist in any doc // Term 3's inverted list is empty, so no docs should be visited - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {3}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -347,7 +347,7 @@ TEST(SeismicIndexSearch, search_returns_results_sorted_by_score) { index.add_docs({{{0, 0.3F}}, {{0, 1.0F}}, {{0, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -371,7 +371,7 @@ TEST(SeismicIndexSearch, search_with_default_parameters) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.5F}; std::vector labels(1, -1); @@ -407,7 +407,7 @@ TEST(SeismicIndexSearch, lambda_prunes_posting_list) { EXPECT_EQ(total_docs, 2); // Search should only find doc0 and doc1, not doc2 or doc3 - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(4, -1); @@ -436,7 +436,7 @@ TEST(SeismicIndexSearch, cut_prunes_query_tokens) { // Query has 3 terms: term0 (0.1), term1 (0.5), term2 (0.9) // With cut=1, only term2 (highest weight 0.9) is used - std::vector query_indptr = {0, 3}; + std::vector query_indptr = {0, 3}; std::vector query_indices = {0, 1, 2}; std::vector query_values = {0.1F, 0.5F, 0.9F}; std::vector labels(1, -1); @@ -473,7 +473,7 @@ TEST(SeismicIndexSearch, large_heap_factor_includes_all_clusters) { index.build(); // Query with terms 0, 1, 2 (weights 1.0, 0.8, 0.5) - std::vector query_indptr = {0, 3}; + std::vector query_indptr = {0, 3}; std::vector query_indices = {0, 1, 2}; std::vector query_values = {1.0F, 0.8F, 0.5F}; std::vector labels(6, -1); @@ -523,7 +523,7 @@ TEST(SeismicIndexSearch, small_heap_factor_prunes_clusters) { index.build(); // Query with terms 0, 1, 2 (weights 1.0, 0.8, 0.5) - std::vector query_indptr = {0, 3}; + std::vector query_indptr = {0, 3}; std::vector query_indices = {0, 1, 2}; std::vector query_values = {1.0F, 0.8F, 0.5F}; @@ -594,7 +594,7 @@ TEST(SeismicIndexSearch, heap_factor_controls_result_count_large_dataset) { index.build(); // Query with 4 terms (similar to OpenSearch test) - std::vector query_indptr = {0, 4}; + std::vector query_indptr = {0, 4}; std::vector query_indices = {1000, 2000, 3000, 4000}; std::vector query_values = {0.12F, 0.64F, 0.87F, 0.53F}; @@ -739,7 +739,7 @@ TEST(SeismicIndexIO, write_and_read_search_produces_same_results) { original.build(); // Search on original - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.8F}; std::vector labels_original(3, -1); @@ -792,7 +792,7 @@ TEST(SeismicIndexIO, write_and_read_multiple_terms) { EXPECT_EQ(loaded->get_vectors()->num_vectors(), 4); // Verify search works on loaded index - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -825,7 +825,7 @@ TEST(SeismicIndexSearch, search_exact_match_with_small_selector) { SeismicSearchParameters params(5, 1.0F); params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(2, -1); @@ -856,7 +856,7 @@ TEST(SeismicIndexSearch, search_with_id_selector_filters_results) { SeismicSearchParameters params(5, 1.0F); params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -911,7 +911,7 @@ std::unique_ptr built_index() { } std::vector search_top(Index* index, term_t term, int k) { - std::vector indptr = {0, 1}; + std::vector indptr = {0, 1}; std::vector indices = {term}; std::vector values = {1.0F}; std::vector labels(k, detail::INVALID_IDX); @@ -962,7 +962,7 @@ void expect_both_reads_rejected(char* path, const char* fragment) { // Raw-array CSR corpus. A reproducible random corpus with distinct, ascending // terms per row (CSR convention) and values in (0, 1]. struct RawCsr { - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; idx_t n = 0; @@ -986,7 +986,7 @@ RawCsr make_raw_corpus(idx_t rows, int dim, unsigned seed) { c.indices.push_back(static_cast(t)); c.values.push_back(val_dist(rng)); } - c.indptr.push_back(static_cast(c.indices.size())); + c.indptr.push_back(static_cast(c.indices.size())); } return c; } @@ -1047,7 +1047,7 @@ TEST(SeismicIndexMmapIO, mapped_read_borrows_from_the_file) { reinterpret_cast(vectors->indices_data()); EXPECT_EQ(indices_bytes - indptr_bytes, static_cast((vectors->num_vectors() + 1) * - sizeof(idx_t))); + sizeof(offset_t))); } // Building from a native CSR borrowed via mmap (read_csr(kMmap)) must match @@ -1175,7 +1175,7 @@ std::pair, std::vector> build_and_search(int seed) { index.add_docs(seed_test_docs(kDocs, kDim)); index.build(); - std::vector query_indptr = {0, 3}; + std::vector query_indptr = {0, 3}; std::vector query_indices = {1, 17, 33}; std::vector query_values = {1.0F, 0.7F, 0.4F}; std::vector distances(kTopK); diff --git a/tests/seismic_scalar_quantized_index_test.cpp b/tests/seismic_scalar_quantized_index_test.cpp index 4961072..765dd32 100644 --- a/tests/seismic_scalar_quantized_index_test.cpp +++ b/tests/seismic_scalar_quantized_index_test.cpp @@ -56,7 +56,7 @@ class TestableSeismicSQIndex : public SeismicScalarQuantizedIndex { // Helper to add docs from map format: {{term: value, ...}, ...} void add_docs(const std::vector>& docs) { - std::vector indptr; + std::vector indptr; std::vector indices; std::vector values; @@ -66,7 +66,7 @@ class TestableSeismicSQIndex : public SeismicScalarQuantizedIndex { indices.push_back(static_cast(term)); values.push_back(value); } - indptr.push_back(static_cast(indices.size())); + indptr.push_back(static_cast(indices.size())); } SeismicScalarQuantizedIndex::add(static_cast(docs.size()), @@ -252,7 +252,7 @@ TEST(SeismicSQIndexSearch, search_returns_empty_when_no_vectors) { 5); Index* idx = &index; - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.5F}; std::vector labels(5, -1); @@ -276,7 +276,7 @@ TEST(SeismicSQIndexSearch, search_finds_matching_doc_8bit) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -298,7 +298,7 @@ TEST(SeismicSQIndexSearch, search_finds_matching_doc_16bit) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -320,7 +320,7 @@ TEST(SeismicSQIndexSearch, search_multiple_queries) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}, {{2, 0.8F}, {3, 0.6F}}}); index.build(); - std::vector query_indptr = {0, 1, 2}; + std::vector query_indptr = {0, 1, 2}; std::vector query_indices = {0, 2}; std::vector query_values = {1.0F, 1.0F}; std::vector labels(2, -1); @@ -343,7 +343,7 @@ TEST(SeismicSQIndexSearch, search_respects_k_limit) { index.add_docs({{{0, 1.0F}}, {{0, 0.9F}}, {{0, 0.8F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(2, -1); @@ -366,7 +366,7 @@ TEST(SeismicSQIndexSearch, search_with_default_parameters) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.5F}; std::vector labels(1, -1); @@ -397,7 +397,7 @@ TEST(SeismicSQIndexSearch, lambda_prunes_posting_list) { } EXPECT_EQ(total_docs, 2); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(4, -1); @@ -420,7 +420,7 @@ TEST(SeismicSQIndexSearch, cut_prunes_query_tokens) { index.add_docs({{{0, 1.0F}}, {{1, 1.0F}}, {{2, 1.0F}}, {{3, 1.0F}}}); index.build(); - std::vector query_indptr = {0, 3}; + std::vector query_indptr = {0, 3}; std::vector query_indices = {0, 1, 2}; std::vector query_values = {0.1F, 0.5F, 0.9F}; std::vector labels(1, -1); @@ -447,7 +447,7 @@ TEST(SeismicSQIndexSearch, large_heap_factor_includes_all_clusters) { {{0, 0.1F}, {3, 0.4F}}}); index.build(); - std::vector query_indptr = {0, 3}; + std::vector query_indptr = {0, 3}; std::vector query_indices = {0, 1, 2}; std::vector query_values = {1.0F, 0.8F, 0.5F}; std::vector labels(6, -1); @@ -475,7 +475,7 @@ TEST(SeismicSQIndexSearch, search_with_sq_search_parameters) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(1, -1); @@ -561,7 +561,7 @@ TEST(SeismicSQIndexIO, write_and_read_search_produces_same_results) { {{1, 0.9F}, {3, 0.7F}}}); original.build(); - std::vector query_indptr = {0, 2}; + std::vector query_indptr = {0, 2}; std::vector query_indices = {0, 1}; std::vector query_values = {1.0F, 0.8F}; std::vector labels_original(3, -1); @@ -665,7 +665,7 @@ TEST(SeismicSQIndexSearch, heap_factor_controls_result_count_large_dataset) { index.add_docs(docs); index.build(); - std::vector query_indptr = {0, 4}; + std::vector query_indptr = {0, 4}; std::vector query_indices = {1000, 2000, 3000, 4000}; std::vector query_values = {0.12F, 0.64F, 0.87F, 0.53F}; @@ -722,7 +722,7 @@ TEST(SeismicSQIndexSearch, search_exact_match_with_small_selector) { SeismicSearchParameters params(5, 1.0F); params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(2, -1); @@ -747,7 +747,7 @@ TEST(SeismicSQIndexSearch, search_exact_match_scores_match_normal_path_scores) { index.add_docs({{{0, 1.0F}}, {{0, 0.5F}}, {{0, 0.8F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; @@ -812,7 +812,7 @@ TEST(SeismicSQIndexSearch, search_with_id_selector_filters_results) { SeismicSearchParameters params(5, 1.0F); params.set_id_selector(&selector); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; std::vector labels(3, -1); @@ -841,7 +841,7 @@ TEST(SeismicSQIndexSearch, search_without_seismic_parameters_uses_defaults) { index.add_docs({{{0, 1.0F}, {1, 0.5F}}, {{0, 0.2F}}}); index.build(); - std::vector query_indptr = {0, 1}; + std::vector query_indptr = {0, 1}; std::vector query_indices = {0}; std::vector query_values = {1.0F}; @@ -902,7 +902,7 @@ std::unique_ptr built_index(QuantizerType qtype) { } std::vector search_top(Index* index, term_t term, int k) { - std::vector indptr = {0, 1}; + std::vector indptr = {0, 1}; std::vector indices = {term}; std::vector values = {1.0F}; std::vector labels(k, detail::INVALID_IDX); @@ -918,7 +918,7 @@ std::vector search_top(Index* index, term_t term, int k) { std::pair, std::vector> search_scored(Index* index, term_t term, int k) { - std::vector indptr = {0, 1}; + std::vector indptr = {0, 1}; std::vector indices = {term}; std::vector values = {1.0F}; std::vector labels(k, detail::INVALID_IDX); @@ -1147,7 +1147,7 @@ TEST_P(SeismicSQIndexMmapIO, mapped_values_are_aligned_for_the_code_width) { EXPECT_EQ( reinterpret_cast(vectors->values_data()) % element_size, 0U); EXPECT_EQ( - reinterpret_cast(vectors->indptr_data()) % alignof(idx_t), + reinterpret_cast(vectors->indptr_data()) % alignof(offset_t), 0U); EXPECT_EQ( reinterpret_cast(vectors->indices_data()) % alignof(term_t), @@ -1247,7 +1247,8 @@ TEST(SeismicSQIndexMmapIOSingle, mapped_read_borrows_from_the_file) { reinterpret_cast(vectors->indices_data()); EXPECT_EQ( indices_bytes - indptr_bytes, - static_cast((vectors->num_vectors() + 1) * sizeof(idx_t))); + static_cast((vectors->num_vectors() + 1) * + sizeof(offset_t))); } // A stream has no file to map, so the flag alone must not send read_index down @@ -1368,7 +1369,7 @@ TEST(SeismicSQIndexMmapIOSingle, mmap_codes_csr_build_matches_add_build) { .lambda = 10, .beta = 2, .alpha = 0.5F, .seed = 42}; // A small reproducible corpus, held as raw CSR arrays so it can be fed both // ways from the same bytes. - const std::vector indptr = {0, 2, 4, 6, 9}; + const std::vector indptr = {0, 2, 4, 6, 9}; const std::vector indices = {0, 2, 1, 3, 0, 4, 2, 3, 4}; const std::vector values = {1.0F, 0.9F, 0.5F, 0.7F, 0.3F, 0.8F, 0.6F, 0.4F, 0.2F}; @@ -1414,7 +1415,7 @@ TEST(SeismicSQIndexMmapIOSingle, mmap_codes_csr_build_matches_add_build) { // 16-bit-wide values (element_size 2) are fed to an 8-bit index. TEST(SeismicSQIndexMmapIOSingle, mmap_codes_csr_wrong_width_is_rejected) { constexpr int kDim = 5; - const std::vector indptr = {0, 2, 4}; + const std::vector indptr = {0, 2, 4}; const std::vector indices = {0, 2, 1, 3}; std::vector wide_codes(indices.size() * 2); csr_test::TempCsrFiles csr("nsparse_sesq_wrongwidth"); diff --git a/tests/sparse_vectors_test.cpp b/tests/sparse_vectors_test.cpp index 7fc0644..01ef8f2 100644 --- a/tests/sparse_vectors_test.cpp +++ b/tests/sparse_vectors_test.cpp @@ -38,7 +38,7 @@ class SparseVectorsResidency : public ::testing::TestWithParam { // result rather than being locals of the caller. template nsparse::SparseVectors make(size_t dimension, - std::vector indptr, + std::vector indptr, std::vector indices, const std::vector& values) { const nsparse::SparseVectorsConfig config = { @@ -61,7 +61,7 @@ class SparseVectorsResidency : public ::testing::TestWithParam { private: struct Storage { - std::vector indptr; + std::vector indptr; std::vector indices; // Words, not bytes: map_vectors rejects a values pointer misaligned for // element_size, and an allocation is only guaranteed aligned for its own @@ -75,7 +75,7 @@ class SparseVectorsResidency : public ::testing::TestWithParam { }; template - const Storage& store(std::vector indptr, + const Storage& store(std::vector indptr, std::vector indices, const std::vector& values) { // A deque keeps earlier elements put, so a second make() in the same @@ -229,7 +229,7 @@ TEST_P(SparseVectorsResidency, get_dense_vector_out_of_range) { TEST_P(SparseVectorsResidency, indptr_data) { auto vectors = make(5, {0, 2, 3}, {0, 1, 4}, {1.0F, 2.0F, 3.0F}); - const nsparse::idx_t* indptr = vectors.indptr_data(); + const nsparse::offset_t* indptr = vectors.indptr_data(); ASSERT_EQ(indptr[0], 0); ASSERT_EQ(indptr[1], 2); ASSERT_EQ(indptr[2], 3); @@ -417,7 +417,7 @@ TEST(SparseVectors, add_vectors_batch_float) { {.element_size = nsparse::U32, .dimension = 10}); // Two vectors: [0,1] and [2,3,4] - std::vector indptr = {0, 2, 5}; + std::vector indptr = {0, 2, 5}; std::vector indices = {0, 1, 2, 3, 4}; std::vector values = {1.0F, 2.0F, 3.0F, 4.0F, 5.0F}; @@ -433,7 +433,7 @@ TEST(SparseVectors, add_vectors_batch_uint8) { nsparse::SparseVectors vectors( {.element_size = nsparse::U8, .dimension = 10}); - std::vector indptr = {0, 2, 4}; + std::vector indptr = {0, 2, 4}; std::vector indices = {0, 1, 2, 3}; std::vector values = {10, 20, 30, 40}; @@ -458,7 +458,7 @@ TEST(SparseVectors, add_vectors_empty_indptr) { nsparse::SparseVectors vectors( {.element_size = nsparse::U32, .dimension = 10}); - std::vector indptr = {0}; // Less than 2 elements + std::vector indptr = {0}; // Less than 2 elements std::vector indices = {}; std::vector values = {}; @@ -471,7 +471,7 @@ TEST(SparseVectors, add_vectors_throws_on_size_mismatch) { nsparse::SparseVectors vectors( {.element_size = nsparse::U32, .dimension = 10}); - std::vector indptr = {0, 2}; + std::vector indptr = {0, 2}; std::vector indices = {0, 1}; std::vector values = {1, 2}; // Should be 8 bytes for 2 floats @@ -497,7 +497,7 @@ TEST(SparseVectors, serialize_deserialize_empty) { // --------------------------------------------------------------------------- TEST(SparseVectorsMapVectors, borrows_without_copying) { - const std::vector indptr = {0, 2}; + const std::vector indptr = {0, 2}; const std::vector indices = {0, 1}; const std::vector values = {1.0F, 2.0F}; @@ -522,7 +522,7 @@ TEST(SparseVectorsMapVectors, an_empty_indptr_maps_nothing) { } TEST(SparseVectorsMapVectors, cannot_be_appended_to) { - const std::vector indptr = {0, 1}; + const std::vector indptr = {0, 1}; const std::vector indices = {0}; const std::vector values = {7}; @@ -537,7 +537,7 @@ TEST(SparseVectorsMapVectors, cannot_be_appended_to) { } TEST(SparseVectorsMapVectors, throws_on_zero_dimension) { - const std::vector indptr = {0, 1}; + const std::vector indptr = {0, 1}; const std::vector indices = {0}; const std::vector values = {7}; @@ -549,7 +549,7 @@ TEST(SparseVectorsMapVectors, throws_on_zero_dimension) { } TEST(SparseVectorsMapVectors, throws_on_unsupported_element_size) { - const std::vector indptr = {0, 1}; + const std::vector indptr = {0, 1}; const std::vector indices = {0}; const std::vector values = {1, 2, 3}; @@ -561,7 +561,7 @@ TEST(SparseVectorsMapVectors, throws_on_unsupported_element_size) { } TEST(SparseVectorsMapVectors, throws_on_null_arrays) { - const std::vector indptr = {0, 1}; + const std::vector indptr = {0, 1}; const std::vector indices = {0}; const std::vector values = {7}; const nsparse::SparseVectorsConfig config = {.element_size = nsparse::U8, @@ -585,7 +585,7 @@ TEST(SparseVectorsMapVectors, throws_on_null_arrays) { } TEST(SparseVectorsMapVectors, throws_on_size_mismatch) { - const std::vector indptr = {0, 2}; + const std::vector indptr = {0, 2}; const std::vector indices = {0, 1}; const std::vector values = {1, 2}; // 2 floats need 8 bytes @@ -597,7 +597,7 @@ TEST(SparseVectorsMapVectors, throws_on_size_mismatch) { } TEST(SparseVectorsMapVectors, throws_on_misaligned_values) { - const std::vector indptr = {0, 2}; + const std::vector indptr = {0, 2}; const std::vector indices = {0, 1}; auto storage = misaligned_storage(2 * sizeof(float)); const uint8_t* values = storage.data() + 1; @@ -611,7 +611,7 @@ TEST(SparseVectorsMapVectors, throws_on_misaligned_values) { } TEST(SparseVectorsMapVectors, throws_when_indptr_does_not_start_at_zero) { - const std::vector indptr = {1, 2}; + const std::vector indptr = {1, 2}; const std::vector indices = {0, 1}; const std::vector values = {7, 8}; @@ -624,7 +624,7 @@ TEST(SparseVectorsMapVectors, throws_when_indptr_does_not_start_at_zero) { TEST(SparseVectorsMapVectors, throws_on_non_monotonic_indptr) { // Row 1 ends before it starts, so its length would underflow. - const std::vector indptr = {0, 2, 1, 2}; + const std::vector indptr = {0, 2, 1, 2}; const std::vector indices = {0, 1}; const std::vector values = {7, 8}; @@ -636,7 +636,7 @@ TEST(SparseVectorsMapVectors, throws_on_non_monotonic_indptr) { } TEST(SparseVectorsMapVectors, throws_when_indptr_does_not_end_at_the_count) { - const std::vector indptr = {0, 1}; + const std::vector indptr = {0, 1}; const std::vector indices = {0, 1}; const std::vector values = {7, 8}; @@ -657,7 +657,7 @@ namespace { constexpr size_t kSerializedHeaderBytes = 3 * sizeof(size_t); nsparse::SparseVectors owned_vectors(size_t dimension, size_t element_size, - const std::vector& indptr, + const std::vector& indptr, const std::vector& indices, const std::vector& values) { nsparse::SparseVectors vectors( @@ -683,7 +683,7 @@ TEST(SparseVectorsLayout, pads_values_to_the_element_width) { const size_t bytes = serialized(vectors).size(); const size_t unpadded = kSerializedHeaderBytes + - 2 * sizeof(nsparse::idx_t) + + 2 * sizeof(nsparse::offset_t) + 3 * sizeof(nsparse::term_t) + 3 * sizeof(float); ASSERT_EQ(bytes, unpadded + 2); ASSERT_EQ(bytes % alignof(float), 0); @@ -694,7 +694,7 @@ TEST(SparseVectorsLayout, pads_nothing_when_already_aligned) { std::vector(2 * sizeof(float), 0)); const size_t expected = kSerializedHeaderBytes + - 2 * sizeof(nsparse::idx_t) + + 2 * sizeof(nsparse::offset_t) + 2 * sizeof(nsparse::term_t) + 2 * sizeof(float); ASSERT_EQ(serialized(vectors).size(), expected); } @@ -705,7 +705,7 @@ TEST(SparseVectorsLayout, pads_nothing_for_byte_wide_values) { owned_vectors(4, nsparse::U8, {0, 3}, {0, 1, 3}, {10, 20, 30}); const size_t expected = kSerializedHeaderBytes + - 2 * sizeof(nsparse::idx_t) + + 2 * sizeof(nsparse::offset_t) + 3 * sizeof(nsparse::term_t) + 3; ASSERT_EQ(serialized(vectors).size(), expected); }