feat(vsjoin): implement VSJoin dual-index architecture (v0.1.3.1)

cmake/enableTest.cmake

@@ -23,10 +23,12 @@ macro(add_gtest TARGET_NAME SOURCE_FILE)
     add_test(NAME ${TARGET_NAME} COMMAND ${TARGET_NAME})
 
     # 让 CLion 能枚举单测用例
+    # 注意: 使用 POST_BUILD 模式避免并行构建时的 race condition
     if(COMMAND gtest_discover_tests)
         gtest_discover_tests(${TARGET_NAME}
             WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
             DISCOVERY_TIMEOUT 30
+            DISCOVERY_MODE PRE_TEST
         )
     endif()
 endmacro()

config/integration_test_cases.toml

@@ -49,7 +49,7 @@ window_state_type = "shared"
 index_strategy = "shared"
 data_sizes = [500]
 window_size_ms = 1000
-parallelism = [1, 2, 4, 8, 16]
+parallelism = [1, 2, 4, 8, 16, 24, 32]
 expected_min_recall = 0.95
 expected_min_precision = 1.0
 
@@ -341,7 +341,7 @@ clustered_training_samples = 50
 clustered_cold_start_enabled = true
 clustered_broadcast_dedup = true
 data_sizes = [2000]
-parallelism = [1, 2, 4, 8, 16]
+parallelism = [1, 2, 4, 8, 16, 24, 32]
 expected_min_recall = 0.90
 enabled = false
 
@@ -551,63 +551,107 @@ expected_min_recall = 0.85
 enabled = false
 
 # ==================== VSJoin 测试 ====================
-# Full VSJoin with LSH partitioning
+# VSJoin 双层索引（Global IVF + Local BruteForce）+ 后台重建
+# 
+# 架构说明：
+# - Global Index: 2 个 IVF 索引（左/右流各一个），存储历史数据快照
+# - Local Index: 2*P 个 BruteForce 索引（每个分区左/右流各一个），存储实时数据
+# - 后台重建: 定期将 WindowState 中的数据重建到 Global Index
+# - LSH 分区: 使用局部敏感哈希进行向量分区
+
+# VSJoin 基线测试 - 验证基本功能
 [[test_case]]
-name = "vsjoin_complete"
-description = "Full VSJoin with LSH partitioning and partitioned index"
+name = "vsjoin_baseline"
+description = "VSJoin baseline test with two-tier index"
 algorithm = "vsjoin"
 partition_strategy = "lsh"
-window_state_type = "partitioned_vector"
+window_state_type = "two_tier"
 index_strategy = "partitioned"
+num_partitions = 4
+# VSJoin 参数
 vsjoin_num_hash_functions = 8
 vsjoin_boundary_threshold = 0.1
-vsjoin_async_threads = 2
-vsjoin_allowed_lateness = 1000
-num_partitions = 8
+vsjoin_rebuild_interval_ms = 3000
+vsjoin_rebuild_threshold = 500
+# Global Index (IVF) 参数
 ivf_nlist = 50
-ivf_nprobes = 5
+ivf_nprobes = 10
+# 测试配置
 data_sizes = [500, 1000]
-parallelism = [2, 4]
+parallelism = [1, 2, 4]
 expected_min_recall = 0.70
-enabled = false
+enabled = true
 
+# VSJoin 高召回测试 - 更多哈希函数提升分区质量
 [[test_case]]
-name = "vsjoin_high_hash"
-description = "VSJoin with more hash functions for better partitioning"
+name = "vsjoin_high_recall"
+description = "VSJoin with higher hash functions for better recall"
 algorithm = "vsjoin"
 partition_strategy = "lsh"
-window_state_type = "partitioned_vector"
+window_state_type = "two_tier"
 index_strategy = "partitioned"
+num_partitions = 4
+# VSJoin 参数 - 更多哈希函数
 vsjoin_num_hash_functions = 16
-vsjoin_boundary_threshold = 0.1
-vsjoin_async_threads = 2
-vsjoin_allowed_lateness = 1000
-num_partitions = 8
+vsjoin_boundary_threshold = 0.15
+vsjoin_rebuild_interval_ms = 2000
+vsjoin_rebuild_threshold = 300
+# Global Index (IVF) 参数 - 更大的 nprobes
 ivf_nlist = 50
-ivf_nprobes = 5
+ivf_nprobes = 15
+# 测试配置
 data_sizes = [500]
-parallelism = [2, 4]
+parallelism = [1, 2, 4]
 expected_min_recall = 0.75
-enabled = false
+enabled = true
 
+# VSJoin 并行扩展测试 - 测试多并行度下的性能
+# 注意: VSJoin 的 LSH 分区在高并行度下召回率会下降（LSH 算法特性）
+#       当前配置下并行度 1-16 能保持 100% 召回率
 [[test_case]]
-name = "vsjoin_low_latency"
-description = "VSJoin optimized for low latency"
+name = "vsjoin_parallelism_scaling"
+description = "VSJoin parallelism scalability test"
 algorithm = "vsjoin"
 partition_strategy = "lsh"
-window_state_type = "partitioned_vector"
+window_state_type = "two_tier"
 index_strategy = "partitioned"
+num_partitions = 16
+# VSJoin 参数
 vsjoin_num_hash_functions = 8
-vsjoin_boundary_threshold = 0.05
-vsjoin_async_threads = 4
-vsjoin_allowed_lateness = 500
+vsjoin_boundary_threshold = 0.1
+vsjoin_rebuild_interval_ms = 3000
+vsjoin_rebuild_threshold = 500
+# Global Index (IVF) 参数
+ivf_nlist = 50
+ivf_nprobes = 10
+# 测试配置 - 并行度 1-16 稳定通过
+data_sizes = [2000]
+parallelism = [1, 2, 4, 8, 16, 24, 32]
+expected_min_recall = 0.90
+enabled = true
+
+# VSJoin 低延迟测试 - 快速重建
+[[test_case]]
+name = "vsjoin_low_latency"
+description = "VSJoin optimized for low latency with fast rebuild"
+algorithm = "vsjoin"
+partition_strategy = "lsh"
+window_state_type = "two_tier"
+index_strategy = "partitioned"
 num_partitions = 4
+# VSJoin 参数 - 快速重建
+vsjoin_num_hash_functions = 8
+vsjoin_boundary_threshold = 0.1
+vsjoin_rebuild_interval_ms = 1000
+vsjoin_rebuild_threshold = 200
+# Global Index (IVF) 参数
 ivf_nlist = 30
-ivf_nprobes = 3
+ivf_nprobes = 8
+# 测试配置
 data_sizes = [500]
-parallelism = [2, 4]
-expected_min_recall = 0.65
-enabled = false
+parallelism = [1, 2, 4]
+expected_min_recall = 0.60
+enabled = true
 allow_approximate_match = true
 
 # ==================== 混合测试（多并行度/多数据规模）====================

config/vsjoin_strategy.toml

@@ -0,0 +1,111 @@
+# VSJoin 配置示例
+# 
+# VSJoin 是一种基于双层索引的向量流 Join 算法，使用 LSH 分区和 Local/Global 索引。
+# 
+# 关键特性：
+# - Local Index: 分区内的轻量级索引（推荐 BruteForce）
+# - Global Index: 跨分区的候选召回索引（推荐 IVF）
+# - 后台重建: 定期重建 Global Index 保持索引新鲜度
+# - 边界向量多播: 处理分区边界的向量以保证召回率
+
+[default]
+# 基础配置
+algorithm = "vsjoin"
+is_eager = true
+similarity_threshold = 0.8
+dimension = 128
+
+# 分区配置
+partition_strategy = "lsh"
+window_state_type = "partitioned"
+index_strategy = "partitioned"
+num_partitions = 4
+
+# 窗口配置
+window_size_ms = 10000
+step_size_ms = 1000
+
+# VSJoin 核心参数
+vsjoin_multicast_k = 2              # 边界向量多播到 k 个分区（推荐 2-3）
+vsjoin_rebuild_interval_ms = 5000   # Global Index 重建间隔（毫秒）
+vsjoin_rebuild_threshold = 1000     # 触发重建的记录数阈值
+
+# VSJoin 索引类型
+vsjoin_local_index_type = "bruteforce"  # Local Index 类型（推荐 bruteforce）
+vsjoin_global_index_type = "ivf"        # Global Index 类型（推荐 ivf）
+
+# LSH 分区器参数
+vsjoin_num_hash_functions = 8       # LSH 哈希函数数量
+vsjoin_boundary_threshold = 0.1     # 边界向量阈值
+
+# IVF 参数（用于 Global Index）
+ivf_nlist = 100
+ivf_nprobes = 10
+
+# HNSW 参数（备选 Global Index）
+hnsw_m = 16
+hnsw_ef_construction = 200
+hnsw_ef_search = 50
+
+# ==================== 策略变体 ====================
+
+[strategies.vsjoin_default]
+# 默认 VSJoin 配置
+algorithm = "vsjoin"
+partition_strategy = "lsh"
+window_state_type = "partitioned"
+index_strategy = "partitioned"
+vsjoin_multicast_k = 2
+vsjoin_rebuild_interval_ms = 5000
+vsjoin_rebuild_threshold = 1000
+vsjoin_local_index_type = "bruteforce"
+vsjoin_global_index_type = "ivf"
+vsjoin_num_hash_functions = 8
+vsjoin_boundary_threshold = 0.1
+
+[strategies.vsjoin_high_recall]
+# 高召回率配置（更多多播）
+algorithm = "vsjoin"
+partition_strategy = "lsh"
+window_state_type = "partitioned"
+index_strategy = "partitioned"
+vsjoin_multicast_k = 3              # 更多多播提高召回
+vsjoin_rebuild_interval_ms = 3000   # 更频繁的重建
+vsjoin_rebuild_threshold = 500      # 更小的阈值
+vsjoin_local_index_type = "bruteforce"
+vsjoin_global_index_type = "ivf"
+vsjoin_num_hash_functions = 12      # 更多哈希函数
+vsjoin_boundary_threshold = 0.15    # 更宽的边界
+ivf_nprobes = 20                    # 更多探测提高召回
+
+[strategies.vsjoin_high_throughput]
+# 高吞吐量配置（减少多播开销）
+algorithm = "vsjoin"
+partition_strategy = "lsh"
+window_state_type = "partitioned"
+index_strategy = "partitioned"
+vsjoin_multicast_k = 1              # 减少多播
+vsjoin_rebuild_interval_ms = 10000  # 较少的重建
+vsjoin_rebuild_threshold = 2000     # 更大的阈值
+vsjoin_local_index_type = "bruteforce"
+vsjoin_global_index_type = "ivf"
+vsjoin_num_hash_functions = 6       # 较少哈希函数
+vsjoin_boundary_threshold = 0.05    # 更窄的边界
+ivf_nprobes = 5                     # 较少探测提高速度
+
+[strategies.vsjoin_hnsw_global]
+# 使用 HNSW 作为 Global Index
+algorithm = "vsjoin"
+partition_strategy = "lsh"
+window_state_type = "partitioned"
+index_strategy = "partitioned"
+vsjoin_multicast_k = 2
+vsjoin_rebuild_interval_ms = 5000
+vsjoin_rebuild_threshold = 1000
+vsjoin_local_index_type = "bruteforce"
+vsjoin_global_index_type = "hnsw"   # 使用 HNSW
+vsjoin_num_hash_functions = 8
+vsjoin_boundary_threshold = 0.1
+hnsw_m = 16
+hnsw_ef_construction = 200
+hnsw_ef_search = 100                # HNSW 搜索参数