Skip to content

Commit 3ce3432

Browse files
MingqiWang-coderMingqiWang-coder
committed
feat: 添加 GTI、IP-DiskANN、PLSH 构建到部署脚本和 CI/CD
1 parent 794e6d3 commit 3ce3432

2 files changed

Lines changed: 151 additions & 57 deletions

File tree

‎.github/workflows/build-test.yml‎

Lines changed: 28 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Build and Test PyCANDYAlgo
1+
name: Build and Test All Modules
22

33
on:
44
push:
@@ -8,7 +8,7 @@ on:
88
workflow_dispatch: # 允许手动触发
99

1010
jobs:
11-
build-pycandyalgo:
11+
build-all:
1212
runs-on: ubuntu-22.04
1313

1414
steps:
@@ -35,9 +35,12 @@ jobs:
3535
with:
3636
path: |
3737
algorithms_impl/build
38-
key: ${{ runner.os }}-cmake-${{ hashFiles('algorithms_impl/CMakeLists.txt') }}
38+
algorithms_impl/gti/GTI/build
39+
algorithms_impl/ipdiskann/build
40+
algorithms_impl/plsh/build
41+
key: ${{ runner.os }}-cmake-all-${{ hashFiles('algorithms_impl/CMakeLists.txt') }}
3942
restore-keys: |
40-
${{ runner.os }}-cmake-
43+
${{ runner.os }}-cmake-all-
4144
4245
- name: Install system dependencies
4346
run: |
@@ -67,50 +70,35 @@ jobs:
6770
run: |
6871
chmod +x deploy.sh
6972
./deploy.sh --skip-system-deps
70-
timeout-minutes: 45
73+
timeout-minutes: 60
7174

7275
- name: Verify PyCANDYAlgo import
7376
run: |
7477
source sage-db-bench/bin/activate
78+
echo "=== Test PyCANDYAlgo Import ==="
79+
python3 -c "import PyCANDYAlgo; print('PyCANDYAlgo:', PyCANDYAlgo.__version__)"
7580
76-
# 设置 MKL 库路径
77-
if [ -d "/opt/intel/oneapi/mkl/latest/lib/intel64" ]; then
78-
export LD_LIBRARY_PATH="/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH"
79-
fi
80-
# 设置 Torch 库路径
81-
TORCH_LIB=$(python3 -c "import torch; import os; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))" 2>/dev/null || true)
82-
if [ -n "$TORCH_LIB" ] && [ -d "$TORCH_LIB" ]; then
83-
export LD_LIBRARY_PATH="$TORCH_LIB:$LD_LIBRARY_PATH"
84-
fi
85-
86-
echo "=== Python Environment ==="
87-
which python3
88-
python3 --version
81+
- name: Verify GTI (gti_wrapper) import
82+
run: |
83+
source sage-db-bench/bin/activate
84+
echo "=== Test gti_wrapper Import ==="
85+
python3 -c "import gti_wrapper; print('gti_wrapper: OK')" || echo "gti_wrapper: SKIPPED (optional)"
8986
90-
echo ""
91-
echo "=== Check .so file ==="
92-
SO_FILE=$(find algorithms_impl -name "PyCANDYAlgo*.so" -type f | head -1)
93-
if [ -n "$SO_FILE" ]; then
94-
echo "Found: $SO_FILE"
95-
echo "Size: $(ls -lh "$SO_FILE" | awk '{print $5}')"
96-
else
97-
echo "❌ PyCANDYAlgo.so not found"
98-
exit 1
99-
fi
87+
- name: Verify IP-DiskANN (ipdiskann) import
88+
run: |
89+
source sage-db-bench/bin/activate
90+
echo "=== Test ipdiskann Import ==="
91+
python3 -c "import ipdiskann; print('ipdiskann: OK')" || echo "ipdiskann: SKIPPED (optional)"
10092
101-
echo ""
102-
echo "=== Test PyCANDYAlgo Import ==="
103-
python3 -c "import PyCANDYAlgo; print('PyCANDYAlgo imported successfully'); print('Version:', PyCANDYAlgo.__version__)"
93+
- name: Verify PLSH (plsh_python) import
94+
run: |
95+
source sage-db-bench/bin/activate
96+
echo "=== Test plsh_python Import ==="
97+
python3 -c "import plsh_python; print('plsh_python: OK')" || echo "plsh_python: SKIPPED (optional)"
10498
10599
- name: Test core dependencies
106100
run: |
107101
source sage-db-bench/bin/activate
108-
109-
# 设置 MKL 库路径
110-
if [ -d "/opt/intel/oneapi/mkl/latest/lib/intel64" ]; then
111-
export LD_LIBRARY_PATH="/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH"
112-
fi
113-
114102
python3 -c "import numpy; import torch; print('numpy:', numpy.__version__); print('torch:', torch.__version__)"
115103
116104
- name: Upload build logs on failure
@@ -122,6 +110,9 @@ jobs:
122110
algorithms_impl/build/cmake_config.log
123111
algorithms_impl/build/CMakeFiles/CMakeError.log
124112
algorithms_impl/build/CMakeFiles/CMakeOutput.log
113+
algorithms_impl/gti/GTI/build/CMakeFiles/CMakeError.log
114+
algorithms_impl/ipdiskann/build/CMakeFiles/CMakeError.log
115+
algorithms_impl/plsh/build/CMakeFiles/CMakeError.log
125116
retention-days: 7
126117

127118
lint:

‎deploy.sh‎

Lines changed: 123 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
# ============================================================================
3-
# SAGE-DB-Bench 部署脚本 (仅 PyCANDYAlgo)
3+
# SAGE-DB-Bench 部署脚本
44
# ============================================================================
55
#
66
# 本脚本会：
@@ -9,8 +9,9 @@
99
# 3. 安装 Python 依赖包
1010
# 4. 初始化 Git submodules
1111
# 5. 构建 PyCANDYAlgo
12-
# 6. 安装 PyCANDYAlgo 到虚拟环境
13-
# 7. 验证 PyCANDYAlgo 导入
12+
# 6. 构建 GTI、IP-DiskANN、PLSH
13+
# 7. 安装所有模块到虚拟环境
14+
# 8. 验证所有模块导入
1415
#
1516
# 使用方法:
1617
# ./deploy.sh
@@ -117,7 +118,7 @@ print_info "Python 版本: $(python3.10 --version)"
117118
# ============================================================================
118119
# 开始部署
119120
# ============================================================================
120-
print_banner "SAGE-DB-Bench 部署 (PyCANDYAlgo)"
121+
print_banner "SAGE-DB-Bench 部署"
121122

122123
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
123124
cd "$SCRIPT_DIR"
@@ -129,7 +130,7 @@ echo ""
129130
# 步骤 1: 系统依赖
130131
# ============================================================================
131132
if [ "$SKIP_SYSTEM_DEPS" = false ]; then
132-
print_header "步骤 1/7: 安装系统依赖"
133+
print_header "步骤 1/8: 安装系统依赖"
133134

134135
if [ -f /etc/os-release ]; then
135136
. /etc/os-release
@@ -163,13 +164,13 @@ if [ "$SKIP_SYSTEM_DEPS" = false ]; then
163164
print_warning "非 Ubuntu/Debian 系统，请手动安装依赖"
164165
fi
165166
else
166-
print_header "步骤 1/7: 跳过系统依赖安装"
167+
print_header "步骤 1/8: 跳过系统依赖安装"
167168
fi
168169

169170
# ============================================================================
170171
# 步骤 2: 创建虚拟环境
171172
# ============================================================================
172-
print_header "步骤 2/7: 创建 Python 虚拟环境"
173+
print_header "步骤 2/8: 创建 Python 虚拟环境"
173174

174175
VENV_DIR="$SCRIPT_DIR/sage-db-bench"
175176

@@ -191,7 +192,7 @@ pip install --upgrade pip setuptools wheel -q
191192
# ============================================================================
192193
# 步骤 3: 安装 Python 依赖
193194
# ============================================================================
194-
print_header "步骤 3/7: 安装 Python 依赖"
195+
print_header "步骤 3/8: 安装 Python 依赖"
195196

196197
print_step "安装 PyTorch (CPU 版本)..."
197198
pip install torch --index-url https://download.pytorch.org/whl/cpu -q
@@ -204,7 +205,7 @@ print_success "Python 依赖安装完成"
204205
# ============================================================================
205206
# 步骤 4: 初始化 Git Submodules
206207
# ============================================================================
207-
print_header "步骤 4/7: 初始化 Git Submodules"
208+
print_header "步骤 4/8: 初始化 Git Submodules"
208209

209210
if [ -f ".gitmodules" ]; then
210211
print_step "初始化 submodules..."
@@ -218,7 +219,7 @@ fi
218219
# 步骤 5: 构建 PyCANDYAlgo
219220
# ============================================================================
220221
if [ "$SKIP_BUILD" = false ]; then
221-
print_header "步骤 5/7: 构建 PyCANDYAlgo"
222+
print_header "步骤 5/8: 构建 PyCANDYAlgo"
222223

223224
cd "$SCRIPT_DIR/algorithms_impl"
224225

@@ -248,13 +249,93 @@ if [ "$SKIP_BUILD" = false ]; then
248249

249250
cd "$SCRIPT_DIR"
250251
else
251-
print_header "步骤 5/7: 跳过构建"
252+
print_header "步骤 5/8: 跳过构建"
252253
fi
253254

254255
# ============================================================================
255-
# 步骤 6: 安装 PyCANDYAlgo
256+
# 步骤 6: 构建 GTI、IP-DiskANN、PLSH
256257
# ============================================================================
257-
print_header "步骤 6/7: 安装 PyCANDYAlgo"
258+
if [ "$SKIP_BUILD" = false ]; then
259+
print_header "步骤 6/8: 构建 GTI、IP-DiskANN、PLSH"
260+
261+
SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
262+
263+
# 计算并行编译数
264+
NPROC=$(nproc 2>/dev/null || echo 4)
265+
JOBS=$((NPROC > 8 ? 8 : NPROC))
266+
267+
# --- 构建 GTI ---
268+
print_step "构建 GTI (gti_wrapper)..."
269+
GTI_DIR="$SCRIPT_DIR/algorithms_impl/gti/GTI"
270+
if [ -d "$GTI_DIR" ]; then
271+
cd "$GTI_DIR"
272+
rm -rf build 2>/dev/null || true
273+
mkdir -p build && cd build
274+
cmake .. -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE=$(which python3) 2>&1 | tail -5
275+
make -j${JOBS} 2>&1 | tail -10
276+
# 查找并复制 .so 文件
277+
SO_FILE=$(find . -name "gti_wrapper*.so" 2>/dev/null | head -1)
278+
if [ -n "$SO_FILE" ]; then
279+
cp "$SO_FILE" "$SITE_PACKAGES/"
280+
print_success "GTI (gti_wrapper) 构建完成"
281+
else
282+
print_warning "gti_wrapper.so 未找到"
283+
fi
284+
else
285+
print_warning "GTI 目录不存在: $GTI_DIR"
286+
fi
287+
288+
# --- 构建 IP-DiskANN ---
289+
print_step "构建 IP-DiskANN (ipdiskann)..."
290+
IPDISKANN_DIR="$SCRIPT_DIR/algorithms_impl/ipdiskann"
291+
if [ -d "$IPDISKANN_DIR" ]; then
292+
cd "$IPDISKANN_DIR"
293+
rm -rf build 2>/dev/null || true
294+
mkdir -p build && cd build
295+
cmake .. -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE=$(which python3) 2>&1 | tail -5
296+
make -j${JOBS} 2>&1 | tail -10
297+
# 查找并复制 .so 文件
298+
SO_FILE=$(find . -name "ipdiskann*.so" 2>/dev/null | head -1)
299+
if [ -n "$SO_FILE" ]; then
300+
cp "$SO_FILE" "$SITE_PACKAGES/"
301+
print_success "IP-DiskANN (ipdiskann) 构建完成"
302+
else
303+
print_warning "ipdiskann.so 未找到"
304+
fi
305+
else
306+
print_warning "IP-DiskANN 目录不存在: $IPDISKANN_DIR"
307+
fi
308+
309+
# --- 构建 PLSH ---
310+
print_step "构建 PLSH (plsh_python)..."
311+
PLSH_DIR="$SCRIPT_DIR/algorithms_impl/plsh"
312+
if [ -d "$PLSH_DIR" ]; then
313+
cd "$PLSH_DIR"
314+
rm -rf build 2>/dev/null || true
315+
mkdir -p build && cd build
316+
cmake .. -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE=$(which python3) 2>&1 | tail -5
317+
make -j${JOBS} 2>&1 | tail -10
318+
# 查找并复制 .so 文件
319+
SO_FILE=$(find . -name "plsh_python*.so" 2>/dev/null | head -1)
320+
if [ -n "$SO_FILE" ]; then
321+
cp "$SO_FILE" "$SITE_PACKAGES/"
322+
print_success "PLSH (plsh_python) 构建完成"
323+
else
324+
print_warning "plsh_python.so 未找到"
325+
fi
326+
else
327+
print_warning "PLSH 目录不存在: $PLSH_DIR"
328+
fi
329+
330+
cd "$SCRIPT_DIR"
331+
else
332+
print_header "步骤 6/8: 跳过构建"
333+
fi
334+
335+
# ============================================================================
336+
# 步骤 7: 安装 PyCANDYAlgo
337+
# ============================================================================
338+
print_header "步骤 7/8: 安装 PyCANDYAlgo"
258339

259340
cd "$SCRIPT_DIR/algorithms_impl"
260341

@@ -284,22 +365,40 @@ fi
284365
cd "$SCRIPT_DIR"
285366

286367
# ============================================================================
287-
# 步骤 7: 验证 PyCANDYAlgo 导入
368+
# 步骤 8: 验证所有模块导入
288369
# ============================================================================
289-
print_header "步骤 7/7: 验证 PyCANDYAlgo"
290-
291-
print_step "测试导入..."
370+
print_header "步骤 8/8: 验证所有模块导入"
292371

293-
# 测试 PyCANDYAlgo 导入
372+
print_step "测试 PyCANDYAlgo..."
294373
if python3 -c "import PyCANDYAlgo; print('VERSION:', PyCANDYAlgo.__version__)" 2>&1; then
295374
print_success "PyCANDYAlgo 导入成功"
296375
else
297376
print_error "PyCANDYAlgo 导入失败"
298-
# 显示详细错误
299377
python3 -c "import PyCANDYAlgo" 2>&1 || true
300378
exit 1
301379
fi
302380

381+
print_step "测试 gti_wrapper..."
382+
if python3 -c "import gti_wrapper; print('gti_wrapper OK')" 2>&1; then
383+
print_success "gti_wrapper 导入成功"
384+
else
385+
print_warning "gti_wrapper 导入失败 (可选模块)"
386+
fi
387+
388+
print_step "测试 ipdiskann..."
389+
if python3 -c "import ipdiskann; print('ipdiskann OK')" 2>&1; then
390+
print_success "ipdiskann 导入成功"
391+
else
392+
print_warning "ipdiskann 导入失败 (可选模块)"
393+
fi
394+
395+
print_step "测试 plsh_python..."
396+
if python3 -c "import plsh_python; print('plsh_python OK')" 2>&1; then
397+
print_success "plsh_python 导入成功"
398+
else
399+
print_warning "plsh_python 导入失败 (可选模块)"
400+
fi
401+
303402
# 测试核心依赖
304403
print_step "测试核心依赖..."
305404
python3 -c "import numpy; print('numpy:', numpy.__version__)" || print_warning "numpy 不可用"
@@ -310,7 +409,11 @@ python3 -c "import torch; print('torch:', torch.__version__)" || print_warning "
310409
# ============================================================================
311410
print_banner "部署完成！"
312411

313-
echo "✅ PyCANDYAlgo 已成功构建并安装"
412+
echo "✅ 已成功构建并安装以下模块:"
413+
echo " - PyCANDYAlgo"
414+
echo " - gti_wrapper (GTI)"
415+
echo " - ipdiskann (IP-DiskANN)"
416+
echo " - plsh_python (PLSH)"
314417
echo ""
315418
echo "使用方法:"
316419
echo " source sage-db-bench/bin/activate"

0 commit comments

Comments
 (0)