diff --git a/.github/actions/complexEnvBuilder/action.yml b/.github/actions/complexEnvBuilder/action.yml new file mode 100644 index 0000000000..cce9b8da47 --- /dev/null +++ b/.github/actions/complexEnvBuilder/action.yml @@ -0,0 +1,36 @@ +name: "complex-env-builder" +description: "complex environment builder" +inputs: + build-type: + description: "the way to test" + required: false + default: SDMF + metadata: + description: "metadata service" + required: false + default: zookeeper +runs: + using: "composite" # Mandatory parameter + steps: + - if: inputs.build-type=='SDMF-Less-Conn' || inputs.build-type=='SDMF-Enough-Conn' + name: Stop and clear Metadata Service + uses: ./.github/actions/context + with: + work-name: stop-iginx-meta + metadata: ${{ inputs.metadata }} + + - if: inputs.build-type=='SDMF-Less-Conn' || inputs.build-type=='SDMF-Enough-Conn' + name: Run Metadata Service + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ inputs.metadata }} + + - if: inputs.build-type=='SDMF-Less-Conn' || inputs.build-type=='SDMF-Enough-Conn' + name: Change IGinX config + uses: ./.github/actions/confWriter + with: + Test-Way: inputs.build-type + + - if: inputs.build-type=='SDMF-Less-Conn' || inputs.build-type=='SDMF-Enough-Conn' + name: Start IGinX + uses: ./.github/actions/iginxRunner diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 117ddbaa2e..42b9731de8 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -200,3 +200,8 @@ runs: echo "$RUNNER_OS is not supported" exit 1 fi + +# - if: inputs.Test-Way=='SDMF' +# name: Change the config for SDMF +# shell: bash +# run: | diff --git a/.github/scripts/test/SDMF/SDMF_insert.sh b/.github/scripts/test/SDMF/SDMF_insert.sh new file mode 100644 index 0000000000..27e45f4656 --- /dev/null +++ b/.github/scripts/test/SDMF/SDMF_insert.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set +e +SCRIPT_COMMAND="/home/lhz/lhz/iginx-client-*/sbin/start_cli.sh" +BATCH=10 +ALL=10 + +for ((k = 1; k <= 19; k++)); do + P="a$k" + for ((i = 0; i <= $ALL-$BATCH; i += $BATCH)); do + COMMAND="" + for ((j = i; j < i + $BATCH && j <= 100000; j++)); do + value=$((j + 1)) + COMMAND+="($j, $value)," + done + + COMMAND=${COMMAND%,} + + COMMAND="insert into a.b(key, $P) values $COMMAND;" + # 创建临时文件 + TEMP_FILE=./tmp/$P_$i.txt + + echo "$COMMAND" > "$TEMP_FILE" + # 使用重定向输入的方式执行命令 + bash -c "cat $TEMP_FILE | $SCRIPT_COMMAND > output.txt 2>&1" +# bash -c "$SCRIPT_COMMAND > output.txt 2>&1" + done +done +echo "insert done" diff --git a/.github/scripts/test/SDMF/SDMF_insert_query.sh b/.github/scripts/test/SDMF/SDMF_insert_query.sh new file mode 100644 index 0000000000..4e4e906c7e --- /dev/null +++ b/.github/scripts/test/SDMF/SDMF_insert_query.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set +e +chmod +x .github/scripts/test/SDMF/*.sh + +.github/scripts/test/SDMF/SDMF_insert.sh + +sleep 10 + +.github/scripts/test/SDMF/SDMF_query.sh + +sleep 10 + +.github/scripts/test/SDMF/result_merge.sh + +trap 'exit 0' ERR \ No newline at end of file diff --git a/.github/scripts/test/SDMF/SDMF_query.sh b/.github/scripts/test/SDMF/SDMF_query.sh new file mode 100644 index 0000000000..f6af6dc00c --- /dev/null +++ b/.github/scripts/test/SDMF/SDMF_query.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set +e +SCRIPT_COMMAND="/home/lhz/lhz/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" +TIMES=$1 +PLL=10 +echo "" > read_output.txt +mkdir tmp +for ((k = 1; k <= $TIMES; k++)); do + for ((i = 1; i <= $PLL; i += 1)); do + COMMAND="select a1,a2,a3,a4,a5,a6,a7,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19 from a.b where key<100000;" + TEMP_FILE=./tmp/tmp_${k}_${i}.txt + echo "$COMMAND" > "$TEMP_FILE" + # 使用重定向输入的方式执行命令 + # bash -c "cat $TEMP_FILE | $SCRIPT_COMMAND >> ./tmp/read_output_${k}_${i}.txt && cat ./tmp/read_output_${k}_${i}.txt >> read_output.txt &" & + bash -c "cat $TEMP_FILE | $SCRIPT_COMMAND >> ./tmp/read_output_${k}_${i}.txt &" & + done +done + +for ((k = 1; k <= $TIMES; k++)); do + for ((i = 1; i <= $PLL; i += 1)); do + rm ./tmp/tmp_${k}_${i}.txt + done +done +echo "query done" + + + diff --git a/.github/scripts/test/SDMF/result_merge.sh b/.github/scripts/test/SDMF/result_merge.sh new file mode 100644 index 0000000000..c144fb575f --- /dev/null +++ b/.github/scripts/test/SDMF/result_merge.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set +e +TIMES=10 +PLL=10 +echo "" > read_output_all.txt +# 将单独的输出文件追加到总的输出文件中 +for ((k = 1; k <= $TIMES; k++)); do + for ((i = 0; i <= $PLL; i += 1)); do + OUTPUT_FILE="./tmp/read_output_${k}_${i}.txt" + cat "$OUTPUT_FILE" >> read_output_all.txt + rm "$OUTPUT_FILE" # 清理单独的输出文件 + done +done diff --git a/.github/scripts/test/SDMF/time_avg_judge.sh b/.github/scripts/test/SDMF/time_avg_judge.sh new file mode 100644 index 0000000000..49c658ce44 --- /dev/null +++ b/.github/scripts/test/SDMF/time_avg_judge.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set +e +# 指定文件路径 +file_path="/home/lhz/lhz/iginx/conn_test/test_shell/read_output_all.txt" # 替换为你的文件路径 + +# 统计匹配到的字符串数量和总耗时 +count=$(grep -o "Time cost: [0-9]\+ ms" "$file_path" | wc -l) +total_time=$(grep -o "Time cost: [0-9]\+ ms" "$file_path" | awk '{ total += $3 } END { print total }') + +# 计算平均耗时 +if [ "$count" -eq 0 ]; then + average_time=0 +else + average_time=$(echo "scale=2; $total_time / $count" | bc) +fi + +# 打印结果 +echo "总共匹配到 $count 个字符串" +echo "平均耗时为 $average_time ms" + +trap 'exit 0' ERR diff --git a/.github/scripts/test/SDMF/time_gather.sh b/.github/scripts/test/SDMF/time_gather.sh new file mode 100644 index 0000000000..83080c2207 --- /dev/null +++ b/.github/scripts/test/SDMF/time_gather.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set +e +# 指定文件路径 +file_path="/home/lhz/lhz/iginx/conn_test/test_shell/read_output_all.txt" # 替换为你的文件路径 + +# 提取匹配到的耗时值,并将其写入临时文件 +grep -o "Time cost: [0-9]\+ ms" "$file_path" | awk '{ print $3 }' > all_time_file.txt + +trap 'exit 0' ERR \ No newline at end of file diff --git a/.github/scripts/test/SDMF/timeout_judge.sh b/.github/scripts/test/SDMF/timeout_judge.sh new file mode 100644 index 0000000000..1f95f001de --- /dev/null +++ b/.github/scripts/test/SDMF/timeout_judge.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set +e +# 检查read_output_all.txt文件是否包含"exception"字段 +if grep -q "Exception" "read_output_all.txt"; then + echo "Error: Exception found in read_output_all.txt." +# exit 1 +else + echo "No exception found in read_output_all.txt." +# exit 0 +fi + +trap 'exit 0' ERR \ No newline at end of file diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml index c4116604e3..4ead8e1498 100644 --- a/.github/workflows/standalone-test-pushdown.yml +++ b/.github/workflows/standalone-test-pushdown.yml @@ -102,3 +102,32 @@ jobs: shell: bash run: | cat iginx-*.log + + # test push down in sigle DU multi fragment + - name: Build the complex test environment + uses: ./.github/actions/complexEnvBuilder + with: + build-type: SDMF-Less-Conn + + - name: Query optimization test for merging fragment + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/SDMF_insert_query.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/SDMF_insert_query.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/timeout_judge.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/timeout_judge.sh" + + # test push down in sigle DU multi fragment + - name: Build the complex test environment + uses: ./.github/actions/complexEnvBuilder + with: + build-type: SDMF-Enough-Conn + + - name: test query time after + shell: bash + run: | + "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/SDMF_insert_query.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/time_gather.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/time_gather.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/time_avg_judge.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/SDMF/time_avg_judge.sh" diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index a4f8234894..c211af50be 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,27 +11,27 @@ concurrency: cancel-in-progress: true jobs: - unit-test: - uses: ./.github/workflows/unit-test.yml - unit-mds: - uses: ./.github/workflows/unit-mds.yml - case-regression: - uses: ./.github/workflows/case-regression.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' +# unit-test: +# uses: ./.github/workflows/unit-test.yml +# unit-mds: +# uses: ./.github/workflows/unit-mds.yml +# case-regression: +# uses: ./.github/workflows/case-regression.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test: +# uses: ./.github/workflows/standalone-test.yml +# with: +# metadata-matrix: '["zookeeper"]' standalone-test-pushdown: uses: ./.github/workflows/standalone-test-pushdown.yml with: metadata-matrix: '["zookeeper"]' - db-ce: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - remote-test: - uses: ./.github/workflows/remote-test.yml - with: - metadata-matrix: '["zookeeper"]' +# db-ce: +# uses: ./.github/workflows/DB-CE.yml +# with: +# metadata-matrix: '["zookeeper"]' +# remote-test: +# uses: ./.github/workflows/remote-test.yml +# with: +# metadata-matrix: '["zookeeper"]'