build topling-jni #41
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # TODO: How to cache make files / speed up build progress here? | |
| name: "build topling-jni" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version_suffix: | |
| required: false | |
| default: '-SNAPSHOT' | |
| use_lto: | |
| required: false | |
| description: Link Use LTO | |
| default: '1' | |
| unit_test: | |
| required: false | |
| type: boolean | |
| description: Unit Test | |
| default: false | |
| deploy_maven: | |
| required: false | |
| type: boolean | |
| description: publish to maven repo | |
| default: true | |
| jmh_test: | |
| required: false | |
| type: boolean | |
| description: JMH Test | |
| default: false | |
| jobs: | |
| build: | |
| # refer https://github.com/actions/runner-images to get the details | |
| runs-on: ubuntu-22.04 | |
| env: | |
| GCC_VER: "11.3" # TODO: better get from the 'gcc --version' | |
| GITHUB_TOKEN: ${{ github.token }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| cache: maven | |
| server-id: github # Value of the distributionManagement/repository/id field of the pom.xml | |
| settings-path: ${{ github.workspace }} # location for the settings.xml file | |
| - name: Install Prerequisites | |
| run: | | |
| set -xe | |
| cat $GITHUB_WORKSPACE/settings.xml | |
| sudo apt-get update -y && sudo apt-get install -y \ | |
| libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev \ | |
| libbz2-dev libcurl4-openssl-dev liburing-dev \ | |
| libsnappy-dev libbz2-dev liblz4-dev libzstd-dev | |
| - name: Init submodule & Setup ssh | |
| run: | | |
| git submodule update --init --recursive | |
| mkdir -p ~/.ssh && mkdir -p /opt/lib | |
| ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts | |
| echo nproc = `nproc` | |
| - name: Git clone SidePlugin(s) | |
| run: | | |
| # this will be auto triggerred in normal build, we defined | |
| # it here is just for seperate steps more clearly. | |
| # there is no dedicated target for this purpose, so we use `clean` | |
| # as the target, it need not any `clean` at this point. | |
| # `clean` is just used for triggers auto git clone | |
| make clean | |
| - name: make depend for Compile Unit Test | |
| if: ${{ inputs.unit_test }} | |
| run: | | |
| # this will be auto triggerred in normal build, we defined | |
| # it here is just for seperate steps more clearly. | |
| # need to explicit set MAKE_UNIT_TEST=1 | |
| echo JAVA_HOME = $JAVA_HOME | |
| set -xe | |
| make depend java/include/java_header_list.mk \ | |
| MAKE_UNIT_TEST=1 \ | |
| -j`nproc` DEBUG_LEVEL=1 UPDATE_REPO=0 \ | |
| DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 | |
| - name: Compile & Run Java Unit Test | |
| if: ${{ inputs.unit_test }} | |
| run: | | |
| echo JAVA_HOME = $JAVA_HOME | |
| set -xe | |
| # Makefile will auto set MAKE_UNIT_TEST=1 for `jtest` | |
| make jtest -j`nproc` DEBUG_LEVEL=1 UPDATE_REPO=0 \ | |
| DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 | |
| - name: make depend for Compile RocksDBJava Release | |
| run: | | |
| echo JAVA_HOME = $JAVA_HOME | |
| set -xe | |
| env USE_LTO=${{inputs.use_lto}} \ | |
| make depend java/include/java_header_list.mk \ | |
| -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0 \ | |
| DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 \ | |
| STRIP_DEBUG_INFO=1 ROCKSDB_JAR_WITH_DYNAMIC_LIBS=1 | |
| - name: Compile RocksDBJava Release | |
| run: | | |
| echo JAVA_HOME = $JAVA_HOME | |
| set -xe | |
| env USE_LTO=${{inputs.use_lto}} \ | |
| make rocksdbjava -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0 \ | |
| DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 \ | |
| STRIP_DEBUG_INFO=1 ROCKSDB_JAR_WITH_DYNAMIC_LIBS=1 | |
| ROCKSDB_VERSION=`build_tools/version.sh full` | |
| ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}} | |
| cd java | |
| # F\?ROCKSDB_JAVA_VERSION for both flink and normal rocksdbjni | |
| sed 's/\${F\?ROCKSDB_JAVA_VERSION}/'"${ROCKSDB_JAVA_VERSION}/" pom.xml.template > pom.xml | |
| - name: Show librocksdbjni dependency | |
| run: ldd java/target/*.so | |
| - name: Move to Local & Publish rocksjni to GitHub Packages | |
| run: | | |
| echo JAVA_HOME = $JAVA_HOME | |
| set -xe | |
| ROCKSDB_VERSION=`build_tools/version.sh full` | |
| ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}} | |
| cd java/target | |
| mvn install:install-file -ntp -e \ | |
| -DpomFile=../pom.xml \ | |
| -Dfile=rocksdbjni-${ROCKSDB_VERSION}-linux64.jar \ | |
| -Dversion=${ROCKSDB_JAVA_VERSION} | |
| # TODO: why 'deploy' doesn't include install step here? if we only use deploy, will lack local jar | |
| if ${{ inputs.deploy_maven }}; then | |
| # TODO: what's the pom file for it? add with '-DpomFile=/xx/pom.xml' | |
| mvn deploy:deploy-file -e -s $GITHUB_WORKSPACE/settings.xml \ | |
| -DpomFile=../pom.xml \ | |
| -Durl=https://maven.pkg.github.com/$GITHUB_REPOSITORY -DrepositoryId=github \ | |
| -Dfile=rocksdbjni-${ROCKSDB_VERSION}-linux64.jar \ | |
| -Dversion=${ROCKSDB_JAVA_VERSION} | |
| fi | |
| # for compile jmh.jar to test the performance | |
| - name: Build SideGetBenchmarks with Maven | |
| run: | | |
| set -xe | |
| echo ${{github.workspace}} && echo $GITHUB_WORKSPACE | |
| ROCKSDB_VERSION=`build_tools/version.sh full` | |
| ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}} | |
| cd java/jmh && ls -l && pwd | |
| db_artifactId=`sed -n 's/.*<artifactId>\(f\?rocksdbjni\)<\/artifactId>.*/\1/p' ../pom.xml` | |
| mvn clean package -e -ntp -f pom.xml \ | |
| -D db.artifactId=${db_artifactId} \ | |
| -D db.version=${ROCKSDB_JAVA_VERSION} | |
| - name: Run JMH SideGetBenchmarks | |
| if: ${{ inputs.jmh_test }} | |
| run: | | |
| set -xe | |
| ROCKSDB_VERSION=`build_tools/version.sh full` | |
| ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}} | |
| mkdir -p /dev/shm/db_bench_community | |
| cd $GITHUB_WORKSPACE/java/jmh | |
| cp -v $GITHUB_WORKSPACE/sideplugin/rockside/src/topling/web/{style.css,index.html} /dev/shm/db_bench_community | |
| export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/java/target:$LD_LIBRARY_PATH | |
| # Note: webserver should visit while running | |
| db_artifactId=`sed -n 's/.*<artifactId>\(f\?rocksdbjni\)<\/artifactId>.*/\1/p' ../pom.xml` | |
| java --add-opens java.base/java.nio=ALL-UNNAMED \ | |
| -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar \ | |
| -p keyCount=1000 -p keySize=128 -p valueSize=32768 \ | |
| -p sideConf=$GITHUB_WORKSPACE/sideplugin/rockside/sample-conf/db_bench_community.yaml SideGetBenchmarks | |
| - name: Publish ToplingDB JMH JAR to GitHub Packages | |
| if: ${{ inputs.deploy_maven }} | |
| run: | | |
| set -xe | |
| ROCKSDB_VERSION=`build_tools/version.sh full` | |
| ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}} | |
| cd $GITHUB_WORKSPACE/java/jmh | |
| db_artifactId=`sed -n 's/.*<artifactId>\(f\?rocksdbjni\)<\/artifactId>.*/\1/p' ../pom.xml` | |
| mvn deploy -e -f pom.xml \ | |
| -D db.artifactId=${db_artifactId} \ | |
| -D db.version=${ROCKSDB_JAVA_VERSION} \ | |
| -s $GITHUB_WORKSPACE/settings.xml \ | |
| -DaltDeploymentRepository=github::default::https://maven.pkg.github.com/$GITHUB_REPOSITORY |