Add verifier tests

.github/workflows/integration_tests_cuda.yml

@@ -0,0 +1,79 @@
+name: Integration Tests - CUDA
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+  schedule:
+    - cron: "0 5 * * *"
+
+jobs:
+  tests:
+    runs-on:
+      group: dahlia
+      labels: RTX5060
+    container:
+      image: daisytuner/docc-run-env-llvm19-ubuntu-24.04:latest-amd64
+      options: >-
+        --cap-add=PERFMON
+        --gpus=all
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
+          submodules: recursive
+          lfs: true
+
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          cmake -G Ninja \
+             -DCMAKE_C_COMPILER=clang-19 \
+             -DCMAKE_CXX_COMPILER=clang++-19 \
+             -DCMAKE_BUILD_TYPE=Debug \
+             -DLLVM_BUILD_FRONTEND=ON \
+             -DLLVM_BUILD_TESTS=ON \
+             -DSDFG_BUILD_TESTS=OFF \
+             -DINSTALL_GTEST=OFF \
+             -DBUILD_TESTS:BOOL=OFF \
+             -DBUILD_BENCHMARKS:BOOL=OFF \
+             -DBUILD_BENCHMARKS_GOOGLE:BOOL=OFF \
+             ..
+          ninja -j$(nproc)
+          cpack -G DEB
+
+          apt-get install -y ./docc*.deb
+          apt-get install -y libhdf5-dev
+
+      - name: Integration tests - CUDA
+        run: |
+          export CPATH=/usr/local/include:$CPATH
+          export LIBRARY_PATH=/usr/local/lib:$LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+          export PATH=/usr/local/bin:$PATH
+          export LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer-19)
+          export CPLUS_INCLUDE_PATH=$(pwd)/rtl/include/daisy_rtl:$CPLUS_INCLUDE_PATH
+
+          pip install pytest==7.1.3 --break-system-packages
+          pip install pytest-parallel --break-system-packages
+
+          cd integration
+
+          nvidia-smi
+
+          # Basic tests
+          pytest -v -rx basic_test.py
+          pytest -v -rx rtl_test.py
+
+          # PolyBench
+          pytest -v -rx polybench_cuda_test.py
+
+          # Rodinia
+          pytest -v -rx rodinia_cuda_test.py
+
+          # Apps
+          pytest -v -rx apps_cuda_test.py

.github/workflows/integration_tests_openmp.yml

@@ -0,0 +1,81 @@
+name: Integration Tests - OpenMP
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+  schedule:
+    - cron: "0 5 * * *"
+
+jobs:
+  tests:
+    runs-on:
+      group: dahlia
+      labels: openmp
+    container:
+      image: daisytuner/docc-run-env-llvm19-ubuntu-24.04:latest-amd64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
+          submodules: recursive
+          lfs: true
+
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          cmake -G Ninja \
+             -DCMAKE_C_COMPILER=clang-19 \
+             -DCMAKE_CXX_COMPILER=clang++-19 \
+             -DCMAKE_BUILD_TYPE=Debug \
+             -DLLVM_BUILD_FRONTEND=ON \
+             -DLLVM_BUILD_TESTS=ON \
+             -DSDFG_BUILD_TESTS=OFF \
+             -DINSTALL_GTEST=OFF \
+             -DBUILD_TESTS:BOOL=OFF \
+             -DBUILD_BENCHMARKS:BOOL=OFF \
+             -DBUILD_BENCHMARKS_GOOGLE:BOOL=OFF \
+             ..
+          ninja -j$(nproc)
+          cpack -G DEB
+
+          apt-get install -y ./docc*.deb
+          apt-get install -y libhdf5-dev
+
+      # - name: Build Plugins # out-of-tree build currently broken, as needed deps are not installed as part of docc-llvm deb
+      #   run: |
+      #     cd integration/tests/plugins/simple
+      #     mkdir build && cd build
+      #     cmake -GNinja ..
+      #     ninja -j$(nproc)
+
+      - name: Integration tests - OpenMP
+        run: |
+          export LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer-19)
+
+          pip install pytest==7.1.3 --break-system-packages
+          pip install pytest-parallel --break-system-packages
+
+          cd integration
+
+          # Plugin
+          # LD_LIBRARY_PATH=../../build/integration/tests/plugins/simple:$LD_LIBRARY_PATH pytest -v -s plugin_test.py
+
+          # Polybench
+          pytest --workers auto -v -rx polybench_none_test.py
+          pytest --workers auto -v -rx polybench_sequential_test.py
+          pytest --workers auto -v -rx polybench_openmp_test.py
+
+          # Rodinia
+          pytest -v -rx rodinia_none_test.py
+          pytest -v -rx rodinia_sequential_test.py
+          pytest -v -rx rodinia_openmp_test.py
+
+          # Apps
+          pytest -v -rx apps_none_test.py
+          pytest -v -rx apps_sequential_test.py
+          pytest -v -rx apps_openmp_test.py

.github/workflows/integration_tests_transfertuning.yml

@@ -0,0 +1,80 @@
+name: Integration Tests - Transfertuning
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+  schedule:
+    - cron: "0 5 * * *"
+
+jobs:
+  tests:
+    runs-on:
+      group: dahlia
+      labels: openmp
+    container:
+      image: daisytuner/docc-run-env-llvm19-ubuntu-24.04:latest-amd64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
+          submodules: recursive
+          lfs: true
+
+      - name: Mark GitHub Actions workdir as safe
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Dependencies
+        run: |
+          pip install pytest==7.1.3 pytest-parallel tqdm
+          pip install pandas numpy scipy
+
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          cmake -G Ninja \
+             -DCMAKE_C_COMPILER=clang-19 \
+             -DCMAKE_CXX_COMPILER=clang++-19 \
+             -DCMAKE_BUILD_TYPE=Debug \
+             -DLLVM_BUILD_FRONTEND=ON \
+             -DLLVM_BUILD_TESTS=ON \
+             -DSDFG_BUILD_TESTS=OFF \
+             -DINSTALL_GTEST=OFF \
+             -DBUILD_TESTS:BOOL=OFF \
+             -DBUILD_BENCHMARKS:BOOL=OFF \
+             -DBUILD_BENCHMARKS_GOOGLE:BOOL=OFF \
+             ..
+          ninja -j$(nproc)
+          cpack -G DEB
+
+          apt-get install -y ./docc*.deb
+
+      - name: Integration tests - Transfertuning
+        env:
+          DOCC_ACCESS_TOKEN: ${{ secrets.DOCC_CI_TOKEN }}
+        run: |
+          export CPATH=/usr/local/include:$CPATH
+          export LIBRARY_PATH=/usr/local/lib:$LIBRARY_PATH
+          export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+          export PATH=$PATH:/usr/local/bin
+          export LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer-19)
+
+          export DOCC_OFFLOAD_REPORT=1
+
+          # End-to-end tests for transfer tuning
+          cd integration/tests/matmul/
+          docc -g -O3 -docc-tune=sequential -docc-transfer-tune -docc-save-temps matmul.c -o matmul.out
+          ./matmul.out
+          cd ../../../
+
+          cd integration
+          pytest -v -rx -s polybench_transfertuning_test.py

.github/workflows/llvm_tests_san.yml

@@ -23,6 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Mark GitHub Actions workdir as safe

.github/workflows/release.yml

@@ -27,6 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - uses: pypa/cibuildwheel@v3.3.1
@@ -64,6 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       # Pin docc-compiler version to match release
@@ -156,6 +158,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Define Version

.github/workflows/sanitizer_tests_asan.yml

@@ -18,6 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Formatting
@@ -55,6 +56,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Install dependencies

.github/workflows/sanitizer_tests_lsan.yml

@@ -18,6 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Formatting

.github/workflows/sanitizer_tests_ubsan.yml

@@ -18,6 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Formatting

.github/workflows/unit_tests_linux.yml

@@ -25,6 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Mark GitHub Actions workdir as safe
@@ -153,6 +154,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Mark GitHub Actions workdir as safe
@@ -237,6 +239,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -321,6 +324,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -403,6 +407,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -477,6 +482,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -549,6 +555,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/unit_tests_macos.yml

@@ -21,6 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Install dependencies

.github/workflows/unit_tests_release.yml

@@ -23,6 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Mark GitHub Actions workdir as safe
@@ -79,6 +80,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          token: ${{ secrets.BENCHMARK_TESTS_TOKEN }}
           submodules: recursive
 
       - name: Install dependencies

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "cmake"]
 	path = cmake
 	url = git@github.com:daisytuner/cmake-scripts.git
+[submodule "integration/tests"]
+	path = integration/tests
+	url = git@github.com:daisytuner/benchmark-tests.git