diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 634ad85..9b68528 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -131,6 +131,7 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include: "${{ fromJson(needs.test-matrix.outputs.tests) }}"
 
@@ -175,6 +176,10 @@ jobs:
       - name: Stagger tests
         run: 'sleep $((RANDOM % 60 + 1))'
 
+      - name: Cleanup Leftovers
+        if: always()
+        run: helpers/cleanup
+
       - name: Create Test Cluster
         run: helpers/run-in-test-cluster
 
diff --git a/helpers/cleanup b/helpers/cleanup
index cacf5de..b8392f4 100755
--- a/helpers/cleanup
+++ b/helpers/cleanup
@@ -6,9 +6,9 @@ set -euo pipefail
 
 export CLUSTER_PREFIX="${CLUSTER_PREFIX-k8test}"
 
-if ! test -f k8test/cluster/inventory.yml; then
-  exit 0
-fi
+# Make sure the k8test/helpers/release-set CLI is available
+source helpers/run-in-test-cluster
+ensure-k8test > /dev/null
 
 k8test/playbooks/destroy-cluster.yml \
   -i k8test/cluster/inventory.yml \
diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster
index 73c6139..2331c5c 100755
--- a/helpers/run-in-test-cluster
+++ b/helpers/run-in-test-cluster
@@ -19,7 +19,12 @@ function ensure-k8test() {
   # on the runners and cloning via SSH does not work.
   #
   # Therefore we run a separate install block that is only meant for GitHub.
-  if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+  if [[ "${GITHUB_ACTIONS:-}" == "true" ]] then
+    if test -d k8test; then
+      source k8test/venv/bin/activate 
+      return
+    fi
+
     mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts
     git clone https://github.com/cloudscale-ch/k8test
     python3 -m venv k8test/venv