tests: fix binary test passing when failed and llvm-cov deprecated flag (#165)

* tests: coverage - fix llvm-cov deprecated --export-prefix

solves [#164](#164)

Signed-off-by: Artie Poole <stuart.poole@canonical.com>

* tests: workflows: fix binary test passing even if the script errored

sets pipefail flag and catches sub-script return state
disable -e only around coverage pipeline

Signed-off-by: Artie Poole <stuart.poole@canonical.com>

* tests: binary - the TF job did not catch the error from _run it seems

Signed-off-by: Artie Poole <stuart.poole@canonical.com>

* tests: coverage-test default RUSTFLAGS to empty in export call

Signed-off-by: Artie Poole <stuart.poole@canonical.com>

---------

Signed-off-by: Artie Poole <stuart.poole@canonical.com>

Author: artiepoole

.github/testflinger-assets/test_binary_device_script.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -x
+set -euxo pipefail
 
 echo "INFO: Installing dependencies"
 sudo env DEBIAN_FRONTEND=noninteractive apt update && sudo env DEBIAN_FRONTEND=noninteractive apt install build-essential -y
@@ -18,5 +18,15 @@ cd fpgad
 mkdir -p artifacts
 sudo journalctl -f -n0 > artifacts/journal.log 2>&1 &
 JOURNAL_PID=$!
+
+# Temporarily disable -e to capture the test exit status even on failure
+set +e
 ./tests/coverage_test.sh 2>&1 | tee artifacts/coverage_test.log
-sudo kill ${JOURNAL_PID}  || true
+TEST_EXIT_STATUS=${PIPESTATUS[0]}
+set -e
+
+# Cleanup regardless of test result
+sudo kill ${JOURNAL_PID} || true
+
+# Exit with the test's exit status
+exit $TEST_EXIT_STATUS

.github/testflinger-assets/testflinger_job.yaml

@@ -34,6 +34,7 @@ test_data:
     - agent: device_script.sh
       local: ./device_script.sh
   test_cmds: |
+    set -e
     # get hw-cert-team helper tools
     echo "INFO: setting up scriptlets"
     export TOOLS_PATH=tools
@@ -47,8 +48,16 @@ test_data:
 
     echo "INFO: Running script on DUT"
     _run 'chmod +x /home/ubuntu/device_script.sh'
+
+    # Run the test script, but always retrieve artifacts even if it fails
+    set +e
     _run '/home/ubuntu/device_script.sh'
+    TEST_SCRIPT_EXIT=$?
+    set -e
 
     echo "INFO: Retrieving artifacts"
     mkdir -p artifacts
     _get fpgad/artifacts/* artifacts
+
+    # Fail the job if the test script failed
+    exit $TEST_SCRIPT_EXIT

tests/coverage_test.sh

@@ -15,9 +15,9 @@
 set -xeu
 
 # Enable llvm-coverage build results
-eval "$(cargo llvm-cov show-env --export-prefix)"
+eval "$(cargo llvm-cov show-env --sh)"
 
-export RUSTFLAGS="$RUSTFLAGS -C llvm-args=-runtime-counter-relocation"
+export RUSTFLAGS="${RUSTFLAGS:-} -C llvm-args=-runtime-counter-relocation"
 
 # build the daemon only, to avoid getting coverage for cli (no tests written)
 cargo build --bin daemon