diff --git a/.github/actions/lava-test-plans/action.yml b/.github/actions/lava-test-plans/action.yml index f7e1318c0..f83d37a8b 100644 --- a/.github/actions/lava-test-plans/action.yml +++ b/.github/actions/lava-test-plans/action.yml @@ -65,6 +65,14 @@ runs: run-id: ${{ inputs.build_id }} pattern: build-url_*_${{ inputs.distro_name }} + - name: 'Download build metadata' + continue-on-error: true + uses: actions/download-artifact@v7 + with: + github-token: ${{ inputs.gh_token }} + run-id: ${{ inputs.build_id }} + pattern: build-meta_*_${{ inputs.distro_name }} + - name: Run lava-test-plans id: lava-test-plans shell: bash @@ -78,6 +86,8 @@ runs: INPUTS_TESTKIT_REF: ${{ inputs.testkit_ref }} INPUTS_PR_NUMBER: ${{ inputs.pr_number }} INPUTS_PR_URL: ${{ inputs.pr_url }} + INPUTS_REF: ${{ inputs.ref }} + INPUTS_BUILD_ID: ${{ inputs.build_id }} run: | set -euo pipefail cd lava-test-plans && pip install . @@ -119,6 +129,21 @@ runs: echo "GITHUB_WORKFLOW_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >> "${VARS_OUT_PATH}/gh-variables.ini" echo "GITHUB_WORKFLOW_RUN_ID=${GITHUB_RUN_ID}" >> "${VARS_OUT_PATH}/gh-variables.ini" echo "GITHUB_WORKFLOW_RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT}" >> "${VARS_OUT_PATH}/gh-variables.ini" + SOURCE_REPO="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" + SOURCE_SHA="${GITHUB_SHA}" + SOURCE_BRANCH="${GITHUB_REF_NAME}" + BUILD_TYPE="${GITHUB_EVENT_NAME^^}" + if [ "${GITHUB_EVENT_NAME}" = "workflow_run" ]; then + BUILD_TYPE="PR" + if [ -f "${GITHUB_EVENT_PATH:-}" ]; then + # github.sha is the target branch here, not the built commit + SOURCE_REPO=$(jq -r --arg d "${SOURCE_REPO}" '.workflow_run.head_repository.html_url // $d' "${GITHUB_EVENT_PATH}") + SOURCE_SHA=$(jq -r '.workflow_run.head_sha // ""' "${GITHUB_EVENT_PATH}") + SOURCE_BRANCH=$(jq -r '.workflow_run.head_branch // ""' "${GITHUB_EVENT_PATH}") + fi + fi + DISTRO="${INPUTS_TESTPLAN%%/*}" + [ "${DISTRO}" != "${INPUTS_TESTPLAN}" ] || DISTRO="" IMAGE_TYPE="core-image-base" case "${INPUTS_DISTRO_NAME}" in "qcom-distro"*) @@ -127,6 +152,25 @@ runs: echo "AUTO_LOGIN_PASSWORD=oelinux123" >> "${VARS_OUT_PATH}/gh-variables.ini" ;; esac + COMMON_META="${VARS_OUT_PATH}/gh-metadata.json" + jq -n \ + --arg build_type "${BUILD_TYPE}" \ + --arg build_run_id "${INPUTS_BUILD_ID}" \ + --arg source_repo "${SOURCE_REPO}" \ + --arg source_branch "${SOURCE_BRANCH}" \ + --arg source_sha "${SOURCE_SHA}" \ + --arg distro "${DISTRO}" \ + --arg build_config "${INPUTS_DISTRO_NAME}" \ + --arg testplan "${INPUTS_TESTPLAN}" \ + --arg testkit_ref "${INPUTS_TESTKIT_REF}" \ + --arg lava_test_plans_ref "${INPUTS_REF}" \ + --arg gh_workflow_name "${GITHUB_WORKFLOW}" \ + --arg image "${IMAGE_TYPE}" \ + '{EXTRA_METADATA: {"build-type": $build_type, "build-run-id": $build_run_id, + "source-repo": $source_repo, "source-branch": $source_branch, "source-sha": $source_sha, + "distro": $distro, "build-config": $build_config, "testplan": $testplan, + "testkit-ref": $testkit_ref, "lava-test-plans-ref": $lava_test_plans_ref, + "gh-workflow-name": $gh_workflow_name, "image": $image}}' > "${COMMON_META}" RESULT_JSON='{"target": []}' for machine in ${INPUTS_MACHINES//,/ }; do @@ -134,15 +178,26 @@ runs: cp "${VARS_OUT_PATH}/gh-variables.ini" "${VARS_OUT_PATH}/${machine}.ini" echo "IMAGE_FILE_NAME=${IMAGE_TYPE}-${machine}.rootfs.qcomflash.tar.gz" >> "${VARS_OUT_PATH}/${machine}.ini" echo "DEVICE_TYPE=${machine}" >> "${VARS_OUT_PATH}/${machine}.ini" + # machines whose artifacts are published under a different build + ARTIFACT_MACHINE="${machine}" + case "${machine}" in + dragonboard-410c|dragonboard-820c) ARTIFACT_MACHINE="qcom-armv8a" ;; + esac + MACHINE_META="${VARS_OUT_PATH}/${machine}-metadata.json" + jq --arg machine "${machine}" --arg build_machine "${ARTIFACT_MACHINE}" \ + '.EXTRA_METADATA += {"machine": $machine, "build-machine": $build_machine}' \ + "${COMMON_META}" > "${MACHINE_META}" + VARIABLES=("${VARS_OUT_PATH}/${machine}.ini" "${MACHINE_META}") + BUILD_META="${GITHUB_WORKSPACE}/build-meta_${ARTIFACT_MACHINE}_${INPUTS_DISTRO_NAME}/build-meta_${ARTIFACT_MACHINE}_${INPUTS_DISTRO_NAME}" + if [ -f "${BUILD_META}" ]; then + VARIABLES+=("${BUILD_META}") + else + echo "No build metadata for ${machine}" + fi + export BUILD_URL="" # dir name and file name are the same as we're downloading multiple files - export BUILD_URL_FILE="$GITHUB_WORKSPACE/build-url_${machine}_${INPUTS_DISTRO_NAME}/build-url_${machine}_${INPUTS_DISTRO_NAME}" - if [ "${machine}" = "dragonboard-410c" ]; then - export BUILD_URL_FILE="$GITHUB_WORKSPACE/build-url_qcom-armv8a_${INPUTS_DISTRO_NAME}/build-url_qcom-armv8a_${INPUTS_DISTRO_NAME}" - fi - if [ "${machine}" = "dragonboard-820c" ]; then - export BUILD_URL_FILE="$GITHUB_WORKSPACE/build-url_qcom-armv8a_${INPUTS_DISTRO_NAME}/build-url_qcom-armv8a_${INPUTS_DISTRO_NAME}" - fi + export BUILD_URL_FILE="$GITHUB_WORKSPACE/build-url_${ARTIFACT_MACHINE}_${INPUTS_DISTRO_NAME}/build-url_${ARTIFACT_MACHINE}_${INPUTS_DISTRO_NAME}" echo "${BUILD_URL_FILE}" if [ -f "$BUILD_URL_FILE" ]; then export BUILD_URL=$(cat "${BUILD_URL_FILE}") @@ -158,19 +213,19 @@ runs: echo "ROOTFS_URL=${BUILD_URL}/${INPUTS_DISTRO_NAME}/${machine}/${IMAGE_TYPE}-${machine}.rootfs.qcomflash.tar.gz" >> "${VARS_OUT_PATH}/${machine}.ini" if [ "${machine}" = "dragonboard-410c" ]; then - echo "ROOTFS_IMG_FILE=${IMAGE_TYPE}-qcom-armv8a.rootfs.ext4" >> "${VARS_OUT_PATH}/${machine}.ini" - echo "BOOT_IMG_FILE=boot-apq8016-sbc-qcom-armv8a.img" >> "${VARS_OUT_PATH}/${machine}.ini" + echo "ROOTFS_IMG_FILE=${IMAGE_TYPE}-${ARTIFACT_MACHINE}.rootfs.ext4" >> "${VARS_OUT_PATH}/${machine}.ini" + echo "BOOT_IMG_FILE=boot-apq8016-sbc-${ARTIFACT_MACHINE}.img" >> "${VARS_OUT_PATH}/${machine}.ini" echo "BUILD_OS=${INPUTS_DISTRO_NAME}${INPUTS_KERNEL}/" >> "${VARS_OUT_PATH}/${machine}.ini" fi if [ "${machine}" = "dragonboard-820c" ]; then - echo "ROOTFS_IMG_FILE=${IMAGE_TYPE}-qcom-armv8a.rootfs.ext4" >> "${VARS_OUT_PATH}/${machine}.ini" - echo "BOOT_IMG_FILE=boot-apq8096-db820c-qcom-armv8a.img" >> "${VARS_OUT_PATH}/${machine}.ini" + echo "ROOTFS_IMG_FILE=${IMAGE_TYPE}-${ARTIFACT_MACHINE}.rootfs.ext4" >> "${VARS_OUT_PATH}/${machine}.ini" + echo "BOOT_IMG_FILE=boot-apq8096-db820c-${ARTIFACT_MACHINE}.img" >> "${VARS_OUT_PATH}/${machine}.ini" echo "BUILD_OS=${INPUTS_DISTRO_NAME}${INPUTS_KERNEL}/" >> "${VARS_OUT_PATH}/${machine}.ini" fi - cat "${VARS_OUT_PATH}/${machine}.ini" + cat "${VARIABLES[@]}" # INPUTS_TESTPLAN contains distro name - lava-test-plans --dry-run --variables "${VARS_OUT_PATH}/${machine}.ini" --test-plan "${PROJECT}/${INPUTS_TESTPLAN}" --device-type "projects/${PROJECT}/devices/${machine}" --dry-run-path "${JOBS_OUT_PATH}/${machine}-${INPUTS_DISTRO_NAME}-${INPUTS_TESTPLAN}" || true + lava-test-plans --dry-run --variables "${VARIABLES[@]}" --test-plan "${PROJECT}/${INPUTS_TESTPLAN}" --device-type "projects/${PROJECT}/devices/${machine}" --dry-run-path "${JOBS_OUT_PATH}/${machine}-${INPUTS_DISTRO_NAME}-${INPUTS_TESTPLAN}" || true # create the JSON file that is used for the job submission matrix find "${JOBS_OUT_PATH}/${machine}-${INPUTS_DISTRO_NAME}-${INPUTS_TESTPLAN}" -name "*.yaml" for JOB_PATH in $( find "${JOBS_OUT_PATH}/${machine}-${INPUTS_DISTRO_NAME}-${INPUTS_TESTPLAN}" -name "*.yaml"); diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 55e9ebe8d..d62f1a046 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -195,6 +195,37 @@ jobs: INPUTS_MACHINE: ${{ inputs.machine }} INPUTS_DISTRO_NAME: ${{ inputs.distro_name }} INPUTS_KERNEL_DIRNAME: ${{ inputs.kernel_dirname }} + - name: Extract build metadata from the image SBOM + id: build-metadata + if: inputs.target + shell: bash + env: + INPUTS_MACHINE: ${{ inputs.machine }} + INPUTS_DISTRO_NAME: ${{ inputs.distro_name }} + INPUTS_KERNEL_DIRNAME: ${{ inputs.kernel_dirname }} + run: | + set -euo pipefail + uploads_dir="./uploads/${INPUTS_DISTRO_NAME}${INPUTS_KERNEL_DIRNAME}/${INPUTS_MACHINE}" + FILENAME="build-meta_${INPUTS_MACHINE}_${INPUTS_DISTRO_NAME}${INPUTS_KERNEL_DIRNAME}" + echo "filename=${FILENAME}" >> "$GITHUB_OUTPUT" + SPDX=() + for f in "${uploads_dir}"/*.spdx.json; do + if [ -e "${f}" ]; then + SPDX+=("${f}") + fi + done + if [ "${#SPDX[@]}" -eq 0 ]; then + echo "No image SBOM found under ${uploads_dir}" + echo '{}' > "${FILENAME}" + elif [ ! -f ci/build-metadata.py ]; then + echo "No ci/build-metadata.py in this repository" + echo '{}' > "${FILENAME}" + elif ! python3 ci/build-metadata.py "${SPDX[@]}" > "${FILENAME}"; then + echo "::warning::could not read build metadata from ${uploads_dir}" + echo '{}' > "${FILENAME}" + fi + cat "${FILENAME}" + - name: Set S3 object tag run: | if [[ "$GITHUB_EVENT_NAME" = "pull_request" ]]; then @@ -248,3 +279,11 @@ jobs: name: ${{ steps.print-output.outputs.filename }} path: ${{ steps.print-output.outputs.filename }} + - name: Upload build metadata + if: inputs.target + uses: actions/upload-artifact@v6 + with: + overwrite: true + name: ${{ steps.build-metadata.outputs.filename }} + path: ${{ steps.build-metadata.outputs.filename }} + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 810bb3753..687eb8c50 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ on: description: "Token used to submit jobs to the LAVA lab" env: # git sha, tag or branch in lava-test-plans repository - LAVA_TEST_PLANS_REF: "6070e41a1472e3fd595ea7ac849f7c30f86a2282" + LAVA_TEST_PLANS_REF: "2f2900cc3e7f2215ce7db6b5bf67955a0782ea31" # CalVer tag in qcom-linux-testkit repository (format: testkit-YYYY.MM.DD) TESTKIT_REF: "testkit-2026.08.14" diff --git a/ci/build-metadata.py b/ci/build-metadata.py new file mode 100755 index 000000000..b2475cab4 --- /dev/null +++ b/ci/build-metadata.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Collect build metadata from the image SPDX documents of a build. + +oe-core inherits create-spdx by default, so every image build already writes +an SBOM recording each recipe's source URI and resolved revision. Reading the +metadata back out of it costs nothing at build time and avoids having the +build record the same facts twice. + +Currently this reports the kernel: its recipe, version, release, repository +and commit. + +Nothing here is specific to a kernel recipe: the anchor is the package named +by KERNEL_PACKAGE_NAME ("kernel" unless a layer overrides it), which +kernel.bbclass produces for every provider, qcom or not. + +A build writes one document per image and not all of them describe a kernel: +an initramfs image installs kernel modules but no kernel package. Every +document given is therefore examined and the most complete answer wins. + +The result is printed as a lava-test-plans variables file, under EXTRA_METADATA. +""" +import json, re, sys + +KNAME = "kernel" + +def provenance(path): + doc = json.load(open(path)) + graph = doc["@graph"] + packages = [e for e in graph if e.get("type") == "software_Package"] + + kernel = next((p for p in packages if p.get("name") == KNAME), None) + if kernel is None: + # An image with no kernel package, such as an initramfs. + return {} + + recipe = recipe_of(kernel) + mine = [p for p in packages if recipe_of(p) == recipe] + return describe(kernel, recipe, mine) + + +def recipe_of(entry): + """The recipe whose SPDX document an entry belongs to.""" + m = re.search(r"/spdxdocs/(.+?)-[0-9a-f]{8}-[0-9a-f]{4}-", entry.get("spdxId", "")) + return m.group(1) if m else None + + +def source_index(p): + """Position of a source within SRC_URI. + + create-spdx numbers sources by their 1-based index in SRC_URI, so a + recipe's own repository is source/1. linux-yocto fetches yocto-kernel-cache + as a second git source; the ordering separates them without having to + recognise the cache by name. + """ + m = re.search(r"/source/(\d+)$", p.get("spdxId", "")) + return int(m.group(1)) if m else 1 << 30 + + +def describe(kernel, recipe, mine): + # Patches carry the purpose "patch" rather than "source", so what is left + # here is a real source and the lowest index is the recipe's own + # repository. Every source is still reported, so a surprise stays visible. + sources = sorted((p for p in mine + if p.get("software_primaryPurpose") == "source"), + key=source_index) + primary = sources[0] if sources else None + + repo = commit = "" + if primary: + loc = primary.get("software_downloadLocation", "") + if loc.startswith("git+"): + # git+@; a tarball kernel has no revision to report. + repo, _, commit = loc[4:].partition("@") + else: + repo = loc + + # kernel.bbclass names one package "-". + # Keying off that rather than "kernel-image-" keeps this independent + # of KERNEL_IMAGETYPE, which differs per machine (Image, Image.gz, zImage). + release = "" + for p in mine: + m = re.fullmatch(re.escape(KNAME) + r"-(\d.*)", p.get("name") or "") + if m: + release = m.group(1) + break + + return { + "kernel-recipe": recipe or "", + "kernel-version": kernel.get("software_packageVersion", ""), + "kernel-release": release, + "kernel-repo": repo, + "kernel-commit": commit, + **({"kernel-all-sources": [p.get("software_downloadLocation", "") for p in sources]} + if len(sources) > 1 else {}), + } + + +def completeness(result): + """A result with a commit beats one with only a release, beats nothing.""" + return (bool(result.get("kernel-commit")), bool(result.get("kernel-release"))) + + +best = {} +for path in sorted(sys.argv[1:]): + try: + result = provenance(path) + except (OSError, ValueError, KeyError): + continue + if completeness(result) > completeness(best): + best = result + if best.get("kernel-commit"): + # Nothing better to find; skip parsing the remaining documents. + break + +print(json.dumps({"EXTRA_METADATA": best} if best else {}, indent=2))