diff --git a/.github/actions/setup-cloudxr-sdk/action.yml b/.github/actions/setup-cloudxr-sdk/action.yml index 0532ec0518..1c50ea02df 100644 --- a/.github/actions/setup-cloudxr-sdk/action.yml +++ b/.github/actions/setup-cloudxr-sdk/action.yml @@ -42,7 +42,12 @@ runs: path: | deps/cloudxr/*.tar.gz deps/cloudxr/*.tgz - key: cloudxr-sdk-${{ runner.os }}-${{ runner.arch }}-rt${{ steps.parse.outputs.runtime_version }}-web${{ steps.parse.outputs.web_version }} + # The URL each tarball came from, written beside it on download. A + # cache hit without it leaves every CloudXR wheel member unattributable: + # an archive explains one only if the repository tracks it or this + # marker says where it was fetched from. + deps/cloudxr/*.source + key: cloudxr-sdk-v2-${{ runner.os }}-${{ runner.arch }}-rt${{ steps.parse.outputs.runtime_version }}-web${{ steps.parse.outputs.web_version }} - name: Download CloudXR Runtime SDK shell: bash @@ -66,4 +71,5 @@ runs: path: | deps/cloudxr/*.tar.gz deps/cloudxr/*.tgz + deps/cloudxr/*.source key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 427e2ed446..c812cde7b8 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -17,8 +17,9 @@ concurrency: env: # Shared apt build dependencies. Each job installs these plus its own extras - # (ccache, clang-format, the OAK camera autotools chain). - CORE_APT_DEPS: build-essential cmake glslang-tools libegl-dev libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols + # (ccache, clang-format, the OAK camera autotools chain). libmagic1: see + # scripts/sbom/pyproject.toml. + CORE_APT_DEPS: build-essential cmake glslang-tools libegl-dev libmagic1 libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols # Pin vcpkg to an immutable commit for reproducible, supply-chain-stable CI # instead of tracking its moving default branch. Port versions are fixed by the # consuming manifest's builtin-baseline (e.g. DepthAI's vcpkg.json), so this @@ -283,6 +284,56 @@ jobs: mv install/wheels-repaired/*.whl install/wheels/ rmdir install/wheels-repaired || true + # Run after auditwheel and before upload so evidence describes the repaired bytes. + - name: Package licenses and contents SBOM into wheels (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + args=() + for wheel in install/wheels/*.whl; do args+=(--wheel "${wheel}"); done + if (( ${#args[@]} == 0 )); then + echo "No wheels to describe" + exit 1 + fi + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + isaaccapture-sbom build \ + --build-dir "${BUILD_DIR}" "${args[@]}" --out-dir sbom + + - name: Check wheel SBOMs (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + for wheel in install/wheels/*.whl; do + stem="$(basename "${wheel}" .whl)" + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + isaaccapture-sbom check --wheel "${wheel}" \ + --manifest sbom/manifest.json \ + --build-evidence "sbom/${stem}.build-evidence.json" + done + for document in sbom/*.spdx.json; do + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + --extra dev pyspdxtools -i "${document}" + done + + - name: Install and import the finished wheel (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + wheels=(install/wheels/*.whl) + # --find-links resolves the sibling transition wheel from this build. + uv venv --python ${{ matrix.python_version }} .venv-wheel-check + uv pip install --python .venv-wheel-check \ + --find-links install/wheels "${wheels[@]}" + .venv-wheel-check/bin/python -c "import isaaccapture; print('imported', isaaccapture.__version__)" + # Exercise standalone verification without build evidence or a manifest. + for wheel in "${wheels[@]}"; do + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + isaaccapture-sbom check --wheel "${wheel}" + done + - name: Create tarball to preserve permissions if: matrix.build_type == 'Release' run: tar -cvf isaacteleop-install.tar -C install . @@ -303,6 +354,74 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Upload wheel SBOM sidecars + if: matrix.build_type == 'Release' + uses: actions/upload-artifact@v6 + with: + name: isaaccapture-sbom-${{ matrix.arch }}-py${{ matrix.python_version }} + path: sbom/ + if-no-files-found: error + retention-days: 7 + + # The SBOM collector's tests need only a configured tree, so they run beside + # the product build; wheel-gate waits on them. + test-sbom-collector: + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-22.04' }} + permissions: + contents: read + timeout-minutes: 20 + strategy: + fail-fast: ${{ github.event_name == 'pull_request' }} + matrix: + arch: ['x64', 'arm64'] + env: + BUILD_DIR: build + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 # configure derives the version from history + persist-credentials: false + + - name: Install uv + uses: ./.github/actions/setup-uv + + - name: Install Apt dependencies + run: | + sudo apt-get update + sudo apt-get install -y $CORE_APT_DEPS + + # BUILD_TESTING=ON: a test target links a static library the project builds, + # which the codemodel test looks for. + - name: Configure CMake + run: | + cmake -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PLUGINS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_VIZ=OFF \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DENABLE_CLANG_FORMAT_CHECK=OFF \ + -DBUILD_TESTING=ON + + # Pythons: build-ubuntu's python_version matrix; keep the two lists in step. + # The named build makes missing inputs fail the tests rather than skip them. + - name: Test the SBOM collector + env: + ISAACCAPTURE_SBOM_BUILD_DIR: ${{ env.BUILD_DIR }} + run: | + set -uo pipefail + failed=() + for python in 3.11 3.12 3.13; do + uv run --project scripts/sbom --python "${python}" \ + --extra dev pytest scripts/sbom/tests || failed+=("${python}") + done + if (( ${#failed[@]} )); then + echo "::error::SBOM collector tests failed under Python ${failed[*]}" + exit 1 + fi + pip-editable-install: # Validates the scikit-build-core (PEP 517/660) from-source install path added # for issue #735: an sdist build and `pip install -e .` over the root @@ -999,6 +1118,7 @@ jobs: permissions: {} needs: - build-ubuntu + - test-sbom-collector - test-viz-gpu - test-viz-sanitizers - test-cloudxr @@ -1027,6 +1147,103 @@ jobs: echo "All wheel dependency jobs completed successfully" + publish-sbom-evidence: # durable copies attached to the GitHub release + # Attach durable evidence to releases; Actions artifacts expire. + runs-on: ubuntu-22.04 + needs: + - wheel-gate + permissions: + contents: write + timeout-minutes: 20 + env: + ARCHIVE: isaaccapture-sbom-${{ github.ref_name }} + + # !cancelled() lets wheel-gate's result decide despite skipped upstream jobs. + if: >- + !cancelled() + && needs.wheel-gate.result == 'success' + && github.repository_id == '1092046207' + && startsWith(github.ref, 'refs/tags/v') + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Install uv + uses: ./.github/actions/setup-uv + + - name: Download wheel artifacts + uses: actions/download-artifact@v7 + with: + pattern: isaacteleop-wheels-* + merge-multiple: true + path: wheels + + - name: Download wheel SBOM sidecars + uses: actions/download-artifact@v7 + with: + pattern: isaaccapture-sbom-* + path: sbom-sidecars + + # Fix manifest and archive timestamps for reproducible reruns. + - name: Pin the build clock to the commit + run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" + + - name: Pack and verify the release evidence archive + run: | + set -euo pipefail + + # Universal wheels repeat across matrix entries. Reject conflicting copies + # and publish only the evidence bound by the merged manifest. + uv run --project scripts/sbom --python 3.12 \ + isaaccapture-sbom merge-manifests \ + --output "${ARCHIVE}/isaaccapture-sbom-manifest.json" \ + --evidence-out "${ARCHIVE}" \ + sbom-sidecars/*/manifest.json + + # ZIP preserves filenames that GitHub renames in loose assets (`!` to `.`). + # Fix times, modes, order, and extra fields for reproducibility; + # ZIP stores local time, so set TZ too. + touch -d "@${SOURCE_DATE_EPOCH}" "${ARCHIVE}"/* + chmod 644 "${ARCHIVE}"/* + printf '%s\n' "${ARCHIVE}"/* | LC_ALL=C sort \ + | TZ=UTC zip -X -q "${ARCHIVE}.zip" -@ + + # Verify the unpacked release using the published consumer procedure. + unzip -q "${ARCHIVE}.zip" -d unpacked + uv run --project scripts/sbom --python 3.12 \ + isaaccapture-sbom check-set \ + --manifest "unpacked/${ARCHIVE}/isaaccapture-sbom-manifest.json" \ + --wheel-dir wheels --evidence-dir "unpacked/${ARCHIVE}" + + unzip -l "${ARCHIVE}.zip" + + - name: Attach evidence to the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + + # The tag can trigger this job before release-tag.yml creates its draft. + # Wait for that release to preserve its draft/approval step. + deadline=$(( SECONDS + 900 )) + until gh release view "${TAG}" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo "::error::No release exists for ${TAG} after 15 minutes; evidence not attached." + exit 1 + fi + echo "Waiting for the ${TAG} release to be created..." + sleep 20 + done + + gh release upload "${TAG}" "${ARCHIVE}.zip" --clobber + + echo "Attached ${ARCHIVE}.zip to ${TAG}" + publish-wheel: # to internal Artifactory runs-on: [self-hosted, linux] permissions: {} @@ -1062,7 +1279,7 @@ jobs: - name: Ensure clean wheels directory run: | - rm -rf wheels + rm -rf wheels sbom-sidecars - name: Download wheel artifacts uses: actions/download-artifact@v7 @@ -1071,6 +1288,76 @@ jobs: merge-multiple: true path: wheels + - name: Download wheel SBOM sidecars + uses: actions/download-artifact@v7 + with: + pattern: isaaccapture-sbom-* + path: sbom-sidecars + + - name: Verify wheel variant coverage and digests + run: | + set -euo pipefail + + # Keep platform and ABI lists aligned with the build-ubuntu Release matrix. + platforms=(x86_64 aarch64) + abis=(cp311 cp312 cp313) + expected=$(( ${#platforms[@]} * ${#abis[@]} )) + + shopt -s nullglob + manifests=(sbom-sidecars/*/manifest.json) + if (( ${#manifests[@]} != expected )); then + echo "Expected ${expected} SBOM manifests, found ${#manifests[@]}" + ls -la sbom-sidecars || true + exit 1 + fi + + declare -A named=() + for manifest in "${manifests[@]}"; do + while IFS=$'\t' read -r filename digest; do + wheel="wheels/${filename}" + if [[ ! -f "${wheel}" ]]; then + echo "Manifest names ${filename}, which is not among the downloaded wheels" + exit 1 + fi + actual="$(sha256sum "${wheel}" | cut -d' ' -f1)" + if [[ "${actual}" != "${digest}" ]]; then + echo "${filename} was modified after its digest was bound" + echo " manifest: ${digest}" + echo " actual: ${actual}" + exit 1 + fi + named["${filename}"]=1 + done < <(jq -r '.wheels[] | [.filename, .sha256] | @tsv' "${manifest}") + done + + # Every downloaded wheel is uploaded below, so require evidence for all of them. + for wheel in wheels/*.whl; do + name="$(basename "${wheel}")" + if [[ -z "${named[${name}]:-}" ]]; then + echo "${name} is among the downloaded wheels, and no manifest names it" + exit 1 + fi + done + + for platform in "${platforms[@]}"; do + for abi in "${abis[@]}"; do + found=0 + for filename in "${!named[@]}"; do + if [[ "${filename}" == *"-${abi}-${abi}-manylinux"*"_${platform}.whl" ]]; then + found=1 + break + fi + done + if (( found == 0 )); then + echo "No wheel was published for ${abi} on ${platform}" + printf ' have: %s\n' "${!named[@]}" + exit 1 + fi + done + done + + echo "All ${expected} wheel variants present and matching their recorded digests" + - name: Upload wheel(s) to Artifactory id: upload-artifactory env: diff --git a/.gitignore b/.gitignore index f939df2136..1355488147 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,10 @@ MUJOCO_LOG.TXT # Runtime device files injected by MCP server infrastructure .mcp.json +# Python coverage databases +.coverage +.coverage.* + # Agent session logs (never committed) tmp/ @@ -84,3 +88,5 @@ tmp/ !/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml /examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/* !/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/follower_arm.xml +# Wheel SBOM and license evidence written by the collector in scripts/sbom/ +/sbom/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71554b4c01..cbf2b318ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - id: debug-statements # SPDX license headers (REUSE): require SPDX-FileCopyrightText and SPDX-License-Identifier - repo: https://github.com/fsfe/reuse-tool - rev: v6.2.0 + rev: v6.2.0 # scripts/sbom/pyproject.toml pins the same reuse to read this metadata hooks: - id: reuse-lint-file name: REUSE licenses (source files and complete EgoPost subtree) @@ -52,8 +52,9 @@ repos: name: Check copyright year (include current year) entry: python3 scripts/check_copyright_year.py --fix language: system - # Include CMakeLists.txt and Dockerfile.xxx by matching those explicitly in the regex - files: \.(py|js|ts|cpp|c|h|hpp|cxx|hxx|cmake|md|yaml|yml|sh|fbs|toml|html)$|CMakeLists\.txt$|Dockerfile(\.[A-Za-z0-9_-]+)?$ + # Include CMakeLists.txt and Dockerfile.xxx by matching those explicitly in the regex. + # Same rule as SOURCE_FILES in scripts/check_licenses.py; change both together. + files: \.(py|pyi|js|ts|cpp|cc|c|h|hpp|cxx|hxx|inc|cu|cuh|frag|vert|cmake|md|yaml|yml|sh|fbs|toml|html)$|CMakeLists\.txt$|Dockerfile(\.[A-Za-z0-9_-]+)?$ - id: check-lfs-pointers name: Ensure Git LFS-marked files are LFS pointers entry: python3 scripts/check_lfs_pointers.py diff --git a/AGENTS.md b/AGENTS.md index 55ebb73014..0113244048 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,6 +198,13 @@ measurement, a trap someone already paid for. They are not a place to narrate. inertial frame, so STL axes need `mesh_pos`/`mesh_quat`" beats three paragraphs re-deriving why. Keep measured numbers and file/line references; cut the prose around them. +- **No counts that drift.** A comment must not state how many files, rows, + components or call sites something currently has. The number is true the day + it is written and wrong after the next build or refactor, and nothing fails + when it goes stale. State the property instead — "these SDKs share build-ids", + not "they share five". A measured number earns its place only when it is a + fixed fact about the world (a protocol limit, a benchmark with its date). + - **Do not narrate history.** "An earlier revision did X and it was wrong" is what `git log` is for. If the wrong approach is tempting enough to warn about, write the warning as a rule — "do not derive this from the mesh" — @@ -271,6 +278,45 @@ pre-commit install --hook-type commit-msg - If a hook failure shows **missing or non-obvious repo policy** (not a one-off typo), you **must** add a **short** reminder under **Mandatory learning loop** rules to the right `AGENTS.md` or adjacent **`//` comments** so the next run does not repeat it—unless it is already documented. +## The wheel SBOM + +Published wheels carry a contents SBOM and license evidence derived from the +build itself. There is **no checked-in dependency registry** and adding one would +be a second source of truth — declare a dependency the way you already would and +it appears in the next wheel's inventory. + +When the SBOM step stops the Release build, fix the cause; never exclude a file or +record an assumed license. Remedies: `scripts/sbom/README.md` ('What stops the Release +build'). + +The collector is a project of its own: `uv run --project scripts/sbom +isaaccapture-sbom ...`. Its dependency pins live in `scripts/sbom/pyproject.toml` +and nowhere else — never restate them as `--with` flags in a workflow or a doc. +Its tests run the same way, through uv; do not register them with the product's +CMake or CTest. + +**Read [`scripts/sbom/README.md`](scripts/sbom/README.md) first** if you are +changing anything under `scripts/sbom/`, changing how a dependency reaches a +wheel, or touching the wheel-packaging steps in `build-ubuntu.yml`. + +## A rule changed in one place is changed in none + +When a fact is stated in more than one place — a predicate in several modules, a +claim in both the code and the docs, a heading over the rows it describes — fixing +one instance and leaving the rest is the most common way a correction fails to +stick here. It has happened repeatedly: a licence rule enforced in four modules, a +design doc contradicting its own collector, a per-row reason corrected under a +heading that still said the opposite. + +- **Grep for the claim, not the line you edited.** Search the phrase, the + predicate, the constant — across `docs/`, `scripts/`, `tests/` and the code — + and fix every hit in the same pass. +- **Prefer one authority to a swept duplicate.** If the same rule has to hold in + several places, give it one definition and call it; the sweep is what you do + when you cannot. +- **Verify against output, not against the test suite.** Several of these passed + every test while producing wrong artifacts. Regenerate and look. + ## Mandatory learning loop (AGENTS.md and comments) **Hard requirement:** When **any** of the following happens, you **must** complete steps 1–3 **before** you end the session or move on as if the work were complete: diff --git a/CMakeLists.txt b/CMakeLists.txt index 498bd53482..ded7797a6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,13 @@ cmake_minimum_required(VERSION 3.24...3.25) +# SBOM discovery needs the codemodel and toolchains replies (CMake >=3.27). +# Feature-test the query: raising the policy version would enable CMP0148, +# removing FindPythonInterp still used by pybind11's legacy path through vcpkg. +if(COMMAND cmake_file_api) + cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2 TOOLCHAINS 1) +endif() + # Workaround for older CMake policy compatibility in dependencies set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum CMake policy version") diff --git a/deps/README.md b/deps/README.md index 0e03882854..9ada231254 100644 --- a/deps/README.md +++ b/deps/README.md @@ -95,6 +95,27 @@ machine and are located via `find_package`. - **Build**: Header-only library - **License**: BSD-style +### OpenXR extension headers (vendored) +- **Location**: `third_party/openxr_extensions/` +- **Contents**: `openxr_extension_helpers.h` (The Khronos Group Inc., Apache-2.0) + and `XR_MNDX_xdev_space.h` (Collabora, Ltd., BSL-1.0) +- **Purpose**: OpenXR extension declarations this project compiles against and + therefore redistributes in the wheel. Checked in rather than fetched, so they + sit here instead of beside NVIDIA's own extension headers in + `cloudxr/openxr_extensions/`. +- **Build**: header-only `Teleop::openxr_extensions_upstream`, carried by + `Teleop::openxr_extensions` so consumers link one target. +- **License**: Apache-2.0 and BSL-1.0 (see `LICENSES/`) + +### SPDX license list data +- **Source**: https://github.com/spdx/license-list-data.git +- **Version**: 31ba1a50e5397e00a304dbadc76531740e89ee48 (v3.29.0) +- **Purpose**: reference license texts. The SBOM collector names a dependency's + license by matching the text that dependency ships against these; nothing from + this checkout is redistributed. See `scripts/sbom/README.md`. +- **Build**: data only, no build step. +- **License**: CC0-1.0 + ### GLM - **Source**: https://github.com/g-truc/glm.git - **Version**: 1.0.1 diff --git a/deps/cloudxr/.gitignore b/deps/cloudxr/.gitignore index 78883dbf52..17b5d1e62a 100644 --- a/deps/cloudxr/.gitignore +++ b/deps/cloudxr/.gitignore @@ -1,5 +1,7 @@ # CloudXR runtime and SDK tarballs *.tar.gz +# Download provenance sidecars +*.tar.gz.source *.tgz # CloudXR environment files diff --git a/deps/cloudxr/openxr_extensions/CMakeLists.txt b/deps/cloudxr/openxr_extensions/CMakeLists.txt index 3d7b69f902..eb0e6412f6 100644 --- a/deps/cloudxr/openxr_extensions/CMakeLists.txt +++ b/deps/cloudxr/openxr_extensions/CMakeLists.txt @@ -1,9 +1,14 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Carry the upstream headers through this target to preserve the public interface. add_library(teleop_openxr_extensions INTERFACE) add_library(Teleop::openxr_extensions ALIAS teleop_openxr_extensions) target_include_directories(teleop_openxr_extensions INTERFACE $ ) + +target_link_libraries(teleop_openxr_extensions INTERFACE + Teleop::openxr_extensions_upstream +) diff --git a/deps/third_party/CMakeLists.txt b/deps/third_party/CMakeLists.txt index 56971d614f..76352a70c6 100644 --- a/deps/third_party/CMakeLists.txt +++ b/deps/third_party/CMakeLists.txt @@ -5,6 +5,9 @@ cmake_minimum_required(VERSION 3.20) include(FetchContent) +# Vendored third-party sources, checked in rather than fetched. +add_subdirectory(openxr_extensions) + # ============================================================================== # CMake sanitizers # ============================================================================== @@ -279,6 +282,37 @@ if(BUILD_VIZ AND BUILD_PYTHON_BINDINGS) include("${CMAKE_CURRENT_LIST_DIR}/Mujoco.cmake") endif() +# ============================================================================== +# SPDX license list data +# ============================================================================== +# Reference texts for license matching; this checkout builds and ships nothing. +# Pin by commit because upstream tags can move. +find_package(Git REQUIRED) + +set(SPDX_LICENSE_DATA_COMMIT "31ba1a50e5397e00a304dbadc76531740e89ee48") # v3.29.0 +set(SPDX_LICENSE_DATA_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/license-list-data-src") + +message(STATUS "Fetching SPDX license list data from GitHub...") +FetchContent_Declare( + license_list_data + SOURCE_DIR "${SPDX_LICENSE_DATA_SOURCE_DIR}" + DOWNLOAD_COMMAND + "${CMAKE_COMMAND}" + "-DGIT_EXECUTABLE=${GIT_EXECUTABLE}" + "-DSPDX_REPOSITORY=https://github.com/spdx/license-list-data.git" + "-DSPDX_COMMIT=${SPDX_LICENSE_DATA_COMMIT}" + "-DSPDX_SOURCE_DIR=${SPDX_LICENSE_DATA_SOURCE_DIR}" + "-DSPDX_SPARSE_DIR=json" + -P "${CMAKE_CURRENT_LIST_DIR}/fetch_license_list_data.cmake" +) +FetchContent_MakeAvailable(license_list_data) + +foreach(_spdx_index licenses.json exceptions.json) + if(NOT EXISTS "${SPDX_LICENSE_DATA_SOURCE_DIR}/json/${_spdx_index}") + message(FATAL_ERROR "SPDX license list data is missing json/${_spdx_index} after fetch") + endif() +endforeach() + # ============================================================================== # Chromium physical key table (vendored source, BSD-3-Clause; see chromium/README.md) # ============================================================================== diff --git a/deps/third_party/fetch_license_list_data.cmake b/deps/third_party/fetch_license_list_data.cmake new file mode 100644 index 0000000000..e9bdcd1e3a --- /dev/null +++ b/deps/third_party/fetch_license_list_data.cmake @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Fetch only the JSON corpus used by the SBOM collector. + +foreach(_required GIT_EXECUTABLE SPDX_REPOSITORY SPDX_COMMIT SPDX_SOURCE_DIR SPDX_SPARSE_DIR) + if(NOT DEFINED ${_required}) + message(FATAL_ERROR "${_required} is required") + endif() +endforeach() + +if(EXISTS "${SPDX_SOURCE_DIR}/.git") + execute_process( + COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + OUTPUT_VARIABLE _current_commit + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _rev_parse_result + ) + if(_rev_parse_result EQUAL 0 AND _current_commit STREQUAL SPDX_COMMIT) + # The download command can change the sparse path without changing the commit. + # Cone mode requires a directory, not a file path. + execute_process( + COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "${SPDX_SPARSE_DIR}" + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY + ) + return() + endif() +endif() + +file(REMOVE_RECURSE "${SPDX_SOURCE_DIR}") +execute_process( + COMMAND "${GIT_EXECUTABLE}" clone --quiet --filter=tree:0 --no-checkout + "${SPDX_REPOSITORY}" "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "${SPDX_SPARSE_DIR}" + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND "${GIT_EXECUTABLE}" checkout --quiet --detach "${SPDX_COMMIT}" + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY +) diff --git a/deps/third_party/openxr_extensions/.clang-format b/deps/third_party/openxr_extensions/.clang-format new file mode 100644 index 0000000000..9d159247d5 --- /dev/null +++ b/deps/third_party/openxr_extensions/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: false diff --git a/deps/third_party/openxr_extensions/CMakeLists.txt b/deps/third_party/openxr_extensions/CMakeLists.txt new file mode 100644 index 0000000000..45376d71c8 --- /dev/null +++ b/deps/third_party/openxr_extensions/CMakeLists.txt @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Keep upstream extension headers separate from NVIDIA headers for license attribution. + +add_library(teleop_openxr_extensions_upstream INTERFACE) +add_library(Teleop::openxr_extensions_upstream ALIAS teleop_openxr_extensions_upstream) + +target_include_directories(teleop_openxr_extensions_upstream INTERFACE + $ +) diff --git a/deps/cloudxr/openxr_extensions/XR_MNDX_xdev_space.h b/deps/third_party/openxr_extensions/XR_MNDX_xdev_space.h similarity index 100% rename from deps/cloudxr/openxr_extensions/XR_MNDX_xdev_space.h rename to deps/third_party/openxr_extensions/XR_MNDX_xdev_space.h diff --git a/deps/cloudxr/openxr_extensions/openxr_extension_helpers.h b/deps/third_party/openxr_extensions/openxr_extension_helpers.h similarity index 100% rename from deps/cloudxr/openxr_extensions/openxr_extension_helpers.h rename to deps/third_party/openxr_extensions/openxr_extension_helpers.h diff --git a/docs/source/development/build_from_source/index.rst b/docs/source/development/build_from_source/index.rst index f62565738a..c3202c792b 100644 --- a/docs/source/development/build_from_source/index.rst +++ b/docs/source/development/build_from_source/index.rst @@ -20,7 +20,8 @@ Prerequisites ------------- - **CMake** 3.24 or higher (Ubuntu 22.04's apt ``cmake`` is 3.22 — install a newer one - from `Kitware's APT repository `_ or with ``pip install cmake``) + from `Kitware's APT repository `_ or with ``pip install cmake``). + A Release wheel with its contents SBOM needs 3.27, which added ``cmake_file_api()``. - **C++20** compatible compiler - **Python** 3.11, 3.12, or 3.13 (default 3.11; see ``ISAAC_TELEOP_PYTHON_VERSION`` in root ``CMakeLists.txt``) - **uv** for Python dependency management and managed Python diff --git a/docs/source/development/license.rst b/docs/source/development/license.rst index 8d6adf63b0..28a33afe7e 100644 --- a/docs/source/development/license.rst +++ b/docs/source/development/license.rst @@ -28,6 +28,7 @@ Pre-commit automatically checks and updates license headers with `REUSE `_. It selects ``Apache-2.0`` by default and ``LicenseRef-OpenMDW-1.1`` for EgoPost, preserving existing attribution and explicit third-party exceptions. Review and stage any automatic edits before -committing. +committing. Checked-in third-party files record upstream's terms; see *What stops +the Release build* in :code-file:`scripts/sbom/README.md`. You can also run ``reuse spdx -o project.spdx`` to generate an SPDX report for the project. diff --git a/docs/source/index.rst b/docs/source/index.rst index b390ebb360..91e26a3885 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -98,6 +98,7 @@ Table of Contents references/requirements references/build + references/sbom references/generated_trackers references/retargeting/index references/device_provider_monitoring diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst new file mode 100644 index 0000000000..6b1af9cbf4 --- /dev/null +++ b/docs/source/references/sbom.rst @@ -0,0 +1,100 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Wheel contents SBOM and license evidence +======================================== + +Every published ``isaaccapture`` wheel lists what it redistributes and carries the license +text behind each obligation. You can check both offline, without installing the wheel. The +Release build refuses to ship a wheel with a gap in either. + +To check a wheel, see `Verifying a wheel`_. To learn how the evidence is worked out, or why +a build stopped, see the collector's developer README, :code-file:`scripts/sbom/README.md`. + +What a wheel carries +-------------------- + +.. code-block:: text + + isaaccapture---.whl + └── isaaccapture-.dist-info/ + ├── METADATA # License-Expression + one License-File per text below + ├── licenses/ + │ ├── LICENSE.md # this project's own license + │ ├── LICENSES/.txt # this project's license texts + │ ├── THIRD-PARTY-NOTICES.md # each component, its license, and where the text is + │ └── third-party// # the component's own license file, or a reference text + └── sboms/ + └── .spdx.json # SPDX 2.3 contents inventory + +- **The SBOM** lists every file in the wheel but ``RECORD`` and itself: where it came from, + under what license, and its digest. It describes the wheel's own contents only. Build + tools, dependencies this wheel does not carry (the build evidence records them with + ``"in_this_wheel": false``), + install-time dependencies (``Requires-Dist`` stays authoritative; the SBOM records the + requirement, never a license for it), source distributions and ``pip install .``, + containers and the web client are not covered. +- **Each third-party component** ships its own license file or, for material checked into + this repository, the reference text for its declared license beside its copyright + notice. +- ``sboms/`` may also hold a CycloneDX document ``auditwheel`` writes for the libraries it + bundled. + +Evidence published with a release +--------------------------------- + +Each tagged GitHub release carries one archive, ``isaaccapture-sbom-.zip``. Unpacked, +its ``isaaccapture-sbom-/`` directory holds, for every wheel: + +``.spdx.json`` + An exact copy of the document embedded in the wheel. + +``.licenses.md`` / ``.licenses.json`` + Each redistributed component, the license concluded for it and the file it was read + from, how it reaches the wheel, and what the installation environment supplies instead. + It covers this wheel only; what the build used but did not ship is in the build evidence. + +``.build-evidence.json`` + Resolved commits, declared constraints, archive digests and build options, from the + build machine, and every component the build resolved, whether or not it reached the + wheel. + +and, once per release: + +``isaaccapture-sbom-manifest.json`` + Each wheel's filename and SHA-256 and its evidence files' digests, recorded after the + last change to the wheel, so a wheel touched since no longer matches. + +Verifying a wheel +----------------- + +You need ``uv`` and a clone of this repository: + +.. code-block:: bash + + git clone --depth 1 https://github.com/NVIDIA/IsaacCapture + uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check \ + --wheel isaaccapture---.whl + +To check a whole release, unpack its archive and put every wheel its manifest lists, for +each Python version and architecture, in a directory of their own. A wheel or evidence file +the manifest does not name fails the check: + +.. code-block:: bash + + unzip isaaccapture-sbom-.zip + uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check-set \ + --manifest isaaccapture-sbom-/isaaccapture-sbom-manifest.json \ + --wheel-dir wheels --evidence-dir isaaccapture-sbom- + +``check`` confirms the SBOM accounts for every file, its digests match the wheel and its +``RECORD``, each license text it cites is packaged where it says, and every +``LicenseRef-`` carries its text; add ``--build-evidence`` or ``--manifest`` to check +against the release evidence too. + +The SBOM describes a wheel; it is not a signature. Integrity is the wheel's ``RECORD``, +which installers check. These releases carry no +`PEP 740 `_ attestations. + +To read the inventory, open ``.dist-info/sboms/*.spdx.json`` with any SPDX 2.3 tool. A +file's ``SPDXID`` changes only when its path does, so two releases can be diffed. diff --git a/pyproject.toml b/pyproject.toml index 767461704b..fca4fb6dde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,15 @@ description = "Isaac Teleop" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" -license-files = ["LICENSE.md"] +# Shared with the setuptools wheel via src/core/python/CMakeLists.txt. +# Avoid a LICENSES/ glob: the pool includes terms for components no wheel carries. +license-files = [ + "LICENSE.md", + "LICENSES/Apache-2.0.txt", + "LICENSES/BSD-3-Clause.txt", + "LICENSES/BSL-1.0.txt", + "LICENSES/MIT.txt", +] authors = [ { name = "NVIDIA" }, ] @@ -28,6 +36,9 @@ dependencies = [ "pyyaml>=6.0.3", ] +[project.urls] +Homepage = "https://github.com/NVIDIA/IsaacCapture" + [project.optional-dependencies] dev = [ "pytest>=7.0.0", @@ -105,11 +116,9 @@ isaacteleop = { path = "src/compat" } [tool.scikit-build] # EgoPost is hosted here but is not part of the Isaac Capture distribution. sdist.exclude = ["src/python/isaaccapture_egopost/**", "LICENSES/LicenseRef-OpenMDW-1.1.txt"] -# Floor matches CMakeLists' cmake_minimum_required (3.24, for `cmake --fresh`). -# Ubuntu 22.04's apt CMake is 3.22 and no longer satisfies it, so scikit-build-core -# fetches CMake from PyPI there; the <4 cap keeps that fetch on the 3.x line rather -# than silently jumping to 4.x. -cmake.version = ">=3.24,<4" +# SBOM file-API queries require CMake 3.27. scikit-build-core fetches a newer +# CMake when the system copy is too old; keep that fetch on the 3.x line. +cmake.version = ">=3.27,<4" cmake.build-type = "Release" # Keep the CMake build tree under build-wheel// (e.g. diff --git a/scripts/check_licenses.py b/scripts/check_licenses.py index 445342fbac..ea5bf577ef 100644 --- a/scripts/check_licenses.py +++ b/scripts/check_licenses.py @@ -17,8 +17,10 @@ EGOPOST_LICENSE = "LicenseRef-OpenMDW-1.1" DEFAULT_LICENSE = "Apache-2.0" COPYRIGHT_HOLDER = "NVIDIA CORPORATION & AFFILIATES. All rights reserved." +# Same rule as check-copyright-year's `files:`. SOURCE_FILES = re.compile( - r"\.(py|js|ts|cpp|c|h|hpp|cxx|hxx|cmake|md|yaml|yml|sh|fbs|toml|html)$" + r"\.(py|pyi|js|ts|cpp|cc|c|h|hpp|cxx|hxx|inc|cu|cuh|frag|vert|cmake|md|yaml|yml" + r"|sh|fbs|toml|html)$" r"|CMakeLists\.txt$|Dockerfile(\.[A-Za-z0-9_-]+)?$" ) diff --git a/scripts/download_cloudxr_runtime_sdk.sh b/scripts/download_cloudxr_runtime_sdk.sh index caa643308f..0b1b7f5a80 100755 --- a/scripts/download_cloudxr_runtime_sdk.sh +++ b/scripts/download_cloudxr_runtime_sdk.sh @@ -112,6 +112,8 @@ download_ngc_file() { rm -f "$out_path" return 1 fi + # Preserve the download URL as SBOM provenance. + printf '%s\n' "$url" > "${out_path}.source" } # Try each remote name in turn; the first that resolves wins. diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md new file mode 100644 index 0000000000..fa98988f76 --- /dev/null +++ b/scripts/sbom/README.md @@ -0,0 +1,390 @@ + + +# Wheel SBOM collector + +`isaaccapture-sbom` derives a wheel's contents SBOM and license evidence from the build +that produced it, packages both into the wheel, and checks the result. What a wheel +carries and how to verify one are on the user page, +[Wheel contents SBOM and license evidence](https://nvidia.github.io/IsaacCapture/main/references/sbom.html) +(source: [`docs/source/references/sbom.rst`](../../docs/source/references/sbom.rst)). +This file is how it works. + +Where to look: running or testing it, [Running it](#running-it); a member the build +cannot explain, [How the inventory is worked out](#how-the-inventory-is-worked-out); a +wrong or missing license, [How licenses are determined](#how-licenses-are-determined); a +library from the build machine or static code, +[Libraries from the build machine](#libraries-from-the-build-machine) and +[Static code on the link line](#static-code-on-the-link-line); bumping the SPDX list, +[Reference texts](#reference-texts); what stops the build and how to fix it, +[What stops the Release build](#what-stops-the-release-build). + +## Running it + +The collector is a project of its own; its dependencies are pinned in +[`pyproject.toml`](pyproject.toml). `reuse` reads files through libmagic, so `build` +needs it (`libmagic1` on Ubuntu); `check` does not. + +Build and install a Release wheel as in +[Building from source](../../docs/source/development/build_from_source/index.rst), then: + +```bash +# `build` reads the SPDX reference texts configuring fetched into +# build/_deps/license-list-data-src/json, unless --license-data says otherwise. +uv run --project scripts/sbom isaaccapture-sbom build \ + --build-dir build --wheel install/wheels/.whl --out-dir sbom +uv run --project scripts/sbom isaaccapture-sbom check --wheel install/wheels/.whl \ + --manifest sbom/manifest.json --build-evidence sbom/.build-evidence.json +``` + +- `build` rewrites the wheel in place, so run it on a repaired wheel and do not touch + the wheel afterwards. Set `SOURCE_DATE_EPOCH` to make the result reproducible. +- `auditwheel` writes its own CycloneDX document into `.dist-info/sboms/`; the collector + finds its document there by name and leaves the rest alone. +- `--project` makes `uv` resolve the collector's pins and run it from anywhere. It + inventories the checkout you are in, or `--repo-root`. + +### What `check` verifies + +The SBOM's per-file digests show that the description belongs to the wheel in front of +you and agrees with its `RECORD`; they are not a second root of trust. The document +lists every member except `RECORD` and itself, since neither can carry its own digest; +`check` reads both against the archive directly. With just a wheel it confirms that + +- the SBOM covers every file in the wheel except `RECORD` and the SBOM itself, +- each digest the SBOM records matches the bytes present, and agrees with `RECORD`, +- `RECORD` itself agrees with the archive, so the wheel is still installable, +- the packaged notices are there, and every `LicenseRef-` has its text embedded. + +`--build-evidence` also checks that the build accounts for every member; `--manifest` +checks the wheel against the digest bound at publication and the sidecars beside it. +`check-set` runs the whole thing over every wheel in a release, and fails a wheel in +`--wheel-dir`, or a file in `--evidence-dir`, that the manifest does not name. +`merge-manifests` refuses a wheel name whose builds produced different bytes, SBOMs or +license reports. Only build evidence may differ; the first manifest's in path order (in +CI, artifact-name order) is bound, and `--evidence-out` copies exactly the bound files. +Any command reading a manifest refuses one that lists no wheel. CI validates each +published document with `spdx-tools` before release. + +### Tests + +From the repository root: + +```bash +uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests +``` + +They read the file API codemodel, and the SPDX list where `build` finds it without +`--license-data`, from the build tree `ISAACCAPTURE_SBOM_BUILD_DIR` names, else `build/`. +An unnamed `build/` that lacks them skips the tests; a named one fails them. A configured +tree is enough, so CI runs this command in a job of its own, beside the product build, and +publishes nothing unless it passes. + +## How the inventory is worked out + +Every inventory entry is derived from the build, from these sources: + +**`build/_deps/*-src`** — FetchContent trees. + +- A tree that is a git repository of its own is asked for its commit and remote, and + from the remote the supplier, homepage and purl. Git is asked of no other tree, since + it would answer for whichever checkout encloses the build directory, or for whatever + `GIT_DIR` names; `GIT_DIR` is therefore cleared. +- Any other tree is identified by its declaration — `GIT_REPOSITORY` and `GIT_TAG`, or + `URL` and `URL_HASH` — from the stamps ExternalProject writes beside the populate + sub-build, or under `CMakeFiles/fc-stamp` once policy `CMP0168` is `NEW`, and from the + clone script the git stamp names. A stamp outlives a change of method, so where both + are present the tree decides: a git clone is a checkout of its own. +- The version is the declared tag, or the ref a GitHub URL names, verbatim, so a tag + gives one version whether declared by tag or as a GitHub archive. Any other archive's + version is read from its name, and its supplier is not asserted: the host serving a + download is not who supplies it. +- A download location is stated only for an `http`, `https`, `ftp` or `ftps` archive + URL, or a git remote reached over `http`, `https`, `ssh` or `git`, and never with its + credentials, query or fragment. + +**vcpkg's install records** — the `vcpkg/info` lists under `VCPKG_INSTALLED_DIR` (by +default the build tree's `vcpkg_installed`): which port installed each file. + +**CMake's file API codemodel** — the build graph as CMake describes it: which sources +were compiled into which binary, which targets it depends on, what its link line names +and, through include paths, which header-only dependencies it was compiled against. +Followed transitively, so a library linked into a library linked into the wheel is still +reported. The top-level `CMakeLists.txt` requests it with `cmake_file_api()`; the +project builds without it, but a wheel with an SBOM needs it, hence the CMake version in +[Prerequisites](../../docs/source/development/build_from_source/index.rst#prerequisites). +Do not read the generator's own output instead: Ninja writes no `link.txt`. + +**The compiler's dependency output** — which headers each object read: the per-object +depfiles and `compiler_depend.make` under the Makefile generators, `ninja -t deps` under +Ninja. A Ninja build is collected where `ninja` is on `PATH`; the collector says so once +when it is not. A target whose records miss a source it compiles falls back to its +listed sources and include directories. + +**Archives in the repository** — every member of every tarball is hashed; a wheel member +that matches one is recorded as unpacked from that archive. + +**The working tree and the build tree** — what remains is matched against the project's +own sources and its generated output. + +**The build machine** — for a shared library none of the above explains; see +[Libraries from the build machine](#libraries-from-the-build-machine). + +A member none of these explains stops the build +([below](#what-stops-the-release-build)). Adding a dependency needs no change here: +declare it as usual — a `FetchContent_Declare`, a tarball under `deps/` — and it appears +in the next wheel's inventory. + +Post-processing is recorded as such. `patchelf`, `auditwheel`'s RPATH rewrite and the +build's MJCF asset stripping leave bytes that no longer hash to their origin, so those +members are reported as *derived from* a named input rather than as an exact copy, with +the input's digest wherever the input was hashed: an archive member is, a file matched +only by its path is not. + +A file's `SPDXID` is its path in the distribution, reduced to the characters an SPDX +identifier allows, plus a short digest of the unreduced path, with the versioned +`.dist-info` and `.data` directories counted as fixed. + +### Libraries from the build machine + +A shared library none of the build's indexes explains did not come from this build, so +the build machine is asked: resolve its SONAME and take its license from the supplying +package's copyright file, or from a license file beside the library for toolchains that +install outside the package manager. A library the machine cannot find is reported as +unexplained, not attributed. + +- The `pkg:deb` purl is namespaced by the archive publishing the installed version: the + `Origin` apt records for it (`ubuntu` for Ubuntu's, `nvidia` for NVIDIA's CUDA + repository). Where no archive lists the installed version — superseded, or a local + rebuild — the archives listing its other versions decide, and the build evidence says + so. +- Where no vendor can be named — no archive lists the package (a local `.deb`); the + deciding archives disagree, ignoring case, or name no `Origin`; or the `Origin` is a + URL or host name, which a purl namespace cannot be — the package is + `pkg:generic/@`. The build machine's own distribution is not + consulted. +- No repair tool's directory name is consulted, so a library the build produced is + attributed to the build wherever it ends up. +- Copyright notices are read only from the `Copyright:` fields of a machine-readable + Debian copyright file, leaving out stanzas that cover only the Debian packaging. Any + other text (an EULA, a free-form copyright file) quotes the notices of what it + bundles, so none is asserted; the packaged text still carries them all. + +### Static code on the link line + +Static code leaves no `DT_NEEDED` entry, so the link line is read: every fragment CMake +records for it, `-Wl,` and `-Xlinker` words included, with the values of options known +to take one set aside. + +- A path is judged by its content, not its name: an archive or object file is static + code, a shared library is left to `DT_NEEDED`, and a GNU `ld` script (`libc.so`, + naming `libc_nonshared.a`) is followed to the files and `-l` names its `INPUT` and + `GROUP` commands give, a relative name looked up beside the script first, as `ld` + does. +- A `-l` name is resolved as `ld` resolves it: the line's `-L` directories, then the + directories the compiler hands the linker (CMake's toolchains reply, or the compiler's + own `-print-search-dirs` for a language that reply does not cover), then + `ld --verbose`'s built-in `SEARCH_DIR` list, taking a shared library before an archive + unless `-Bstatic` is in force. A leading `=` in either names the sysroot (the target's + `CMAKE_SYSROOT`, else the compiler's), as does an absolute name in a script inside it. + `--push-state` and `--pop-state` are honoured, and the driver's `-static` governs the + whole line. +- The lines read are the member's own and those of shared libraries it depends on. An + executable or module it merely waits for, such as a code generator, links nothing into + it, nor does anything reached only through one. + +What is found is attributed: + +- an archive or object a target of this build produced is that target's contribution; +- one inside a fetched component's tree is that component's; +- one vcpkg installed belongs to the port whose install record + (`vcpkg/info/__.list`) names it. The port is identified by the + upstream source its `share//vcpkg.spdx.json` says it built — a `pkg:github` purl + where its one source is on GitHub, `pkg:generic` with no supplier where it is not or + where several sources name no single upstream, since purl has no vcpkg type — at the + upstream version, without vcpkg's `#` recipe revision. Its license is read from + `share//copyright`; +- one tracked in this repository belongs to the holder its REUSE info names, and is this + project's own where that is this project; +- one from the build machine is asked about as a shared library is: the owning package + and its copyright file or, for an archive under a CUDA toolkit, the license file the + toolkit ships; +- an archive with no members links no code — glibc 2.34 left `librt.a`, `libpthread.a` + and `libdl.a` as such stubs — and is recorded in the build evidence rather than as a + component. + +Each component found this way is recorded with `STATIC_LINK`; anything else stops the +build ([below](#what-stops-the-release-build)). + +Not inventoried: what the compiler driver adds without naming it on the link line +(`libgcc` and the C runtime's start files among it), and a vcpkg port's headers read +through an include directory other ports also install into. Every port shares one +include root, so an include directory credits a port only where each file vcpkg +installed under it is that port's; headers read from the shared root are credited only +through a linked archive of the same port. + +## How licenses are determined + +Licenses are never inferred from this project's license, and never read from a list +maintained by hand. + +### Where terms are read from + +- **Files in this repository** — from their REUSE info + ([License headers](../../docs/source/development/license.rst), + [REUSE specification](https://reuse.software/spec/)). The pre-commit hook lints it for + source files; the collector reads it for every file REUSE covers, whatever its type, + and for every source here the build compiled or included. A file identified as a third + party's (below) is held to the per-file gate even where the hook does not lint it; a + file naming no holder at all counts as this project's own. +- **Fetched components** — from the license and notice files the component ships: + `LICENSE`, `COPYING`, `BSD-LICENSE`, a `LICENSES/` pool or, where none of those at its + root is a grant, the license section of its README. Each text is matched against the + SPDX reference texts. Its source headers, `.license` sidecars and `REUSE.toml` are not + read. +- **Archives in the repository** — only the license files at the archive's root. + +### Reference texts + +[`deps/third_party/CMakeLists.txt`](../../deps/third_party/CMakeLists.txt) fetches +`spdx/license-list-data` at a pinned commit into `_deps/license-list-data-src`, like any +other dependency, so the build evidence records it with `"in_this_wheel": false`. Each +report records the commit its identifications were made against. + +Every current identifier is available to match against. Naming one in the document is a +separate question: the document states the license list it was built against, and +`license-expression` — which consumers validate with and `pyproject.toml` pins — ships +its own, older copy. An id the fetched list knows and that grammar does not cannot be +published as an id, so the text it was read from travels as a `LicenseRef-`. A REUSE +declaration naming one travels whole, since an `OR` or a `WITH` cannot be kept in part. +The component is listed under *License evidence gaps* with that reason. + +To move to a newer list, change `SPDX_LICENSE_DATA_COMMIT` in +`deps/third_party/CMakeLists.txt`, with its release comment, and the entry in +[`deps/README.md`](../../deps/README.md#spdx-license-list-data). There is no separate +list of identifiers to extend. + +### What ships + +- **A fetched component** ships its own license file verbatim, copyright holders and + all. Do not substitute a reference text: it would drop the notice — for MIT and the + BSD licenses the difference is the line naming who licensed it to you — and replace + the component's actual grant with our reading of it. +- **Third-party material checked in here** is identified by the copyright holder it is + declared under: a `.license` sidecar or `REUSE.toml` annotation where REUSE applies + one, and the file's own first copyright line unless REUSE applies those in its place. + Its terms are the identifiers REUSE gives each of its files, under REUSE's own + precedence. The reference text for each license it names ships, labelled in the + document as a substitution, since a reader cannot tell a template from an author's + notice; the file's copyright notice travels with it. The expression keeps its + structure, so a choice stays a choice; an exception's text ships too where the + expression travels as a `LicenseRef-`. +- **Terms stated for part of a component** — a vendored asset set or bundled library + with its own `LICENSE` and holder — are read and packaged only where something under + that directory reached the wheel: a file copied or derived from it (including a copy + the build made into the component's own build tree), a source compiled from it, or a + header the compiler's dependency output says it read. Where a target compiles + something its dependency output does not account for, the sources it lists count + instead, and so does any include directory above or below that directory. A linked + archive of the component keeps all of them, since what it holds is unknown. A + `.license` sidecar covers only the file it annotates. The build evidence says which of + those kept each text. +- **A bundled library compiled in** ships its texts and holders, each listed as terms + stated for part of the component; the component's concluded license stays its own + grant. +- **The distribution's own declared license** is an obligation like any other. Where the + wheel ships no text for it, the reference text for each identifier is packaged from + the pinned list and declared under PEP 639. +- **The notices file** opens with the distribution's own license and copyright holder, + read from the packaging metadata where no file it ships states one. Otherwise a wheel + shipping only metadata would carry a template naming nobody. +- **Never a file from `LICENSES/`** as a third party's terms: that pool licenses *this* + project and carries its notices. Substituted texts come from the pinned list. Under + `.dist-info/licenses/LICENSES/` a wheel carries the project's own pool files where it + has them, and otherwise the reference text for the license it declares. + +`licenseDeclared` on the wheel is what the distribution says of itself; +`licenseConcluded` is what the document found in it. A wheel carrying a component whose +terms the build could not name states that component's `LicenseRef-` alongside the +declaration. Where nothing at all is concluded for a component, the wheel's conclusion +is `NOASSERTION`. + +### Matching + +A match needs a canonical license text to be almost entirely present *and* to account +for a real share of the file, so a license reproduced in an appendix of a long agreement +is not that agreement's terms. The second threshold is the looser: it bounds padding +around a text, so a short file quoting a license in full still matches. A file carrying +two *distinct* licenses is reported as carrying both; where one is a near-duplicate of +the other, only the match survives. A disjunctive grant is not expressible. + +Every grant a component ships is accounted for in its conclusion: + +- A text that matches nothing travels as the component's `LicenseRef-`, AND-ed with what + its other texts are identified as: an MIT `LICENSE` beside an unmatched EULA concludes + `LicenseRef- AND MIT`. +- A text matching identifiers that share one reference text names the one the others + extend by a suffix — `MPL-2.0` beside `MPL-2.0-no-copyleft-exception`, `OFL-1.1` + beside its `-RFN` variants — SPDX's name for the unmodified text. Where there is none, + as for `GPL-3.0-only` and `GPL-3.0-or-later`, the choice is the licensor's, stated in + a notice rather than the text, so the text is carried too, and the evidence lists it + as matching either, not both. +- A text its identifiers account for less than four fifths of, copyright notices aside, + is carried too, beside the identifiers it names: the rest may be terms no identifier + names, as in fmt's MIT with its exception paragraph. + +The `LicenseRef-` carries exactly the texts carried, and those components are listed +under *License evidence gaps*. So is a component whose only texts are a REUSE +`LICENSES/` pool: it ships them, which meets the obligation, but a pool says what each +file is under, not what the component is, so its license is `NOASSERTION`. So is a +component whose grants name nothing the grammar accepts. + +## What stops the Release build + +The build stops rather than ship a gap, and its error names the files or components +involved. None of these is fixed by excluding a file or recording an assumed license. +Other errors — an unconfigured build tree, no file API codemodel, an unreadable +archive — are not gaps, and name their own cause. + +**A wheel member the build cannot explain** — none of the +[inventory's sources](#how-the-inventory-is-worked-out) accounts for it, including a +shared library the build machine cannot resolve. It reached the wheel by a route the +collector cannot see: teach the collector that route rather than excluding the file. + +**Static code on a link line the collector cannot trace** — a name that resolves to +nothing, a fragment that does not parse, a path that is none of the +[kinds above](#static-code-on-the-link-line) (which is also what an unknown option's +value looks like), or an archive inside the build tree that no target produced and no +install record names, which is where ExternalProject leaves its output. Link something +the collector can trace — a target of this build, a fetched dependency, a system package +or a known toolkit — or teach it the route that input took. + +**A redistributed component with no license text** — packaging the text is the +obligation. For a fetched component, that is none of the texts +[Where terms are read from](#where-terms-are-read-from) lists; a license file in a +subdirectory covers only that part. For any other component it is the source named above +for its kind: an archive's root license files, a vcpkg port's `share//copyright`, +a build-machine library's package copyright file or the license file beside it. Obtain +the terms; the component's REUSE info and source headers do not count. + +**A declared license with no text shipped** — the wheel's `License-Expression` does not +parse, or names an identifier the pinned list cannot supply a text for, such as a +`LicenseRef-`, and the wheel packages none. A text counts only as `.txt` +outside `third-party/`: add it to `license-files` in `pyproject.toml` under that name, or +drop the declaration. + +**A checked-in third-party file whose REUSE info fails the per-file gate** — the file's +REUSE info names no license; names one with no current SPDX reference text (a +`LicenseRef-` among them, even beside one that has a text); states an expression that +does not parse; or names an id that is neither SPDX-listed nor a `LicenseRef-` or +`DocumentRef-`. An exception needs only to be listed as an exception. The build names +each such file and why; another file of the same holder does not lend it its terms. + +- For real upstream terms, record them as REUSE info naming current SPDX-listed + identifiers ([License headers](../../docs/source/development/license.rst), + [REUSE specification](https://reuse.software/spec/)). +- A tag that is not a real declaration — quoted in a string, or mentioned in prose — + goes between `REUSE-IgnoreStart` and `REUSE-IgnoreEnd`, or is set aside by a + `.license` sidecar or a `REUSE.toml` annotation with `precedence = "override"`. diff --git a/scripts/sbom/isaaccapture_sbom/__init__.py b/scripts/sbom/isaaccapture_sbom/__init__.py new file mode 100644 index 0000000000..11895f10c7 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/__init__.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Wheel contents SBOM and license evidence collector.""" + +TOOL_NAME = "isaaccapture-sbom" +# Bump on any change that alters the shape of an emitted document or report. +TOOL_VERSION = "1.0.0" +SPDX_VERSION = "SPDX-2.3" + + +def stamped_now(): + """Return the publication time, honoring SOURCE_DATE_EPOCH.""" + import os + from datetime import datetime, timezone + + epoch = os.environ.get("SOURCE_DATE_EPOCH") + if epoch and epoch.isdigit(): + return datetime.fromtimestamp(int(epoch), timezone.utc) + return datetime.now(timezone.utc) diff --git a/scripts/sbom/isaaccapture_sbom/build.py b/scripts/sbom/isaaccapture_sbom/build.py new file mode 100644 index 0000000000..64ef28d2b5 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/build.py @@ -0,0 +1,538 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Package a wheel's contents SBOM and license evidence from its build tree.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import tempfile +from dataclasses import replace +from pathlib import Path + +from . import TOOL_NAME, TOOL_VERSION, stamped_now +from . import document as document_module +from . import evidence as evidence_module +from . import licensing +from . import inventory as inventory_module +from . import report as report_module +from . import validate as validate_module +from . import wheelfile +from packaging.utils import canonicalize_name + +from . import discovery as discovery_module +from .discovery import ( + PREBUILT, + Reach, + covers, + project_copyright, + project_supplier, + reach_rank, + sha256_file, +) +from .document import NOASSERTION +from .inventory import Attribution +from .wheelfile import Entry + +# Only the document namespace is fixed; distribution identity comes from metadata. +PROJECT = {"document_namespace": "https://github.com/NVIDIA/IsaacCapture/spdx"} + + +def _project_facts(repo_root: Path, metadata: dict) -> dict: + urls = metadata.get("project_urls") or {} + homepage = next( + ( + value + for key, value in urls.items() + if key.lower() in ("homepage", "home-page", "source", "repository") + ), + metadata.get("home_page") or NOASSERTION, + ) + # A sibling distribution may have a different author than the root project. + author = (metadata.get("author") or "").strip() + return { + **PROJECT, + "supplier": f"Organization: {author}" + if author + else project_supplier(repo_root), + # The legacy License field is free text, not an SPDX expression. + "license": metadata.get("license_expression") or NOASSERTION, + "homepage": homepage, + # Metadata-only wheels have no source file carrying the project notice. + "copyright": project_copyright(repo_root), + } + + +class BuildError(Exception): + """The wheel cannot be published with the evidence available.""" + + +_REMEDIES = "\nSee 'What stops the Release build' in scripts/sbom/README.md." + + +def _entry(name: str, data: bytes) -> Entry: + return Entry( + name=name, + size=len(data), + sha256=hashlib.sha256(data).hexdigest(), + sha1=hashlib.sha1(data).hexdigest(), # noqa: S324 - SPDX 2.3 requires SHA1 + ) + + +def default_license_data(build_dir: Path) -> Path: + """Where deps/third_party materializes the SPDX license list for this build.""" + return build_dir / "_deps" / "license-list-data-src" / "json" + + +def _with_reached_nested(inventory, components: dict) -> dict: + """Keep nested terms only for files or compiled code that reached this wheel.""" + # Deduplicate reach shared by several members, retaining the first member as evidence. + reached: dict[str, dict[tuple[str, str], Reach]] = {} + # A prebuilt archive conservatively retains all nested terms of its component. + prebuilt: dict[str, str] = {} + for member, attribution in inventory.attributions.items(): + for key, paths in attribution.read.items(): + into = reached.setdefault(key, {}) + for path, how in paths.items(): + known = into.get((path, how)) + if known is None or member < known.member: + into[(path, how)] = Reach(how, path, member) + for key, roles in attribution.components.items(): + if PREBUILT in roles: + prebuilt[key] = min(member, prebuilt.get(key, member)) + + def strength(hit: Reach): + return reach_rank(hit.how), hit.path, hit.member + + updated = {} + for key, component in components.items(): + if not any(item.kind == "nested" for item in component.evidence): + continue + evidence = [] + for item in component.evidence: + if item.kind != "nested": + evidence.append(item) + continue + hits = [ + hit + for hit in reached.get(key, {}).values() + if covers(hit.path, hit.how, item.scope) + ] + if key in prebuilt: + whole = component.root.name if component.root else key + hits.append(Reach(PREBUILT, whole, prebuilt[key])) + if hits: + evidence.append(replace(item, reached_by=min(hits, key=strength))) + updated[key] = replace( + component, + evidence=tuple(evidence), + copyright_text=discovery_module.notice_from(evidence), + ) + return updated + + +def _declared_without_text(declared: str, shipped: set[str]) -> list[str]: + """Identifiers in the declared expression with no packaged text to match.""" + if declared == NOASSERTION: + return [] + try: + identifiers = licensing.identifiers_in(declared) + except licensing.ExpressionReadError: + # Malformed declarations must fail the missing-text gate. + return [declared] + return sorted( + item + for item in identifiers + # Exceptions modify a license; this gate requires texts only for license operands. + if not licensing.is_exception(item) and f"{item.lower()}.txt" not in shipped + ) + + +def _own_texts(shipped: set[str]) -> set[str]: + """Return paths relative to .dist-info/licenses/, excluding third-party terms. + + A component's text must not discharge the distribution's own obligation: removing + that component would also remove the distribution's license. + """ + return {item for item in shipped if not item.startswith("third-party/")} + + +def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: + """Supply reference texts missing from the distribution's declared licenses. + + Third-party terms must instead come from the components' own files. + """ + if declared == NOASSERTION: + return {} + names = {Path(item).name.lower() for item in _own_texts(shipped)} + wanted = {} + for item in licensing.canonical_evidence("this-distribution", {declared}): + identifier = item.identified + if identifier and f"{identifier.lower()}.txt" not in names: + wanted[identifier] = item.text + return wanted + + +def siblings_of(wheels: list[Path], repo_root: Path) -> dict[str, dict]: + """Read sibling wheel metadata before rendering, indexed by canonical name. + + Metadata supports cyclic requirements without cyclic document hashes. + """ + known: dict[str, dict] = {} + for wheel in wheels: + _, parsed = wheelfile.read_metadata(wheel, wheelfile.dist_info_of(wheel)) + facts = _project_facts(repo_root, parsed) + known[canonicalize_name(parsed["name"])] = { + "version": parsed.get("version") or NOASSERTION, + "supplier": facts["supplier"], + "license_declared": facts["license"], + } + return known + + +def build( + repo_root: Path, + build_dir: Path, + wheel_path: Path, + out_dir: Path, + license_data: Path | None = None, + system_resolver=None, + discovery: evidence_module.Discovery | None = None, + siblings: dict[str, dict] | None = None, +) -> dict: + """Package evidence into `wheel_path` in place and write the sidecars. + + `discovery` is the build tree read once. Reading it per wheel walks and + hashes the whole tree, and decompresses every fetched archive, again. + """ + licensing.load_corpus(license_data or default_license_data(build_dir)) + if discovery is None: + discovery = evidence_module.discover(repo_root, build_dir) + wheel = wheelfile.scan(wheel_path, analyze_elf=True) + sbom_name = wheelfile.sbom_member(wheel.dist_info, wheel_path.name) + if any(item.name == sbom_name for item in wheel.entries): + raise BuildError( + f"{wheel_path.name} already carries {sbom_name}. Run this on a freshly " + "repaired wheel; rewriting one twice would duplicate its members." + ) + metadata_raw, wheel_metadata = wheelfile.read_metadata(wheel_path, wheel.dist_info) + + resolver = inventory_module.Resolver(discovery, system_resolver) + inventory = inventory_module.build( + wheel, metadata_raw, resolver, discovery.staged_root + ) + + # Resolve nested terms and shared LicenseRefs before rendering any output, + # so notices, reports, and evidence use the same terms and identifiers. + settled = _with_reached_nested(inventory, inventory.components_present) + present = {**inventory.components_present, **settled} + renamed = licensing.shared_refs(present.values()) + for key, component in list(present.items()): + present[key] = replace( + component, + license_concluded=licensing.rename_refs( + component.license_concluded, renamed + ), + license_declared=licensing.rename_refs(component.license_declared, renamed), + ) + inventory.components_present.update(present) + resolver.components.update(present) + + if inventory.unattributed: + raise BuildError( + "these wheel members could not be traced to anything this build " + "produced or fetched:\n " + + "\n ".join(inventory.unattributed) + + "\nThey reached the wheel by a route the collector cannot see; teach " + "the collector that route rather than excluding the file." + _REMEDIES + ) + + if inventory.untraced: + raise BuildError( + "the link lines of wheel members name these, and nothing says whose " + "code they hold:\n " + + "\n ".join(inventory.untraced) + + "\nLink only what the collector can trace -- a target of this build, a " + "fetched dependency's tree, a system package or a known toolkit -- or " + "teach it the route this input took. ExternalProject installs into the " + "build tree by a route it does not know." + _REMEDIES + ) + + unlicensed = sorted( + f"{path} ({key}): {reason}" + for key, component in inventory.components_present.items() + for path, reason in component.unlicensed + ) + if unlicensed: + raise BuildError( + "the REUSE licence info of these checked-in third-party files fails " + "the per-file gate:\n " + + "\n ".join(unlicensed) + + "\nRecord upstream's terms as REUSE info: a header tag, a .license " + "sidecar or a REUSE.toml annotation. A stray tag is set aside instead." + + _REMEDIES + ) + + missing = [ + key + for key, component in inventory.components_present.items() + if not licensing.obligation_texts(component.evidence) + ] + if missing: + raise BuildError( + "no license text was found in the files of redistributed component(s): " + + ", ".join(missing) + + "\nThe component ships no LICENSE/COPYING file and states no grant in " + "its README; obtain the terms before publishing it." + _REMEDIES + ) + + # Compute the final member set before rendering its SBOM. + additions: dict[str, bytes] = {} + packaged_paths: list[str] = [] + for key, component in inventory.components_present.items(): + for item in component.evidence: + target = report_module.packaged_license_path(wheel.dist_info, key, item) + if target in additions: + continue + additions[target] = item.text.encode("utf-8") + packaged_paths.append( + report_module.declared_license_path(wheel.dist_info, target) + ) + + # Metadata-only wheels still need their own declared license texts. + declared = _project_facts(repo_root, wheel_metadata)["license"] + shipped = { + report_module.declared_license_path(wheel.dist_info, item.name) + for item in wheel.entries + if report_module.is_packaged_license(wheel.dist_info, item.name) + } | set(packaged_paths) + missing_own = _own_license_texts(declared, shipped) + for identifier, text in sorted(missing_own.items()): + target = ( + f"{report_module.licenses_root(wheel.dist_info)}LICENSES/{identifier}.txt" + ) + additions[target] = text.encode("utf-8") + packaged_paths.append( + report_module.declared_license_path(wheel.dist_info, target) + ) + + notices_path = report_module.notices_path(wheel.dist_info) + additions[notices_path] = report_module.notices_markdown( + inventory, + wheel.dist_info, + wheel_path.name, + _project_facts(repo_root, wheel_metadata), + ).encode("utf-8") + packaged_paths.append( + report_module.declared_license_path(wheel.dist_info, notices_path) + ) + + unmet = _declared_without_text( + declared, + {Path(item).name.lower() for item in _own_texts(set(packaged_paths) | shipped)}, + ) + if unmet: + raise BuildError( + f"{wheel_path.name} declares {declared} and packages no text for " + + ", ".join(unmet) + + ".\nA distribution has to ship the terms it declares, the same as " + "anything it redistributes: add the text to license-files in " + "pyproject.toml as .txt, or drop the declaration." + _REMEDIES + ) + + patched_metadata = wheelfile.add_license_files(metadata_raw, sorted(packaged_paths)) + metadata_declared = patched_metadata != metadata_raw + replacements = {f"{wheel.dist_info}/METADATA": patched_metadata} + + # Exclude the SBOM and RECORD to avoid circular digests. SPDX files require + # checksums; the verifier checks these excluded members against the archive. + excluded = [f"{wheel.dist_info}/RECORD", sbom_name] + + projected = [item for item in wheel.entries if item.name not in replacements] + [ + _entry(name, data) for name, data in {**replacements, **additions}.items() + ] + projected_wheel = replace(wheel, entries=tuple(projected)) + for name in [*additions, *replacements]: + # Do not scrape our generated summaries as upstream copyright notices. + packaged_text = name in additions and report_module.is_third_party_license( + wheel.dist_info, name + ) + body = additions.get(name, b"").decode("utf-8", "replace") + inventory.attributions[name] = Attribution( + path=name, + origin="metadata", + exact=True, + detail="license evidence packaged by the SBOM collector" + if name in additions + else "distribution metadata, updated to declare the packaged license files", + spdx_tag=licensing.normalized_tag(body) if packaged_text else None, + # Retain all holders, including those in multi-component copyright files. + copyright_text="\n".join( + licensing.fold_notices(licensing.read_notices(body)) + ) + if packaged_text + else None, + ) + # Record the SBOM before rendering so the evidence covers the final member set. + inventory.attributions[sbom_name] = Attribution( + path=sbom_name, + origin="metadata", + detail="the contents SBOM this collector embedded", + ) + + evidence_doc = evidence_module.document( + discovery, wheel_path.name, inventory, resolver.components + ) + evidence_doc["ci"] = environment_summary() + evidence_doc["system_libraries"] = resolver.system_libraries + evidence_doc["static_inputs"] = resolver.static_inputs + + spdx = document_module.build_document( + _project_facts(repo_root, wheel_metadata), + evidence_doc, + projected_wheel, + inventory, + excluded, + siblings, + ) + spdx_bytes = json.dumps(spdx, indent=2, sort_keys=True).encode("utf-8") + b"\n" + additions[sbom_name] = spdx_bytes + + with tempfile.TemporaryDirectory(dir=str(wheel_path.parent)) as scratch: + staged = Path(scratch) / wheel_path.name + wheelfile.rewrite( + wheel_path, + staged, + additions=additions, + replacements=replacements, + dist_info=wheel.dist_info, + ) + shutil.move(str(staged), str(wheel_path)) + + payload = report_module.report(inventory, evidence_doc, wheel_path.name) + payload["packaged_license_files"] = sorted(packaged_paths) + payload["metadata_license_files_declared"] = metadata_declared + + out_dir.mkdir(parents=True, exist_ok=True) + stem = wheel_path.name.removesuffix(".whl") + sidecars = { + "spdx": out_dir / f"{stem}.spdx.json", + "license_report_json": out_dir / f"{stem}.licenses.json", + "license_report_md": out_dir / f"{stem}.licenses.md", + "build_evidence": out_dir / f"{stem}.build-evidence.json", + } + sidecars["spdx"].write_bytes(spdx_bytes) + sidecars["license_report_json"].write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + sidecars["license_report_md"].write_text( + report_module.report_markdown(payload), encoding="utf-8" + ) + sidecars["build_evidence"].write_text( + json.dumps(evidence_doc, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + # Bind the wheel digest only after the final mutation. + return { + "schema": "isaaccapture-sbom-manifest/1", + "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), + "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, + "wheels": [ + { + "filename": wheel_path.name, + "sha256": sha256_file(wheel_path), + "size": wheel_path.stat().st_size, + "sbom_in_wheel": sbom_name, + **{key: payload[key] for key in report_module.GAP_LISTS}, + "sidecars": { + kind: { + "filename": path.name, + "sha256": sha256_file(path), + "size": path.stat().st_size, + } + for kind, path in sidecars.items() + }, + } + ], + } + + +def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict: + """Merge variant manifests, optionally copying exactly their bound sidecars.""" + found = [ + (path.parent, item) + for path in sorted(paths) + for item in validate_module.manifest_records(path) + ] + merged = merge_records([item for _, item in found]) + if evidence_out is None: + return merged + + # Copy from the selected record's directory, not another build of the same wheel. + beside = {id(item): parent for parent, item in found} + bound = [(beside[id(record)], record) for record in merged["wheels"]] + # Validate the entire set before copying any files. + failures = [ + f"{record['filename']}: {failure}" + for parent, record in bound + for failure in validate_module.sidecar_failures(parent, record) + ] + if failures: + raise BuildError("; ".join(failures)) + evidence_out.mkdir(parents=True, exist_ok=True) + for parent, record in bound: + if parent.resolve() == evidence_out.resolve(): + continue + for name in validate_module.bound_names([record]): + shutil.copyfile(parent / name, evidence_out / name) + return merged + + +def merge_records(wheels: list[dict]) -> dict: + # Universal wheels repeat across matrix entries. Their bytes and wheel-derived + # sidecars must agree; build evidence may differ, so bind the first entry's. + identity = ("filename", "sha256", "size", "sbom_in_wheel") + unique: dict[str, dict] = {} + conflicting: set[str] = set() + disagreeing: set[str] = set() + for item in wheels: + validate_module.checked_record(item) + seen = unique.setdefault(item["filename"], item) + if seen is item: + continue + if [seen.get(key) for key in identity] != [item.get(key) for key in identity]: + conflicting.add(item["filename"]) + disagreeing.update( + f"{item['filename']} ({kind})" + for kind in validate_module.CONTENT_SIDECARS + if seen["sidecars"][kind]["sha256"] != item["sidecars"][kind]["sha256"] + ) + if conflicting: + raise BuildError( + "the same wheel filename was published with different contents: " + + ", ".join(sorted(conflicting)) + ) + if disagreeing: + raise BuildError( + "builds of one wheel described it differently: " + + ", ".join(sorted(disagreeing)) + ) + return { + "schema": "isaaccapture-sbom-manifest/1", + "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), + "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, + "wheels": sorted(unique.values(), key=lambda item: item["filename"]), + } + + +def environment_summary() -> dict: + """Record CI/cache context for reproducibility.""" + return { + "runner_os": os.environ.get("RUNNER_OS"), + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + "github_sha": os.environ.get("GITHUB_SHA"), + "github_ref": os.environ.get("GITHUB_REF"), + } diff --git a/scripts/sbom/isaaccapture_sbom/cli.py b/scripts/sbom/isaaccapture_sbom/cli.py new file mode 100644 index 0000000000..84fdf82caa --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/cli.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command line for the wheel SBOM and license evidence collector.""" + +from __future__ import annotations + +import argparse +import json +import sys +import zipfile +from pathlib import Path + +from . import TOOL_NAME, TOOL_VERSION +from . import build as build_module +from . import discovery +from . import wheelfile +from . import evidence as evidence_module +from . import licensing +from . import report as report_module +from . import validate as validate_module + + +def _repo_root(value: str | None) -> Path: + """Resolve the requested checkout, defaulting to the caller's working tree. + + The installed collector's location is unrelated to the source checkout. + """ + if value: + return Path(value).resolve() + top = discovery.git(Path.cwd(), "rev-parse", "--show-toplevel") + return Path(top).resolve() if top else Path.cwd() + + +def _report(failures: list[str], subject: str) -> int: + if failures: + print(f"{subject}: {len(failures)} problem(s)", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + return 1 + print(f"{subject}: OK") + return 0 + + +def _cmd_build(args: argparse.Namespace) -> int: + repo_root = _repo_root(args.repo_root) + build_dir = Path(args.build_dir) + out_dir = Path(args.out_dir) + + license_data = ( + Path(args.license_data) + if args.license_data + else build_module.default_license_data(build_dir) + ) + + licensing.load_corpus(license_data) + discovery = evidence_module.discover(repo_root, build_dir) + + manifests: list[dict] = [] + wheels = sorted(Path(item) for item in args.wheel) + siblings = build_module.siblings_of(wheels, repo_root) + for wheel in wheels: + manifest = build_module.build( + repo_root, + build_dir, + wheel, + out_dir, + license_data, + discovery=discovery, + siblings=siblings, + ) + manifests.append(manifest) + print(f"packaged evidence into {wheel.name}") + + merged = build_module.merge_records( + [item for manifest in manifests for item in manifest["wheels"]] + ) + manifest_path = out_dir / "manifest.json" + manifest_path.write_text( + json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"manifest: {manifest_path}") + + for item in merged["wheels"]: + for key in report_module.GAP_LISTS: + for entry in item[key]: + print( + f"::warning::{item['filename']}: {entry['component']}: " + f"{entry['reason']}" + ) + return 0 + + +def _cmd_check(args: argparse.Namespace) -> int: + status = 0 + for wheel in sorted(Path(item) for item in args.wheel): + failures = validate_module.check( + wheel, + Path(args.build_evidence) if args.build_evidence else None, + Path(args.manifest) if args.manifest else None, + Path(args.evidence_dir) if args.evidence_dir else None, + ) + status |= _report(failures, wheel.name) + return status + + +def _cmd_check_set(args: argparse.Namespace) -> int: + failures = validate_module.check_set( + Path(args.manifest), Path(args.wheel_dir), Path(args.evidence_dir) + ) + return _report(failures, Path(args.manifest).name) + + +def _cmd_merge_manifests(args: argparse.Namespace) -> int: + merged = build_module.merge_manifests( + [Path(item) for item in args.manifest], + Path(args.evidence_out) if args.evidence_out else None, + ) + Path(args.output).write_text( + json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"merged {len(merged['wheels'])} wheel record(s) -> {args.output}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog=TOOL_NAME, description=__doc__) + parser.add_argument( + "--version", action="version", version=f"{TOOL_NAME} {TOOL_VERSION}" + ) + parser.add_argument( + "--repo-root", + help="IsaacCapture checkout (default: the git checkout you are standing in)", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + build = subparsers.add_parser( + "build", + help="discover what a build put in its wheels, then package the evidence into them", + ) + build.add_argument("--build-dir", required=True, help="configured CMake build tree") + build.add_argument( + "--wheel", + required=True, + action="append", + help="wheel to describe; rewritten in place, repeatable", + ) + build.add_argument( + "--out-dir", required=True, help="receives the sidecars and manifest.json" + ) + build.add_argument( + "--license-data", + help=( + "SPDX license list json/ directory fetched by deps/third_party " + "(default: /_deps/license-list-data-src/json)" + ), + ) + build.set_defaults(handler=_cmd_build) + + check = subparsers.add_parser( + "check", + help="check a wheel against the SBOM it carries, plus any evidence you supply", + ) + check.add_argument( + "--wheel", required=True, action="append", help="wheel to check; repeatable" + ) + check.add_argument( + "--build-evidence", help="also check that the build accounts for every member" + ) + check.add_argument( + "--manifest", help="also check the wheel's bound digest and sidecars" + ) + check.add_argument( + "--evidence-dir", help="where the sidecars live (default: beside the manifest)" + ) + check.set_defaults(handler=_cmd_check) + + check_set = subparsers.add_parser( + "check-set", help="run check over every wheel a merged manifest advertises" + ) + check_set.add_argument( + "--manifest", required=True, help="merged manifest of the release" + ) + check_set.add_argument( + "--wheel-dir", + required=True, + help="the release's wheels; one the manifest does not list fails the set", + ) + check_set.add_argument( + "--evidence-dir", + required=True, + help="the release's sidecars; a file the manifest does not bind fails the set", + ) + check_set.set_defaults(handler=_cmd_check_set) + + merge = subparsers.add_parser( + "merge-manifests", help="fold per-variant manifests into one" + ) + merge.add_argument("--output", required=True, help="merged manifest to write") + merge.add_argument( + "--evidence-out", help="copy the sidecars the merged manifest binds here" + ) + merge.add_argument("manifest", nargs="+", help="per-variant manifests") + merge.set_defaults(handler=_cmd_merge_manifests) + + args = parser.parse_args(argv) + try: + return args.handler(args) + except ( + evidence_module.EvidenceError, + build_module.BuildError, + licensing.CorpusError, + licensing.ExpressionReadError, + discovery.ArchiveError, + discovery.SupplierError, + discovery.FileApiError, + wheelfile.WheelError, + validate_module.ManifestError, + # Report malformed inputs as CLI errors (exit 2), without a traceback. + ValueError, + OSError, + KeyError, + zipfile.BadZipFile, + ) as error: + print(f"error: {error}", file=sys.stderr) + return 2 diff --git a/scripts/sbom/isaaccapture_sbom/discovery.py b/scripts/sbom/isaaccapture_sbom/discovery.py new file mode 100644 index 0000000000..f8fd6648ab --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/discovery.py @@ -0,0 +1,2389 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Discover components and provenance from build outputs. + +Indexes cover fetched trees, the CMake graph, archive members, and repository +and generated files. See scripts/sbom/README.md for attribution rules. +""" + +from __future__ import annotations + +import functools +import glob +import hashlib +import json +import logging +import os +import re +import shlex +import subprocess +import tomllib +import tarfile +import zipfile +from collections.abc import Iterable +from dataclasses import dataclass, field, replace +from pathlib import Path, PurePosixPath +from typing import NamedTuple +from urllib.parse import urlparse + +from packageurl import PackageURL + +from . import elf, licensing + +_SKIP_DIRS = { + ".git", + ".venv", + "venv", + "node_modules", + "__pycache__", + ".mypy_cache", + ".ruff_cache", +} +ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".tar.xz", ".tar.bz2", ".tar", ".zip") +_MAX_ARCHIVE_BYTES = 1 << 31 +# Retain ELF prefixes even for nested archive members to read their build IDs. +_ELF_HEAD_BYTES = 1 << 16 + +_GITHUB_REMOTE = re.compile( + r"github\.com[:/](?P[^/]+)/(?P[^/]+?)(?:\.git)?/?$" +) +_GITHUB_REPO = r"(?P[^/]+)/(?P[^/]+)/" +_GITHUB_DOWNLOADS = ( + re.compile( + rf"^https?://github\.com/{_GITHUB_REPO}releases/download/(?P[^/]+)/[^/]+$" + ), + re.compile( + rf"^https?://github\.com/{_GITHUB_REPO}archive/(?:refs/(?:tags|heads)/)?" + rf"(?P.+?)(?:{'|'.join(map(re.escape, ARCHIVE_SUFFIXES))})$" + ), + re.compile( + rf"^https?://codeload\.github\.com/{_GITHUB_REPO}(?:legacy\.)?(?:tar\.gz|zip)/" + r"(?:refs/(?:tags|heads)/)?(?P.+)$" + ), +) +# Local download paths identify only the builder, not a public origin. +_DOWNLOAD_SCHEMES = ("http", "https", "ftp", "ftps") +_GIT_TRANSPORTS = ("http", "https", "ssh", "git") +# git's scp-like `[user@]host:path`, which SPDX knows only as `ssh://`. +_SCP_REMOTE = re.compile(r"^(?P[^@/:]+@)?(?P[^@/:]{2,}):(?!//)/?") +_VERSION_IN_NAME = re.compile(r"(\d+\.\d+[\w.+-]*?)(?=-|\.tar|\.tgz|\.zip|$)") +# The ref ExternalProject's clone script checks out; `clone --no-checkout` is not one. +_GITCLONE_TAG = re.compile(r'(?[^"]+)"') +_DEPS_SOURCE = re.compile(r"_deps/(?P[A-Za-z0-9_.+-]+)-src(?:/|$)") + + +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() + +_LOG = logging.getLogger("isaaccapture.sbom.discovery") + + +def component_of(display: str) -> str: + """Recover the component key from FileIndex's -src/ display path.""" + return display.split("/", 1)[0].removesuffix("-src") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _walk(root: Path): + """Every regular file under `root`, never following a symlink.""" + stack = [root] + while stack: + current = stack.pop() + try: + entries = list(os.scandir(current)) + except OSError: + continue + for entry in entries: + if entry.is_symlink(): + continue + if entry.is_dir(follow_symlinks=False): + if entry.name not in _SKIP_DIRS: + stack.append(Path(entry.path)) + elif entry.is_file(follow_symlinks=False): + yield Path(entry.path) + + +# Source trees + + +@dataclass(frozen=True) +class Component: + """Something a wheel can contain, identified from where it came from.""" + + key: str + name: str + # "source-tree" | "archive" | "system-library" | "vendored-source" | "vcpkg-port" + kind: str + supplier: str + homepage: str + purl: str + version: str + download_location: str + source_info: str + root: Path | None = None + license_concluded: str = "NOASSERTION" + license_declared: str = "NOASSERTION" + copyright_text: str = "NOASSERTION" + evidence: tuple[licensing.LicenseEvidence, ...] = () + # (path, reason) for per-file REUSE failures; a holder's other files cannot + # lend their licenses to an unlicensed file. + unlicensed: tuple[tuple[str, str], ...] = () + + +def own_git(directory: Path, *args: str) -> str | None: + """Query git only when directory is the checkout root. + + An archive extracted beneath the project must not inherit the enclosing + repository's identity. + """ + if not _is_own_checkout(directory.resolve()): + return None + return git(directory, *args) + + +@functools.cache +def _is_own_checkout(resolved: Path) -> bool: + """Cached per resolved path: one collector run does not create checkouts.""" + top = git(resolved, "rev-parse", "--show-toplevel") + return top is not None and Path(top).resolve() == resolved + + +def git(repo: Path, *args: str) -> str | None: + """Run a read-only git query, returning None on failure. + + Clear repository-local variables (including GIT_DIR and GIT_CONFIG) that + override -C. Preserve GIT_CONFIG_PARAMETERS/COUNT for settings such as + safe.directory, following git's submodule behavior. + """ + local = set(_git_local_variables()) + env = {name: value for name, value in os.environ.items() if name not in local} + return _ask("git", "-C", str(repo), *args, env=env).strip() or None + + +@functools.cache +def _git_local_variables() -> tuple[str, ...]: + listed = _ask("git", "rev-parse", "--local-env-vars").split() + kept = {"GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT"} + return tuple(name for name in listed if name not in kept) + + +def version_in_name(name: str) -> str | None: + """The version an archive's file name states, as in `SDL2-2.32.10.tar.gz`.""" + match = _VERSION_IN_NAME.search(name) + return match.group(1) if match else None + + +def notice_from(evidence) -> str: + """Read copyright notices from the component's license grants.""" + # Pool texts can name the license author (for example WTFPL's), not the component. + notices = [ + found + for item in evidence + if item.kind in ("grant", "nested") + for found in licensing.read_notices(item.text) + if not licensing.notice_is_the_licence_authors( + found, [license_id for license_id, _, _ in item.matches] + ) + ] + return "\n".join(licensing.fold_notices(notices)) if notices else "NOASSERTION" + + +class _Identity(NamedTuple): + name: str + supplier: str + homepage: str + purl: str + download_location: str + version: str + + +def _github_identity(org: str, repo: str, ref: str | None) -> tuple[str, str, str]: + purl = PackageURL(type="github", namespace=org, name=repo, version=ref) + return f"Organization: {org}", f"https://github.com/{org}/{repo}", purl.to_string() + + +def _identity( + remote: str | None, name: str, commit: str | None, tag: str | None = None +) -> _Identity: + """Identify a git source by remote URL and commit, falling back to its declared tag.""" + ref = commit or tag + version = ref or "NOASSERTION" + match = _GITHUB_REMOTE.search(remote) if remote else None + if match: + org, repo = match.group("org"), match.group("repo") + download = f"git+https://github.com/{org}/{repo}.git" + ( + f"@{ref}" if ref else "" + ) + return _Identity(repo, *_github_identity(org, repo, ref), download, version) + if remote: + purl = PackageURL(type="generic", name=name, version=ref).to_string() + location = _SCP_REMOTE.sub(r"ssh://\g\g/", remote) + if urlparse(location).scheme.lower() not in _GIT_TRANSPORTS: + download = "NOASSERTION" + else: + download = f"git+{location}@{ref}" if ref else location + return _Identity(name, "NOASSERTION", remote, purl, download, version) + return _Identity( + name, + "NOASSERTION", + "NOASSERTION", + f"pkg:generic/{name}", + "NOASSERTION", + version, + ) + + +def _archive_identity(url: str, digest: str | None, name: str) -> tuple[_Identity, str]: + """Identify an archive from its declared URL and record its sourceInfo.""" + parsed = urlparse(url) + file_name = PurePosixPath(parsed.path).name + downloadable = parsed.scheme.lower() in _DOWNLOAD_SCHEMES + download = url if downloadable else "NOASSERTION" + detail = f"Declared archive: {url if downloadable else f'local file {file_name}'}" + detail += f" ({digest})." if digest else "." + + match = next(filter(None, (form.match(url) for form in _GITHUB_DOWNLOADS)), None) + if match: + org, repo, ref = match.group("org", "repo", "ref") + github = _github_identity(org, repo, ref) + return _Identity(repo, *github, download, ref), detail + + qualifiers = {"download_url": url} if downloadable else {} + if digest and "=" in digest: + algorithm, value = digest.split("=", 1) + algorithm = algorithm.lower().replace("_", "-") + qualifiers["checksum"] = f"{algorithm}:{value.lower()}" + version = version_in_name(file_name) + purl = PackageURL( + type="generic", name=name, version=version, qualifiers=qualifiers or None + ) + identity = _Identity( + name, + "NOASSERTION", + "NOASSERTION", + purl.to_string(), + download, + version or "NOASSERTION", + ) + return identity, detail + + +@dataclass(frozen=True) +class _Declaration: + """What a FetchContent declaration asked for.""" + + repository: str | None = None + tag: str | None = None + url: str | None = None + hash: str | None = None + + +def _stamps(deps_dir: Path) -> dict[str, dict[str, dict[str, str]]]: + """Index FetchContent stamps by tree and download method.""" + # Under the populate sub-build, or, with CMP0168 NEW, the build's own CMakeFiles. + roots = ( + (deps_dir, "*-subbuild/*-populate-prefix/src/*-populate-stamp"), + (deps_dir.parent / "CMakeFiles" / "fc-stamp", "*"), + ) + newest: dict[tuple[str, str], tuple[int, dict[str, str]]] = {} + for base, pattern in roots: + for method in ("url", "git"): + for stamp in base.glob(f"{pattern}/*-{method}info.txt"): + fields = {} + text = stamp.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + key, separator, value = line.partition("=") + if separator and not line.startswith("#"): + fields[key.strip()] = value.strip() + if not fields.get("source_dir"): + continue + # Both layouts can hold one after a policy change; the newer wins. + slot = (Path(fields["source_dir"]).name, method) + written = stamp.stat().st_mtime_ns + if slot not in newest or written > newest[slot][0]: + newest[slot] = (written, fields) + by_tree: dict[str, dict[str, dict[str, str]]] = {} + for (tree, method), (_, fields) in newest.items(): + by_tree.setdefault(tree, {})[method] = fields + return by_tree + + +def _declaration(deps_dir: Path, fields: dict[str, str]) -> _Declaration: + tag = None + # The git stamp's command runs the clone script, which names the ref; the + # stamp's paths are absolute, so a moved build tree is rebased onto this one. + built_in = Path(fields["source_dir"]).parent.parent + for word in fields.get("command", "").split(";"): + if not word.endswith("-gitclone.cmake"): + continue + script = Path(word) + if not script.is_file() and script.is_relative_to(built_in): + script = deps_dir.parent / script.relative_to(built_in) + if script.is_file(): + text = script.read_text(encoding="utf-8", errors="replace") + match = _GITCLONE_TAG.search(text) + tag = match.group("tag") if match else None + return _Declaration( + repository=fields.get("repository") or None, + tag=tag, + # A CMake list of mirrors, tried in order. + url=fields.get("url(s)", "").split(";")[0] or None, + hash=fields.get("hash") or None, + ) + + +def _public_url(url: str) -> str: + """`url` without credentials in its authority, and without the query and + fragment, where a signed download carries its token.""" + parsed = urlparse(url) + if "@" not in parsed.netloc and not parsed.query and not parsed.fragment: + return url + netloc = parsed.netloc.rpartition("@")[2] + return parsed._replace(netloc=netloc, query="", fragment="").geturl() + + +def discover_source_trees(deps_dir: Path) -> dict[str, Component]: + """Every FetchContent tree under the build's ``_deps``.""" + components: dict[str, Component] = {} + stamps = _stamps(deps_dir) + for root in sorted(deps_dir.glob("*-src")): + if not root.is_dir(): + continue + key = root.name.removesuffix("-src") + # A stamp outlives a switch of method, and an unchanged one keeps its old + # mtime, so the tree decides: a git clone is a checkout of its own. + methods = stamps.get(root.name, {}) + method = "git" if _is_own_checkout(root.resolve()) else "url" + fields = methods.get(method) or next(iter(methods.values()), None) + declared = _declaration(deps_dir, fields) if fields else _Declaration() + commit = own_git(root, "rev-parse", "HEAD") + remote = ( + own_git(root, "config", "--get", "remote.origin.url") or declared.repository + ) + remote = _public_url(remote) if remote else None + describe = own_git(root, "describe", "--tags", "--always") + + details = [f"Fetched into {root.name}."] + if remote or not declared.url: + identity = _identity(remote, key, commit, declared.tag) + if declared.tag: + details.append(f"Declared constraint: {declared.tag}.") + else: + identity, detail = _archive_identity( + _public_url(declared.url), declared.hash, key + ) + details.append(detail) + if describe: + details.append(f"git describe: {describe}.") + + evidence = licensing.discover_in_tree(key, root, root.name) + concluded, declared_expression = licensing.expression(evidence, key) + components[key] = Component( + key=key, + name=identity.name, + kind="source-tree", + supplier=identity.supplier, + homepage=identity.homepage, + purl=identity.purl, + version=identity.version, + download_location=identity.download_location, + source_info=" ".join(details), + root=root, + copyright_text=notice_from(evidence), + license_concluded=concluded, + license_declared=declared_expression, + evidence=tuple(evidence), + ) + return components + + +# Build graph + + +@dataclass(frozen=True) +class Artifact: + """One file a target produces, and the target that produced it.""" + + output: Path + target: str + target_id: str + + +@dataclass(frozen=True) +class _Node: + """A target as the CMake file API describes it.""" + + name: str + kind: str + sources: tuple[Path, ...] + includes: tuple[Path, ...] + dependencies: tuple[str, ...] + # What its link line names. Only linkable targets have one; a static + # library's own requirements land on its consumers' lines. + linked: tuple[Path, ...] = () # outputs of targets in this graph + inputs: tuple[LinkInput, ...] = () # everything else that can carry code + # Sources with compile groups; merely listed sources are not necessarily compiled. + compiled: tuple[Path, ...] = () + objects: Path | None = None + + +# Evidence strength for nested-license reach. Missing compiler records fall +# back to listed sources and include directories. Prebuilt archives retain +# the whole component's terms because their source contents are unknown. +REACH = ( + "copied-file", + "derived-file", + "compiled-source", + "header-read", + "listed-source", + "include-dir", +) +PREBUILT = "prebuilt-library" + + +class Reach(NamedTuple): + """Evidence retaining a nested license: method, source path, and wheel member.""" + + how: str + path: str + member: str + + +def reach_rank(how: str) -> int: + """Strongest first; a linked archive ranks below every path.""" + return REACH.index(how) if how in REACH else len(REACH) + + +def note_reach(found: dict, path, how: str) -> None: + """Record `how` for `path` unless stronger evidence is already there.""" + current = found.get(path) + if current is None or reach_rank(how) < reach_rank(current): + found[path] = how + + +def covers(path: str, how: str, scope: str) -> bool: + """Whether a reached path puts something under a nested text's scope in + the wheel. An include directory may have been read anywhere beneath it, + so one above the scope keeps it too.""" + reached, scope_path = PurePosixPath(path), PurePosixPath(scope) + return reached.is_relative_to(scope_path) or ( + how == "include-dir" and scope_path.is_relative_to(reached) + ) + + +@dataclass(frozen=True) +class LinkInput: + """An archive or object a link line names, or a name it could not resolve.""" + + named: str # as the link line spells it + path: Path | None + kind: str # "archive" | "object" | "unresolved" + reason: str = "" + members: int | None = None # an archive's object files + + @property + def key(self) -> str: + return str(self.path) if self.path else self.named + + +@dataclass +class _Line: + """One target's link line while it is read.""" + + link_dir: Path + directories: list[Path] + sysroot: str = "" + linked: list[Path] = field(default_factory=list) + inputs: list[LinkInput] = field(default_factory=list) + + +@dataclass(frozen=True) +class _Search: + """Implicit linker search directories and the sysroot used for leading = paths.""" + + compiler_dirs: list[Path] + compiler: str | None + sysroot: str + + @property + def directories(self) -> list[Path]: + linker = ( + _linker_search_dirs(self.compiler, self.sysroot) if self.compiler else [] + ) + return self.compiler_dirs + linker + + +# Driver and linker options whose value is the next word. Skipping the value +# keeps a version script or an rpath from being read as an input. +_DRIVER_TAKES_VALUE = frozenset( + """ + -o -x -T -u -e -z -I -B -isystem -include -imacros -MF -MT -MQ + -Xassembler -Xpreprocessor --param + """.split() +) +_LINKER_TAKES_VALUE = frozenset( + """ + -rpath --rpath -rpath-link --rpath-link --version-script -soname --soname -h + -z -T --script -Map --Map --dynamic-list -e --entry -u --undefined --defsym + -o --output --sysroot -y --trace-symbol --wrap -m -R --just-symbols + -dynamic-linker --dynamic-linker -I -F -f --filter --auxiliary --exclude-libs + -plugin --plugin -plugin-opt --plugin-opt --section-start -Ttext -Tdata -Tbss + --retain-symbols-file -Y -G --gpsize -a -A --architecture -b --format -assert + -init -fini -c --mri-script -P --depaudit --audit --require-defined + --export-dynamic-symbol --export-dynamic-symbol-list --dependency-file + --sort-section --section-ordering-file + """.split() +) +_STATIC_MODE = frozenset({"-Bstatic", "-dn", "-non_shared", "-static"}) +_DYNAMIC_MODE = frozenset({"-Bdynamic", "-dy", "-call_shared"}) +# GNU ld script commands that name inputs; AS_NEEDED nests inside them. +_SCRIPT_INPUTS = re.compile(r"\b(?:INPUT|GROUP)\s*\(") +_SCRIPT_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +_SCRIPT_DEPTH = 8 +_NOT_FOUND = ( + "not found; if it is the value of a linker option, that option is one the " + "collector does not know takes a value" +) +_UNKNOWN_KIND = "not an archive, an object, a shared library or a linker script" +_UNSEARCHED = ( + "found in no -L directory, compiler link directory or linker search directory" +) +_NO_COMPILER = ( + "found in no -L directory, and the build records no compiler to ask for its " + "search directories; reconfigure so the file API writes its toolchains reply" +) + + +def _linker_words(fragments: list[dict]) -> tuple[list[str], list[str]]: + """Unpack linker words and retain unparseable fragments. + + Read every CMake role: paths passed as link options appear under flags. + Unpack -Wl, and -Xlinker, and apply the driver's -static to the whole line. + """ + stream: list[str] = [] + unparsed: list[str] = [] + static = False + for fragment in fragments: + try: + words = iter(shlex.split(fragment["fragment"])) + except ValueError: + unparsed.append(fragment["fragment"]) + continue + if fragment.get("role") == "libraryPath": + stream.extend(f"-L{word.removeprefix('-L')}" for word in words) + continue + for word in words: + if word.startswith("-Wl,"): + stream.extend(word[4:].split(",")) + elif word == "-Xlinker": + stream.append(next(words, "")) + elif word in _DRIVER_TAKES_VALUE: + next(words, None) + elif word == "-static": + static = True + elif word.startswith(("-l", "-L")) or not word.startswith("-"): + stream.append(word) + return (["-Bstatic"] if static else []) + _joined(stream), unparsed + + +def _joined(stream: list[str]) -> list[str]: + """Each search option as one word: `-l x`, `--library x` and `--library=x` + all become `-lx`, and the same for `-L`.""" + words: list[str] = [] + pending = iter(word for word in stream if word) + for word in pending: + option, equals, value = word.partition("=") + if word in ("-L", "--library-path", "-l", "--library"): + option, value = word, next(pending, "") + elif not (equals and option in ("--library-path", "--library")): + words.append(word) + continue + words.append(("-L" if option in ("-L", "--library-path") else "-l") + value) + return words + + +def _search_library(name: str, directories: list[Path], static: bool) -> Path | None: + """The file ld takes for `-l`: `.so` before `.a` in each directory.""" + if name.startswith(":"): + candidates = [name[1:]] + else: + candidates = ([] if static else [f"lib{name}.so"]) + [f"lib{name}.a"] + for directory in directories: + for candidate in candidates: + if (directory / candidate).is_file(): + return directory / candidate + return None + + +def _sysrooted(text: str, sysroot: str, base: Path) -> Path: + """A search directory as ld reads it: a leading `=` stands for the sysroot.""" + if text.startswith("="): + return Path(sysroot + text[1:]).resolve() + return (base / text).resolve() + + +def _inspect(path: Path) -> tuple[str | None, int | None, list[str] | None]: + """(kind, archive members, script entries), judged by content. + + The kind is "archive", "object", "shared", "script", "missing", or None for + a file that is none of them. + """ + if not path.is_file(): + return "missing", None, None + members = archive_members(path) + if members is not None: + return "archive", members, None + kind = {"ET_REL": "object", "ET_DYN": "shared"}.get(elf.file_type(path) or "") + if kind: + return kind, None, None + entries = _script_inputs(path) + return ("script" if entries is not None else None), None, entries + + +def _script_inputs(path: Path) -> list[str] | None: + """Return INPUT/GROUP paths and -l names, or None for a non-linker-script file.""" + try: + with path.open("rb") as handle: + head = handle.read(1 << 16) + except OSError: + return None + if b"\0" in head: + return None + body = _SCRIPT_COMMENT.sub(" ", head.decode("utf-8", "replace")) + commands = list(_SCRIPT_INPUTS.finditer(body)) + if not commands: + return None + found: list[str] = [] + for match in commands: + depth, end = 1, match.end() + while end < len(body) and depth: + depth += {"(": 1, ")": -1}.get(body[end], 0) + end += 1 + found += [ + item + for item in re.split(r"[\s,()]+", body[match.end() : end - 1]) + if item and item != "AS_NEEDED" + ] + return found + + +def owned_input(item: LinkInput, ownership: Ownership | None) -> str | None: + """The component a linked archive or object belongs to by the ownership + index -- a fetched tree, a vcpkg port, vendored material -- if it holds code.""" + if ownership is None or item.path is None or item.kind == "unresolved": + return None + if item.members == 0: + return None + return ownership.owner_of(item.path) + + +def _owner(path: Path, ownership: Ownership | None) -> str | None: + """The component a compiled source belongs to, if any.""" + match = _DEPS_SOURCE.search(path.as_posix()) + if match: + return match.group("name") + return ownership.owner_of(path) if ownership is not None else None + + +def _include_owners(include: Path, ownership: Ownership | None) -> set[str]: + """Credit every owner under an include directory when individual headers are unknown.""" + match = _DEPS_SOURCE.search(include.as_posix()) + if match: + return {match.group("name")} + return ownership.owners_under(include) if ownership is not None else set() + + +class SupplierError(Exception): + """The project does not name exactly one author to record as supplier.""" + + +class ArchiveError(Exception): + """An archive the build fetched cannot be read.""" + + +class FileApiError(Exception): + """The build tree carries no CMake file API reply to read.""" + + +def _within(repo_root: Path, build_dir: Path): + """Test checkout membership outside the build tree; both roots must be resolved.""" + + def check(path: Path) -> bool: + resolved = path if path.is_absolute() else path.resolve() + return resolved.is_relative_to(repo_root) and not resolved.is_relative_to( + build_dir + ) + + return check + + +def _reply_objects(reply: Path) -> tuple[dict[str, Path], str]: + """Read objects from the lexicographically latest index. + + Other reply filenames contain unordered content hashes, and stale codemodels + may survive reconfiguration. Do not select them by glob or filename order. + """ + indexes = sorted(reply.glob("index-*.json")) if reply.is_dir() else [] + if not indexes: + return {}, "" + index = json.loads(indexes[-1].read_text(encoding="utf-8")) + generator = index.get("cmake", {}).get("generator", {}).get("name", "") + wanted = {"codemodel": 2, "toolchains": 1} + objects: dict[str, Path] = {} + for item in index.get("objects", []): + version = item.get("version") or {} + if item.get("kind") in wanted and wanted[item["kind"]] == version.get("major"): + found = reply / item["jsonFile"] + if found.is_file(): + objects[item["kind"]] = found + return objects, generator + + +_CACHE_COMPILER = re.compile(r"^CMAKE_(?P[A-Z]+)_COMPILER$") +_SEARCH_DIR = re.compile(r'SEARCH_DIR\("(?P[^"]+)"\)') + + +def cmake_cache(build_dir: Path) -> dict[str, str]: + """Every entry of a build tree's CMakeCache.txt, by name; empty if it has none.""" + cache = build_dir / "CMakeCache.txt" + if not cache.is_file(): + return {} + entries: dict[str, str] = {} + for line in cache.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith(("#", "//")) or "=" not in line or ":" not in line: + continue + entries[line.split(":", 1)[0]] = line.split("=", 1)[1] + return entries + + +def _search_directories(toolchains: Path | None, build_dir: Path) -> dict[str, _Search]: + """Find implicit linker directories by language. + + CMake records only top-level enabled languages; query other compilers directly. + Append the linker's built-in SEARCH_DIRs after the compiler's directories. + """ + compilers = { + match["language"]: value + for name, value in cmake_cache(build_dir).items() + if (match := _CACHE_COMPILER.match(name)) + } + recorded: dict[str, list[str] | None] = {} + if toolchains is not None: + payload = json.loads(toolchains.read_text(encoding="utf-8")) + for item in payload.get("toolchains", []): + compiler = item.get("compiler", {}) + recorded[item["language"]] = compiler.get("implicit", {}).get( + "linkDirectories" + ) + if compiler.get("path"): + compilers[item["language"]] = compiler["path"] + + found: dict[str, _Search] = {} + for language in sorted(set(compilers) | set(recorded)): + compiler = compilers.get(language) + listed = recorded.get(language) + if listed is not None: + directories = [Path(item) for item in listed] + elif compiler: + directories = _compiler_search_dirs(compiler) + else: + directories = None + if directories is None: + continue + sysroot = _ask(compiler, "-print-sysroot").strip() if compiler else "" + found[language] = _Search(directories, compiler, sysroot) + return found + + +def _ask(*command: str, env: dict[str, str] | None = None) -> str: + """Return command stdout, or an empty string on failure.""" + try: + result = subprocess.run( + list(command), + capture_output=True, + text=True, + check=False, + timeout=60, + env=env, + ) + except (OSError, subprocess.SubprocessError): + return "" + return result.stdout if result.returncode == 0 else "" + + +# GCC/Clang depfiles and CMake compiler_depend.make use make rule syntax. +_MAKE_CONTINUATION = re.compile(r"\\\r?\n") +_MAKE_RULE = re.compile(r"^(?:\\.|[^:\\#])+:(?:\s+(?P.*))?$") +_MAKE_WORD = re.compile(r"(?:\\.|[^\s\\])+") +_MAKE_ESCAPE = re.compile(r"\\(.)") +_NINJA_DEPS = re.compile(r"^(?P\S.*): #deps \d+") +# The `CMakeFiles/.dir/` an output sits in, from the top of the build. +_OBJECTS_DIR = re.compile(r"^(?:.*?/)?CMakeFiles/[^/]+\.dir/") + + +def _make_rules(text: str) -> list[list[str]]: + """The prerequisites of each rule of a depfile that names any, unescaped. + + An object's rule names its source first; `-MP`'s rules for headers name + nothing. + """ + found = [] + for line in _MAKE_CONTINUATION.sub(" ", text).splitlines(): + match = _MAKE_RULE.match(line) + words = _MAKE_WORD.findall(match["prerequisites"] or "") if match else [] + if words: + found.append( + [_MAKE_ESCAPE.sub(r"\1", word).replace("$$", "$") for word in words] + ) + return found + + +def _ninja_deps(build_root: Path) -> dict[str, list[str]]: + """`ninja -t deps`: each output's recorded dependencies, by output path.""" + found: dict[str, list[str]] = {} + current: list[str] | None = None + for line in _ask("ninja", "-C", str(build_root), "-t", "deps").splitlines(): + match = _NINJA_DEPS.match(line) + if match: + current = found.setdefault(match["output"], []) + elif current is not None and line.startswith(" ") and line.strip(): + current.append(line.strip()) + return found + + +class DependencyLog: + """What the compiler recorded reading, per target objects directory.""" + + def __init__(self, generator: str, build_root: Path) -> None: + self._ninja_generator = "Ninja" in generator + self._build_root = build_root + self._ninja: dict[str, list[list[str]]] | None = None + self._resolved: dict[str, Path] = {} + self._read: dict[Path, tuple[set[Path], bool]] = {} + + def read(self, objects: Path, compiled: tuple[Path, ...]) -> tuple[set[Path], bool]: + """Return source/header prerequisites and whether every compiled source has its own record. + + A source included by another object does not count as its own compile record. + """ + if objects not in self._read: + read: set[Path] = set() + recorded: set[Path] = set() + wanted, own = set(compiled), objects.resolve() + for base, prerequisites in self._records(objects): + paths = [self._resolve_under(base, item) for item in prerequisites] + # A record for a source the target no longer compiles is stale; + # one CMake generated into the objects directory (a PCH) is + # current, except a unity source the codemodel no longer lists. + source = paths[0] + generated = source.is_relative_to(own) and not source.is_relative_to( + own / "Unity" + ) + if source not in wanted and not generated: + continue + recorded.add(source) + read.update(paths) + self._read[objects] = (read, set(compiled) <= recorded) + return self._read[objects] + + def _records(self, objects: Path) -> list[tuple[Path, list[str]]]: + """One record per object, its source first, with the directory its + relative paths are relative to.""" + if self._ninja_generator: + return self._ninja_records(objects) + return self._make_records(objects) if objects.is_dir() else [] + + def _ninja_records(self, objects: Path) -> list[tuple[Path, list[str]]]: + """Ninja's deps log names objects and paths from the top of the build.""" + if self._ninja is None: + self._ninja = {} + for output, items in _ninja_deps(self._build_root).items(): + match = _OBJECTS_DIR.match(output) + if match and items: + self._ninja.setdefault(match.group(0), []).append(items) + if not self._ninja: + _LOG.warning( + "`ninja -t deps` in %s recorded nothing; nested licence " + "texts are kept by what targets list and include instead. " + "Collect where `ninja` is on PATH.", + self._build_root, + ) + try: + prefix = objects.relative_to(self._build_root).as_posix() + "/" + except ValueError: + return [] + return [(self._build_root, items) for items in self._ninja.get(prefix, [])] + + def _make_records(self, objects: Path) -> list[tuple[Path, list[str]]]: + """A depfile is written where make runs the compiler, the target's own + build directory; compiler_depend.make is relative to the top.""" + files = [ + *((objects.parent.parent, path) for path in sorted(objects.rglob("*.d"))), + (self._build_root, objects / "compiler_depend.make"), + ] + return [ + (base, rule) + for base, path in files + if path.is_file() + for rule in _make_rules(path.read_text(encoding="utf-8", errors="replace")) + ] + + def _resolve_under(self, base: Path, item: str) -> Path: + joined = os.path.join(base, item) + if joined not in self._resolved: + self._resolved[joined] = Path(joined).resolve() + return self._resolved[joined] + + +@functools.lru_cache(maxsize=None) +def _compiler_search_dirs(compiler: str) -> list[Path] | None: + """The `libraries:` list of ` -print-search-dirs`, GCC's and Clang's.""" + for line in _ask(compiler, "-print-search-dirs").splitlines(): + if line.startswith("libraries:"): + listed = line.split(":", 1)[1].strip().removeprefix("=") + return [ + Path(item).resolve() + for item in listed.split(":") + if Path(item).is_dir() + ] + return None + + +@functools.lru_cache(maxsize=None) +def _linker_search_dirs(compiler: str, sysroot: str) -> list[Path]: + """The SEARCH_DIRs of the linker the compiler runs, from `ld --verbose`.""" + linker = _ask(compiler, "-print-prog-name=ld").strip() or "ld" + found = [] + for match in _SEARCH_DIR.finditer(_ask(linker, "--verbose")): + directory = _sysrooted(match["path"], sysroot, Path("/")) + if directory.is_dir(): + found.append(directory) + return found + + +class BuildGraph: + """Read the compile/link graph from CMake's file API. + + Do not scrape link.txt: Ninja does not produce it. + """ + + def __init__(self, build_dir: Path, config: str | None = None) -> None: + self.build_dir = build_dir + self.artifacts: dict[Path, Artifact] = {} + self._nodes: dict[str, _Node] = {} + self._by_name: dict[str, list[Artifact]] = {} + self._closures: dict[str, tuple[_Node, ...]] = {} + self._static: dict[str, dict[LinkInput, set[str]]] = {} + self._inspected: dict[ + Path, tuple[str | None, int | None, list[str] | None] + ] = {} + self._dependencies = DependencyLog("", build_dir) + self._linked: dict[str, tuple[_Node, ...]] = {} + self._node_reach: dict[ + tuple[str, Ownership | None], dict[str, dict[Path, str]] + ] = {} + self._load(config) + + def _load(self, config: str | None) -> None: + reply = self.build_dir / ".cmake" / "api" / "v1" / "reply" + objects, generator = _reply_objects(reply) + if "codemodel" not in objects: + raise FileApiError( + f"no CMake file API codemodel under {reply}. Configure the project " + "with CMake 3.27 or newer, which is where cmake_file_api() asks for it." + ) + model_file = objects["codemodel"] + search = _search_directories(objects.get("toolchains"), self.build_dir) + # A relative link-line path is relative to where the link runs: the + # target's own build directory under the Makefile generators, the top + # of the build tree under Ninja. + per_directory = "Makefiles" in generator or "WMake" in generator + + codemodel = json.loads(model_file.read_text(encoding="utf-8")) + source_root = Path(codemodel["paths"]["source"]) + build_root = Path(codemodel["paths"]["build"]) + self._dependencies = DependencyLog(generator, build_root) + + configurations = codemodel["configurations"] + chosen = next( + (item for item in configurations if item["name"] == (config or "")), None + ) + if chosen is None: + # Do not attribute a wheel to a different build configuration. + if config: + raise FileApiError( + f"codemodel under {reply} describes no {config!r} configuration, " + f"only {[item['name'] for item in configurations]}" + ) + chosen = configurations[0] + + def resolve(raw: str, root: Path) -> Path: + path = Path(raw) + return path if path.is_absolute() else (root / path) + + lines: dict[str, tuple[list[str], list[str], Path, _Search | None]] = {} + for entry in chosen["targets"]: + # Target JSON paths are relative to the codemodel file. + target = json.loads( + (model_file.parent / entry["jsonFile"]).read_text(encoding="utf-8") + ) + includes = { + resolve(item["path"], source_root).resolve() + for group in target.get("compileGroups", []) + for item in group.get("includes", []) + } + target_dir = resolve(target.get("paths", {}).get("build", "."), build_root) + link_dir = target_dir if per_directory else build_root + link = target.get("link", {}) + words, unparsed = _linker_words(link.get("commandFragments", [])) + found = search.get(link.get("language", "")) + # CMAKE_SYSROOT reaches the linker as --sysroot, which the compiler's + # own -print-sysroot does not report. + sysroot = link.get("sysroot", {}).get("path") + if found is not None and sysroot: + found = replace(found, sysroot=sysroot) + lines[entry["id"]] = (words, unparsed, link_dir, found) + self._nodes[entry["id"]] = _Node( + name=target["name"], + kind=target["type"], + sources=tuple( + resolve(item["path"], source_root).resolve() + for item in target.get("sources", []) + ), + includes=tuple(sorted(includes)), + dependencies=tuple( + item["id"] for item in target.get("dependencies", []) + ), + compiled=tuple( + resolve(item["path"], source_root).resolve() + for item in target.get("sources", []) + if "compileGroupIndex" in item + ), + # Where the Makefile and Ninja generators put a target's objects. + objects=target_dir / "CMakeFiles" / f"{target['name']}.dir", + ) + for item in target.get("artifacts", []): + output = resolve(item["path"], build_root).resolve() + artifact = Artifact( + output=output, target=target["name"], target_id=entry["id"] + ) + self.artifacts[output] = artifact + self._by_name.setdefault(output.name, []).append(artifact) + + # Resolve link inputs after indexing every target output. + for target_id, (words, unparsed, link_dir, found) in lines.items(): + line = self._read_line(words, link_dir, found) + line.inputs += [ + LinkInput(item, None, "unresolved", "does not parse as shell words") + for item in unparsed + ] + self._nodes[target_id] = replace( + self._nodes[target_id], + linked=tuple(dict.fromkeys(line.linked)), + inputs=tuple(dict.fromkeys(line.inputs)), + ) + + def _read_line( + self, words: list[str], link_dir: Path, search: _Search | None + ) -> _Line: + """Resolve linker inputs using all -L paths, then implicit search paths. + + Shared libraries are handled separately through DT_NEEDED. + """ + sysroot = search.sysroot if search else "" + line = _Line( + link_dir, + [ + _sysrooted(word[2:], sysroot, link_dir) + for word in words + if word.startswith("-L") + ] + + (search.directories if search else []), + sysroot, + ) + static = False + saved: list[bool] = [] + pending = iter(words) + for word in pending: + if word in _STATIC_MODE: + static = True + elif word in _DYNAMIC_MODE: + static = False + elif word == "--push-state": + saved.append(static) + elif word == "--pop-state": + static = saved.pop() if saved else False + elif word.startswith("-l"): + hit = _search_library(word[2:], line.directories, static) + if hit is None: + reason = _UNSEARCHED if search else _NO_COMPILER + line.inputs.append(LinkInput(word, None, "unresolved", reason)) + else: + self._classify(word, hit, line, static) + elif word.startswith("-"): + if word in _LINKER_TAKES_VALUE: + next(pending, None) + else: + self._classify(word, link_dir / word, line, static) + return line + + def _classify( + self, spelling: str, path: Path, line: _Line, static: bool, depth: int = 0 + ) -> None: + """Classify inputs, following linker scripts under the current -Bstatic state.""" + # ld looks beside the name it was given, not beside a symlink's target. + beside = path.parent + path = path.resolve() + if path in self.artifacts: + line.linked.append(path) + return + if path not in self._inspected: + self._inspected[path] = _inspect(path) + kind, members, entries = self._inspected[path] + if kind in ("archive", "object"): + line.inputs.append(LinkInput(spelling, path, kind, members=members)) + elif kind == "missing": + line.inputs.append(LinkInput(spelling, path, "unresolved", _NOT_FOUND)) + elif kind is None: + line.inputs.append(LinkInput(spelling, path, "unresolved", _UNKNOWN_KIND)) + elif kind == "script" and depth > _SCRIPT_DEPTH: + line.inputs.append( + LinkInput( + spelling, path, "unresolved", "linker scripts nest too deeply here" + ) + ) + elif kind == "script": + # An absolute name in a script inside the sysroot, or one opening + # with `=`, is read under the sysroot. + in_sysroot = bool(line.sysroot) and path.is_relative_to( + Path(line.sysroot).resolve() + ) + for entry in entries or []: + if entry.startswith("-l"): + hit = _search_library(entry[2:], line.directories, static) + else: + if entry.startswith("=") or (in_sysroot and entry.startswith("/")): + entry = line.sysroot + entry.removeprefix("=") + # Beside the script, then where the link runs, then the + # search path; an absolute name is itself in each. + hit = _search_library( + ":" + entry, [beside, line.link_dir, *line.directories], static + ) + if hit is None: + named = f"{spelling} ({path.name}: {entry})" + line.inputs.append(LinkInput(named, None, "unresolved", _NOT_FOUND)) + else: + self._classify(spelling, hit, line, static, depth + 1) + + def by_name(self, name: str) -> list[Artifact]: + return self._by_name.get(name, []) + + def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: + """Find compiled/included files belonging to the checkout. + + Exclude the build tree: fetched sources and vcpkg headers have separate + ownership, and generated headers are not checked-in third-party sources. + """ + inside = _within(repo_root.resolve(), build_dir.resolve()) + found: set[Path] = set() + for node in self._nodes.values(): + found.update(item for item in node.sources if inside(item)) + for include in node.includes: + if inside(include) and include.is_dir(): + found.update( + item + for item in include.rglob("*") + if item.is_file() and inside(item) + ) + return found + + def _reach(self, target_id: str, prune=None) -> tuple[_Node, ...]: + """Traverse dependencies and linked target outputs, even without declared edges. + + Pruned nodes and paths reachable only through them are excluded; the root remains. + """ + seen: set[str] = set() + stack = [target_id] + found: list[_Node] = [] + while stack: + current = stack.pop() + if current in seen or current not in self._nodes: + continue + seen.add(current) + node = self._nodes[current] + if prune is not None and current != target_id and prune(node): + continue + found.append(node) + stack.extend(node.dependencies) + stack.extend(self.artifacts[item].target_id for item in node.linked) + return tuple(found) + + def _closure(self, target_id: str) -> tuple[_Node, ...]: + """`_reach`, memoised per instance, not with `lru_cache`: a cache on the + class would hold every graph ever built alive for the life of the process.""" + if target_id not in self._closures: + self._closures[target_id] = self._reach(target_id) + return self._closures[target_id] + + def static_inputs(self, artifact: Artifact) -> dict[LinkInput, set[str]]: + """Map external static inputs to the targets that link them. + + Exclude build-order executables/modules and their dependencies. + """ + if artifact.target_id not in self._static: + found: dict[LinkInput, set[str]] = {} + for node in self._linked_closure(artifact.target_id): + for item in node.inputs: + found.setdefault(item, set()).add(node.name) + self._static[artifact.target_id] = found + return self._static[artifact.target_id] + + def _linked_closure(self, target_id: str) -> tuple[_Node, ...]: + """`_reach` without other executables and modules, nor what only they + reach: what can end up inside the artifact.""" + if target_id not in self._linked: + self._linked[target_id] = self._reach( + target_id, lambda node: node.kind in ("EXECUTABLE", "MODULE_LIBRARY") + ) + return self._linked[target_id] + + def contributions( + self, artifact: Artifact, ownership: Ownership | None = None + ) -> dict[str, set[str]]: + """Map contributing components to compiled-source, header-include, or prebuilt roles.""" + found: dict[str, set[str]] = {} + + def note(key: str, how: str) -> None: + found.setdefault(key, set()).add(how) + + for node in self._closure(artifact.target_id): + for source in node.sources: + owner = _owner(source, ownership) + if owner: + note(owner, "compiled-source") + for include in node.includes: + for owner in _include_owners(include, ownership): + note(owner, "header-include") + for item in self.static_inputs(artifact): + owner = owned_input(item, ownership) + if owner: + note(owner, PREBUILT) + return found + + def reached( + self, artifact: Artifact, ownership: Ownership | None = None + ) -> dict[str, dict[Path, str]]: + """The resolved paths of each component that went into an artifact, + each with its strongest `REACH`.""" + reached: dict[str, dict[Path, str]] = {} + for node in self._linked_closure(artifact.target_id): + for key, paths in self._reach_of(node, ownership).items(): + into = reached.setdefault(key, {}) + for path, how in paths.items(): + note_reach(into, path, how) + return reached + + def _reach_of( + self, node: _Node, ownership: Ownership | None + ) -> dict[str, dict[Path, str]]: + """One target's owned paths by `REACH`; what it lists and includes only + where it compiles something its dependency output does not cover.""" + if (node.name, ownership) in self._node_reach: + return self._node_reach[(node.name, ownership)] + found: dict[str, dict[Path, str]] = {} + + def mark(owner: str | None, path: Path, how: str) -> None: + if owner: + note_reach(found.setdefault(owner, {}), path, how) + + if node.objects is None: + return found + for source in node.compiled: + mark(_owner(source, ownership), source, "compiled-source") + read, covered = self._dependencies.read(node.objects, node.compiled) + for path in read: + mark(_owner(path, ownership), path, "header-read") + if not covered: + for source in node.sources: + mark(_owner(source, ownership), source, "listed-source") + for include in node.includes: + for owner in _include_owners(include, ownership): + mark(owner, include, "include-dir") + self._node_reach[(node.name, ownership)] = found + return found + + def first_party( + self, artifact: Artifact, repo_root: Path, ownership: Ownership | None = None + ) -> bool: + """True when this build compiled the artifact from the project's own sources.""" + for node in self._closure(artifact.target_id): + if node.name != artifact.target: + continue + for source in node.sources: + if _owner(source, ownership): + continue + try: + source.relative_to(repo_root) + except ValueError: + continue + return True + return False + + +# ============================================================================== +# Ownership +# ============================================================================== + + +class Ownership: + """Resolve ownership across registered component roots and individual files.""" + + def __init__(self) -> None: + # path -> (owner, whether an include directory credits it outright) + self._files: dict[Path, tuple[str, bool]] = {} + self._roots: list[tuple[Path, str]] = [] + # directory -> (owners credited outright, owners credited if alone) + self._index: dict[Path, tuple[set[str], set[str]]] | None = None + # Cache repeated header lookups across compilation units. + self._owners: dict[Path, str | None] = {} + + def register_root(self, root: Path, key: str) -> None: + self._roots.append((root.resolve(), key)) + self._owners.clear() + + def roots_of(self, key: str) -> list[Path]: + """The resolved trees registered for a component.""" + return [root for root, owner in self._roots if owner == key] + + def register_file(self, path: Path, key: str, *, by_directory: bool = True) -> None: + """Register a file's owner. + + With by_directory=False, an include directory credits this owner only if + it owns every registered file beneath that directory. + """ + self._files[path.resolve()] = (key, by_directory) + self._index = None + self._owners.clear() + + def owner_of(self, path: Path) -> str | None: + if path not in self._owners: + self._owners[path] = self._find_owner(path) + return self._owners[path] + + def _find_owner(self, path: Path) -> str | None: + resolved = path.resolve() + entry = self._files.get(resolved) + if entry is not None: + return entry[0] + # Longest root wins, so a vendored subdirectory beats its parent. + best: tuple[int, str] | None = None + for root, key in self._roots: + text = str(root) + if str(resolved) == text or str(resolved).startswith(text + "/"): + if best is None or len(text) > best[0]: + best = (len(text), key) + return best[1] if best else None + + def owners_under(self, directory: Path) -> set[str]: + """Components with files inside a directory, for include-path evidence.""" + resolved = directory.resolve() + outright, alone = self._by_directory().get(resolved, (set(), set())) + found = set(outright) + prefix = str(resolved) + found.update( + key + for root, key in self._roots + if str(root) == prefix or str(root).startswith(prefix + "/") + ) + if len(alone) == 1: + found |= alone + return found + + def _by_directory(self) -> dict[Path, tuple[set[str], set[str]]]: + if self._index is None: + self._index = {} + for path, (key, outright) in self._files.items(): + for directory in (path, *path.parents): + entry = self._index.setdefault(directory, (set(), set())) + entry[0 if outright else 1].add(key) + return self._index + + +# Vendored sources: identify third parties by copyright holder, not directory. + +# Accept REUSE tags and plain upstream copyright notices. +_COPYRIGHT_TAG = re.compile( + r"SPDX-FileCopyrightText:\s*(?P[^\n\r]+)" + r"|^[^\w\n]*Copyright\b(?P[^\n\r]+)", + re.MULTILINE, +) +# Comment markup around a copyright line: `//`, `#`, ` * `, ``, `/* ... */`. +_COMMENT_MARKUP = re.compile(r"^(?:|\*/)\s*$") +_YEARS = re.compile(r"(copyright|\(c\)|©|\d{4}(\s*-\s*\d{4})?|,|\.)", re.IGNORECASE) + + +def project_authors(repo_root: Path) -> set[str]: + """Read author names verbatim; supplier output must retain their casing.""" + pyproject = repo_root / "pyproject.toml" + if not pyproject.is_file(): + return set() + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return set() + return { + str(author["name"]) + for author in data.get("project", {}).get("authors", []) + if author.get("name") + } + + +def project_supplier(repo_root: Path) -> str: + """Require exactly one declared author for SPDX's single supplier field.""" + authors = sorted(project_authors(repo_root)) + if len(authors) != 1: + raise SupplierError( + f"{repo_root}/pyproject.toml names {len(authors)} project authors " + f"({', '.join(authors) or 'none'}); SPDX records one supplier for " + "the distribution" + ) + return f"Organization: {authors[0]}" + + +def project_copyright(repo_root: Path) -> str: + """Read the project notice from its packaging file's REUSE header. + + Metadata-only wheels have no source notice, and reference license texts + do not identify the distribution's copyright holder. + """ + pyproject = repo_root / "pyproject.toml" + if not pyproject.is_file(): + return "NOASSERTION" + try: + head = pyproject.read_text(encoding="utf-8", errors="replace")[:4096] + except OSError: + return "NOASSERTION" + return licensing.read_copyright(head) or "NOASSERTION" + + +@dataclass(frozen=True) +class _Stated: + """Who holds a file, and the notices that say so.""" + + holder: str + notices: tuple[str, ...] + + +def _holder_in(text: str) -> str: + """The holder a copyright statement names, without years or boilerplate.""" + holder = " ".join(_YEARS.sub(" ", _COMMENT_MARKUP.sub("", text)).split()) + return holder.removesuffix(" All rights reserved").strip() + + +def _notice_line(text: str) -> str: + notice = _COMMENT_MARKUP.sub("", " ".join(text.split())) + return re.sub(r"^SPDX-(?:File|Snippet)CopyrightText:\s*", "", notice) + + +def _stated_by(path: Path) -> _Stated | None: + """Holder and notice, from the first copyright line in a text file's head.""" + from reuse.extract import HEURISTICS_CHUNK_SIZE, detect_encoding + + try: + with path.open("rb") as handle: + raw = handle.read(4096) + except OSError: + return None + # REUSE's own text test: a PNG's `Copyright` text chunk is not a header. + encoding = detect_encoding(raw[:HEURISTICS_CHUNK_SIZE]) + if encoding is None: + return None + match = _COPYRIGHT_TAG.search(raw.decode(encoding, "replace")) + if not match: + return None + holder = _holder_in(match.group("holder") or match.group("plain") or "") + return _Stated(holder, (_notice_line(match.group(0)),)) if holder else None + + +def _header_applies(project, used: Path, infos) -> bool: + """Whether REUSE applies the file's own text beside its declared info. + + It does where it took info from the header, and under `aggregate` whatever + it parsed -- unless a `.license` sidecar exists or any `override` applies, + the conditions `Project.reuse_info_of` tests before reading the file. + """ + from reuse.copyright import SourceType + from reuse.global_licensing import PrecedenceType + + if any(info.source_type is SourceType.FILE_HEADER for info in infos): + return True + if project.global_licensing is None or Path(f"{used}.license").exists(): + return False + found = project.global_licensing.reuse_info_of(project.relative_from_root(used)) + return PrecedenceType.AGGREGATE in found and PrecedenceType.OVERRIDE not in found + + +def _held_by(project, used: Path, infos, authors: set[str]) -> _Stated | None: + """Who other than the project holds a file, as a `_Stated`, if anyone does. + + A sidecar or REUSE.toml notice naming someone else makes the file theirs, + under the first such holder by name. The file's first copyright line counts + as well unless REUSE applied declared notices that all name a holder in the + file's place: a sidecar, `override`, or `closest` where REUSE found nothing + in the file. Then the file is ours if those all name the project. + """ + from reuse.copyright import SourceType + + def ours(holder: str) -> bool: + return any(author.lower() in holder.lower() for author in authors) + + # (holder, notice, whether it names the project), once per declared notice. + declared = [ + ( + _holder_in(notice.name), + _notice_line(notice.to_string(original=True)), + ours(notice.name), + ) + for info in infos + if info.source_type is not SourceType.FILE_HEADER + for notice in info.copyright_notices + ] + set_aside = ( + bool(declared) + and all(holder for holder, _, _ in declared) + and not _header_applies(project, used, infos) + ) + stated = None if set_aside else _stated_by(used) + scraped = stated if stated is not None and not ours(stated.holder) else None + foreign = [(holder, line) for holder, line, mine in declared if holder and not mine] + if not foreign: + return scraped + holder = min(foreign, key=lambda item: (item[0].casefold(), item))[0] + lines = [line for _, line in foreign] + list(scraped.notices if scraped else ()) + return _Stated(holder, tuple(lines)) + + +def _covered(project, path: Path, *, built: bool) -> bool: + """Whether REUSE covers a file. + + For a file the build read, only what REUSE excludes by name counts: a link + is judged by its target, and a submodule, a meson subproject or a + VCS-ignored file it compiled still ships. + """ + from reuse.covered_files import is_path_ignored + + if not built and path.is_symlink(): + return False + real = path.resolve() + if not real.is_relative_to(project.root) or not real.is_file(): + return False + # With no VCS strategy REUSE checks neither ignore rules nor submodules. + return not any( + is_path_ignored( + item, + include_submodules=project.include_submodules, + include_meson_subprojects=built or project.include_meson_subprojects, + vcs_strategy=None if built else project.vcs_strategy, + ) + for item in (real, *real.parents) + if item != project.root and item.is_relative_to(project.root) + ) + + +def _info_for(project, own: Path, target: Path): + """REUSE info for a file's holder and for its licence, and the holder's path. + + A link's own declarations are its sidecar and the annotations on its path. + Where they give a field, REUSE's view of the link decides it, as for a file + in its place; otherwise REUSE's view of the target, which applies the + target's precedence to the bytes they share. + """ + from reuse.copyright import SourceType + + infos = project.reuse_info_of(own) + if target == own: + return infos, infos, own + mine = [info for info in infos if info.source_type is not SourceType.FILE_HEADER] + theirs = project.reuse_info_of(target) + licensed = infos if any(info.spdx_expressions for info in mine) else theirs + if any(info.copyright_notices for info in mine): + return infos, licensed, own + return theirs, licensed, target + + +def discover_vendored( + repo_root: Path, + candidates: set[Path], + authors: set[str], + *, + tracked: Iterable[str], +) -> tuple[dict[str, Component], dict[str, list[Path]]]: + """Group third-party files from build inputs and REUSE-covered tracked files. + + Tracked files include data and Python absent from the compile graph. + Licenses follow REUSE precedence; holders follow _held_by. + """ + # Here, not at module level: only discovery needs reuse, and reuse needs libmagic. + from reuse.project import Project + + project = Project.from_directory(repo_root.resolve()) + built = {path for path in candidates if _covered(project, path, built=True)} + candidates = built | { + path + for path in (repo_root / item for item in tracked) + if _covered(project, path, built=False) + } + grouped: dict[str, list[Path]] = {} + licences: dict[str, set[str]] = {} + names: dict[str, str] = {} + notices: dict[str, set[str]] = {} + unlicensed: dict[str, list[tuple[str, str]]] = {} + + for path in sorted(candidates): + under_root = path.relative_to(repo_root) + own = project.root / under_root + infos, licensed, used = _info_for(project, own, path.resolve()) + stated = _held_by(project, used, infos, authors) + if stated is None: + continue + reuse_declared = [ + expression for info in licensed for expression in info.spdx_expressions + ] + key = f"vendored:{licensing.spdx_safe(stated.holder).lower()}" + grouped.setdefault(key, []).append(path) + names[key] = stated.holder + notices.setdefault(key, set()).update(stated.notices) + reason = licensing.why_unlicensed(reuse_declared) + if reason: + unlicensed.setdefault(key, []).append((str(under_root), reason)) + else: + licences.setdefault(key, set()).update(map(str, reuse_declared)) + + components: dict[str, Component] = {} + for key, paths in grouped.items(): + concluded, evidence = licensing.declared_terms(key, licences.get(key, set())) + declared = concluded + # List build inputs first and mark truncation explicitly. + named = [ + name + for _, name in sorted( + (item not in built, str(item.relative_to(repo_root))) for item in paths + ) + ] + relative = ", ".join(named[:4]) + ( + f", and {len(named) - 4} more" if len(named) > 4 else "" + ) + components[key] = Component( + key=key, + name=f"Vendored source ({names[key]})", + kind="vendored-source", + supplier=f"Organization: {names[key]}", + homepage="NOASSERTION", + # A copyright holder is not a package identity; do not mint a purl from it. + purl="NOASSERTION", + version="NOASSERTION", + download_location="NOASSERTION", + source_info=( + f"Checked into this repository rather than fetched; its REUSE info " + f"or header names its copyright holder. Files: {relative}." + ), + license_concluded=concluded, + license_declared=declared, + copyright_text="\n".join( + licensing.fold_notices(notices.get(key, {names[key]})) + ), + evidence=tuple(evidence), + unlicensed=tuple(unlicensed.get(key, ())), + ) + return components, grouped + + +# vcpkg install records identify ports whose outputs are outside the CMake graph. + + +def discover_vcpkg( + installed: Path, target_triplet: str | None = None +) -> tuple[dict[str, Component], dict[str, list[Path]]]: + """Every port vcpkg installed under `installed`, and the files each owns. + + A port installed for the host as well is described from its install for + `target_triplet` -- what the build links -- where there is one. + """ + info = installed / "vcpkg" / "info" + components: dict[str, Component] = {} + owned: dict[str, list[Path]] = {} + for listing in sorted(info.glob("*.list")) if info.is_dir() else []: + # __.list; port names carry no underscore. + port, _, rest = listing.stem.partition("_") + version, _, triplet = rest.rpartition("_") + key = f"vcpkg:{port}" + owned.setdefault(key, []).extend( + installed / line + for line in listing.read_text(encoding="utf-8").splitlines() + if line and not line.endswith("/") + ) + if key not in components or triplet == target_triplet: + components[key] = _vcpkg_component(key, port, version, triplet, installed) + return components, owned + + +def _vcpkg_component( + key: str, port: str, version: str, triplet: str, installed: Path +) -> Component: + """A port, identified by the upstream source its own SBOM says it built. + + vcpkg writes `share//vcpkg.spdx.json` with the recipe and the + downloaded sources, and copies upstream's licence to `share//copyright`. + """ + share = installed / triplet / "share" / port + try: + sbom = json.loads((share / "vcpkg.spdx.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + sbom = {} + packages = {item.get("SPDXID"): item for item in sbom.get("packages", [])} + recipe = packages.get("SPDXRef-port", {}) + sources = [ + item["downloadLocation"] + for spdx_id, item in packages.items() + if str(spdx_id).startswith("SPDXRef-resource-") and item.get("downloadLocation") + ] + port_version = recipe.get("versionInfo") or version + # `#` is the recipe's revision, not anything upstream released. + version = port_version.partition("#")[0] + # One source names the upstream; several (or none) name no single one. + upstream = sources[0] if len(sources) == 1 else None + location = (upstream or "").removeprefix("git+") + remote, _, ref = ( + location.rpartition("@") + if "@" in location.rsplit("/", 1)[-1] + else (location, "", "") + ) + if remote and _GITHUB_REMOTE.search(remote): + identity = _identity(remote, port, ref or None) + supplier, homepage, purl = identity.supplier, identity.homepage, identity.purl + download = identity.download_location + else: + supplier, homepage, download = "NOASSERTION", "NOASSERTION", upstream + purl = PackageURL(type="generic", name=port, version=version).to_string() + homepage = recipe.get("homepage") or homepage + + evidence = [] + licence = share / "copyright" + if licence.is_file(): + evidence.append( + licensing.evidence_for( + key, + "vcpkg-port", + licence.relative_to(installed.parent).as_posix(), + licence.read_text(encoding="utf-8", errors="replace"), + "grant", + ) + ) + concluded, declared = licensing.expression(evidence, key) + details = [ + f"Built and installed by vcpkg port {port} {port_version} for {triplet}." + ] + if upstream: + details.append(f"Source: {upstream}.") + if recipe.get("downloadLocation"): + details.append(f"Port recipe: {recipe['downloadLocation']}.") + return Component( + key=key, + name=port, + kind="vcpkg-port", + supplier=supplier, + homepage=homepage, + purl=purl, + version=version, + download_location=download or "NOASSERTION", + source_info=" ".join(details), + license_concluded=concluded, + license_declared=declared, + copyright_text=notice_from(evidence), + evidence=tuple(evidence), + ) + + +# Build-host libraries: resolve inputs not explained by build or source indexes. + +_SEARCH_ROOTS = ( + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + "/usr/lib", + "/lib/x86_64-linux-gnu", + "/lib/aarch64-linux-gnu", + "/usr/local/lib", +) + + +def _host_library_paths() -> list[Path]: + """Every shared library the loader can see, plus the CUDA trees it cannot.""" + paths: set[Path] = set() + for line in _ask("ldconfig", "-p").splitlines(): + _, _, target = line.partition("=>") + if target.strip(): + paths.add(Path(target.strip())) + for root in _library_roots(): + paths.update(item for item in root.glob("*.so*") if item.is_file()) + return sorted(paths) + + +@functools.lru_cache(maxsize=1) +def _host_build_id_index() -> dict[str, str]: + """Index host libraries by build ID, which survives SONAME/RPATH repair. + + Cache the index to avoid rescanning host libraries for every wheel member. + """ + index: dict[str, str] = {} + for path in _host_library_paths(): + build_id = elf.read_build_id(path) + if not build_id: + continue + # A regular file beats a symlink to it: dpkg-query knows the real path. + current = index.get(build_id) + if current is None or ( + path.is_symlink() is False and Path(current).is_symlink() + ): + index[build_id] = str(path) + return index + + +def _host_origin(library: Path | None, **known) -> dict: + """Resolve a host file and identify its supplying package and terms.""" + origin: dict = {**known, "resolved_path": None, "package": None, "copyright": None} + if library is None: + return origin + library = library.resolve() + origin["resolved_path"] = str(library) + return _identify_host_package(library, origin) + + +def resolve_system_library_by_build_id(build_id: str) -> dict: + """Trace a vendored library back by build-id, whatever it was renamed to.""" + found = _host_build_id_index().get(build_id) + library = Path(found).resolve() if found else None + return _host_origin( + library, soname=library.name if library else None, build_id=build_id + ) + + +def resolve_system_library(soname: str) -> dict: + """Trace an auditwheel-vendored library back to the package that supplied it.""" + library = next( + (root / soname for root in _library_roots() if (root / soname).exists()), None + ) + return _host_origin(library, soname=soname) + + +def resolve_static_archive(archive: Path) -> dict: + """Identify the package or toolkit supplying a linked archive or object. + + The link line already supplies its path; existence alone proves no ownership. + """ + if not archive.is_file(): + return _host_origin(None, toolkit_root=None) + root = _toolkit_root(archive) + return _host_origin(archive, toolkit_root=str(root) if root else None) + + +def archive_members(path: Path) -> int | None: + """How many object files a `!` archive holds; None if it is not one. + + glibc 2.34 folded librt, libpthread and libdl into libc and left each as an + archive with no members, so naming one on a link line links nothing. + """ + try: + with path.open("rb") as handle: + magic = handle.read(8) + if magic not in (b"!\n", b"!\n"): + return None + count = 0 + while header := handle.read(60): + if len(header) < 60 or header[58:60] != b"`\n": + return None + name = header[:16].rstrip() + size = int(header[48:58]) + skip = size + size % 2 + # BSD writes a long name at the head of the member's data. + if name.startswith(b"#1/"): + length = int(name[3:]) + name = handle.read(length).rstrip(b"\0") + skip -= length + # The symbol table and the long-name table are not members. + index = name in (b"/", b"//", b"/SYM64/") or name.startswith( + b"__.SYMDEF" + ) + count += not index + # A thin archive stores only its index; members stay outside it. + if index or magic == b"!\n": + handle.seek(skip, os.SEEK_CUR) + return count + except (OSError, ValueError): + return None + + +def _identify_host_package(library: Path, origin: dict) -> dict: + # dpkg names a multiarch package with its architecture, `libc6:i386`, and + # dpkg-query -W and apt answer for one architecture only when asked so. + qualified = _dpkg_owner(library) + if not qualified: + return origin + package = qualified.partition(":")[0] + origin["package"] = package + origin["supplier"] = _package_supplier(qualified) + origin["version"] = _package_field(qualified, "Version") + origin["apt_origin"], origin["apt_origin_from"] = _apt_origin(qualified) + copyright_file = Path("/usr/share/doc") / package / "copyright" + if copyright_file.is_file(): + origin["copyright"] = str(copyright_file) + return origin + + +_DPKG_PACKAGE = re.compile(r"^[a-z0-9][a-z0-9+.-]+(?::[a-z0-9-]+)?$") + + +def _dpkg_owner(library: Path) -> str | None: + """The first package `dpkg-query -S` names as owning `library`. + + Its last line answers, `[, ...]: `; diversion lines + come before it, translated. Where `library` is diverted, the file there may + not be the listed package's, so none is named. + """ + lines = _ask("dpkg-query", "-S", str(library)).splitlines() + suffix = f": {library}" + if not lines or any(line.endswith(suffix) for line in lines[:-1]): + return None + if not lines[-1].endswith(suffix): + return None + owner = lines[-1].removesuffix(suffix).split(",", 1)[0].strip() + return owner if _DPKG_PACKAGE.match(owner) else None + + +def _package_field(package: str, field: str) -> str | None: + return _ask("dpkg-query", "-W", f"-f=${{{field}}}", package).strip() or None + + +# `apt-cache policy` lines. Its headings are translated; these are not. With no +# package named: a source (`%4i `) and, directly under it, its +# Release fields. In a version table: a version (`***` when installed, perhaps +# with a note such as `(phased 20%)`) and, indented further, a source listing it. +_APT_SOURCE = re.compile(r"^(?P *)-?\d+ (?P\S.*?)\s*$") +_APT_RELEASE = re.compile(r"^ {5}release (?P.*)$") +_APT_RELEASE_FIELD = re.compile(r",(?=[a-z]=)") +_APT_VERSION = re.compile(r"^ (?P\*\*\*| ) \S+ -?\d+(?: .*)?$") + + +@functools.lru_cache(maxsize=1) +def _apt_releases() -> dict[str, dict[str, str]]: + """Each source apt knows, with the fields of its Release file (`o` is Origin).""" + releases: dict[str, dict[str, str]] = {} + source = None + for line in _ask("apt-cache", "policy").splitlines(): + listed = _APT_SOURCE.match(line) + if listed and len(listed["indent"]) <= 3: + source = listed["source"] + releases[source] = {} + elif source and (release := _APT_RELEASE.match(line)): + releases[source] = dict( + item.partition("=")[::2] + for item in _APT_RELEASE_FIELD.split(release["fields"]) + ) + source = None + else: + source = None + return releases + + +@functools.lru_cache(maxsize=None) +def _apt_origin(package: str) -> tuple[str | None, str | None]: + """`(Origin, "installed version")` where the archives listing the installed + version agree on one, ignoring case; else `(Origin, "other versions")` where + none lists it and those listing other versions agree; else `(None, None)`. + """ + releases = _apt_releases() + # Installed or not: lowercased Origin ("" for none) -> first spelling. + origins: dict[bool, dict[str, str]] = {True: {}, False: {}} + installed = False + for line in _ask("apt-cache", "policy", package).splitlines(): + if version := _APT_VERSION.match(line): + installed = version["installed"] == "***" + elif (listed := _APT_SOURCE.match(line)) and len(listed["indent"]) >= 5: + fields = releases.get(listed["source"], {}) + # dpkg's status file, release `a=now`, says what is installed, not + # where it came from. + if fields.get("a") != "now": + origin = fields.get("o", "") + origins[installed].setdefault(origin.lower(), origin) + decided_by_installed = bool(origins[True]) + found = origins[decided_by_installed] + if len(found) == 1 and "" not in found: + decided = "installed version" if decided_by_installed else "other versions" + return next(iter(found.values())), decided + return None, None + + +def _package_supplier(package: str) -> str | None: + """Format the package Maintainer as an SPDX supplier; the package name is not one.""" + maintainer = _package_field(package, "Maintainer") + if not maintainer: + return None + # "Name " is Debian's form; SPDX wants "Organization: Name (email)". + match = re.match(r"^(?P.+?)\s*<(?P[^>]+)>$", maintainer) + if match: + return f"Organization: {match['name']} ({match['email']})" + return f"Organization: {maintainer}" + + +def _library_roots() -> list[Path]: + roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] + return roots + [root / "lib64" for root in _toolkit_roots()] + + +@functools.lru_cache(maxsize=None) +def _toolkit_roots() -> list[Path]: + """Toolkits that may install outside the package manager: CUDA's.""" + return sorted( + { + Path(item).resolve() + for item in glob.glob("/usr/local/cuda*") + if Path(item).is_dir() + } + ) + + +def _toolkit_root(path: Path) -> Path | None: + resolved = path.resolve() + return next( + (root for root in _toolkit_roots() if resolved.is_relative_to(root)), None + ) + + +def _license_near(library: Path) -> Path | None: + """Find terms beside a library, searching up to its toolkit root. + + CUDA can install outside dpkg, with targets//lib below the root EULA.txt. + """ + root = _toolkit_root(library) + current = library.resolve().parent + levels = 3 if root is None else len(current.relative_to(root).parts) + 1 + for _ in range(levels): + for entry in sorted(current.iterdir()) if current.is_dir() else []: + if entry.is_file() and licensing.classify( + entry.name, licensing.files_in(current) + ): + return entry + if current.parent == current: + break + current = current.parent + return None + + +def _system_notice(evidence: list[licensing.LicenseEvidence]) -> str: + """Read holders from machine-readable Debian copyright fields, or NOASSERTION. + + Do not scrape free-form agreements: bundled notices can name other components. + """ + for item in evidence: + fields = licensing.dep5_copyright(item.text) + if fields: + return "\n".join(licensing.fold_notices(fields)) + return "NOASSERTION" + + +def _system_purl(soname: str, origin: dict) -> str: + """`pkg:deb//...` for a package an archive's Origin names, else generic. + + The deb type requires its namespace, the vendor, lowercased. ECMA-427 5.6.3 + keeps `/` out of a namespace segment and a URL host out of a namespace, so + an Origin that is a URL or a host name names no vendor either. + """ + package = origin.get("package") + vendor = (origin.get("apt_origin") or "").lower() if package else "" + if "/" in vendor or ("." in vendor and not re.search(r"\s", vendor)): + vendor = "" + return PackageURL( + type="deb" if vendor else "generic", + namespace=vendor or None, + name=package or soname, + version=origin.get("version"), + ).to_string() + + +def system_component( + soname: str, origin: dict, route: str | None = None, key: str | None = None +) -> Component: + """Describe a host library; route defaults to auditwheel and key to system:.""" + evidence: list[licensing.LicenseEvidence] = [] + key = key or f"system:{soname}" + license_path = origin.get("copyright") + if not license_path and origin.get("resolved_path"): + found = _license_near(Path(origin["resolved_path"])) + license_path = str(found) if found else None + origin["license_file"] = license_path + if license_path and Path(license_path).is_file(): + text = Path(license_path).read_text(encoding="utf-8", errors="replace") + evidence.append( + licensing.evidence_for(key, "build-host", license_path, text, "grant") + ) + # Use the component key: different libraries can share a SONAME but not terms. + concluded, declared = licensing.expression(evidence, key) + + package = origin.get("package") + details = [ + route + or f"Vendored into the wheel by auditwheel from {origin.get('resolved_path') or soname}." + ] + if package: + details.append(f"Supplied by system package {package}.") + return Component( + key=key, + name=soname, + kind="system-library", + supplier=origin.get("supplier") or "NOASSERTION", + copyright_text=_system_notice(evidence), + homepage="NOASSERTION", + purl=_system_purl(soname, origin), + version=origin.get("version") or "NOASSERTION", + download_location="NOASSERTION", + source_info=" ".join(details), + license_concluded=concluded, + license_declared=declared, + evidence=tuple(evidence), + ) + + +# Content indexes + + +@dataclass(frozen=True) +class Member: + """A file inside an archive, or a file on disk.""" + + container: str + path: str + sha256: str + size: int + build_id: str | None = None + + +class ArchiveIndex: + """Members of every archive in the repo, so extracted bytes can be traced back.""" + + def __init__(self) -> None: + self.by_hash: dict[str, Member] = {} + self.all_by_hash: dict[str, list[Member]] = {} + self.by_name: dict[str, list[Member]] = {} + # Build IDs survive auditwheel/patchelf RPATH and SONAME changes. + self.all_by_build_id: dict[str, list[Member]] = {} + self.archives: dict[str, dict] = {} + + def add_archive(self, path: Path, display: str) -> None: + size = path.stat().st_size + if size > _MAX_ARCHIVE_BYTES: + return + members = self._read_members(path, display) + self.archives[display] = { + "path": display, + "sha256": sha256_file(path), + "size": size, + "member_count": len(members), + } + at_root = frozenset( + Path(member.path).name + for member, _ in members + if Path(member.path).parent.as_posix() in {".", ""} + ) + for member, data in members: + # Nested members retain only a prefix; read their build ID, not full ELF metadata. + build_id = elf.build_id_of(data) + if build_id: + member = replace(member, build_id=build_id) + self.all_by_build_id.setdefault(build_id, []).append(member) + self.by_hash.setdefault(member.sha256, member) + self.all_by_hash.setdefault(member.sha256, []).append(member) + self.by_name.setdefault(Path(member.path).name, []).append(member) + self._maybe_license(display, member, data, at_root) + + def _maybe_license( + self, display: str, member: Member, data: bytes, at_root: frozenset[str] + ) -> None: + name = Path(member.path).name + if Path(member.path).parent.as_posix() not in {".", ""}: + return # only the archive root states the bundle's own terms + if name.upper() == "VERSION" and len(data) < 256: + # Version files may contain extra lines that are not part of the version. + text = data.decode("utf-8", "replace").strip().splitlines() + self.archives[display]["version_text"] = text[0].strip() if text else "" + return + if not licensing.classify(name, at_root.__contains__): + return + self.archives[display].setdefault("license_members", []).append((member, data)) + + @staticmethod + def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]]: + """Stream member hashes; retain root files and ELF prefixes only.""" + collected: list[tuple[Member, bytes]] = [] + + def take(name: str, handle) -> None: + # Retain root license texts and nested ELF prefixes without buffering large members. + at_root = Path(name).parent.as_posix() in {".", ""} + digest = hashlib.sha256() + size = 0 + kept = bytearray() + head = b"" + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + if at_root: + kept += chunk + elif not size and elf.is_elf(chunk): + head = chunk[:_ELF_HEAD_BYTES] + size += len(chunk) + collected.append( + ( + Member(display, name, digest.hexdigest(), size), + bytes(kept) if at_root else head, + ) + ) + + try: + if path.suffix == ".zip": + with zipfile.ZipFile(path) as archive: + for info in archive.infolist(): + if not info.is_dir(): + with archive.open(info) as handle: + take(info.filename, handle) + else: + with tarfile.open(path) as archive: + for info in archive: + if not info.isfile(): + continue + handle = archive.extractfile(info) + if handle is not None: + take(info.name.removeprefix("./"), handle) + except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: + # Report unreadable fetched archives directly, before attribution failures obscure them. + raise ArchiveError(f"{display} cannot be read: {error}") from error + return collected + + +def tracked_files(repo_root: Path) -> list[str]: + """List tracked files; a directory walk would also treat install output as source.""" + listing = git(repo_root, "ls-files", "-z", "--cached") + if listing is None: + raise FileApiError( + f"{repo_root} is not a git checkout this build can query; the SBOM " + "distinguishes repository source from build output by what git tracks" + ) + return [item for item in listing.split("\0") if item] + + +def discover_archives( + repo_root: Path, held: frozenset[str], skip: tuple[Path, ...] = () +) -> ArchiveIndex: + """Index tracked or explicitly fetched archives, excluding build outputs. + + Require held explicitly; an omitted tracked-file list silently loses provenance. + """ + index = ArchiveIndex() + skipped = tuple(item.resolve() for item in skip) + for path in _walk(repo_root): + # Compare path segments: skipping dist must not skip distribution. + if any(path.is_relative_to(item) for item in skipped): + continue + name = path.name.lower() + if not name.endswith(ARCHIVE_SUFFIXES): + continue + relative = path.relative_to(repo_root).as_posix() + # Untracked archives need a download marker; staged copies cannot prove origin. + if relative not in held and not path.with_name(path.name + ".source").is_file(): + continue + index.add_archive(path, relative) + return index + + +class FileIndex: + """Files on disk by content hash, for tracing a copied wheel member back.""" + + def __init__(self) -> None: + self.by_hash: dict[str, str] = {} + # Keep all candidates: identical license texts or source files do not prove ownership. + self.all_by_hash: dict[str, list[str]] = {} + self.by_suffix: dict[str, list[str]] = {} + self._digests: dict[str, str] = {} + self._by_display: dict[str, str] = {} + + def digest_of(self, path: Path) -> str | None: + """The digest this index already computed for a path, if it indexed it.""" + return self._digests.get(str(path.resolve())) + + def add_tree( + self, root: Path, display_root: str, skip: tuple[Path, ...] = () + ) -> None: + # Resolve relative skip paths and compare segments, not string prefixes. + inside = [] + for item in skip: + try: + inside.append(item.resolve().relative_to(root.resolve())) + except ValueError: + continue + self._add( + root, + display_root, + ( + path + for path in _walk(root) + if not any( + path.relative_to(root).is_relative_to(item) for item in inside + ) + ), + ) + + def add_paths(self, root: Path, display_root: str, paths) -> None: + """Index an explicit list rather than whatever the tree happens to hold.""" + self._add(root, display_root, (root / item for item in paths)) + + def _add(self, root: Path, display_root: str, paths) -> None: + # Resolve once per tree: _walk does not cross symlinks. + resolved_root = root.resolve() + for path in paths: + if not path.is_file() or path.is_symlink(): + continue + relative = path.relative_to(root).as_posix() + display = f"{display_root}/{relative}" + try: + digest = sha256_file(path) + except OSError: + continue + self._digests[str(resolved_root / relative)] = digest + self.by_hash.setdefault(digest, display) + self.all_by_hash.setdefault(digest, []).append(display) + self._by_display[display] = digest + # Include the parent directory to distinguish common names such as __init__.py. + self.by_suffix.setdefault( + "/".join(relative.rsplit("/", 2)[-2:]), [] + ).append(display) + + def digest_of_display(self, display: str) -> str | None: + """The digest this index recorded for one of its own display paths.""" + return self._by_display.get(display) + + def under(self, display_prefix: str): + """Every (display, digest) this index holds beneath a display path.""" + prefix = display_prefix.rstrip("/") + "/" + return ( + (display, digest) + for display, digest in self._by_display.items() + if display.startswith(prefix) + ) + + def path_suffix_match(self, wheel_path: str) -> str | None: + key = "/".join(wheel_path.rsplit("/", 2)[-2:]) + matches = self.by_suffix.get(key, []) + return matches[0] if len(matches) == 1 else None diff --git a/scripts/sbom/isaaccapture_sbom/document.py b/scripts/sbom/isaaccapture_sbom/document.py new file mode 100644 index 0000000000..656e816e49 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/document.py @@ -0,0 +1,574 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render the discovered inventory as an SPDX 2.3 JSON document.""" + +from __future__ import annotations + +import hashlib +import re + +from packageurl import PackageURL +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version + +from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing, report, stamped_now +from .discovery import Component +from .inventory import Inventory +from .wheelfile import WheelInfo + +NOASSERTION = "NOASSERTION" +_LINK_HOWS = {"compiled-source", "header-include", "prebuilt-library"} +# Shared content identifies candidates, not proven CONTAINS/COPY_OF relationships. +_AMBIGUOUS_HOWS = {"shared-content"} + + +WHEEL_PACKAGE_ID = "SPDXRef-Package-wheel" + + +def _spdx_id(prefix: str, value: str) -> str: + return f"SPDXRef-{prefix}-{licensing.spdx_safe(value)}" + + +def component_package_id(key: str) -> str: + """The id this document gives a component. The verifier joins on it.""" + return _spdx_id("Package", key) + + +def file_id(path: str, dist_info: str) -> str: + """Derive a stable ID from the path, normalizing versioned .dist-info/.data roots. + + Include a path digest to distinguish names that sanitize alike. + """ + stem = dist_info.removesuffix(".dist-info") + hashed = path + for directory, token in ((dist_info, "dist-info"), (f"{stem}.data", "data")): + if path.startswith(f"{directory}/"): + path = token + path.removeprefix(directory) + # No member name holds a NUL, so a real top-level `data/` hashes apart. + hashed = "\0" + path + break + return _spdx_id("File", f"{path}-{licensing.sha256_text(hashed)[:8]}") + + +def verification_code(sha1_digests: list[str]) -> str: + """SPDX package verification code: SHA1 over the sorted file SHA1s.""" + return hashlib.sha1("".join(sorted(sha1_digests)).encode("ascii")).hexdigest() # noqa: S324 + + +def _scored(match) -> str: + license_id, containment, coverage = match + return f"{license_id} (containment {containment:.2f}, coverage {coverage:.2f})" + + +def _attribution_texts(component: Component, dist_info: str) -> list[str]: + texts = [ + f"License evidence: {item.path} (sha256:{item.sha256}, {item.size} bytes, " + f"{item.origin}, {item.kind}) packaged at " + # Whitespace after the path: validate.py reads it up to the first one. + f"{report.packaged_license_path(dist_info, component.key, item)}" + + ( + " " + + licensing.matched_prose( + item.matches, name=_scored, join="; ", lead="identified as " + ) + if item.matches + else f" — the SPDX reference text for {item.identified}, an identifier " + "REUSE declares for this component's files; not the component's own file" + if item.origin == licensing.SPDX_REFERENCE + else "" + ) + + (f" — {report.NESTED_ROLE}" if item.kind == "nested" else "") + for item in component.evidence + ] + if not licensing.grant_texts(component.evidence): + texts.append( + "No license grant was found in this component's own files; see the " + "component-to-license report." + ) + return texts + + +def _component_package(component: Component, roles: set[str], dist_info: str) -> dict: + package = { + "SPDXID": component_package_id(component.key), + "name": component.name, + "versionInfo": component.version, + "supplier": component.supplier, + "downloadLocation": component.download_location, + "filesAnalyzed": False, + "licenseConcluded": component.license_concluded, + "licenseDeclared": component.license_declared, + "copyrightText": component.copyright_text, + **( + { + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": component.purl, + } + ] + } + if component.purl != NOASSERTION + else {} + ), + "attributionTexts": _attribution_texts(component, dist_info), + "sourceInfo": f"{component.source_info} Reaches this wheel as: {', '.join(sorted(roles))}.", + } + if component.homepage != NOASSERTION: + package["homepage"] = component.homepage + return package + + +def _extracted_entries(components: list[Component]) -> list[dict]: + """One entry per distinct text, using the shared component LicenseRef.""" + entries: dict[str, dict] = {} + for component in components: + carried = licensing.carried_texts(component.evidence) + if not carried: + continue + body = licensing.verbatim_terms(component.evidence) + reason = licensing.why_unnamed(component.evidence) + for ref in licensing.license_refs( + component.license_concluded, component.license_declared + ): + if ref in entries and entries[ref]["extractedText"] != body: + raise ValueError( + f"{ref} was minted for two components with different terms; " + "a LicenseRef identifies one text" + ) + # A shared text must cite all components and evidence paths that supply it. + covers = set(entries.get(ref, {}).get("_for", ())) | {component.name} + cited = sorted( + set(entries.get(ref, {}).get("_from", ())) + | {item.path for item in carried} + ) + entries[ref] = { + "licenseId": ref, + "name": "License terms shipped with " + ", ".join(sorted(covers)), + "_for": covers, + "_from": cited, + "extractedText": body, + "comment": ( + reason[:1].upper() + + reason[1:] + + "; reproduced here verbatim from " + + ", ".join(cited) + + "." + ), + } + for entry in entries.values(): + entry.pop("_for", None) + entry.pop("_from", None) + return sorted(entries.values(), key=lambda item: item["licenseId"]) + + +_EXTRA_MARKER = re.compile(r"\bextra\s*==") + + +def _unanalyzed_package( + package_id: str, name: str, comment: str, external_refs: list[dict] | None = None +) -> dict: + """Describe an external package without asserting its contents.""" + package = { + "SPDXID": package_id, + "name": name, + "versionInfo": NOASSERTION, + "supplier": NOASSERTION, + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": NOASSERTION, + "copyrightText": NOASSERTION, + "comment": comment, + } + if external_refs: + package["externalRefs"] = external_refs + return package + + +def _is_extra_gated(requirement: Requirement) -> bool: + return requirement.marker is not None and bool( + _EXTRA_MARKER.search(str(requirement.marker)) + ) + + +def _by_version(specifier): + """Specifier clauses in version order, or as given if one will not parse.""" + try: + return sorted(specifier, key=lambda item: Version(item.version)) + except InvalidVersion: + return list(specifier) + + +def _dependency_refs( + name: str, requirement: Requirement, version: str | None = None +) -> list[dict]: + """Represent requirements with an unversioned purl and a VERS constraint. + + A purl version must be exact; it cannot encode a requirement range. + """ + refs = [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": PackageURL( + type="pypi", name=name, version=version + ).to_string(), + } + ] + # VERS requires version ordering; sorting operators puts <1.9 before >=1.6. + constraints = [ + f"{item.operator}{item.version}" for item in _by_version(requirement.specifier) + ] + if constraints: + refs.append( + { + "referenceCategory": "OTHER", + "referenceType": "vers", + "referenceLocator": "vers:pypi/" + "|".join(constraints), + } + ) + return refs + + +def _source_info(project: dict, evidence_doc: dict, build: dict, wheel) -> str: + """Describe source and build facts applicable to this wheel. + + Universal wheels must not inherit the build matrix's native architecture. + """ + source = evidence_doc.get("project", {}) + parts = [] + if project["homepage"] != NOASSERTION: + parts.append(f"Built from {project['homepage']}") + commit = source.get("commit") + if commit: + described = source.get("describe") or "" + parts.append( + f"at commit {commit}" + + (" with uncommitted changes" if described.endswith("-dirty") else "") + ) + if "none-any" not in wheel.path.name: + parts.append(f"arch {build.get('arch')}") + parts.append(f"CMAKE_BUILD_TYPE {build.get('config')}") + return "; ".join(parts) + "." if parts else NOASSERTION + + +def _wheel_conclusion(declared: str, inventory: Inventory) -> str: + """Conjoin the distribution's declaration and every redistributed component's terms. + + Any unresolved component makes the result NOASSERTION, which cannot be AND-joined. + """ + present = inventory.components_present.values() + if any(item.license_concluded == NOASSERTION for item in present): + return NOASSERTION + # Flatten AND operands for deduplication while preserving OR choices. + operands: set[str] = set() + for component in present: + operands.update(licensing.operands_of(component.license_concluded)) + if declared != NOASSERTION: + operands.update(licensing.operands_of(declared)) + return licensing.combine(sorted(operands)) if operands else declared + + +def build_document( + project: dict, + evidence_doc: dict, + wheel: WheelInfo, + inventory: Inventory, + excluded_files: list[str], + siblings: dict[str, dict] | None = None, +) -> dict: + """Render SPDX, rejecting file-ID collisions. + + The caller validates the result. siblings maps canonical distribution names + to metadata from other wheels in this run. + """ + files: list[dict] = [] + file_ids: dict[str, str] = {} + sha1_digests: list[str] = [] + + stated_in_files: set[str] = set() + # Only first-party file notices contribute to the wheel package's own copyright. + stated_copyright: set[str] = set() + paths_by_id: dict[str, str] = {} + for entry in sorted(wheel.entries, key=lambda item: item.name): + if entry.name in excluded_files: + continue + spdx_id = file_id(entry.name, wheel.dist_info) + # Paths that sanitise alike and share a 32-bit digest prefix. + if spdx_id in paths_by_id: + raise ValueError( + f"{wheel.path.name}: {paths_by_id[spdx_id]} and {entry.name} " + f"would both be {spdx_id}" + ) + paths_by_id[spdx_id] = entry.name + file_ids[entry.name] = spdx_id + sha1_digests.append(entry.sha1) + + attribution = inventory.attributions.get(entry.name) + record = { + "SPDXID": spdx_id, + "fileName": f"./{entry.name}", + "checksums": [ + {"algorithm": "SHA1", "checksumValue": entry.sha1}, + {"algorithm": "SHA256", "checksumValue": entry.sha256}, + ], + "licenseConcluded": NOASSERTION, + "copyrightText": (attribution.copyright_text if attribution else None) + or NOASSERTION, + } + if attribution and attribution.spdx_tag: + record["licenseConcluded"] = attribution.spdx_tag + record["licenseInfoInFiles"] = [attribution.spdx_tag] + stated_in_files.add(attribution.spdx_tag) + else: + # Missing file declarations must remain visible in the package roll-up. + stated_in_files.add(NOASSERTION) + if ( + attribution + and attribution.copyright_text + and not attribution.primary + # Packaged terms may name a component or license author, not this distribution. + and not report.is_packaged_license(wheel.dist_info, entry.name) + ): + stated_copyright.add(attribution.copyright_text) + + comment = [] + if attribution: + comment.append(f"Origin: {attribution.origin}; {attribution.detail}") + if entry.is_elf: + comment.append( + f"ELF {entry.machine}; SONAME {entry.soname or 'none'}; " + f"DT_NEEDED: {', '.join(entry.needed) or 'none'}" + ) + if comment: + record["comment"] = " | ".join(comment) + files.append(record) + + code = verification_code(sha1_digests) + wheel_id = WHEEL_PACKAGE_ID + build = evidence_doc.get("build", {}) + + # The distribution declares its license; the conclusion also includes bundled terms. + wheel_package = { + "SPDXID": wheel_id, + "name": wheel.distribution, + "versionInfo": wheel.version, + "packageFileName": wheel.path.name, + "supplier": project["supplier"], + "originator": project["supplier"], + "downloadLocation": NOASSERTION, + "filesAnalyzed": True, + "homepage": project["homepage"], + "licenseConcluded": _wheel_conclusion(project["license"], inventory), + "licenseDeclared": project["license"], + "licenseInfoFromFiles": sorted(stated_in_files) or [NOASSERTION], + # Metadata-only wheels need the project notice as a fallback. + "copyrightText": "\n".join(licensing.fold_notices(stated_copyright)) + or project.get("copyright", NOASSERTION), + "packageVerificationCode": { + "packageVerificationCodeValue": code, + "packageVerificationCodeExcludedFiles": [ + f"./{name}" for name in sorted(excluded_files) + ], + }, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": PackageURL( + type="pypi", name=wheel.distribution, version=wheel.version + ).to_string(), + } + ], + "hasFiles": [file_ids[name] for name in sorted(file_ids)], + "sourceInfo": _source_info(project, evidence_doc, build, wheel), + } + + packages = [wheel_package] + relationships = [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": wheel_id, + "relationshipType": "DESCRIBES", + } + ] + + for key, component in inventory.components_present.items(): + roles = inventory.component_roles.get(key, set()) + package = _component_package(component, roles, wheel.dist_info) + packages.append(package) + package_id = package["SPDXID"] + + if roles & _LINK_HOWS: + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "STATIC_LINK", + } + ) + if roles - _LINK_HOWS - _AMBIGUOUS_HOWS: + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "CONTAINS", + } + ) + + for name in inventory.files_of(key): + if name not in file_ids: + continue + attribution = inventory.attributions[name] + exact = attribution.exact + relationships.append( + { + "spdxElementId": file_ids[name], + "relatedSpdxElement": package_id, + "relationshipType": "COPY_OF" if exact else "GENERATED_FROM", + } + ) + + # Ambiguous files have no primary owner; encode their candidate relationships too. + for name in inventory.files_sharing(key): + if name in file_ids: + relationships.append( + { + "spdxElementId": file_ids[name], + "relatedSpdxElement": package_id, + "relationshipType": "OTHER", + } + ) + + for soname, consumers in inventory.external_runtime.items(): + package_id = _spdx_id("Package-external", soname) + packages.append( + _unanalyzed_package( + package_id, + soname, + "Supplied by the installation environment, not redistributed in " + f"this wheel. Required by: {', '.join(consumers)}.", + ) + ) + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "DEPENDS_ON", + } + ) + + # Group repeated requirements by package and version range; markers belong + # to relationships, not separate package nodes. + wanted: dict[tuple[str, str], list[Requirement]] = {} + for requirement in inventory.requires_dist: + parsed = Requirement(requirement) + wanted.setdefault((parsed.name, str(parsed.specifier)), []).append(parsed) + + for (name, _), group in sorted(wanted.items()): + package_id = _spdx_id( + "Package-pypi", + f"{name}-{hashlib.sha256(str(sorted(map(str, group))).encode()).hexdigest()[:8]}", + ) + extras = sorted({extra for item in group for extra in item.extras}) + declared = ( + "Consumer requirement declared in wheel metadata: " + # " | ", not "; ": a requirement string contains its own semicolon + # before the marker. + + " | ".join(sorted(str(item) for item in group)) + + "." + + (f" Requested with: {', '.join(extras)}." if extras else "") + ) + sibling = (siblings or {}).get(canonicalize_name(name)) + if sibling is None: + element = package_id + comment = ( + f"{declared} Resolved at install time; this document states no " + "license for it." + ) + packages.append( + _unanalyzed_package( + package_id, name, comment, _dependency_refs(name, group[0]) + ) + ) + else: + # Use sibling metadata, not document hashes: universal wheels must be identical + # across matrix entries, and cyclic requirements cannot have cyclic hashes. + element = package_id + comment = ( + f"{declared} Built by this same build; its own document " + "describes its contents." + ) + packages.append( + { + "SPDXID": package_id, + "name": name, + "versionInfo": sibling["version"], + "supplier": sibling["supplier"], + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": sibling["license_declared"], + "copyrightText": NOASSERTION, + "externalRefs": _dependency_refs( + name, group[0], version=sibling["version"] + ), + "comment": comment, + } + ) + # Optional only if every line that asks for it is gated on an extra. One + # unconditional line makes the dependency unconditional. + if all(_is_extra_gated(item) for item in group): + relationships.append( + { + "spdxElementId": element, + "relatedSpdxElement": wheel_id, + "relationshipType": "OPTIONAL_DEPENDENCY_OF", + "comment": comment, + } + ) + else: + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": element, + "relationshipType": "DEPENDS_ON", + "comment": comment, + } + ) + + stamp = stamped_now() + + extracted = _extracted_entries(list(inventory.components_present.values())) + + return { + "spdxVersion": SPDX_VERSION, + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": wheel.path.name.removesuffix(".whl"), + "documentNamespace": f"{project['document_namespace']}/{wheel.path.name}/{code}", + "creationInfo": { + "created": stamp.strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": [ + f"Tool: {TOOL_NAME}-{TOOL_VERSION}", + f"Organization: {project['supplier'].split(': ', 1)[-1]}", + ], + # Record the corpus actually used for identification, not a default version. + "licenseListVersion": ".".join( + str(evidence_doc["license_list_version"]).split(".")[:2] + ), + "comment": ( + "Contents inventory for one built wheel, discovered from the build " + "that produced it. Build-time-only inputs, toolchains and " + "source-distribution contents are out of scope here." + ), + }, + "packages": packages, + "files": files, + "relationships": relationships, + "hasExtractedLicensingInfos": extracted, + } diff --git a/scripts/sbom/isaaccapture_sbom/elf.py b/scripts/sbom/isaaccapture_sbom/elf.py new file mode 100644 index 0000000000..fcc6701b76 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/elf.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read SONAME, NEEDED, and GNU build IDs with pyelftools. + +Only collection parses ELF; consumer verification hashes bytes without +invoking a binary parser. +""" + +from __future__ import annotations + +import io +import struct +from dataclasses import dataclass +from pathlib import Path + +from elftools.common.exceptions import ELFError +from elftools.elf.elffile import ELFFile + +ELF_MAGIC = b"\x7fELF" + +# pyelftools' arch names, mapped to what a wheel's platform tag calls them. +_ARCH = {"x64": "x86_64", "x86": "i386", "AArch64": "aarch64", "ARM": "arm"} + + +@dataclass(frozen=True) +class DynamicInfo: + soname: str | None + needed: tuple[str, ...] + machine: str + # The linker's build ID survives patchelf SONAME/RPATH rewrites. + build_id: str | None = None + + +class NotAnElf(Exception): + """The blob does not start with the ELF magic.""" + + +def is_elf(data: bytes) -> bool: + return data[:4] == ELF_MAGIC + + +def _build_id_from(elffile: ELFFile) -> str | None: + # Segment, not section: a stripped library keeps its program headers. + for segment in elffile.iter_segments(type="PT_NOTE"): + for note in segment.iter_notes(): + if note["n_type"] == "NT_GNU_BUILD_ID": + return note["n_desc"] + return None + + +def build_id_of(data: bytes) -> str | None: + """Read a build ID from an ELF prefix; return None if the note lies beyond it.""" + if not is_elf(data): + return None + try: + return _build_id_from(ELFFile(io.BytesIO(data))) + except (ELFError, OSError, struct.error): + return None + + +def _from_file(path: Path, read): + """Apply read to an ELFFile without loading the whole binary. + + Unreadable or truncated ELF returns None, including parser struct.error failures. + """ + try: + with path.open("rb") as handle: + if handle.read(4) != ELF_MAGIC: + return None + handle.seek(0) + return read(ELFFile(handle)) + except (OSError, ELFError, struct.error): + return None + + +def read_build_id(path: Path) -> str | None: + """The build-id of a file on disk, read without loading it.""" + return _from_file(path, _build_id_from) + + +def read_dynamic(data: bytes) -> DynamicInfo: + """Parse SONAME, NEEDED and the build-id out of an in-memory ELF image.""" + if not is_elf(data): + raise NotAnElf("missing ELF magic") + + elffile = ELFFile(io.BytesIO(data)) + machine = elffile.get_machine_arch() + + soname: str | None = None + needed: list[str] = [] + + # Read the dynamic segment: stripped libraries can lack section headers. + for segment in elffile.iter_segments(type="PT_DYNAMIC"): + for tag in segment.iter_tags(): + kind = tag.entry.d_tag + if kind == "DT_SONAME": + soname = tag.soname + elif kind == "DT_NEEDED": + needed.append(tag.needed) + + return DynamicInfo( + soname=soname, + needed=tuple(needed), + machine=_ARCH.get(machine, machine), + build_id=_build_id_from(elffile), + ) + + +def file_type(path: Path) -> str | None: + """A file's ELF `e_type` -- `ET_REL` for an object, `ET_DYN` for a shared + library -- or None if it is not an ELF file.""" + return _from_file(path, lambda elffile: elffile.header["e_type"]) diff --git a/scripts/sbom/isaaccapture_sbom/evidence.py b/scripts/sbom/isaaccapture_sbom/evidence.py new file mode 100644 index 0000000000..30f7b59e50 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/evidence.py @@ -0,0 +1,379 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assemble every discovery index for one configured build tree.""" + +from __future__ import annotations + +import hashlib +import platform +import re +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlparse + +from packageurl import PackageURL + +from . import TOOL_NAME, TOOL_VERSION, licensing, stamped_now +from .discovery import ( + ARCHIVE_SUFFIXES, + ArchiveIndex, + component_of, + Ownership, + git, + BuildGraph, + Component, + cmake_cache, + FileIndex, + discover_archives, + discover_vendored, + project_authors, + discover_source_trees, + discover_vcpkg, + notice_from, + tracked_files, + version_in_name, + EMPTY_SHA256, +) + +# Record reproducibility settings without unrelated builder-local cache paths. +_CACHE_KEYS = re.compile( + r"^(CMAKE_BUILD_TYPE|CMAKE_CXX_COMPILER|CMAKE_C_COMPILER|CMAKE_SYSTEM_PROCESSOR" + r"|CMAKE_CXX_COMPILER_VERSION|ISAAC_TELEOP_PYTHON_VERSION|BUILD_[A-Z0-9_]+" + r"|ENABLE_[A-Z0-9_]+|VCPKG_[A-Z0-9_]+)$" +) + + +class EvidenceError(Exception): + """The build tree does not carry the evidence a published wheel needs.""" + + +@dataclass +class Discovery: + """Everything one build tree can tell us, indexed for attribution.""" + + repo_root: Path + build_dir: Path + config: str + cache: dict[str, str] + components: dict[str, Component] + graph: BuildGraph + archives: ArchiveIndex + source_files: FileIndex + repo_files: FileIndex + build_files: FileIndex + ownership: Ownership + # build_files display prefix -> component whose tree the build staged there. + staged_component_dirs: dict[str, str] + + @property + def staged_root(self) -> Path | None: + staged = self.build_dir / "python_package" / self.config + return staged if staged.is_dir() else None + + +def _cmake_cache(build_dir: Path) -> dict[str, str]: + cache = build_dir / "CMakeCache.txt" + if not cache.is_file(): + raise EvidenceError( + f"{cache} not found; point --build-dir at a configured build tree" + ) + return { + name: value + for name, value in cmake_cache(build_dir).items() + if _CACHE_KEYS.match(name) + } + + +def _fetched_from(archive: Path) -> str | None: + """Read the download URL preserved beside an archive as a .source marker.""" + marker = archive.with_name(archive.name + ".source") + if not marker.is_file(): + return None + url = marker.read_text(encoding="utf-8", errors="replace").strip() + return url.splitlines()[0].strip() if url else None + + +def _supplier_of(url: str | None) -> str: + """Infer a supplier from the URL's penultimate host label when available.""" + host = urlparse(url).hostname if url else None + if not host: + return "NOASSERTION" + labels = host.split(".") + # For api.ngc.nvidia.com this yields nvidia; this is not a public-suffix parser. + name = labels[-2] if len(labels) >= 2 else "" + return f"Organization: {name}" if len(name) > 2 else "NOASSERTION" + + +def _archive_components( + archives: ArchiveIndex, repo_root: Path +) -> dict[str, Component]: + """Describe each indexed archive with its root license evidence.""" + components: dict[str, Component] = {} + for display, record in archives.archives.items(): + name = Path(display).name + for suffix in ARCHIVE_SUFFIXES: + name = name.removesuffix(suffix) + + evidence = [ + licensing.evidence_for( + display, + "archive-member", + f"{display}!{member.path}", + data.decode("utf-8", "replace"), + # Sidecars were set aside when the member was kept. + licensing.classify(Path(member.path).name, lambda _: False) or "grant", + ) + for member, data in record.get("license_members", []) + ] + # Use the full display path to distinguish archives with the same basename. + concluded, declared = licensing.expression(evidence, display) + + source_url = _fetched_from(repo_root / display) + held = ( + f"fetched from {source_url}" if source_url else "present in the source tree" + ) + components[display] = Component( + key=display, + name=name, + kind="archive", + supplier=_supplier_of(source_url), + homepage="NOASSERTION", + purl=PackageURL(type="generic", name=name).to_string(), + version=_archive_version(archives, display) or "NOASSERTION", + download_location=source_url or "NOASSERTION", + source_info=( + f"Unpacked from {display} (sha256:{record['sha256']}, " + f"{record['member_count']} files), {held}." + ), + license_concluded=concluded, + license_declared=declared, + copyright_text=notice_from(evidence), + evidence=tuple(evidence), + ) + return components + + +def _archive_version(archives: ArchiveIndex, display: str) -> str | None: + """SDK tarballs state their version in a root VERSION member, or in the name.""" + stated = archives.archives.get(display, {}).get("version_text") + if stated: + return stated + return version_in_name(Path(display).name) + + +def discover(repo_root: Path, build_dir: Path) -> Discovery: + """Index a configured build tree.""" + cache = _cmake_cache(build_dir) + deps_dir = build_dir / "_deps" + if not deps_dir.is_dir(): + raise EvidenceError( + f"{deps_dir} not found; configure the build before collecting evidence" + ) + + # Require the configuration that produced the graph and staged package tree. + config = cache.get("CMAKE_BUILD_TYPE") + if not config: + raise EvidenceError( + f"{build_dir}/CMakeCache.txt states no CMAKE_BUILD_TYPE; the build " + "graph and the staged tree are both read per configuration" + ) + graph = BuildGraph(build_dir, config) + # Exclude other build trees too; their staged copies create false source matches. + not_source = ( + build_dir, + repo_root / "build-wheel", + repo_root / "dist", + *( + path.parent + for pattern in ("*/CMakeCache.txt", "*/*/CMakeCache.txt") + for path in repo_root.glob(pattern) + ), + ) + + tracked = tracked_files(repo_root) + components = discover_source_trees(deps_dir) + archives = discover_archives(repo_root, frozenset(tracked), skip=not_source) + components.update(_archive_components(archives, repo_root)) + + source_files = FileIndex() + for root in sorted(deps_dir.glob("*-src")): + if root.is_dir(): + source_files.add_tree(root, root.name) + + # Only tracked files are repository sources; install prefixes have no identifying marker. + repo_files = FileIndex() + repo_files.add_paths(repo_root, ".", tracked) + + # Index generated output, including _deps/*-build, separately from fetched sources. + # Exclude staging so the wheel cannot explain itself. Preserve the actual build + # directory name in evidence paths. + build_files = FileIndex() + build_files.add_tree( + build_dir, + build_dir.name, + skip=(*sorted(deps_dir.glob("*-src")), build_dir / "python_package"), + ) + + # Infer staged-tree ownership from unchanged siblings after transformations. + # Require a unique owner to avoid inventing a license obligation. + staged_component_dirs: dict[str, str] = {} + for root in sorted(deps_dir.iterdir()): + if not root.is_dir() or root.name.endswith("-src"): + continue + display = f"{build_dir.name}/{root.relative_to(build_dir).as_posix()}" + owners = set() + for _, digest in build_files.under(display): + # Empty files cannot identify an owner. + if digest == EMPTY_SHA256: + continue + match = source_files.by_hash.get(digest) + if match: + owners.add(component_of(match)) + if len(owners) == 1: + staged_component_dirs[display] = owners.pop() + + ownership = Ownership() + for root in sorted(deps_dir.glob("*-src")): + if root.is_dir(): + ownership.register_root(root, root.name.removesuffix("-src")) + # Generated dependency headers belong to their component, not the root project. + for root in sorted(deps_dir.glob("*-build")): + if root.is_dir(): + ownership.register_root(root, root.name.removesuffix("-build")) + + ports, port_files = discover_vcpkg( + Path(cache.get("VCPKG_INSTALLED_DIR") or build_dir / "vcpkg_installed"), + cache.get("VCPKG_TARGET_TRIPLET"), + ) + components.update(ports) + for key, paths in port_files.items(): + for path in paths: + ownership.register_file(path, key, by_directory=False) + + # Include tracked data and Python as well as native build inputs. + vendored, owned_paths = discover_vendored( + repo_root, + graph.repo_inputs(repo_root, build_dir), + project_authors(repo_root), + tracked=tracked, + ) + components.update(vendored) + for key, paths in owned_paths.items(): + for path in paths: + ownership.register_file(path, key) + + return Discovery( + repo_root=repo_root, + build_dir=build_dir, + config=config, + cache=cache, + components=components, + graph=graph, + archives=archives, + source_files=source_files, + repo_files=repo_files, + build_files=build_files, + ownership=ownership, + staged_component_dirs=staged_component_dirs, + ) + + +def document( + discovery: Discovery, wheel_name: str, inventory, components: dict[str, Component] +) -> dict: + """Render build evidence using this wheel's resolved components. + + Keep host libraries per wheel; Discovery is shared across the run. + """ + repo_root = discovery.repo_root + arch = { + "x86_64": "amd64", + "AMD64": "amd64", + "aarch64": "arm64", + "arm64": "arm64", + }.get( + discovery.cache.get("CMAKE_SYSTEM_PROCESSOR", platform.machine()), + platform.machine(), + ) + + artifacts = [] + for artifact in discovery.graph.artifacts.values(): + contributions = discovery.graph.contributions(artifact, discovery.ownership) + # Resolved to components only where a wheel member links them; see + # `static_inputs`. + inputs = sorted({item.key for item in discovery.graph.static_inputs(artifact)}) + if not contributions and not inputs: + continue + try: + output = artifact.output.relative_to(discovery.build_dir).as_posix() + except ValueError: + output = artifact.output.as_posix() + artifacts.append( + { + "target": artifact.target, + "output": output, + "first_party": discovery.graph.first_party( + artifact, repo_root, discovery.ownership + ), + "contributions": { + key: sorted(how) for key, how in sorted(contributions.items()) + }, + **({"static_inputs": inputs} if inputs else {}), + } + ) + + return { + "schema": "isaaccapture-build-evidence/2", + "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, + "collected_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), + "wheel": wheel_name, + "project": { + "commit": git(repo_root, "rev-parse", "HEAD"), + "describe": git(repo_root, "describe", "--tags", "--always", "--dirty"), + "version_file": (repo_root / "VERSION").read_text(encoding="utf-8").strip() + if (repo_root / "VERSION").is_file() + else None, + }, + "build": { + "build_dir": str(discovery.build_dir), + "staged_root": ( + str(discovery.staged_root) if discovery.staged_root else None + ), + "config": discovery.config, + "arch": arch, + "platform": platform.platform(), + "cmake_cache": discovery.cache, + }, + "license_data": licensing.corpus_provenance(), + "license_list_version": licensing.corpus()[0], + "components": { + key: { + "name": component.name, + "kind": component.kind, + "supplier": component.supplier, + "homepage": component.homepage, + "purl": component.purl, + "version": component.version, + "download_location": component.download_location, + "source_info": component.source_info, + "license_concluded": component.license_concluded, + "license_declared": component.license_declared, + "evidence": [item.as_json() for item in component.evidence], + # Bind the combined terms as well as individual files so LicenseRef text is verifiable. + "verbatim_terms_sha256": hashlib.sha256( + licensing.verbatim_terms(component.evidence).encode("utf-8") + ).hexdigest(), + "in_this_wheel": key in inventory.components_present, + } + for key, component in sorted(components.items()) + }, + "archives": { + display: { + key: value for key, value in record.items() if key != "license_members" + } + for display, record in sorted(discovery.archives.archives.items()) + }, + "link_artifacts": sorted(artifacts, key=lambda item: item["output"]), + "attributions": [item.as_json() for item in inventory.attributions.values()], + } diff --git a/scripts/sbom/isaaccapture_sbom/inventory.py b/scripts/sbom/isaaccapture_sbom/inventory.py new file mode 100644 index 0000000000..3807399ca1 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/inventory.py @@ -0,0 +1,784 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Attribute wheel members to build inputs; unexplained members block publication.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import hashlib +import re +from pathlib import Path +from typing import TYPE_CHECKING + +from . import elf, licensing +from .discovery import ( + Component, + EMPTY_SHA256, + LinkInput, + note_reach, + component_of, + owned_input, + resolve_static_archive, + resolve_system_library, + resolve_system_library_by_build_id, + sha256_file, + system_component, +) +from . import wheelfile +from .wheelfile import Entry, WheelInfo + +if TYPE_CHECKING: # pragma: no cover + from .evidence import Discovery + +# File types whose contents may carry REUSE tags. +_TAGGABLE = { + ".py", + ".pyi", + ".txt", + ".cfg", + ".toml", + ".yaml", + ".yml", + ".xml", + ".json", + ".md", +} + + +@dataclass +class Attribution: + """One wheel member and the evidence that explains it.""" + + path: str + origin: str + detail: str + components: dict[str, set[str]] = field(default_factory=dict) + primary: str | None = None + spdx_tag: str | None = None + copyright_text: str | None = None + # Byte identity is independent of origin; do not infer it from the attribution route. + exact: bool = False + # Path within the component, when known. + source_path: str | None = None + # Component -> source path -> discovery.REACH method. + read: dict[str, dict[str, str]] = field(default_factory=dict) + + def as_json(self) -> dict: + return { + "path": self.path, + "origin": self.origin, + "detail": self.detail, + "primary": self.primary, + "source_path": self.source_path, + "components": { + key: sorted(how) for key, how in sorted(self.components.items()) + }, + "spdx_tag": self.spdx_tag, + } + + +@dataclass +class Inventory: + attributions: dict[str, Attribution] + components_present: dict[str, Component] + component_roles: dict[str, set[str]] + external_runtime: dict[str, list[str]] + requires_dist: list[str] + unattributed: list[str] = field(default_factory=list) + # Link-line inputs nothing explains, as ` (linked by ...): `. + untraced: list[str] = field(default_factory=list) + _indexed: tuple[dict, dict] | None = field(default=None, repr=False) + + def files_of(self, key: str) -> list[str]: + return self._by_component()[0].get(key, []) + + def files_sharing(self, key: str) -> list[str]: + """Files naming this component as one candidate among several.""" + return self._by_component()[1].get(key, []) + + def _by_component(self) -> tuple[dict[str, list[str]], dict[str, list[str]]]: + """Cache the component index to avoid rescanning members for each lookup.""" + if self._indexed is None: + owned: dict[str, list[str]] = {} + shared: dict[str, list[str]] = {} + for path in sorted(self.attributions): + item = self.attributions[path] + if item.primary: + owned.setdefault(item.primary, []).append(path) + continue + for key, how in item.components.items(): + if "shared-content" in how: + shared.setdefault(key, []).append(path) + self._indexed = (owned, shared) + return self._indexed + + +class Resolver: + """Probes a candidate's bytes and path against every discovered index.""" + + def __init__(self, discovery: Discovery, system_resolver=None) -> None: + self._system_resolver = system_resolver or resolve_system_library + self.system_libraries: dict[str, dict] = {} + # External static inputs by path; unexplained entries fail the build gate. + self.static_inputs: dict[str, dict] = {} + self._input_keys: dict[str, str] = {} + self.repo_root = discovery.repo_root + self.build_dir = discovery.build_dir + self.graph = discovery.graph + # Copy before adding host libraries: Discovery is shared across wheels. + self.components = dict(discovery.components) + self.archives = discovery.archives + self.repo_files = discovery.repo_files + self.build_files = discovery.build_files + self.ownership = discovery.ownership + self.source_files = discovery.source_files + self.staged_component_dirs = discovery.staged_component_dirs + self._artifacts_by_hash: dict[str, list] = {} + self._artifacts_by_build_id: dict[str, list] = {} + self._build_id_of: dict[Path, str | None] = {} + for artifact in self.graph.artifacts.values(): + if artifact.output.is_file(): + # build_files already hashed everything under the build tree. + digest = self.build_files.digest_of(artifact.output) or sha256_file( + artifact.output + ) + self._artifacts_by_hash.setdefault(digest, []).append(artifact) + build_id = elf.read_build_id(artifact.output) + self._build_id_of[artifact.output] = build_id + if build_id: + self._artifacts_by_build_id.setdefault(build_id, []).append( + artifact + ) + + def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: + contributions = self.graph.contributions(artifact, self.ownership) + linked = self.graph.static_inputs(artifact) + for item in sorted(linked, key=lambda item: (item.key, item.named)): + if owned_input(item, self.ownership): + continue + key = self._static_input(item, linked[item]) + if key: + contributions.setdefault(key, set()).add("prebuilt-library") + first_party = self.graph.first_party(artifact, self.repo_root, self.ownership) + primary = None + if not first_party: + # The component that supplied the compiled sources owns the binary; + # the rest are linked into it. + sources = [ + key for key, how in contributions.items() if "compiled-source" in how + ] + primary = min( + sources, key=lambda key: (key != artifact.target, key), default=None + ) + return Attribution( + path=path, + origin="built", + detail=f"{detail}; linked by target {artifact.target}", + components={key: set(how) for key, how in contributions.items()}, + primary=primary, + read=self._displayed(self.graph.reached(artifact, self.ownership)), + ) + + def _displayed( + self, reached: dict[str, dict[Path, str]] + ) -> dict[str, dict[str, str]]: + """Render reached paths as /. + + Build-tree files retain their position and any byte-identical source paths. + An include directory enclosing the tree represents the component root. + """ + shown: dict[str, dict[str, str]] = {} + for key, paths in reached.items(): + component = self.components.get(key) + if component is None or component.root is None: + continue + name = component.root.name + source = component.root.resolve() + trees = [ + source, + *(root for root in self.ownership.roots_of(key) if root != source), + ] + into = shown.setdefault(key, {}) + for path, how in paths.items(): + tree = next((root for root in trees if path.is_relative_to(root)), None) + if tree is None: + if how == "include-dir" and any( + root.is_relative_to(path) for root in trees + ): + note_reach(into, name, how) + continue + for display in self._names_in(name, tree != source, tree, path): + note_reach(into, display, how) + return shown + + def _names_in(self, name: str, built: bool, tree: Path, path: Path) -> list[str]: + """`path` as named in the source tree `name`; see `_displayed`.""" + positional = (Path(name) / path.relative_to(tree)).as_posix() + digest = self.build_files.digest_of(path) if built else None + # An empty file matches every other one, which says nothing. + if digest is None or digest == EMPTY_SHA256: + return [positional] + same = [ + display + for display in self.source_files.all_by_hash.get(digest, []) + if display.startswith(name + "/") + ] + return [*same, positional] + + def _static_input(self, item: LinkInput, targets: set[str]) -> str | None: + record = self.static_inputs.get(item.key) + if record is None: + record = self.static_inputs[item.key] = self._input_record(item) + record["named_as"] = sorted({*record["named_as"], item.named}) + record["linked_by"] = sorted({*record["linked_by"], *targets}) + return record["component"] + + def _input_record(self, item: LinkInput) -> dict: + """Resolve an unowned static input against the repository and build host.""" + record: dict = { + "kind": item.kind, + "named_as": [], + "linked_by": [], + "component": None, + "explained": True, + } + + def refused(outcome: str) -> dict: + record["explained"] = False + record["outcome"] = outcome + return record + + if item.path is None or item.kind == "unresolved": + return refused(item.reason) + path = item.path + if item.kind == "archive": + record["members"] = item.members + if item.members == 0: + record["outcome"] = "contributes no code: the archive has no members" + return record + if path.is_relative_to(self.build_dir.resolve()): + return refused( + "inside the build tree, but no target produced it and nothing " + "records which component installed it" + ) + if self.repo_files.digest_of(path): + # The vendored scan registers third-party holders; remaining tracked inputs are ours. + record["outcome"] = "tracked in this repository, held by this project" + return record + + origin = resolve_static_archive(path) + record.update(origin) + if not (origin["package"] or origin["toolkit_root"]): + return refused( + "no package on the build machine owns it and no known toolkit holds it" + if origin["resolved_path"] + else "not found on the build machine" + ) + component = system_component( + path.name, + origin, + route=f"Statically linked from {path}.", + key=self._input_key(path), + ) + self.components[component.key] = component + record["component"] = component.key + record["outcome"] = ( + f"supplied by system package {origin['package']}" + if origin["package"] + else f"part of the toolkit at {origin['toolkit_root']}" + ) + return record + + def _input_key(self, path: Path) -> str: + """Mint a system key, distinguishing static files with the same basename.""" + key = f"system:{path.name}" + if self._input_keys.get(key, str(path)) != str(path): + key += "@" + hashlib.sha256(str(path).encode()).hexdigest()[:8] + self._input_keys[key] = str(path) + return key + + def resolve( + self, + wheel_path: str, + digest: str, + probe: Path | None, + soname: str | None = None, + build_id: str | None = None, + ) -> Attribution | None: + """Explain one member: by content, then by path, then by the machine.""" + candidate_digest = digest + source = "wheel member" + patched = False + if probe is not None and probe.is_file(): + candidate_digest = sha256_file(probe) + # Staged bytes reveal origin; repaired wheel bytes may differ and must be + # reported as derived rather than copied. + patched = candidate_digest != digest + relative = probe.relative_to(self.build_dir).as_posix() + # Show the staged path only when it differs from the wheel member's path. + source = ( + "staged" + if relative.endswith(f"/{wheel_path}") + else f"staged at {relative}" + ) + + found = self._trace( + wheel_path, candidate_digest, digest, source, soname, build_id + ) + if found is not None and patched: + found.origin = { + "copied": "derived", + "archive-copy": "archive-derived", + }.get(found.origin, found.origin) + found.exact = False + found.detail += ( + "; the member differs from that staged file, so it was rewritten " + "after staging" + ) + return found + + def _trace( + self, + wheel_path: str, + candidate_digest: str, + digest: str, + source: str, + soname: str | None, + build_id: str | None, + ) -> Attribution | None: + # Empty content matches every empty file; only a path can establish its origin. + if candidate_digest == EMPTY_SHA256: + return self._resolve_by_path(wheel_path, source, build_id, candidate_digest) + + artifacts = self._artifacts_by_hash.get(candidate_digest) + if artifacts: + return self._from_artifact( + artifacts[0], wheel_path, f"{source}; exact build output" + ) + + # Shared bytes (for example a standard license text) do not prove ownership. + # Use the path to disambiguate or retain every candidate. + shared = self._shared_content(candidate_digest) + if shared is not None: + by_path = self._resolve_by_path( + wheel_path, source, build_id, candidate_digest + ) + if by_path is not None: + return by_path + return shared(wheel_path, source) + + component_file = self.source_files.by_hash.get(candidate_digest) + if component_file: + key = component_of(component_file) + return Attribution( + path=wheel_path, + origin="copied", + exact=True, + detail=f"{source}; byte-identical to {component_file}", + components={key: {"copied-file"}}, + primary=key, + source_path=component_file, + read={key: {component_file: "copied-file"}}, + ) + + member = self.archives.by_hash.get(candidate_digest) + if member: + return Attribution( + path=wheel_path, + origin="archive-copy", + exact=True, + detail=f"{source}; byte-identical to {member.path} in {member.container}", + components={member.container: {"extracted-file"}}, + primary=member.container, + ) + + repo_file = self.repo_files.by_hash.get(candidate_digest) + if repo_file: + return self._from_repo_file(wheel_path, repo_file, f"{source}; ", True) + + build_file = self.build_files.by_hash.get(candidate_digest) + if build_file: + # Prefer the wheel's path when several build files contain identical bytes. + if Path(build_file).name != Path(wheel_path).name: + by_path = self.build_files.path_suffix_match(wheel_path) + if by_path: + build_file = by_path + owner = self._staged_component_of(build_file) + # A build-tree match proves a copy, but does not identify the source it was made from. + detail = f"{source}; byte-identical to {build_file}" + if owner: + detail += ( + f", in a tree the build staged for {owner}; the input it was" + " produced from is not recorded" + ) + return Attribution( + path=wheel_path, + # Generated output owned by a component still carries that component's terms. + origin="derived" if owner else "generated", + detail=detail, + components={owner: {"derived-file"}} if owner else {}, + primary=owner, + ) + + # Build IDs survive repair and provide stronger evidence than a matching name. + if build_id: + by_build_id = self._resolve_by_build_id(wheel_path, source, build_id) + if by_build_id is not None: + return by_build_id + + by_path = self._resolve_by_path(wheel_path, source, build_id, candidate_digest) + if by_path is not None: + return by_path + + if build_id: + from_host = self._from_build_host_by_build_id(wheel_path, build_id) + if from_host is not None: + return from_host + + # Consult the host last, regardless of the directory auditwheel used. + if soname: + return self._from_build_host(wheel_path, soname) + return None + + def _from_repo_file( + self, wheel_path: str, repo_file: str, lead: str, same: bool + ) -> Attribution: + """Attribute repository files while retaining any third-party ownership.""" + owner = self.ownership.owner_of(self.repo_root / repo_file.removeprefix("./")) + return Attribution( + path=wheel_path, + origin="repo-source", + exact=same, + detail=f"{lead}{repo_file}", + components={owner: {"copied-file" if same else "derived-file"}} + if owner + else {}, + primary=owner, + ) + + def _staged_component_of(self, build_file: str) -> str | None: + """The component whose tree this build staged under _deps//.""" + for prefix, key in self.staged_component_dirs.items(): + if build_file == prefix or build_file.startswith(prefix + "/"): + return key + return None + + def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: + """Require a resolved host library before claiming host origin.""" + origin = self._system_resolver(soname) + if not origin.get("resolved_path"): + return None + self.system_libraries[soname] = origin + component = system_component(soname, origin) + self.components[component.key] = component + + detail = "no index explains these bytes; supplied by the build machine" + if origin.get("package"): + detail += f" as {origin['package']}" + if origin.get("resolved_path"): + detail += f" ({origin['resolved_path']})" + return Attribution( + path=wheel_path, + origin="build-host-library", + detail=detail, + components={component.key: {"vendored-library"}}, + primary=component.key, + ) + + def _shared_content(self, digest: str): + """Return an attribution builder for ambiguous content, otherwise None. + + Retain every candidate's terms without claiming one as the source. + """ + owners: dict[str, str] = {} + for display in self.source_files.all_by_hash.get(digest, []): + owners[component_of(display)] = display + for member in self.archives.all_by_hash.get(digest, []): + owners[member.container] = f"{member.path} in {member.container}" + # Include first-party matches so identical headers cannot imply third-party origin. + ours = self.repo_files.all_by_hash.get(digest, []) + if len(owners) + bool(ours) < 2: + return None + + def build(wheel_path: str, source: str) -> Attribution: + where = ", ".join(sorted([*owners.values(), *ours])) + return Attribution( + path=wheel_path, + origin="shared-content", + detail=( + f"{source}; these bytes are held by more than one component " + f"({where}); the build does not record which supplied this file" + ), + components={key: {"shared-content"} for key in owners}, + ) + + return build + + def _resolve_by_build_id( + self, wheel_path: str, source: str, build_id: str + ) -> Attribution | None: + """The pristine bytes this member was patched from, named by build-id.""" + # SDK archives can share build IDs under different terms; retain all candidates. + members = self.archives.all_by_build_id.get(build_id, []) + containers = {item.container for item in members} + member = members[0] if len(containers) == 1 else None + if member: + return Attribution( + path=wheel_path, + origin="archive-derived", + detail=( + f"{source}; build-id {build_id} matches {member.path} in " + f"{member.container} (origin sha256:{member.sha256})" + ), + components={member.container: {"extracted-file"}}, + primary=member.container, + ) + + artifacts = self._artifacts_by_build_id.get(build_id) + if artifacts: + return self._from_artifact( + artifacts[0], wheel_path, f"{source}; build-id {build_id}, patched" + ) + return None + + def _from_build_host_by_build_id( + self, wheel_path: str, build_id: str + ) -> Attribution | None: + """The machine's own copy, found by build-id after a SONAME rewrite.""" + origin = resolve_system_library_by_build_id(build_id) + if not origin.get("resolved_path"): + return None + soname = origin["soname"] + self.system_libraries[soname] = origin + component = system_component(soname, origin) + self.components[component.key] = component + + detail = ( + f"build-id {build_id} matches the build machine's {origin['resolved_path']}" + ) + if origin.get("package"): + detail += f", supplied by {origin['package']}" + return Attribution( + path=wheel_path, + origin="build-host-library", + detail=detail, + components={component.key: {"vendored-library"}}, + primary=component.key, + ) + + def _resolve_by_path( + self, + wheel_path: str, + source: str, + build_id: str | None = None, + digest: str | None = None, + ) -> Attribution | None: + """Use path evidence when content hashes cannot identify an origin. + + Repaired binaries and stripped MJCF assets can differ from their sources. + Conflicting known build IDs override a name match; equal bytes remain copies. + """ + name = Path(wheel_path).name + + members = [ + member + for member in self.archives.by_name.get(name, []) + if not _build_ids_disagree(build_id, member.build_id) + ] + containers = {member.container for member in members} + if len(containers) == 1: + member = members[0] + return Attribution( + path=wheel_path, + origin="archive-derived", + detail=( + f"{source}; modified copy of {member.path} in {member.container} " + f"(origin sha256:{member.sha256})" + ), + components={member.container: {"extracted-file"}}, + primary=member.container, + ) + + artifacts = [ + artifact + for artifact in self.graph.by_name(name) + if not _build_ids_disagree(build_id, self._build_id_of.get(artifact.output)) + ] + if len(artifacts) == 1: + return self._from_artifact( + artifacts[0], wheel_path, f"{source}; modified build output" + ) + + # Prefer tracked repository paths over coincident dependency suffixes. + # Third-party ownership of tracked files is still preserved. + repo_file = self.repo_files.path_suffix_match(wheel_path) + if repo_file: + same = self.repo_files.digest_of_display(repo_file) == digest + lead = f"{source}; {'identical to' if same else 'derived from'} " + return self._from_repo_file(wheel_path, repo_file, lead, same) + + component_file = self.source_files.path_suffix_match(wheel_path) + if component_file: + key = component_of(component_file) + origin_digest = self.source_files.digest_of_display(component_file) + # Path disambiguation does not imply modification; compare the bytes. + same = origin_digest == digest + return Attribution( + path=wheel_path, + origin="copied" if same else "derived", + exact=same, + detail=( + f"{source}; " + + ("identical to " if same else "derived from ") + + component_file + + ( + "" + if same or not origin_digest + else f" (origin sha256:{origin_digest})" + ) + ), + components={key: {"copied-file" if same else "derived-file"}}, + primary=key, + source_path=component_file, + read={key: {component_file: "copied-file" if same else "derived-file"}}, + ) + + build_file = self.build_files.path_suffix_match(wheel_path) + if build_file: + owner = self._staged_component_of(build_file) + return Attribution( + path=wheel_path, + origin="derived" if owner else "generated", + detail=f"{source}; {build_file}", + components={owner: {"derived-file"}} if owner else {}, + primary=owner, + ) + + return None + + +def build( + wheel: WheelInfo, + metadata_bytes: bytes, + resolver: Resolver, + staged_root: Path | None, +) -> Inventory: + attributions: dict[str, Attribution] = {} + unattributed: list[str] = [] + provided_sonames = {entry.soname for entry in wheel.entries if entry.soname} + + for entry in wheel.entries: + attribution = _attribute(entry, wheel, resolver, staged_root) + if attribution is None: + unattributed.append(entry.name) + continue + attribution.spdx_tag, attribution.copyright_text = _head_facts(entry) + attributions[entry.name] = attribution + + _resolve_shared_by_sibling(attributions) + + roles: dict[str, set[str]] = {} + for attribution in attributions.values(): + for key, how in attribution.components.items(): + roles.setdefault(key, set()).update(how) + + present = { + key: resolver.components[key] + for key in sorted(roles) + if key in resolver.components + } + + external: dict[str, list[str]] = {} + for entry in wheel.entries: + for soname in entry.needed: + if soname not in provided_sonames: + external.setdefault(soname, []).append(entry.name) + + metadata = wheelfile.parse_metadata(metadata_bytes) + return Inventory( + attributions=attributions, + components_present=present, + component_roles=roles, + external_runtime={ + name: sorted(set(files)) for name, files in sorted(external.items()) + }, + requires_dist=sorted(metadata.get("requires_dist") or []), + unattributed=sorted(unattributed), + untraced=sorted( + f"{path} (linked by {', '.join(record['linked_by'])}): {record['outcome']}" + for path, record in resolver.static_inputs.items() + if not record["explained"] + ), + ) + + +def _build_ids_disagree(left: str | None, right: str | None) -> bool: + """True only when both are known and differ -- that is evidence, not absence.""" + return bool(left and right and left != right) + + +def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: + """Resolve identical content using a staged sibling with one known owner. + + For example, SDKs may share libraries but have different libcloudxr.so build + IDs. The uniquely attributed sibling identifies which SDK was staged. + """ + settled: dict[str, set[str]] = {} + for name, item in attributions.items(): + if item.primary: + settled.setdefault(name.rsplit("/", 1)[0], set()).add(item.primary) + + for name, item in attributions.items(): + if item.primary or item.origin != "shared-content": + continue + owners = settled.get(name.rsplit("/", 1)[0], set()) & set(item.components) + if len(owners) != 1: + continue + key = owners.pop() + item.origin = "copied" + item.exact = True + item.detail = ( + f"{item.detail.split(';')[0]}; these bytes are held by more than one " + f"component, and the directory it is staged into is explained by {key}" + ) + item.components = {key: {"copied-file"}} + item.primary = key + + +# Only known backend/tool output is metadata. License files must still trace +# to their sources or to the collector that added them. +_DIST_INFO_WRITTEN = re.compile( + r"^(METADATA|WHEEL|RECORD|INSTALLER|REQUESTED|entry_points\.txt|top_level\.txt" + r"|direct_url\.json|zip-safe|namespace_packages\.txt)$|^sboms/" +) + + +def _attribute( + entry: Entry, + wheel: WheelInfo, + resolver: Resolver, + staged_root: Path | None, +) -> Attribution | None: + if entry.name.startswith(f"{wheel.dist_info}/"): + relative = entry.name[len(wheel.dist_info) + 1 :] + # Do not exempt arbitrary .dist-info files from provenance checks. + if _DIST_INFO_WRITTEN.match(relative): + return Attribution( + path=entry.name, + origin="metadata", + exact=True, + detail="distribution metadata written by the build", + ) + + probe = staged_root / entry.name if staged_root is not None else None + return resolver.resolve( + entry.name, entry.sha256, probe, entry.soname, entry.build_id + ) + + +def _head_facts(entry: Entry) -> tuple[str | None, str | None]: + if Path(entry.name).suffix.lower() not in _TAGGABLE: + return None, None + head = entry.head.decode("utf-8", "replace") + return licensing.normalized_tag(head), licensing.read_copyright(head) diff --git a/scripts/sbom/isaaccapture_sbom/licensing.py b/scripts/sbom/isaaccapture_sbom/licensing.py new file mode 100644 index 0000000000..a4f5308f39 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/licensing.py @@ -0,0 +1,1351 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Identify component license texts against the SPDX corpus. + +Preserve unmatched terms verbatim and report them as unidentified. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Callable +from functools import lru_cache +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING + +from license_expression import ( + ExpressionError, + LicenseWithExceptionSymbol, + combine_expressions, + get_spdx_licensing, +) +from pathlib import Path + +if TYPE_CHECKING: # pragma: no cover + from .discovery import Reach + + +# Match standalone grant names: BSD-LICENSE and LICENSE.md, not license indexes. +# Notices are packaged separately and do not determine the license expression. +_GRANT_NAMES = re.compile( + r"(^|[-_.])(LICEN[CS]E|COPYING|UNLICENSE|EULA)([-_.]|$)", re.IGNORECASE +) +_NOTICE_NAMES = re.compile(r"^(NOTICE|COPYRIGHT|AUTHORS|PATENTS)", re.IGNORECASE) +_README_NAMES = re.compile(r"^README", re.IGNORECASE) +# Skip machine-readable siblings; the text beside them is the evidence. +_SKIP_SUFFIXES = {".py", ".cmake", ".in", ".json", ".yaml", ".yml", ".spdx"} + +_WORD = re.compile(r"[^a-z0-9]+") +_NGRAM = 6 +# A canonical text has to be almost entirely present before it is named. +_CONTAINMENT_THRESHOLD = 0.90 +# Coverage rejects an OSS license quoted in a larger agreement's appendix +# (for example CUDA's EULA) as the agreement's own terms. +_COVERAGE_THRESHOLD = 0.20 +# Carry partially explained grants too: unmatched prose may add terms, as +# in fmt's MIT exception. Exclude copyright notices from this coverage measure. +_EXPLAINED_THRESHOLD = 0.80 +# Two licenses are one family when either text is nearly inside the other. +_FAMILY_THRESHOLD = 0.90 +# REUSE-IgnoreStart +_SPDX_TAG = re.compile(r"SPDX-License-Identifier:\s*(?P[^\n\r*/#]+)") +_FILE_COPYRIGHT = re.compile(r"SPDX-FileCopyrightText:\s*(?P[^\n\r]+)") +# REUSE-IgnoreEnd +# Require a licensing heading; a title such as "License List Data" is not a grant. +_README_LICENSE_SECTION = re.compile( + r"^#{1,6}\s*(licen[cs]e|licen[cs]es|licensing|licen[cs]e information|copyright|legal)" + r"\s*$(?P.*?)(?=^#{1,6}\s|\Z)", + re.IGNORECASE | re.MULTILINE | re.DOTALL, +) + + +@dataclass(frozen=True) +class LicenseEvidence: + """A license text that was read out of the component, not assumed.""" + + component: str + # "component-file" | "archive-member" | "readme-section" | "build-host" + # | "vcpkg-port" + origin: str + path: str + sha256: str + size: int + text: str + # "grant" is the component's own license, "notice" an attribution file, + # "pool" a REUSE LICENSES/ entry that applies to individual files only, and + # "nested" terms stated for one subdirectory of a larger component. + kind: str + # Display text for a reader, not an expression; the ids are in `matches`. + identified: str | None = None + matches: tuple[tuple[str, float, float], ...] = () + spdx_tag: str | None = None + # Nested terms cover a directory or one sidecar-annotated file. reached_by + # records the strongest evidence that this scope contributed to the wheel. + scope: str | None = None + reached_by: Reach | None = None + + def as_json(self) -> dict: + found = { + "component": self.component, + "origin": self.origin, + "path": self.path, + "sha256": self.sha256, + "size": self.size, + "kind": self.kind, + "identified": self.identified, + "matches": [ + { + "license": license_id, + "containment": round(containment, 4), + "coverage": round(coverage, 4), + } + for license_id, containment, coverage in self.matches + ], + "spdx_tag": self.spdx_tag, + } + if self.reached_by: + found["reached_by"] = self.reached_by._asdict() + return found + + +# A REUSE pool satisfies text packaging even when no component-wide license +# can be concluded. Share this predicate across build, report, and verification. +OBLIGATION_KINDS = frozenset({"grant", "pool"}) + + +def satisfies_obligation(kind: str) -> bool: + return kind in OBLIGATION_KINDS + + +def obligation_texts(evidence) -> list[LicenseEvidence]: + return [item for item in evidence if satisfies_obligation(item.kind)] + + +def grant_texts(evidence) -> list[LicenseEvidence]: + return [item for item in evidence if item.kind == "grant"] + + +def pool_texts(evidence) -> list[LicenseEvidence]: + return [item for item in evidence if item.kind == "pool"] + + +def pool_names(paths) -> str: + return ", ".join(sorted(Path(path).stem for path in paths)) + + +def carried_texts(evidence) -> list[LicenseEvidence]: + """The grants a component's LicenseRef stands for.""" + return [item.text for item in _reading(evidence).carried] + + +def carried_and_named(evidence) -> list[LicenseEvidence]: + """Carried grants whose ids the conclusion names as well.""" + return _reading(evidence).also_named + + +def verbatim_terms(evidence) -> str: + """Join carried grants, or all grants if none need carrying. + + Build and verification must use identical joins when binding these terms by hash. + """ + texts = carried_texts(evidence) or grant_texts(evidence) + return "\n\n".join(item.text for item in texts) + + +_DIGITS = re.compile(r"\d+") + + +def _normalize(text: str) -> list[str]: + """Normalize years and clause numbers for matching. + + Retain copyright lines: BSD and MIT clauses contain that word, and deleting + whole lines would remove terms differently depending on upstream wrapping. + """ + return [ + word for word in _WORD.sub(" ", _DIGITS.sub(" ", text.lower())).split() if word + ] + + +def _ngrams(words: list[str]) -> set[str]: + if len(words) < _NGRAM: + return {" ".join(words)} if words else set() + return { + " ".join(words[index : index + _NGRAM]) + for index in range(len(words) - _NGRAM + 1) + } + + +@dataclass(frozen=True) +class Reference: + """An SPDX reference with n-grams for matching and verbatim text for packaging.""" + + name: str + grams: frozenset[str] + text: str = "" + + @property + def size(self) -> int: + return len(self.grams) + + +class CorpusError(Exception): + """The SPDX reference texts this build fetched are missing or unreadable.""" + + +_CORPUS: tuple[str, dict[str, Reference]] | None = None +_CORPUS_META: dict[str, object] = {} +_LISTED: "_Listed | None" = None +# Cache parsing and n-gram construction by corpus directory within this process. +_PARSED: dict[str, tuple[str, dict[str, Reference], "_Listed"]] = {} + + +@dataclass(frozen=True) +class _Listed: + """Every id the pinned list names, deprecated ones included.""" + + licences: frozenset[str] + exceptions: frozenset[str] + # Exception reference texts, from json/exceptions/.json. + exception_texts: dict[str, str] + + +def corpus_provenance() -> dict[str, object]: + """Which reference texts an identification was made against.""" + return dict(_CORPUS_META) + + +def load_corpus(json_dir: Path) -> str: + """Load the pinned SPDX corpus from its json/ directory. + + Retain all identifiers and exception texts for declaration checks; exclude + deprecated license IDs from matching candidates. + """ + global _CORPUS, _LISTED + details = json_dir / "details" + index_file = json_dir / "licenses.json" + if not details.is_dir() or not index_file.is_file(): + raise CorpusError( + f"SPDX license list data not found at {json_dir}. It is fetched by the " + "build (deps/third_party); configure and build, or pass --license-data." + ) + + cached = _PARSED.get(str(details.resolve())) + if cached is not None: + version, entries, listed = cached + else: + entries = {} + for path in sorted(details.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise CorpusError(f"{path} could not be read: {error}") from error + if payload.get("isDeprecatedLicenseId"): + continue + grams = _ngrams(_normalize(payload.get("licenseText", ""))) + if grams: + entries[payload["licenseId"]] = Reference( + name=payload.get("name", payload["licenseId"]), + grams=frozenset(grams), + text=payload.get("licenseText", ""), + ) + + if not entries: + raise CorpusError(f"no usable license texts under {details}") + + index = json.loads(index_file.read_text(encoding="utf-8")) + version = index.get("licenseListVersion", "unknown") + exceptions = json_dir / "exceptions.json" + try: + exception_ids = frozenset( + item["licenseExceptionId"] + for item in json.loads(exceptions.read_text(encoding="utf-8"))[ + "exceptions" + ] + ) + except (OSError, json.JSONDecodeError, KeyError, TypeError) as error: + raise CorpusError( + f"{exceptions} could not be read: {error}. Re-configure the build " + "so deps/third_party fetches it." + ) from error + texts: dict[str, str] = {} + for path in sorted((json_dir / "exceptions").glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise CorpusError(f"{path} could not be read: {error}") from error + if payload.get("licenseExceptionText"): + texts[payload["licenseExceptionId"]] = payload["licenseExceptionText"] + listed = _Listed( + frozenset(item["licenseId"] for item in index["licenses"]), + exception_ids, + texts, + ) + _PARSED[str(details.resolve())] = (version, entries, listed) + + _CORPUS, _LISTED = (version, entries), listed + _CORPUS_META.clear() + _CORPUS_META.update( + { + "license_list_version": version, + "source": str(json_dir), + "commit": _checkout_commit(json_dir.parent), + "licenses": len(entries), + } + ) + return version + + +def _checkout_commit(source_dir: Path) -> str: + """The commit deps/third_party pinned, read back from the checkout itself.""" + + from .discovery import own_git + + return own_git(source_dir, "rev-parse", "HEAD") or "NOASSERTION" + + +def corpus() -> tuple[str, dict[str, Reference]]: + """Return (license-list version, references indexed by identifier).""" + if _CORPUS is None: + raise CorpusError( + "SPDX license list data has not been loaded; it is fetched by the build " + "into _deps/license-list-data-src (see deps/third_party)" + ) + return _CORPUS + + +def identify(text: str) -> list[tuple[str, float, float]]: + """Return (id, canonical-text containment, input-text coverage) matches. + + Both thresholds must pass. Keep distinct matches after family resolution; + do not prune by length, which can miss matches as acceptance thresholds change. + """ + candidate = _ngrams(_normalize(text)) + if not candidate: + return [] + + entries = corpus()[1] + scored: dict[str, tuple[float, float, float]] = {} + for license_id, reference in entries.items(): + overlap = len(reference.grams & candidate) + if not overlap: + continue + containment = overlap / reference.size + coverage = overlap / len(candidate) + if containment >= _CONTAINMENT_THRESHOLD and coverage >= _COVERAGE_THRESHOLD: + scored[license_id] = ( + containment, + coverage, + overlap / len(reference.grams | candidate), + ) + + dominated = _dominated(scored, entries) + return sorted( + (license_id, containment, coverage) + for license_id, (containment, coverage, _) in scored.items() + if license_id not in dominated + ) + + +def _dominated( + scored: dict[str, tuple[float, float, float]], + entries: dict[str, Reference], +) -> set[str]: + """Keep the highest Jaccard match within near-identical license families. + + Length alone would favor ECL-2.0 over Apache-2.0, or CC-BY-NC over CC-BY, + even when the input contains no additional terms. + """ + dropped: set[str] = set() + for left in scored: + for right in scored: + if left == right or right in dropped: + continue + if _family(entries, left, right) and scored[right][2] > scored[left][2]: + dropped.add(left) + break + return dropped + + +def _family(entries: dict[str, Reference], left: str, right: str) -> bool: + if left not in entries or right not in entries: + return False + shared = len(entries[left].grams & entries[right].grams) + return shared / min(entries[left].size, entries[right].size) >= _FAMILY_THRESHOLD + + +def alternatives(matches) -> list[tuple[str, ...]]: + """Group tied matches that their texts cannot distinguish. + + For example, GPL-3.0-only and GPL-3.0-or-later share a reference text. + """ + entries = corpus()[1] + groups: list[list[str]] = [] + for license_id, _, _ in matches: + group = next( + ( + group + for group in groups + if any(_family(entries, license_id, other) for other in group) + ), + None, + ) + if group is None: + groups.append([license_id]) + else: + group.append(license_id) + return [tuple(group) for group in groups] + + +def _name_for(group: tuple[str, ...]) -> str | None: + """Select an unsuffixed base ID when every alternative extends it. + + MPL/OFL variants need a selecting notice; GPL -only/-or-later has no base + ID, so choosing either from the text alone would assert an unsupported grant. + """ + plain = [ + base + for base in group + if all(other == base or other.startswith(f"{base}-") for other in group) + ] + return plain[0] if len(plain) == 1 else None + + +def _either(names) -> str: + return "matches the reference text of " + " or ".join(names) + + +def matched_prose(matches, name=lambda match: match[0], join=" AND ", lead=""): + """Render matches with tied alternatives as prose, not a license conjunction. + + Separate alternative clauses with semicolons so AND cannot join them. + """ + if not matches: + return None + by_id = {match[0]: match for match in matches} + groups = [(group, _name_for(group)) for group in alternatives(matches)] + single = [name(by_id[plain]) for _, plain in groups if plain] + prose = [ + _either(name(by_id[item]) for item in group) + for group, plain in groups + if not plain + ] + parts = ([lead + join.join(single)] if single else []) + prose + return ("; " if prose else join).join(parts) + + +def _read(path: Path) -> str | None: + try: + data = path.read_bytes() + except OSError: + return None + if not data.strip() or b"\0" in data[:4096]: + return None + return data.decode("utf-8", "replace") + + +SPDX_REFERENCE = "spdx-reference" + + +def evidence_for( + component: str, origin: str, display: str, text: str, kind: str +) -> LicenseEvidence: + # Do not score reference texts against their own corpus. Nested matches + # identify license-author notices; they do not conclude the component license. + matchable = kind in ("grant", "nested") and origin != SPDX_REFERENCE + tag = _SPDX_TAG.search(text[:4096]) if matchable else None + matches = tuple(identify(text)) if matchable else () + identified = matched_prose(matches) + if origin == SPDX_REFERENCE: + identified = Path(display).stem + return LicenseEvidence( + component=component, + origin=origin, + path=display, + sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(), + size=len(text.encode("utf-8")), + text=text, + kind=kind, + identified=identified, + matches=matches, + spdx_tag=tag.group("expression").strip() if tag else None, + ) + + +def files_in(directory: Path) -> Callable[[str], bool]: + return lambda name: (directory / name).is_file() + + +def is_sidecar(name: str, beside: Callable[[str], bool]) -> bool: + """A .license file is a sidecar only when its annotated file exists beside it.""" + annotated = name.removesuffix(".license") + return annotated not in {name, ""} and beside(annotated) + + +def classify(name: str, beside: Callable[[str], bool]) -> str | None: + if Path(name).suffix.lower() in _SKIP_SUFFIXES or is_sidecar(name, beside): + return None + if _GRANT_NAMES.search(name): + return "grant" + if _NOTICE_NAMES.search(name): + return "notice" + return None + + +def discover_in_tree( + component: str, root: Path, display_prefix: str +) -> list[LicenseEvidence]: + """Collect root grants/notices, reference pools, and scoped nested terms.""" + found: list[LicenseEvidence] = [] + candidates: list[tuple[Path, str]] = [] + + for entry in sorted(root.iterdir()) if root.is_dir() else []: + if entry.is_file() and (kind := classify(entry.name, files_in(root))): + candidates.append((entry, kind)) + elif entry.is_dir() and entry.name.upper() in {"LICENSES", "LICENSE"}: + candidates.extend( + (item, "pool") for item in sorted(entry.rglob("*")) if item.is_file() + ) + + # Nested terms apply only to reached files (build._with_reached_nested); + # they neither satisfy the root grant requirement nor set the component license. + for path in sorted(root.rglob("*")) if root.is_dir() else []: + if ( + path.is_file() + and path.parent != root + and path.parent.name.upper() not in {"LICENSES", "LICENSE"} + and ( + classify(path.name, files_in(path.parent)) + or _NOTICE_NAMES.match(path.name) + or is_sidecar(path.name, files_in(path.parent)) + ) + ): + candidates.append((path, "nested")) + + for path, kind in candidates: + text = _read(path) + if text is None: + continue + display = f"{display_prefix}/{path.relative_to(root).as_posix()}" + item = evidence_for(component, "component-file", display, text, kind) + if kind == "nested": + covered = ( + path.with_name(path.name.removesuffix(".license")) + if is_sidecar(path.name, files_in(path.parent)) + else path.parent + ) + scope = f"{display_prefix}/{covered.relative_to(root).as_posix()}" + item = replace(item, scope=scope) + found.append(item) + + if not grant_texts(found): + found.extend(_readme_fallback(component, root, display_prefix)) + return found + + +def _readme_fallback( + component: str, root: Path, display_prefix: str +) -> list[LicenseEvidence]: + """Some upstreams state the grant in a README section and ship no license file.""" + for entry in sorted(root.iterdir()) if root.is_dir() else []: + if ( + not entry.is_file() + or not _README_NAMES.match(entry.name) + or is_sidecar(entry.name, files_in(root)) + ): + continue + text = _read(entry) + if text is None: + continue + section = _README_LICENSE_SECTION.search(text) + if section is None or not section.group("body").strip(): + continue + # Hash and package only the licensing section. The path fragment makes its + # digest reproducible and keeps unrelated README prose out of the terms. + display = f"{display_prefix}/{entry.name}#license-section" + return [ + evidence_for( + component, "readme-section", display, section.group(0), "grant" + ) + ] + return [] + + +NOASSERTION = "NOASSERTION" +_ID_SAFE = re.compile(r"[^A-Za-z0-9.-]+") + + +def spdx_safe(value: str) -> str: + """Reduce a string to what an SPDX identifier may contain.""" + return _ID_SAFE.sub("-", value).strip("-") + + +def license_ref(component_key: str) -> str: + """Identifier for a grant that is real but matches no SPDX reference text.""" + return f"LicenseRef-{spdx_safe(component_key)}" + + +def expressible(license_id: str) -> bool: + """Check identifiers against the pinned expression grammar. + + The fetched SPDX list may be newer than license-expression's bundled list; + newer IDs must travel as LicenseRefs until the grammar supports them. + """ + if license_id.startswith("LicenseRef-"): + return True + try: + return not _spdx_licensing().validate(license_id).errors + except (ExpressionError, ValueError, TypeError): + return False + + +def grammar_names(identifiers) -> bool: + return all(expressible(item) or is_exception(item) for item in identifiers) + + +def _explained(text: str, groups) -> float: + """How much of a text its matched `alternatives` groups account for, each + counted once and the text's copyright notices left out.""" + candidate = _ngrams(_normalize(_without_notices(text))) + if not candidate: + return 1.0 + entries = corpus()[1] + return sum( + max(len(entries[key].grams & candidate) for key in group) / len(candidate) + for group in groups + ) + + +def _without_notices(text: str) -> str: + notices = [_normalize(notice) for notice in read_notices(text)] + return "\n".join( + line + for line in text.splitlines() + if not (words := _normalize(line)) + or not any( + notice[index : index + len(words)] == words + for notice in notices + for index in range(len(notice) - len(words) + 1) + ) + ) + + +_NEWER = "newer than the SPDX grammar this document ships with" + + +def why_unnamed(evidence) -> str: + """Why a component's carried grants travel verbatim, as lower-case clauses.""" + carried = _reading(evidence).carried + reasons = [] + if any(grant.unmatched for grant in carried): + reasons.append("matched no text in the SPDX reference corpus") + reasons += [ + f"{_either(group)}, and the text alone does not say which applies" + for group in sorted({group for grant in carried for group in grant.tied}) + ] + reasons += [ + # Round down so a below-threshold result cannot display as meeting the threshold. + "what matched in the SPDX reference corpus accounts for only " + f"{math.floor(grant.explained * 100)}% of {grant.text.path}, its " + "copyright notices aside" + for grant in carried + if grant.partial + ] + declared = sorted( + {key for grant in carried if grant.declared for key in grant.newer} + ) + matched = sorted( + {key for grant in carried if not grant.declared for key in grant.newer} + - set(declared) + ) + if declared: + reasons.append(f"declares {', '.join(declared)}, {_NEWER}") + if matched: + reasons.append(f"matched {', '.join(matched)}, {_NEWER}") + return "; ".join(reasons) + + +def why_unresolved(evidence) -> str: + """Explain NOASSERTION: either a reference pool without a grant, or + grants whose identifiers could not form an expression. + """ + if grants := grant_texts(evidence): + paths = ", ".join(sorted(item.path for item in grants)) + return f"its grants ({paths}) yield no SPDX expression" + pooled = pool_names(item.path for item in pool_texts(evidence)) + return ( + f"only REUSE pool texts ship ({pooled}); no grant states which of them " + "are the component's terms" + ) + + +def _listed() -> _Listed: + if _LISTED is None: + raise CorpusError("SPDX license list data has not been loaded") + return _LISTED + + +def listed_exception(identifier: str) -> bool: + return identifier in _listed().exceptions + + +def exception_evidence(component_key: str, expression: str) -> list[LicenseEvidence]: + """The SPDX reference text for each exception an expression names.""" + texts = _listed().exception_texts + found = [] + for key in sorted(set(identifiers_in(expression)) & _listed().exceptions): + if key not in texts: + raise CorpusError( + f"json/exceptions/{key}.json is missing from the SPDX license list " + "data; re-configure the build so deps/third_party fetches it." + ) + found.append( + evidence_for( + component_key, SPDX_REFERENCE, f"{key}.txt", texts[key], "grant" + ) + ) + return found + + +def why_unlicensed(expressions) -> str | None: + """Explain a vendored file's REUSE declaration failure, if any. + + Every license operand needs its own reference text: MIT cannot supply the + terms for an adjacent LicenseRef. Exceptions need a listed exception ID. + """ + entries = corpus()[1] + licences: set[str] = set() + exceptions: set[str] = set() + for item in expressions: + try: + parsed = ( + _spdx_licensing().parse(str(item), validate=False) + if item.is_valid + else None + ) + except (ExpressionError, ValueError, TypeError): + parsed = None + if parsed is None: + return f"{str(item)!r} does not parse" + for symbol in _spdx_licensing().license_symbols(parsed, decompose=False): + if isinstance(symbol, LicenseWithExceptionSymbol): + licences.add(symbol.license_symbol.key) + exceptions.add(symbol.exception_symbol.key) + else: + licences.add(symbol.key) + listed = _listed() + # `+` is SPDX's or-later operator, not part of the id. Each slot is checked + # against its own list: after WITH, SPDX takes only a listed exception. + for ids, one, many in ( + ( + [key for key in licences if key in listed.exceptions], + "is an SPDX exception, not a licence", + "are SPDX exceptions, not licences", + ), + ( + [ + key + for key in licences + if key.removesuffix("+") not in listed.licences + and key not in listed.exceptions + and not key.startswith(("LicenseRef-", "DocumentRef-")) + ], + "is not an SPDX identifier", + "are not SPDX identifiers", + ), + ( + [key for key in exceptions if key not in listed.exceptions], + "is not an SPDX exception", + "are not SPDX exceptions", + ), + ): + if ids: + return f"{', '.join(sorted(ids))} " + (one if len(ids) == 1 else many) + textless = sorted( + key for key in licences if key not in entries or not entries[key].text + ) + if textless or not licences: + return "no current SPDX-listed licence with a reference text" + ( + f" for {', '.join(textless)}" if textless else "" + ) + return None + + +@dataclass(frozen=True) +class _Grant: + """A grant's supported/newer IDs, tied alternatives, and text coverage.""" + + text: LicenseEvidence + named: tuple[str, ...] = () + newer: tuple[str, ...] = () + tied: tuple[tuple[str, ...], ...] = () + explained: float = 0.0 + + @property + def declared(self) -> bool: + """A reference text for an id REUSE declares, not the component's own.""" + return self.text.origin == SPDX_REFERENCE + + @property + def unmatched(self) -> bool: + return not self.declared and not self.text.matches + + @property + def partial(self) -> bool: + return bool(self.text.matches) and self.explained < _EXPLAINED_THRESHOLD + + +@dataclass(frozen=True) +class _Reading: + named: list[str] + carried: list[_Grant] + # Carried grants whose ids the conclusion names too. + also_named: list[LicenseEvidence] + + +def _read_grant(item: LicenseEvidence) -> _Grant: + if item.origin == SPDX_REFERENCE: + # The declared ID identifies this text; matching it to its own corpus is circular. + ids, tied, explained = [item.identified] if item.identified else [], (), 1.0 + else: + groups = alternatives(item.matches) + names = {group: _name_for(group) for group in groups} + ids = [plain for plain in names.values() if plain] + tied = tuple(group for group, plain in names.items() if not plain) + explained = _explained(item.text, groups) + return _Grant( + item, + tuple(key for key in ids if grammar_names([key])), + tuple(key for key in ids if not grammar_names([key])), + tied, + explained, + ) + + +def _reading(evidence) -> _Reading: + """Separate named grants from terms that must travel verbatim. + + If a REUSE declaration contains an unsupported ID, carry all of its texts + together to preserve OR/WITH semantics. + """ + read = [_read_grant(item) for item in grant_texts(evidence)] + whole = any(grant.declared and grant.newer for grant in read) + carried = [ + grant + for grant in read + if (whole and grant.declared) + or ( + not grant.declared + and (grant.newer or grant.tied or grant.unmatched or grant.partial) + ) + ] + + def names(grant: _Grant) -> tuple[str, ...]: + return () if whole and grant.declared else grant.named + + return _Reading( + sorted({key for grant in read for key in names(grant)}), + carried, + [grant.text for grant in carried if names(grant)], + ) + + +def declared_terms( + component_key: str, expressions +) -> tuple[str, list[LicenseEvidence]]: + """Preserve REUSE expressions and supply their reference texts. + + Reconstructing from texts would turn OR into AND. Unsupported declarations + travel whole as a LicenseRef, including exception texts. + """ + declared = combine(sorted(expressions)) + evidence = canonical_evidence(component_key, set(expressions)) + if declared == NOASSERTION: + return declared, evidence + exceptions = exception_evidence(component_key, declared) + if _reading(evidence + exceptions).carried: + return license_ref(component_key), evidence + exceptions + return declared, evidence + + +def expression(evidence: list[LicenseEvidence], component_key: str) -> tuple[str, str]: + """Conclude matched IDs plus a LicenseRef for any unresolved grant text.""" + reading = _reading(evidence) + identified = reading.named + ( + [license_ref(component_key)] if reading.carried else [] + ) + concluded = _combine(identified) if identified else NOASSERTION + + # Declared and concluded both derive from the component's grant text. + # Do not use its REUSE tag: OpenXR's COPYING.adoc is tagged CC-BY-4.0 + # for the documentation, not for the SDK it describes. + return concluded, concluded + + +def _combine(expressions: list[str]) -> str: + """Join with AND through the SPDX grammar rather than by string join.""" + try: + combined = combine_expressions( + sorted(expressions), relation="AND", licensing=_spdx_licensing() + ) + except (ExpressionError, ValueError, TypeError): + return "NOASSERTION" + # `is None`, not truthiness: a parsed expression raises on __bool__. + return "NOASSERTION" if combined is None else str(combined) + + +def normalized_expression(expression: str) -> str: + """Normalize an SPDX expression, returning NOASSERTION when parsing fails.""" + try: + return str(_spdx_licensing().parse(expression, validate=False)) + except (ExpressionError, ValueError, TypeError): + return "NOASSERTION" + + +def read_spdx_tag(text: str) -> str | None: + """The REUSE tag a file states about itself, if any.""" + match = _SPDX_TAG.search(text) + return match.group("expression").strip() if match else None + + +_NOTICE_YEARS = re.compile(r"\b(\d{4})(?:\s*[-\u2013]\s*(\d{4}))?\b") + + +def fold_notices(notices) -> list[str]: + """Remove notices subsumed by another notice for the same holder. + + Do not merge disjoint year ranges: that would claim years no input states. + """ + parsed = [] + for notice in notices: + years: set[int] = set() + for start, end in _NOTICE_YEARS.findall(notice): + years.update(range(int(start), int(end or start) + 1)) + holder = re.sub(r"[^A-Za-z0-9]+", " ", _NOTICE_YEARS.sub("", notice)) + parsed.append((holder.strip().lower(), frozenset(years), notice)) + + return sorted( + { + notice + for holder, years, notice in parsed + if not any( + other == holder and years < covered for other, covered, _ in parsed + ) + } + ) + + +# Require a copyright marker or year, allowing an unbounded prose lead-in +# such as "Box collision code (...) is Copyright 2016 ...". +_PLAIN_COPYRIGHT = re.compile( + r"^.*?(?PCopyright\b\s*(?:\(c\)|©|\d{4})[^\n\r]*)$", + re.MULTILINE | re.IGNORECASE, +) +# Connectives that indicate a notice was cut off mid-clause. +_DANGLING_WORDS = frozenset( + {"and", "or", "is", "was", "are", "were", "by", "under", "the", "a"} +) +# Distinguish explanatory prose from holder names in Debian Copyright fields. +_PROSE = re.compile( + r"\b(derived|contributed|reserved|portions?|redistribut\w+|written|" + r"modified|see|provided|permission|granted|warrant\w*|software|hereby|" + r"this|these|those|some|all rights)\b", + re.IGNORECASE, +) +_YEARS_AND_MARKS = re.compile(r"\(c\)|©|\d{4}(?:\s*-\s*\d{2,4})?|[,;]", re.I) +# Lines of a licence body that mention copyright without stating one. +_NOT_A_NOTICE = re.compile( + r"\bshall be\b|\bowner or entity\b|\bbe liable\b|\bmeans?\b|" + r"\bsubject to\b|\bdefined as\b", + re.IGNORECASE, +) +_PLACEHOLDER = re.compile( + r"\[yyyy\]||\[name of copyright owner\]| str | None: + """A file's own REUSE tag, normalised, or None if it names nothing usable.""" + tag = read_spdx_tag(text) + if tag is None: + return None + expression = normalized_expression(tag) + # A LicenseRef here has no text to define it with, as for any other file. + if expression == NOASSERTION or license_refs(expression): + return None + # Parsing without validation accepts unknown IDs; validate before emitting. + if not all(map(expressible, _spdx_licensing().license_keys(expression))): + return None + return expression + + +def notice_is_the_licence_authors(notice: str, identified: list[str]) -> bool: + """Identify notices belonging to the license author, not the component. + + Compare against the corpus rather than maintaining a steward list + (for example FSF in GPL, or Sam Hocevar in WTFPL). + """ + entries = corpus()[1] + collapsed = _steward_form(notice) + return any( + (reference := entries.get(license_id)) + and collapsed in _steward_form(reference.text) + for license_id in identified + ) + + +def _steward_form(text: str) -> str: + """Folded the way copies of one steward's line differ: the GPL's reads + `(C)` and `http://` as upstreams ship it, `©` and `https://` in the list.""" + folded = " ".join(text.split()).lower().replace("©", "(c)") + return folded.replace("https://", "http://") + + +def read_notices(text: str) -> list[str]: + """Read all concrete copyright notices, excluding unfilled reference templates.""" + lines = text.splitlines() + found: list[str] = _debian_copyright_fields(lines) + for index, line in enumerate(lines): + match = _PLAIN_COPYRIGHT.match(line) + if not match: + continue + notice = " ".join(match.group("notice").split()).rstrip("*/ ") + # Holders may continue below a year-only line or trailing comma (Debian, qhull, libccd). + step = index + while step + 1 < len(lines) and step - index < 3: + if not (notice.endswith(",") or not _names_a_holder(notice)): + break + ahead = step + 1 + # One blank line may sit between the years and the holder; qhull + # writes `Qhull, Copyright (c) 1993-2020`, a blank, then the holder. + if not lines[ahead].strip() and not _names_a_holder(notice): + ahead += 1 + if ahead >= len(lines): + break + following = " ".join(lines[ahead].split()) + # Stop before a new notice, field, or license clause is mistaken for a holder. + if ( + not following + or _PLAIN_COPYRIGHT.match(following) + or ":" in following + or _PROSE.search(following) + or _NOT_A_NOTICE.search(following) + ): + break + step = ahead + notice = f"{notice} {following}".strip().rstrip("*/ ") + notice = _without_dangling(notice) + if _PLACEHOLDER.search(notice) or _NOT_A_NOTICE.search(notice): + continue + if not _names_a_holder(notice) and not re.search(r"\d{4}", notice): + continue + if notice not in found: + found.append(notice) + return found + + +_NO_HOLDER = frozenset({"none", "n/a", "unknown"}) + + +def _copyright_fields(lines: list[str]) -> list[list[list[str]]]: + """Each Debian `Copyright:` field's paragraphs, each a list of its lines. + + The field runs on over indented lines and ` .` stands for a blank one, + which separates paragraphs. Debian writes `None` where a package states no + holder, and what follows that is commentary, not a holder. + """ + fields: list[list[list[str]]] = [] + for index, line in enumerate(lines): + if not line.lower().startswith("copyright:"): + continue + raw = [line[len("copyright:") :]] + for following in lines[index + 1 :]: + if not following[:1].isspace() or not following.strip(): + break + raw.append(following) + paragraphs: list[list[str]] = [[]] + for value in (" ".join(item.split()) for item in raw): + if value.lower() in _NO_HOLDER: + break + if value == ".": + paragraphs.append([]) + elif value: + paragraphs[-1].append(value) + fields.append([paragraph for paragraph in paragraphs if paragraph]) + return fields + + +def _debian_copyright_fields(lines: list[str]) -> list[str]: + """Read bare holders from Debian Copyright fields without requiring the word Copyright.""" + found: list[str] = [] + for field in _copyright_fields(lines): + values = [value for paragraph in field for value in paragraph] + for position, value in enumerate(values): + holder = value.rstrip(".") + # Full notices are handled by read_notices; stop here when bare holders end. + if _PLAIN_COPYRIGHT.match(holder) or _PROSE.search(holder): + break + if position and len(holder) > 60: + break + if _names_a_holder(f"Copyright {holder}") and holder not in found: + found.append(holder) + return found + + +def dep5_copyright(text: str) -> list[str] | None: + """Read DEP5 Copyright fields, or return None without an opening Format field. + + Ignore quoted notices in license prose and packaging-only stanzas. + """ + lines = text.splitlines() + first = next((line for line in lines if line.strip()), "") + if not first.lower().startswith("format:"): + return None + return [ + notice + for stanza in _stanzas(lines) + if not _packaging_only(stanza) + for field in _copyright_fields(stanza) + for paragraph in field + for notice in _notices_in(paragraph) + ] + + +_OPENS_NOTICE = re.compile(r"(?:copyright\b|©|\(c\))", re.IGNORECASE) +_LEADING_YEAR = re.compile(r"\d{4}") +# Markers, years and punctuation only: a notice that names no holder yet. +_NO_HOLDER_YET = re.compile(r"^(?:copyright\b|\(c\)|©|\d{4}|[\s,\-–;/.])*$", re.I) + + +def _notices_in(paragraph: list[str]) -> list[str]: + """A field paragraph's lines, grouped into the notices they state. + + `Copyright`, © or (c) starts a notice, and so does a year -- unless the + notice above names no holder yet, or is a sentence left unfinished -- and + a bare name after a bare name. Any other line continues the notice above. + """ + + def marked(text: str) -> bool: + return bool(_OPENS_NOTICE.match(text) or _LEADING_YEAR.match(text)) + + def bare(text: str) -> bool: + return not ("\n" in text or marked(text) or _PROSE.search(text)) + + def unfinished(current: str) -> bool: + first = current.split("\n", 1)[0] + return ( + not marked(first) + and bool(_PROSE.search(first)) + and not current.endswith(".") + ) + + def opens(line: str, current: str) -> bool: + if _OPENS_NOTICE.match(line): + return True + if _LEADING_YEAR.match(line): + return not (_NO_HOLDER_YET.match(current) or unfinished(current)) + if current.endswith(",") and line[:1].islower(): + return False + return bare(current) and bare(line) + + notices: list[str] = [] + for line in paragraph: + if not notices or opens(line, notices[-1]): + notices.append(line) + else: + notices[-1] += "\n" + line + return notices + + +def _stanzas(lines: list[str]) -> list[list[str]]: + stanzas: list[list[str]] = [[]] + for line in lines: + if line.strip(): + stanzas[-1].append(line) + elif stanzas[-1]: + stanzas.append([]) + return [stanza for stanza in stanzas if stanza] + + +def _packaging_only(stanza: list[str]) -> bool: + """Identify stanzas limited to debian/; their holders are packagers. + + A header without Files is not packaging-only. + """ + patterns: list[str] = [] + for index, line in enumerate(stanza): + if line.lower().startswith("files:"): + patterns += line[len("files:") :].split() + for following in stanza[index + 1 :]: + if not following[:1].isspace(): + break + patterns += following.split() + patterns = [item.removeprefix("./") for item in patterns] + return bool(patterns) and all( + item == "debian" or item.startswith("debian/") for item in patterns + ) + + +def _without_dangling(notice: str) -> str: + """Drop the connectives a notice ends on because its line broke mid-clause. + + By token, not by regex: a repeated or nested match over the whole string is + quadratic on a long line that ends in many of them. + """ + words = notice.split() + while words and words[-1].rstrip(",;").lower() in _DANGLING_WORDS: + words.pop() + return " ".join(words).rstrip("*/,; ") + + +def _names_a_holder(notice: str) -> bool: + rest = _YEARS_AND_MARKS.sub(" ", notice[len("Copyright") :]) + return bool(re.search(r"[A-Za-z]{2}", rest)) + + +def read_copyright(text: str) -> str | None: + """The notice a file states about itself, beside its REUSE tag.""" + match = _FILE_COPYRIGHT.search(text) + return " ".join(match.group("notice").split()).rstrip("*/ ") if match else None + + +def combine(expressions: list[str]) -> str: + return _combine(expressions) + + +def canonical_evidence( + component_key: str, expressions: set[str] +) -> list[LicenseEvidence]: + """Fetch reference texts for vendored declarations from the pinned SPDX list. + + Do not use the repository's REUSE pool: its texts can carry our own notices. + """ + found: list[LicenseEvidence] = [] + seen: set[str] = set() + entries = corpus()[1] + for expression in sorted(expressions): + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError): + # The per-file build gate reports malformed declarations with their source paths. + continue + for token in _spdx_licensing().license_keys(parsed): + reference = entries.get(token) + if token in seen or reference is None or not reference.text: + continue + seen.add(token) + found.append( + evidence_for( + component_key, + SPDX_REFERENCE, + f"{token}.txt", + reference.text, + "grant", + ) + ) + return found + + +class ExpressionReadError(Exception): + """An expression in a document cannot be read, so what it names is unknown.""" + + +def shared_refs(components) -> dict[str, str]: + """Share a LicenseRef among components with byte-identical terms. + + Resolve IDs before rendering so notices, reports, and SPDX agree. + """ + by_text: dict[str, list[str]] = {} + for component in components: + terms = verbatim_terms(component.evidence) + if terms: + by_text.setdefault(terms, []).append(component.key) + + renamed: dict[str, str] = {} + for text, keys in by_text.items(): + if len(keys) < 2: + continue + canonical = shared_ref(text) + for key in keys: + renamed[license_ref(key)] = canonical + return renamed + + +SHARED_REF = "LicenseRef-shared-text-" + + +def shared_ref(text: str) -> str: + return SHARED_REF + sha256_text(text)[:12] + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def rename_refs(expression: str, renamed: dict[str, str]) -> str: + """Replace whole identifiers: LicenseRef-alpha must not alter LicenseRef-alpha-extra.""" + for old_id, new_id in renamed.items(): + expression = re.sub( + rf"(? bool: + """Check whether an identifier names an SPDX license exception.""" + try: + parsed = _spdx_licensing().parse(identifier, validate=False) + except (ExpressionError, ValueError, TypeError): + return False + return identifier in {str(item) for item in getattr(parsed, "symbols", ())} and any( + getattr(item, "is_exception", False) for item in parsed.symbols + ) + + +def operands_of(expression: str) -> list[str]: + """Split expressions into operands that can be AND-joined without changing meaning. + + Keep OR choices whole; splitting them would require both alternatives. + """ + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError): + return [expression] + if " OR " in str(parsed).upper(): + return [f"({parsed})"] + return sorted(str(key) for key in _spdx_licensing().license_symbols(parsed)) + + +def identifiers_in(expression: str) -> list[str]: + """Every licence identifier an expression names, LicenseRef included.""" + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError) as error: + raise ExpressionReadError(f"{expression!r}: {error}") from error + return sorted(_spdx_licensing().license_keys(parsed)) + + +def license_refs(*expressions: str) -> list[str]: + """The LicenseRef- identifiers an expression uses, per the SPDX grammar.""" + found: set[str] = set() + for expression in expressions: + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError) as error: + # Malformed expressions must fail closed; skipping one could hide undefined references. + raise ExpressionReadError(f"{expression!r}: {error}") from error + found.update( + key + for key in _spdx_licensing().license_keys(parsed) + if key.startswith("LicenseRef-") + ) + return sorted(found) + + +@lru_cache(maxsize=1) +def _spdx_licensing(): + """Building this parses ScanCode's licence database; do it once.""" + return get_spdx_licensing() diff --git a/scripts/sbom/isaaccapture_sbom/report.py b/scripts/sbom/isaaccapture_sbom/report.py new file mode 100644 index 0000000000..d747a27b67 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/report.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Component-to-license report, and the notices packaged inside the wheel.""" + +from __future__ import annotations + +import re + +from . import licensing +from .inventory import Inventory + +NOASSERTION = "NOASSERTION" + +_HOW_PROSE = { + "compiled-source": "its sources were compiled into a shipped binary", + "header-include": "its headers were compiled into a shipped binary", + "prebuilt-library": "a pre-built library of its was linked in", + "copied-file": "files were copied in unchanged", + "derived-file": "files were copied in and modified by the build", + "extracted-file": "files were unpacked from its archive", + "vendored-library": "the library was vendored in by auditwheel", + "shared-content": ( + "a shipped file holds bytes this component also holds, and so do others" + ), +} +_SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") + +# Distinguish vendored copyright holders from installable packages. +_KIND_PROSE = { + "source-tree": "fetched source", + "archive": "fetched archive", + "system-library": "build machine", + "vendored-source": "vendored source", + "vcpkg-port": "vcpkg port", +} + + +_COMMIT = re.compile(r"^[0-9a-f]{40}$") + +# Shared gap categories for JSON, Markdown, manifests, and CI warnings. +GAP_LISTS = { + "components_without_license_evidence": "No license text found for:", + "components_with_unresolved_license": "No license concluded for:", + "components_with_unidentified_license": "License text carried verbatim:", +} + + +def _short(version: str) -> str: + """Abbreviate commit hashes only; truncating a package version changes its identity.""" + return version[:12] if _COMMIT.match(version) else version + + +def licenses_root(dist_info: str) -> str: + """Where a wheel carries licence texts. PEP 639 names them relative to this.""" + return f"{dist_info}/licenses/" + + +def notices_path(dist_info: str) -> str: + return f"{licenses_root(dist_info)}THIRD-PARTY-NOTICES.md" + + +def is_packaged_license(dist_info: str, member: str) -> bool: + return member.startswith(licenses_root(dist_info)) + + +def is_third_party_license(dist_info: str, member: str) -> bool: + return member.startswith(f"{licenses_root(dist_info)}third-party/") + + +def declared_license_path(dist_info: str, member: str) -> str: + """PEP 639 License-File paths are relative to .dist-info/licenses/.""" + return member.removeprefix(licenses_root(dist_info)) + + +# Nested texts may repeat the root grant; describe their scope without implying a difference. +NESTED_ROLE = "terms stated for part of this component" + + +def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: + """Place terms under the component's full evidence path to avoid basename collisions.""" + folder = _SAFE_NAME.sub("_", component_key) + relative = evidence.path.rsplit("!", 1)[-1].split("/", 1)[-1] + # Preserve path separators when sanitizing to avoid flattening distinct paths. + safe = "/".join(_SAFE_NAME.sub("_", part) for part in relative.split("/")) + return f"{licenses_root(dist_info)}third-party/{folder}/{safe}" + + +def _roles(inventory: Inventory, key: str) -> str: + hows = sorted(inventory.component_roles.get(key, set())) + return "; ".join(_HOW_PROSE.get(how, how) for how in hows) or "unrecorded" + + +def notices_markdown( + inventory: Inventory, + dist_info: str, + wheel_name: str, + project: dict | None = None, +) -> str: + """Render notices with the distribution's own license and copyright first. + + Reference templates do not name the holder of a metadata-only wheel. + """ + lines = ["# Notices", ""] + if project: + supplier = project["supplier"].split(": ", 1)[-1] + lines += [f"`{wheel_name}` is distributed by {supplier}."] + if project.get("license", NOASSERTION) != NOASSERTION: + lines.append(f"It is licensed under `{project['license']}`.") + if project.get("copyright", NOASSERTION) != NOASSERTION: + lines.append(f"{project['copyright']}") + lines.append("") + lines += ["## Third-party notices", ""] + if not inventory.components_present: + lines += [ + f"`{wheel_name}` redistributes no third-party component, so no " + "license text is packaged here.", + "", + ] + return "\n".join(lines) + + lines += [ + f"Components redistributed in `{wheel_name}`, with the license text each", + "obligation was read from. Texts are packaged under", + f"`{licenses_root(dist_info)}third-party/`.", + "", + ] + for key, component in inventory.components_present.items(): + lines.append(f"### {component.name}") + lines.append("") + lines.append(f"- License: `{component.license_concluded}`") + if component.copyright_text != "NOASSERTION": + # Reference texts omit component holders. Render their notices separately, + # with explicit breaks so Markdown does not merge them into one paragraph. + lines.append( + "- Copyright: " + "
".join(component.copyright_text.splitlines()) + ) + if component.license_declared != component.license_concluded: + lines.append(f"- Declared: `{component.license_declared}`") + if component.homepage != "NOASSERTION": + lines.append(f"- Homepage: {component.homepage}") + if component.version != "NOASSERTION": + lines.append(f"- Version: `{component.version}`") + lines.append(f"- Reaches this wheel because {_roles(inventory, key)}") + for item in component.evidence: + packaged = packaged_license_path(dist_info, key, item) + lines.append( + f"- Text: `{packaged}`" + + ( + " — the SPDX reference text for this " + + ( + "exception" + if licensing.listed_exception(item.identified or "") + else "licence" + ) + + ", not this component's own file" + if item.origin == licensing.SPDX_REFERENCE + else f" — {NESTED_ROLE}" + if item.kind == "nested" + else "" + ) + ) + if not component.evidence: + lines.append("- No license text was found in this component's own files.") + lines.append("") + return "\n".join(lines) + + +def _carried_subject(carried: int, total: int) -> str: + """How many of a component's grants travel verbatim, as a sentence subject.""" + if carried == total == 1: + return "the shipped grant is" + if carried == total: + return f"its {total} shipped grants are" + return f"{carried} of its {total} shipped grants {'is' if carried == 1 else 'are'}" + + +def _carried_reason(evidence) -> str: + """Which of a component's grants travel verbatim, whether their ids are + named too, and why they travel.""" + carried = licensing.carried_texts(evidence) + named = len(licensing.carried_and_named(evidence)) + subject = _carried_subject(len(carried), len(licensing.grant_texts(evidence))) + manner = ( + " rather than named" + if not named + else " as well as named" + if named == len(carried) + else f", {named} of them as well as named" + ) + return f"{subject} carried verbatim{manner}: {licensing.why_unnamed(evidence)}" + + +def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: + """Machine-readable component-to-license report, including what is missing.""" + components = [] + gaps: dict[str, list[dict]] = {key: [] for key in GAP_LISTS} + + for key, component in inventory.components_present.items(): + obligations = licensing.obligation_texts(component.evidence) + components.append( + { + "component": key, + "name": component.name, + "kind": component.kind, + "version": component.version, + "how_it_reaches_the_wheel": sorted( + inventory.component_roles.get(key, set()) + ), + "license_concluded": component.license_concluded, + "license_declared": component.license_declared, + "supplier": component.supplier, + "homepage": component.homepage, + "purl": component.purl, + "source_info": component.source_info, + "wheel_files": inventory.files_of(key), + "evidence": [item.as_json() for item in component.evidence], + } + ) + if not obligations: + gaps["components_without_license_evidence"].append( + { + "component": key, + "reason": "no license grant found in the component's own files", + } + ) + elif component.license_concluded == NOASSERTION: + # Packaged text can satisfy the gate without identifying a license. + gaps["components_with_unresolved_license"].append( + { + "component": key, + "reason": licensing.why_unresolved(component.evidence), + "evidence": [item.path for item in obligations], + } + ) + elif carried := licensing.carried_texts(component.evidence): + # A partially matched grant still carries unresolved terms. + gaps["components_with_unidentified_license"].append( + { + "component": key, + "reason": _carried_reason(component.evidence), + "evidence": [item.path for item in carried], + } + ) + + return { + "schema": "isaaccapture-license-report/2", + "wheel": wheel_name, + "generated_from_commit": evidence_doc.get("project", {}).get("commit"), + "source_tree_modified": str( + evidence_doc.get("project", {}).get("describe") or "" + ).endswith("-dirty"), + "spdx_license_list_version": evidence_doc.get("license_list_version"), + # Keep builder-local corpus paths in build evidence only. + "license_data": { + key: value + for key, value in evidence_doc.get("license_data", {}).items() + if key != "source" + }, + "components": components, + "externally_supplied_runtime": [ + {"soname": soname, "required_by": consumers} + for soname, consumers in inventory.external_runtime.items() + ], + "consumer_requirements": inventory.requires_dist, + **gaps, + } + + +def report_markdown(payload: dict) -> str: + """Human-readable twin of :func:`report`.""" + lines = [ + f"# Component-to-license report — `{payload['wheel']}`", + "", + f"Source commit: `{payload['generated_from_commit'] or 'unknown'}`" + + ( + " (built from a tree with uncommitted changes)" + if payload.get("source_tree_modified") + else "" + ) + + " ", + f"SPDX license list: `{payload['spdx_license_list_version'] or 'unknown'}` " + f"(spdx/license-list-data `{payload.get('license_data', {}).get('commit', '?')[:12]}`)", + "", + "Every row was discovered from the build that produced this wheel. Nothing", + "here is declared in a checked-in list.", + "", + "## Redistributed components", + "", + "| Component | Kind | Version | License | How it reaches the wheel | Evidence |", + "| --- | --- | --- | --- | --- | --- |", + ] + for item in payload["components"]: + # Summarize reference pools for readability; JSON and packaged notices retain all paths. + shown = [record for record in item["evidence"] if record["kind"] != "pool"] + pooled = [record for record in item["evidence"] if record["kind"] == "pool"] + parts = [ + f"`{record['path']}`" + + (f" → {record['identified']}" if record["identified"] else "") + + (f" ({NESTED_ROLE})" if record["kind"] == "nested" else "") + + ( + " (SPDX reference text, substituted)" + if record["origin"] == licensing.SPDX_REFERENCE + else "" + ) + for record in shown + ] + if pooled: + names = licensing.pool_names(record["path"] for record in pooled) + parts.append(f"REUSE pool ({len(pooled)}): {names}") + evidence = "
".join(parts) or "—" + how = "; ".join( + _HOW_PROSE.get(value, value) for value in item["how_it_reaches_the_wheel"] + ) + version = item["version"] + lines.append( + f"| {item['name']} | {_KIND_PROSE.get(item['kind'], item['kind'])} | " + f"`{_short(version)}` | `{item['license_concluded']}` | " + f"{how or '—'} | {evidence} |" + ) + + lines += ["", "## Externally supplied at runtime", ""] + if payload["externally_supplied_runtime"]: + lines += [ + "Resolved from the installation environment; not redistributed here.", + "", + ] + lines += [ + f"- `{item['soname']}` — required by {', '.join(item['required_by'])}" + for item in payload["externally_supplied_runtime"] + ] + else: + lines.append("None recorded.") + + lines += ["", "## License texts packaged in this wheel", ""] + lines += [ + f"- `{item}`" for item in payload.get("packaged_license_files") or [] + ] or ["None."] + if payload.get("packaged_license_files"): + lines += [ + "", + "Declared in METADATA as `License-File`: " + + ("yes" if payload.get("metadata_license_files_declared") else "no"), + ] + + lines += ["", "## Consumer requirements (wheel metadata)", ""] + lines += [f"- `{item}`" for item in payload["consumer_requirements"]] or [ + "None declared." + ] + + lines += ["", "## License evidence gaps", ""] + listed = [key for key in GAP_LISTS if payload[key]] + for key in listed: + lines += [GAP_LISTS[key], ""] + lines += [ + f"- `{item['component']}` — {item['reason']}" for item in payload[key] + ] + lines.append("") + if not listed: + lines.append( + "None; every redistributed component ships an identified license." + if payload["components"] + else "This distribution redistributes no third-party component." + ) + + return "\n".join(lines) + "\n" diff --git a/scripts/sbom/isaaccapture_sbom/validate.py b/scripts/sbom/isaaccapture_sbom/validate.py new file mode 100644 index 0000000000..eb7288e7a2 --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/validate.py @@ -0,0 +1,685 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify wheel contents and optional build/release evidence. + +verify_wheel needs only the archive and is shared by CI and consumers. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import zipfile +from collections import Counter +from pathlib import Path + +from . import SPDX_VERSION, licensing, report +from .discovery import sha256_file +from . import document as document_module +from . import wheelfile + + +# Derived from the wheel alone, so every build of one wheel writes the same bytes. +CONTENT_SIDECARS = ("spdx", "license_report_json", "license_report_md") +_SIDECAR_KINDS = (*CONTENT_SIDECARS, "build_evidence") + + +class ManifestError(Exception): + """A manifest lacks the structure required for verification.""" + + +def _sidecar(evidence_dir: Path, record: dict, kind: str) -> Path | None: + named = record.get("sidecars", {}).get(kind, {}).get("filename") + if not named: + return None + path = evidence_dir / named + return path if path.is_file() else None + + +def manifest_records(path: Path) -> list[dict]: + """A manifest's wheel records; one listing none checks nothing, so it fails.""" + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except ValueError as error: + raise ManifestError(f"{path}: the manifest is not JSON ({error})") from error + if not isinstance(manifest, dict): + raise ManifestError(f"{path}: the manifest is not a JSON object") + records = manifest.get("wheels") + if not isinstance(records, list): + raise ManifestError(f"{path}: the manifest has no 'wheels' list") + if not records: + raise ManifestError(f"{path}: the manifest lists no wheels") + for item in records: + if not isinstance(item, dict): + raise ManifestError(f"{path}: a 'wheels' entry is not an object") + if not isinstance(item.get("filename"), str) or not item["filename"]: + raise ManifestError(f"{path}: a 'wheels' entry names no wheel") + if not _is_bare_name(item["filename"]): + raise ManifestError( + f"{path}: {item['filename']!r} is not a bare wheel file name" + ) + return records + + +def bound_names(records) -> list[str]: + return [ + sidecar["filename"] + for record in records + for sidecar in record["sidecars"].values() + ] + + +def sidecar_failures(beside: Path, record: dict) -> list[str]: + """Return missing-file or digest failures for a checked record's sidecars.""" + failures = [] + for kind, sidecar in record["sidecars"].items(): + path = beside / sidecar["filename"] + if not path.is_file(): + failures.append(f"sidecar {kind} is missing: {sidecar['filename']}") + elif sha256_file(path) != sidecar["sha256"]: + failures.append(f"sidecar {kind} does not match its recorded digest") + return failures + + +def _is_bare_name(name: str) -> bool: + """A file name joined onto a directory must stay inside that directory.""" + return name not in ("", ".", "..") and not any(sep in name for sep in "/\\:") + + +def _record_for(records: list[dict], filename: str) -> dict: + """Find exactly one record for a wheel, or raise ManifestError.""" + seen = [item for item in records if item.get("filename") == filename] + if len(seen) > 1: + raise ManifestError(f"{filename} is advertised {len(seen)} times") + return checked_record(seen[0]) if seen else {} + + +def checked_record(record: dict) -> dict: + """A record holding every field the checks read, or ManifestError.""" + filename = record["filename"] + for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): + if key not in record: + raise ManifestError(f"{filename}: manifest record has no {key!r}") + sidecars = record["sidecars"] + if not isinstance(sidecars, dict) or not all( + isinstance(item, dict) + and isinstance(item.get("filename"), str) + and isinstance(item.get("sha256"), str) + for item in sidecars.values() + ): + raise ManifestError( + f"{filename}: manifest record's sidecars must each name a file and its sha256" + ) + for kind, item in sidecars.items(): + if not _is_bare_name(item["filename"]): + raise ManifestError( + f"{filename}: {kind} sidecar {item['filename']!r} is not a bare " + "file name" + ) + # Require every sidecar kind; omissions must not bypass evidence checks. + missing = sorted(set(_SIDECAR_KINDS) - set(record["sidecars"])) + if missing: + raise ManifestError( + f"{filename}: manifest record names no {', '.join(missing)} sidecar" + ) + return record + + +_SHARED_ID = re.compile(re.escape(licensing.SHARED_REF) + "[0-9a-f]{12}") +_EVIDENCE_DIGEST = re.compile( + r"\(sha256:(?P[0-9a-f]{64})[^)]*\) packaged at (?P\S+)" +) + +_REQUIRED_KEYS = ( + "spdxVersion", + "dataLicense", + "SPDXID", + "name", + "documentNamespace", + "creationInfo", + "packages", + "files", + "relationships", +) + + +def _embedded_sbom( + archive: zipfile.ZipFile, dist_info: str, wheel_name: str +) -> tuple[str, dict]: + name = wheelfile.sbom_member(dist_info, wheel_name) + if name not in archive.namelist(): + found = [ + item + for item in archive.namelist() + if item.startswith(f"{dist_info}/sboms/") + ] + raise LookupError( + f"expected {name}, found {found or 'nothing'} under {dist_info}/sboms/" + ) + return name, json.loads(archive.read(name)) + + +def verify_wheel( + wheel_path: Path, + scanned: wheelfile.WheelInfo | None = None, + carried: list[dict] | None = None, +) -> list[str]: + """Verify an archive against its embedded SBOM without a repository. + + Append the parsed document to carried for reuse by evidence checks. + """ + failures: list[str] = [] + wheel = scanned or wheelfile.scan(wheel_path) + + with zipfile.ZipFile(wheel_path) as archive: + try: + sbom_name, spdx = _embedded_sbom(archive, wheel.dist_info, wheel_path.name) + except (LookupError, json.JSONDecodeError) as error: + return [f"embedded SBOM: {error}"] + if carried is not None: + carried.append(spdx) + + for key in _REQUIRED_KEYS: + if key not in spdx: + failures.append(f"SPDX document is missing required field {key!r}") + if spdx.get("spdxVersion") != SPDX_VERSION: + failures.append( + f"spdxVersion is {spdx.get('spdxVersion')!r}, expected {SPDX_VERSION!r}" + ) + if failures: + return failures + + identifiers = [spdx["SPDXID"]] + identifiers += [item["SPDXID"] for item in spdx["packages"]] + identifiers += [item["SPDXID"] for item in spdx["files"]] + duplicates = sorted(item for item, seen in Counter(identifiers).items() if seen > 1) + if duplicates: + failures.append(f"duplicate SPDXIDs: {duplicates}") + known = set(identifiers) + # Cross-document element IDs must name a declared DocumentRef. + declared_documents = { + item["externalDocumentId"] for item in spdx.get("externalDocumentRefs", []) + } + for relationship in spdx["relationships"]: + for side in ("spdxElementId", "relatedSpdxElement"): + element = relationship[side] + prefix, _, _ = element.partition(":") + if prefix.startswith("DocumentRef-"): + if prefix not in declared_documents: + failures.append( + f"relationship {relationship['relationshipType']} references " + f"{element!r}, and no externalDocumentRef declares {prefix!r}" + ) + continue + if element not in known: + failures.append( + f"relationship {relationship['relationshipType']} references unknown " + f"element {element!r}" + ) + for item in spdx.get("externalDocumentRefs", []): + checksum = item.get("checksum") or {} + if checksum.get("algorithm") != "SHA1" or not checksum.get("checksumValue"): + failures.append( + f"{item.get('externalDocumentId')!r} names a document with no SHA1 " + "to check it by" + ) + + wheel_package = next( + ( + item + for item in spdx["packages"] + if item["SPDXID"] == "SPDXRef-Package-wheel" + ), + None, + ) + if wheel_package is None: + return failures + ["SPDXRef-Package-wheel is missing"] + if wheel_package.get("packageFileName") != wheel_path.name: + failures.append( + f"packageFileName {wheel_package.get('packageFileName')!r} does not name this wheel" + ) + if ( + f"{wheel_package.get('name')}-{wheel_package.get('versionInfo')}" + != wheel.dist_info.removesuffix(".dist-info") + ): + failures.append( + "wheel package name/version do not match the wheel's .dist-info" + ) + + excluded = { + item.removeprefix("./") + for item in wheel_package.get("packageVerificationCode", {}).get( + "packageVerificationCodeExcludedFiles", [] + ) + } + # Do not trust the document to choose exclusions: arbitrary entries here + # could hide added files from coverage checks. + permitted = {f"{wheel.dist_info}/RECORD", sbom_name} + if excluded != permitted: + failures.append( + "the excluded-files list must be exactly " + f"{sorted(permitted)}, not {sorted(excluded)}" + ) + excluded = permitted + + documented = {item["fileName"].removeprefix("./"): item for item in spdx["files"]} + actual = {item.name: item for item in wheel.entries} + + for name in sorted(set(actual) - set(documented) - excluded): + failures.append(f"wheel file not covered by the SBOM: {name}") + for name in sorted(set(documented) - set(actual)): + failures.append(f"SBOM documents a file the wheel does not contain: {name}") + + sha1_digests = [] + for name, record in sorted(documented.items()): + entry = actual.get(name) + if entry is None: + continue + checksums = { + item["algorithm"]: item["checksumValue"] for item in record["checksums"] + } + if checksums.get("SHA256") != entry.sha256: + failures.append(f"SHA256 mismatch for {name}") + if checksums.get("SHA1") != entry.sha1: + failures.append(f"SHA1 mismatch for {name}") + sha1_digests.append(entry.sha1) + + expected_code = document_module.verification_code(sha1_digests) + actual_code = wheel_package.get("packageVerificationCode", {}).get( + "packageVerificationCodeValue" + ) + if expected_code != actual_code: + failures.append( + f"packageVerificationCode is {actual_code!r} but the documented files hash to {expected_code!r}" + ) + + failures.extend(_check_record(wheel_path, wheel)) + failures.extend(_check_licenses(wheel, spdx)) + failures.extend(_check_license_texts(wheel, spdx)) + return failures + + +def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: + """Check that cited license paths contain the bytes the document describes. + + Per-file hashes alone can agree with a substituted or emptied license file. + """ + failures: list[str] = [] + by_name = {entry.name: entry.sha256 for entry in wheel.entries} + packaged = { + digest + for name, digest in by_name.items() + if report.is_packaged_license(wheel.dist_info, name) + } + if not packaged: + return failures + + for package in spdx.get("packages", []): + for text in package.get("attributionTexts", []): + match = _EVIDENCE_DIGEST.search(text) + if not match: + continue + # Check the cited path too: another component can contain identical license bytes. + where = match.group("where") + if by_name.get(where) != match.group("digest"): + failures.append( + f"{package['name']}: {where} does not hold the licence text " + f"this document records for it (sha256:{match.group('digest')})" + ) + + return failures + + +def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: + failures: list[str] = [] + record = wheelfile.read_record(wheel_path, wheel.dist_info) + record_name = f"{wheel.dist_info}/RECORD" + for entry in wheel.entries: + if entry.name == record_name: + continue + row = record.get(entry.name) + if row is None: + failures.append(f"RECORD does not list {entry.name}") + continue + # Reuse scan hashes; RECORD encodes the same digest in base64. + if row[0] != wheelfile.record_hash_from_sha256(entry.sha256): + failures.append(f"RECORD digest mismatch for {entry.name}") + if row[1] != str(entry.size): + failures.append(f"RECORD size mismatch for {entry.name}") + for name in sorted(set(record) - {item.name for item in wheel.entries}): + failures.append(f"RECORD lists a file the wheel does not contain: {name}") + return failures + + +def _check_attribution( + evidence_doc: dict, wheel: wheelfile.WheelInfo, spdx: dict | None +) -> list[str]: + """Require attribution for every member and packaged terms for each component.""" + stated = evidence_doc.get("wheel") + if stated and stated != wheel.path.name: + return [f"build evidence describes {stated}, not {wheel.path.name}"] + + failures: list[str] = [] + attributed = {item["path"] for item in evidence_doc.get("attributions", [])} + packaged = { + entry.sha256 + for entry in wheel.entries + if report.is_packaged_license(wheel.dist_info, entry.name) + } + recorded: set[str] = set() + + for entry in wheel.entries: + if entry.name not in attributed: + failures.append( + f"build evidence does not explain where {entry.name} came from" + ) + + for key, component in evidence_doc.get("components", {}).items(): + if not component.get("in_this_wheel"): + continue + # Include NOTICE and other non-grant evidence cited by attributionTexts. + recorded.update(item["sha256"] for item in component.get("evidence", [])) + texts = [ + item + for item in component.get("evidence", []) + if licensing.satisfies_obligation(item["kind"]) + ] + if not texts: + failures.append( + f"{key} is redistributed but no license text was found for it" + ) + continue + # Existence at build time is insufficient; the recorded terms must still be packaged. + if not any(item["sha256"] in packaged for item in texts): + failures.append( + f"{key} is redistributed but the license text recorded for it is " + "not packaged in this wheel" + ) + failures.extend(_document_agrees_with_evidence(wheel, recorded, evidence_doc, spdx)) + return failures + + +def _document_agrees_with_evidence( + wheel: wheelfile.WheelInfo, + recorded: set[str], + evidence_doc: dict, + spdx: dict | None, +) -> list[str]: + """Compare license claims with separately recorded build evidence. + + A self-consistent rewrite of the SBOM and wheel can pass _check_license_texts; + the evidence supplies independent digests and conclusions. + """ + if spdx is None: + return [] + + failures = [] + for package in spdx.get("packages", []): + for text in package.get("attributionTexts", []): + match = _EVIDENCE_DIGEST.search(text) + if match and match.group("digest") not in recorded: + failures.append( + f"{package['name']}: the document names a licence text " + f"(sha256:{match.group('digest')}) the build never read" + ) + + # Compare expressions as well as text hashes; an unchanged EULA could otherwise + # be relabeled Apache-2.0. + components = evidence_doc.get("components", {}) + # Join by component ID: source trees, archives, and libraries can share display names. + by_id = { + document_module.component_package_id(key): (key, component) + for key, component in components.items() + } + for package in spdx.get("packages", []): + found = by_id.get(package["SPDXID"]) + if found is None: + continue + key, component = found + for field, stated in ( + ("licenseConcluded", "license_concluded"), + ("licenseDeclared", "license_declared"), + ): + expected = component.get(stated) + actual = package.get(field) + # Shared LicenseRefs are resolved before rendering, so require exact ID agreement. + if not expected or actual == expected: + continue + failures.append( + f"{key}: the document says {field} {actual!r}, " + f"the build recorded {expected!r}" + ) + + # Bind each LicenseRef to the combined terms recorded for its owner, or to + # a recorded shared text. Individual file hashes cannot verify a joined body. + by_ref = {licensing.license_ref(key): key for key in components} + stated = { + key: component.get("verbatim_terms_sha256") + for key, component in components.items() + } + any_stated = {digest for digest in stated.values() if digest} + for extracted in spdx.get("hasExtractedLicensingInfos", []): + ref, text = extracted["licenseId"], extracted.get("extractedText", "") + # A shared ID must match its text hash, not just any component's terms. + if _SHARED_ID.fullmatch(ref) and ref != licensing.shared_ref(text): + failures.append(f"{ref}: the text it carries is not the one its id names") + continue + if not any_stated: + # Legacy evidence without this digest cannot verify the combined text. + continue + owner = by_ref.get(ref) + expected = {stated[owner]} if owner and stated.get(owner) else any_stated + if hashlib.sha256(text.encode("utf-8")).hexdigest() not in expected: + failures.append( + f"{ref}: the text it carries is not the terms the build read" + + (f" for {owner}" if owner else "") + ) + return failures + + +def _refs_in(value) -> set[str]: + """Read LicenseRefs from an expression or list, failing on malformed expressions.""" + if value is None: + return set() + items = value if isinstance(value, list) else [value] + found: set[str] = set() + for item in items: + found.update(licensing.license_refs(str(item))) + return found + + +def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: + failures: list[str] = [] + names = {item.name for item in wheel.entries} + notices = report.notices_path(wheel.dist_info) + if notices not in names: + failures.append(f"{notices} is not packaged") + + declared_refs = set() + for item in spdx.get("files", []): + for field in ("licenseConcluded", "licenseInfoInFiles"): + declared_refs.update(_refs_in(item.get(field))) + for package in spdx["packages"]: + for field in ("licenseConcluded", "licenseDeclared", "licenseInfoFromFiles"): + declared_refs.update(_refs_in(package.get(field))) + file_ids = {item["SPDXID"] for item in spdx.get("files", [])} + for package in spdx["packages"]: + for held in package.get("hasFiles", []): + if held not in file_ids: + failures.append(f"{package['name']} claims {held}, which is not here") + + extracted = { + item["licenseId"] for item in spdx.get("hasExtractedLicensingInfos", []) + } + for missing in sorted(declared_refs - extracted): + failures.append( + f"{missing} is used but has no hasExtractedLicensingInfos entry" + ) + for item in spdx.get("hasExtractedLicensingInfos", []): + if not item.get("extractedText", "").strip(): + failures.append(f"{item['licenseId']} has an empty extractedText") + return failures + + +def check( + wheel: Path, + build_evidence: Path | None = None, + manifest: Path | None = None, + evidence_dir: Path | None = None, +) -> list[str]: + """Verify a wheel, adding checks for each supplied evidence or manifest input.""" + scanned = wheelfile.scan(wheel) + carried: list[dict] = [] + failures = verify_wheel(wheel, scanned, carried) + + if build_evidence is not None: + evidence_doc = json.loads(build_evidence.read_text(encoding="utf-8")) + failures.extend( + _check_attribution(evidence_doc, scanned, carried[0] if carried else None) + ) + + if manifest is not None: + failures.extend(_check_manifest(manifest, wheel, evidence_dir)) + + return failures + + +def _check_manifest( + manifest_path: Path, wheel: Path, evidence_dir: Path | None +) -> list[str]: + """The manifest binds a wheel's digest and names the sidecars beside it.""" + failures: list[str] = [] + record = _record_for(manifest_records(manifest_path), wheel.name) + if not record: + return [f"{manifest_path.name} has no entry for {wheel.name}"] + + digest = sha256_file(wheel) + if record["sha256"] != digest: + failures.append( + "the wheel was modified after its digest was bound: manifest says " + f"{record['sha256']}, the file hashes to {digest}" + ) + + beside = evidence_dir if evidence_dir is not None else manifest_path.parent + failures.extend(sidecar_failures(beside, record)) + + published = beside / record["sidecars"]["spdx"]["filename"] + if published.is_file(): + with zipfile.ZipFile(wheel) as archive: + embedded = record["sbom_in_wheel"] + if embedded not in archive.namelist(): + failures.append( + f"the manifest names {embedded} as this wheel's SBOM, and the " + "wheel carries no such member" + ) + elif archive.read(embedded) != published.read_bytes(): + failures.append( + "the published SBOM copy differs from the one embedded in the wheel" + ) + return failures + + +def _cross_document_refs( + spdx: dict, documents: dict[str, bytes], filename: str +) -> list[str]: + """Resolve external references against this release, checking document hashes + and referenced elements to reject stale sibling references. + """ + failures: list[str] = [] + by_namespace = { + json.loads(body).get("documentNamespace"): (name, body) + for name, body in documents.items() + } + for ref in spdx.get("externalDocumentRefs", []): + found = by_namespace.get(ref.get("spdxDocument")) + if found is None: + failures.append( + f"{filename}: {ref.get('externalDocumentId')} names a document " + f"this release does not publish ({ref.get('spdxDocument')})" + ) + continue + name, body = found + stated = (ref.get("checksum") or {}).get("checksumValue") + actual = hashlib.sha1(body).hexdigest() # noqa: S324 - SPDX 2.3 says SHA1 + if stated != actual: + failures.append( + f"{filename}: {ref.get('externalDocumentId')} states sha1 {stated} " + f"for {name}, which hashes to {actual}" + ) + ids = { + ref["externalDocumentId"]: json.loads(by_namespace[ref["spdxDocument"]][1]) + for ref in spdx.get("externalDocumentRefs", []) + if ref.get("spdxDocument") in by_namespace + } + for relationship in spdx.get("relationships", []): + for side in ("spdxElementId", "relatedSpdxElement"): + prefix, _, element = str(relationship[side]).partition(":") + target = ids.get(prefix) + if target is None or not element: + continue + known = {target["SPDXID"]} + known |= {item["SPDXID"] for item in target.get("packages", [])} + known |= {item["SPDXID"] for item in target.get("files", [])} + if element not in known: + failures.append( + f"{filename}: {relationship[side]} names an element the " + "referenced document does not define" + ) + return failures + + +def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[str]: + """Verify the release and reject wheels or evidence omitted from its manifest.""" + failures: list[str] = [] + records = manifest_records(manifest_path) + for name, seen in sorted(Counter(item["filename"] for item in records).items()): + if seen > 1: + raise ManifestError(f"{name} is advertised {seen} times") + by_name = {item["filename"]: checked_record(item) for item in records} + + # Every downloaded wheel is published, so unlisted wheels cannot be left unchecked. + for stray in sorted({item.name for item in wheel_dir.glob("*.whl")} - set(by_name)): + failures.append( + f"{stray} is in {wheel_dir}, and {manifest_path.name} does not list it" + ) + # Reject unbound evidence too; the wheel check above handles shared-directory wheels. + expected = set(bound_names(by_name.values())) + if manifest_path.parent.resolve() == evidence_dir.resolve(): + expected.add(manifest_path.name) + if evidence_dir.resolve() == wheel_dir.resolve(): + expected |= {item.name for item in wheel_dir.glob("*.whl")} + present = {item.name for item in evidence_dir.glob("*") if item.is_file()} + for stray in sorted(present - expected): + failures.append( + f"{stray} is in {evidence_dir}, and {manifest_path.name} does not bind it" + ) + + published: dict[str, bytes] = {} + for record in by_name.values(): + sidecar = _sidecar(evidence_dir, record, "spdx") + if sidecar: + published[sidecar.name] = sidecar.read_bytes() + + for name, body in sorted(published.items()): + try: + failures.extend(_cross_document_refs(json.loads(body), published, name)) + except json.JSONDecodeError as error: + failures.append(f"{name}: not readable as a document ({error})") + + for record in by_name.values(): + wheel = wheel_dir / record["filename"] + if not wheel.is_file(): + failures.append(f"{record['filename']} is missing from {wheel_dir}") + continue + failures.extend( + f"{record['filename']}: {item}" + for item in check( + wheel, + build_evidence=_sidecar(evidence_dir, record, "build_evidence"), + manifest=manifest_path, + evidence_dir=evidence_dir, + ) + ) + return failures diff --git a/scripts/sbom/isaaccapture_sbom/wheelfile.py b/scripts/sbom/isaaccapture_sbom/wheelfile.py new file mode 100644 index 0000000000..79d4eaecdb --- /dev/null +++ b/scripts/sbom/isaaccapture_sbom/wheelfile.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read and rewrite wheels with packaged evidence. + +Use installer for RECORD parsing and packaging for core metadata. +""" + +from __future__ import annotations + +import base64 +import csv +import hashlib +import io +import os +import time +import zipfile +from collections import Counter +from dataclasses import dataclass +from email import message_from_bytes +from email.generator import BytesGenerator +from email.policy import compat32 +from elftools.common.exceptions import ELFError +from installer.records import Hash, RecordEntry, parse_record_file +from packaging.metadata import RawMetadata, parse_email +from packaging.version import Version +from pathlib import Path + +from . import elf + +# Deterministic default timestamp for added members. +_ADDED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) + + +def _reproducible_timestamp() -> tuple[int, int, int, int, int, int] | None: + """Convert SOURCE_DATE_EPOCH to a ZIP timestamp. + + Apply it to copied members too, so input timestamps cannot affect reproducibility. + """ + epoch = os.environ.get("SOURCE_DATE_EPOCH") + if not epoch or not epoch.isdigit(): + return None + moment = time.gmtime(int(epoch)) + if moment.tm_year < 1980: # zip cannot record anything earlier + return _ADDED_TIMESTAMP + return ( + moment.tm_year, + moment.tm_mon, + moment.tm_mday, + moment.tm_hour, + moment.tm_min, + moment.tm_sec, + ) + + +class WheelError(Exception): + """The wheel is not shaped the way the packaging specs require.""" + + +@dataclass(frozen=True) +class Entry: + """One member of the wheel's zip archive.""" + + name: str + size: int + sha256: str + sha1: str + soname: str | None = None + needed: tuple[str, ...] = () + machine: str | None = None + build_id: str | None = None + # Retain a prefix for REUSE tags to avoid reopening the archive. + head: bytes = b"" + + @property + def is_elf(self) -> bool: + return self.machine is not None + + +@dataclass(frozen=True) +class WheelInfo: + path: Path + distribution: str + version: str + dist_info: str + entries: tuple[Entry, ...] + + +def sbom_member(dist_info: str, wheel_name: str) -> str: + """Name this collector's document in the shared PEP 770 sboms directory. + + Do not select by directory alone: auditwheel.cdx.json may coexist there. + """ + return f"{dist_info}/sboms/{wheel_name}.spdx.json" + + +def record_hash_from_sha256(digest: str) -> str: + """RECORD's hash field, as PEP 376 spells it, from a digest already computed.""" + encoded = ( + base64.urlsafe_b64encode(bytes.fromhex(digest)).rstrip(b"=").decode("ascii") + ) + return str(Hash.parse(f"sha256={encoded}")) + + +def record_hash(data: bytes) -> str: + return record_hash_from_sha256(hashlib.sha256(data).hexdigest()) + + +def _dist_info(names: list[str]) -> str: + candidates = sorted( + { + name.split("/", 1)[0] + for name in names + if name.endswith(".dist-info/METADATA") + } + ) + if len(candidates) != 1: + raise WheelError( + f"expected exactly one .dist-info with METADATA, found {candidates}" + ) + return candidates[0] + + +def dist_info_of(wheel_path: Path) -> str: + """Find .dist-info via the ZIP index without hashing every member.""" + with zipfile.ZipFile(wheel_path) as archive: + return _dist_info(archive.namelist()) + + +def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: + """Hash members, optionally reading ELF metadata for attribution. + + Consumer verification leaves analyze_elf off to avoid parsing binaries. + """ + with zipfile.ZipFile(wheel_path) as archive: + names = archive.namelist() + # Duplicate names make reads and extraction ambiguous; reject them before hashing. + repeated = sorted(item for item, seen in Counter(names).items() if seen > 1) + if repeated: + raise WheelError( + f"more than one member is named {repeated}; a wheel member name " + "has to identify one file" + ) + dist_info = _dist_info(names) + entries: list[Entry] = [] + carrying = [ + item.filename + for item in archive.infolist() + if item.is_dir() and item.file_size + ] + if carrying: + raise WheelError( + f"these members are named as directories but carry bytes: {carrying}" + ) + for info in archive.infolist(): + if info.is_dir(): + continue + with archive.open(info) as handle: + data = handle.read() + soname = needed = machine = build_id = None + if analyze_elf and elf.is_elf(data[:4]): + try: + dynamic = elf.read_dynamic(data) + except ELFError as error: + raise WheelError( + f"{info.filename} begins with the ELF magic but cannot be " + f"read as one: {error}" + ) from error + soname, needed, machine, build_id = ( + dynamic.soname, + dynamic.needed, + dynamic.machine, + dynamic.build_id, + ) + entries.append( + Entry( + name=info.filename, + size=len(data), + sha256=hashlib.sha256(data).hexdigest(), + sha1=hashlib.sha1(data).hexdigest(), # noqa: S324 - SPDX 2.3 requires SHA1 + soname=soname, + needed=needed or (), + machine=machine, + build_id=build_id, + head=data[:4096], + ) + ) + + stem = dist_info.removesuffix(".dist-info") + distribution, _, version = stem.partition("-") + if not version: + raise WheelError(f"cannot read distribution and version from {dist_info!r}") + return WheelInfo( + path=wheel_path, + distribution=distribution, + version=version, + dist_info=dist_info, + entries=tuple(entries), + ) + + +def read_metadata(wheel_path: Path, dist_info: str) -> tuple[bytes, RawMetadata]: + """Core metadata, parsed by packaging rather than as loose email headers.""" + with zipfile.ZipFile(wheel_path) as archive: + raw = archive.read(f"{dist_info}/METADATA") + return raw, parse_metadata(raw) + + +def parse_metadata(raw: bytes) -> RawMetadata: + parsed, _ = parse_email(raw) + return parsed + + +def add_license_files(raw_metadata: bytes, relative_paths: list[str]) -> bytes: + """Declare packaged license files, which PEP 639 requires of Metadata 2.4. + + Older metadata cannot express License-File, so the files still ship but go + undeclared; the caller reports that rather than emitting invalid metadata. + """ + metadata = parse_metadata(raw_metadata) + # Compare as a version: "2.10" sorts before "2.4" as a string. + if Version(metadata.get("metadata_version") or "0") < Version("2.4"): + return raw_metadata + + existing = set(metadata.get("license_files") or []) + additions = [path for path in relative_paths if path not in existing] + if not additions: + return raw_metadata + + # Use the email parser to preserve the long-description payload. + message = message_from_bytes(raw_metadata, policy=compat32) + for path in additions: + message.add_header("License-File", path) + + # Disable header folding so License-File paths remain on one line. + buffer = io.BytesIO() + BytesGenerator(buffer, policy=compat32, maxheaderlen=0).flatten(message) + return buffer.getvalue() + + +def read_record(wheel_path: Path, dist_info: str) -> dict[str, tuple[str, str]]: + """Parse RECORD with installer, which owns the format's escaping rules.""" + with zipfile.ZipFile(wheel_path) as archive: + lines = archive.read(f"{dist_info}/RECORD").decode("utf-8").splitlines() + entries = (RecordEntry.from_elements(*row) for row in parse_record_file(lines)) + return { + # Preserve size 0 for empty files; `entry.size or ""` would erase it. + entry.path: ( + str(entry.hash_) if entry.hash_ else "", + "" if entry.size is None else str(entry.size), + ) + for entry in entries + } + + +def _members(path: Path) -> list[zipfile.ZipInfo]: + with zipfile.ZipFile(path) as archive: + return [item for item in archive.infolist() if not item.is_dir()] + + +def rewrite( + wheel_path: Path, + output_path: Path, + *, + additions: dict[str, bytes], + replacements: dict[str, bytes], + dist_info: str, +) -> None: + """Rewrite members and regenerate RECORD from the resulting bytes.""" + record_name = f"{dist_info}/RECORD" + collisions = sorted( + set(additions) & {item.filename for item in _members(wheel_path)} + ) + if collisions: + raise WheelError( + f"these members already exist and would be added again: {collisions}" + ) + if record_name in additions or record_name in replacements: + raise WheelError("RECORD is regenerated here; do not pass it in") + + pinned = _reproducible_timestamp() + rows: list[tuple[str, str, str]] = [] + with ( + zipfile.ZipFile(wheel_path) as source, + zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as target, + ): + for info in source.infolist(): + if info.is_dir() or info.filename == record_name: + continue + data = replacements.get(info.filename) + if data is None: + data = source.read(info.filename) + member = zipfile.ZipInfo(info.filename, date_time=pinned or info.date_time) + member.external_attr = info.external_attr + member.compress_type = zipfile.ZIP_DEFLATED + target.writestr(member, data) + rows.append((info.filename, record_hash(data), str(len(data)))) + + for name in sorted(additions): + data = additions[name] + member = zipfile.ZipInfo(name, date_time=pinned or _ADDED_TIMESTAMP) + member.external_attr = 0o644 << 16 + member.compress_type = zipfile.ZIP_DEFLATED + target.writestr(member, data) + rows.append((name, record_hash(data), str(len(data)))) + + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + for row in sorted(rows): + writer.writerow(row) + writer.writerow((record_name, "", "")) + member = zipfile.ZipInfo(record_name, date_time=pinned or _ADDED_TIMESTAMP) + member.external_attr = 0o644 << 16 + member.compress_type = zipfile.ZIP_DEFLATED + target.writestr(member, buffer.getvalue()) diff --git a/scripts/sbom/pyproject.toml b/scripts/sbom/pyproject.toml new file mode 100644 index 0000000000..004a8d7b25 --- /dev/null +++ b/scripts/sbom/pyproject.toml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Standalone collector project; usage and design are in README.md. + +[project] +name = "isaaccapture-sbom" +version = "1.0.0" +description = "Discover and package the contents SBOM for an Isaac Capture wheel" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{ name = "NVIDIA" }] + +# Pin format libraries to keep published documents reproducible across builds. +dependencies = [ + "installer==1.0.1", # RECORD + "license-expression==30.4.4", # SPDX expression grammar + "packageurl-python==0.17.6", # purls + "packaging==26.3", # core metadata, requirements + "pyelftools==0.33", # ELF dynamic section (auditwheel uses it too) + # Match reuse-tool in .pre-commit-config.yaml; build also needs libmagic1. + "reuse==6.2.0", +] + +[project.optional-dependencies] +dev = ["pytest", "spdx-tools==0.8.3"] + +[project.scripts] +isaaccapture-sbom = "isaaccapture_sbom.cli:main" + +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["isaaccapture_sbom*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests"] # `import synth`, whatever --import-mode is diff --git a/scripts/sbom/tests/conftest.py b/scripts/sbom/tests/conftest.py new file mode 100644 index 0000000000..8954abb097 --- /dev/null +++ b/scripts/sbom/tests/conftest.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fixtures for the SBOM collector tests.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import synth +from isaaccapture_sbom import build, discovery, licensing + +BUILD_DIR_ENV = "ISAACCAPTURE_SBOM_BUILD_DIR" + + +def repo_root() -> Path | None: + for parent in Path(__file__).resolve().parents: + if (parent / "VERSION").is_file() and (parent / "CMakeLists.txt").is_file(): + return parent + return None + + +def _located(usable, missing: str, within=lambda build_dir: build_dir) -> Path: + """Use the selected build tree or default build/. + + Missing explicit inputs fail; missing defaults skip. + """ + if os.environ.get(BUILD_DIR_ENV): + named = within(Path(os.environ[BUILD_DIR_ENV]).resolve()) + if usable(named): + return named + pytest.fail(f"{BUILD_DIR_ENV}: {missing} at {named}") + root = repo_root() + if root is not None and usable(found := within(root / "build")): + return found + pytest.skip(f"{missing}; configure this project, or set {BUILD_DIR_ENV}") + + +def _corpus_loads(path: Path) -> bool: + try: + licensing.load_corpus(path) + except licensing.CorpusError: + return False + return True + + +def _has_codemodel(path: Path) -> bool: + return any((path / ".cmake" / "api" / "v1" / "reply").glob("codemodel-v2-*.json")) + + +@pytest.fixture(scope="session") +def license_data(): + """The SPDX license list deps/third_party fetched into this build's _deps.""" + return _located( + _corpus_loads, "no SPDX license list data", build.default_license_data + ) + + +@pytest.fixture(autouse=True) +def _license_data_loaded(license_data): + """Restore the full corpus after tests that load a fixture subset.""" + licensing.load_corpus(license_data) + synth.set_license_data(license_data) + + +@pytest.fixture(autouse=True) +def _fresh_checkout_answers(): + """Clear cached checkout and host answers between synthetic workspaces.""" + discovery._is_own_checkout.cache_clear() + discovery._apt_releases.cache_clear() + discovery._apt_origin.cache_clear() + + +@pytest.fixture(scope="session") +def checkout(): + root = repo_root() + if root is None: + pytest.skip("not inside this repository's source tree") + return root + + +@pytest.fixture(scope="session") +def live_build_dir(): + """Use real CMake output to validate the synthetic codemodel assumptions.""" + return _located(_has_codemodel, "no CMake file API codemodel") + + +@pytest.fixture +def workspace(tmp_path): + """A generated project, its build tree, and the wheel that build produced.""" + space = synth.create(tmp_path / "project") + synth.write_wheel(space.wheel, synth.wheel_payload(space)) + return space + + +@pytest.fixture +def built(workspace): + return synth.collect_and_bind(workspace) diff --git a/scripts/sbom/tests/synth.py b/scripts/sbom/tests/synth.py new file mode 100644 index 0000000000..7fdd89279d --- /dev/null +++ b/scripts/sbom/tests/synth.py @@ -0,0 +1,1095 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generate synthetic build trees and wheels, including malformed gate inputs. + +Check reader assumptions separately against real CMake replies, ELF files, +and the SPDX corpus. +""" + +from __future__ import annotations + +import base64 +import contextlib +import csv +import gzip +import hashlib +import io +import os +import json +import struct +import subprocess +import tarfile +import zipfile +from pathlib import Path + +from isaaccapture_sbom import build as build_module + +CONFIG = "Release" +WHEEL_NAME = "isaaccapture-0.4.0-cp312-cp312-manylinux_2_35_x86_64.whl" +STEM = WHEEL_NAME.removesuffix(".whl") +DIST_INFO = "isaaccapture-0.4.0.dist-info" +SDK_ARCHIVE = "deps/vendor/demosdk-1.2.0-linux-amd64.tar.gz" +SDK_DEMO_BUILD_ID = "1111111111111111111111111111111111111111" + +# ELF + +_PT_LOAD = 1 +_PT_DYNAMIC = 2 +_PT_NOTE = 4 +_NT_GNU_BUILD_ID = 3 +_DT_NULL = 0 +_DT_NEEDED = 1 +_DT_STRTAB = 5 +_DT_STRSZ = 10 +_DT_SONAME = 14 +_EM_X86_64 = 62 +_EHDR_SIZE = 64 +_PHDR_SIZE = 56 +_BASE_VADDR = 0x400000 + + +def _build_id_note(build_id: str) -> bytes: + """A .note.gnu.build-id in the form the linker writes it.""" + name = b"GNU\0" + desc = bytes.fromhex(build_id) + padding = b"\0" * (-len(desc) % 4) + return ( + struct.pack(" bytes: + """Build a little-endian ELF64 dynamic segment with an optional GNU build-ID note.""" + strings = [b"\0"] # index 0 is the empty string, per the ELF string-table format + offsets: dict[str, int] = {} + cursor = 1 + for value in (soname, *needed): + if value in offsets: + continue + offsets[value] = cursor + encoded = value.encode("utf-8") + b"\0" + strings.append(encoded) + cursor += len(encoded) + strtab = b"".join(strings) + + entries = [(_DT_SONAME, offsets[soname])] + entries += [(_DT_NEEDED, offsets[name]) for name in needed] + + note = _build_id_note(build_id) if build_id else b"" + segments = 3 if note else 2 + header_size = _EHDR_SIZE + segments * _PHDR_SIZE + dynamic_offset = header_size + dynamic_size = (len(entries) + 3) * 16 + strtab_offset = dynamic_offset + dynamic_size + note_offset = strtab_offset + len(strtab) + total = note_offset + len(note) + len(padding) + + entries.append((_DT_STRTAB, _BASE_VADDR + strtab_offset)) + entries.append((_DT_STRSZ, len(strtab))) + entries.append((_DT_NULL, 0)) + + image = bytearray(total) + image[0:4] = b"\x7fELF" + image[4] = 2 # ELFCLASS64 + image[5] = 1 # ELFDATA2LSB + image[6] = 1 # EV_CURRENT + struct.pack_into(" None: + base = _EHDR_SIZE + index * _PHDR_SIZE + struct.pack_into(" None: + global _LICENSE_DATA + _LICENSE_DATA = path + + +def license_data() -> Path: + if _LICENSE_DATA is None: + raise RuntimeError("set_license_data() has not been called") + return _LICENSE_DATA + + +def spdx_text(license_id: str) -> str: + """The canonical text of a license, from the data the collector reads.""" + payload = json.loads( + (license_data() / "details" / f"{license_id}.json").read_text(encoding="utf-8") + ) + return payload["licenseText"] + + +PROPRIETARY_LICENSE = """\ +DEMO CORP SOFTWARE LICENSE AGREEMENT + +This agreement governs your use of the Demo SDK. You may redistribute the +runtime libraries with your application provided this notice accompanies them. +The software is provided as is, without warranty of any kind. +""" + + +# Workspace + +_DEPS = { + # key: (remote, license file name, SPDX id whose canonical text it ships) + "alpha": ("https://github.com/example-org/alpha.git", "LICENSE", "MIT"), + "beta": ("https://github.com/example-org/beta.git", "LICENSE", "Apache-2.0"), + "gamma": ("https://github.com/example-org/gamma.git", "LICENSE", "Zlib"), + "delta": ( + "https://github.com/example-org/delta.git", + "BSD-LICENSE", + "BSD-3-Clause", + ), + "omega": ("https://github.com/example-org/omega.git", "LICENSE.txt", "BSL-1.0"), + "vendorpy": ( + "https://github.com/example-org/vendorpy.git", + "LICENSE", + "Apache-2.0", + ), +} + +_CACHE = """\ +# This is the CMakeCache file. +CMAKE_BUILD_TYPE:STRING=Release +CMAKE_SYSTEM_PROCESSOR:STRING=x86_64 +CMAKE_CXX_COMPILER:FILEPATH=/usr/bin/c++ +ISAAC_TELEOP_PYTHON_VERSION:STRING=3.12 +BUILD_VIZ:BOOL=ON +BUILD_PYTHON_BINDINGS:BOOL=ON +""" + +_AUTHORED_INIT = """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +__version__ = "0.4.0" +""" +_GENERATED = "# generated at configure time\nTRACKERS = ()\n" +_VENDOR_MODULE = """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 Example Org +# SPDX-License-Identifier: Apache-2.0 + +def retarget(): + return None +""" +# Checked in rather than fetched, and says whose it is the way upstream files +# usually do: a plain copyright line, no REUSE tag. +_VENDORED_HEADER = """\ +// Copyright 2024, Upstream Widgets Ltd. +// SPDX-License-Identifier: BSL-1.0 +#pragma once +""" + +# Upstream's header names no identifier; this repository declares it through +# REUSE instead, as it does for Chromium's key table. +UNTAGGED_HEADER = """\ +// Copyright 2013 The Keys Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +""" +UNTAGGED_KEY = "vendored:the-keys-authors" + +_PROJECT_PYPROJECT = """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[project] +name = "isaaccapture" +version = "0.4.0" +authors = [{ name = "NVIDIA" }] +""" + +_VENDOR_ASSET_SOURCE = ( + '\n' +) +_VENDOR_ASSET_STAGED = '\n' + + +class Workspace: + """Paths into a generated project, and the wheel its build would produce.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.build = root / "build" + self.staged = self.build / "python_package" / CONFIG + self.wheel = root / "dist" / WHEEL_NAME + + +def git(root: Path, *args: str) -> str: + # Fix identity and timestamps for reproducible synthetic commits and wheel provenance. + env = { + **os.environ, + "GIT_AUTHOR_NAME": "synth", + "GIT_AUTHOR_EMAIL": "synth@example.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00+00:00", + "GIT_COMMITTER_NAME": "synth", + "GIT_COMMITTER_EMAIL": "synth@example.invalid", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00+00:00", + } + return subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + env=env, + text=True, + ).stdout.strip() + + +def _commit_all(root: Path, message: str) -> None: + git(root, "add", "-A") + git(root, "commit", "-q", "--allow-empty", "-m", message) + + +def clone_of(root: Path, remote: str, tag: str, git_dir: Path | None = None) -> str: + """Create a dependency checkout and return its commit ID. + + git_dir uses a separate repository behind a .git file, as in a submodule. + """ + root.mkdir(parents=True, exist_ok=True) + separate = [] + if git_dir: + git_dir.parent.mkdir(parents=True, exist_ok=True) + separate = ["--separate-git-dir", str(git_dir)] + git(root, "init", "-q", "-b", "main", *separate) + _commit_all(root, "upstream") + git(root, "tag", tag) + git(root, "remote", "add", "origin", remote) + return git(root, "rev-parse", "HEAD") + + +def _git_checkout(root: Path, initialise: bool = True) -> None: + """Create a real checkout to distinguish tracked sources from build output.""" + write(root / ".gitignore", "build/\ndist/\n") + if initialise: + git(root, "init", "-q", "-b", "main") + _commit_all(root, "synthetic project") + + +def annotation( + path: str, + holder: str | None = None, + *, + licence: str | None = "MIT", + precedence: str = "closest", + year: int = 2020, +) -> str: + """One REUSE.toml annotation; it states no copyright where `holder` is None.""" + lines = ["[[annotations]]", f'path = "{path}"', f'precedence = "{precedence}"'] + if holder is not None: + lines.append(f'SPDX-FileCopyrightText = "Copyright {year} {holder}"') + if licence is not None: + lines.append("SPDX-" + f'License-Identifier = "{licence}"') + return "\n".join(lines) + "\n" + + +def reuse_toml(*annotations: str) -> str: + return "version = 1\n\n" + "\n".join(annotations) + + +def track(root: Path) -> None: + _git_checkout(root, initialise=False) + + +def write(path: Path, data: bytes | str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(data, str): + path.write_text(data, encoding="utf-8") + else: + path.write_bytes(data) + + +def _stage_license_data(target: Path) -> None: + """Mirror what deps/third_party leaves in _deps, with just the licenses used here.""" + source = license_data() + for index in ("licenses.json", "exceptions.json"): + write(target / index, (source / index).read_text(encoding="utf-8")) + for license_id in FIXTURE_LICENSES: + detail = source / "details" / f"{license_id}.json" + if detail.is_file(): + write(target / "details" / detail.name, detail.read_text(encoding="utf-8")) + for exception_id in FIXTURE_EXCEPTIONS: + detail = source / "exceptions" / f"{exception_id}.json" + write(target / "exceptions" / detail.name, detail.read_text(encoding="utf-8")) + + +def declare( + deps: Path, + key: str, + *, + urls: tuple[str, ...] = (), + digest: str | None = None, + repository: str | None = None, + tag: str | None = None, + direct: bool = False, + source: str | None = None, +) -> None: + """Write the stamps a FetchContent declaration leaves in the build tree. + + Laid out as the populate sub-build does, or, with `direct`, as CMP0168 NEW + does. `source` names the tree, `-src` by default. + """ + source_dir = deps / (source or f"{key}-src") + if direct: + files = deps.parent / "CMakeFiles" + stamps, scripts, prefix = files / "fc-stamp" / key, files / "fc-tmp" / key, key + else: + prefix = f"{key}-populate" + scripts = deps / f"{key}-subbuild" / f"{prefix}-prefix" / "tmp" + stamps = scripts.parent / "src" / f"{prefix}-stamp" + + def stamp(kind: str, **fields: str) -> None: + body = "".join(f"{name}={value}\n" for name, value in fields.items()) + write( + stamps / f"{prefix}-{kind}info.txt", + "# This is a generated file and its contents are an internal " + f"implementation detail.\n\nmethod={kind}\n{body}", + ) + + if urls: + stamp( + "url", + source_dir=str(source_dir), + **{"url(s)": ";".join(urls)}, + hash=digest or "", + ) + if repository: + clone = scripts / f"{prefix}-gitclone.cmake" + write( + clone, + # The URL straight after `--no-checkout`, which is not a checkout. + f' clone --no-checkout "{repository}" "{source_dir.name}"\n' + + (f' checkout "{tag}" --\n' if tag else ""), + ) + stamp( + "git", + command=f"/usr/bin/cmake;-P;{clone}", + source_dir=str(source_dir), + repository=repository, + ) + + +def _sdk_tarball(path: Path) -> dict[str, bytes]: + """An SDK archive whose libraries are redistributed, one of them patched.""" + members = { + "LICENSE.txt": PROPRIETARY_LICENSE.encode("utf-8"), + "VERSION": b"1.2.0\n", + "libdemo.so": elf_shared_object( + "libdemo.so", ("libc.so.6",), build_id=SDK_DEMO_BUILD_ID + ), + "libextra.so": elf_shared_object("libextra.so", ("libssl.so.3", "libc.so.6")), + "include/demo.h": b"#pragma once\n", + } + # Fix member and gzip timestamps for reproducible archive bytes. + path.parent.mkdir(parents=True, exist_ok=True) + with ( + gzip.GzipFile(filename=path, mode="wb", mtime=0) as compressed, + tarfile.open(fileobj=compressed, mode="w") as archive, + ): + for name, data in sorted(members.items()): + info = tarfile.TarInfo(f"./{name}") + info.size = len(data) + info.mtime = 0 + archive.addfile(info, io.BytesIO(data)) + return members + + +def create(root: Path) -> Workspace: + workspace = Workspace(root) + build = workspace.build + deps = build / "_deps" + + write(root / "VERSION", "0.4.x\n") + write(root / "CMakeLists.txt", "cmake_minimum_required(VERSION 3.24)\n") + write(root / "LICENSE.md", spdx_text("Apache-2.0")) + for pool_id in ("Apache-2.0", "BSL-1.0"): + write(root / "LICENSES" / f"{pool_id}.txt", spdx_text(pool_id)) + write(root / "src/python/isaaccapture/__init__.py", _AUTHORED_INIT) + write(root / "src/vendor/upstream_helper.h", _VENDORED_HEADER) + write(root / "pyproject.toml", _PROJECT_PYPROJECT) + write(build / "CMakeCache.txt", _CACHE) + + _stage_license_data(deps / "license-list-data-src" / "json") + + for key, (remote, license_name, license_id) in _DEPS.items(): + source = deps / f"{key}-src" + write(source / license_name, spdx_text(license_id)) + write(source / f"{key}.cpp", f"// {key}\n") + declare(deps, key, repository=remote, tag="v1.2.3") + + write(deps / "vendorpy-src/pkg/mod.py", _VENDOR_MODULE) + write(deps / "vendorpy-src/pkg/assets/hand.xml", _VENDOR_ASSET_SOURCE) + + sdk_members = _sdk_tarball(root / SDK_ARCHIVE) + + # Build outputs + extension = "isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so" + extension_bytes = elf_shared_object( + "_ext.cpython-312-x86_64-linux-gnu.so", ("libstdc++.so.6", "libc.so.6") + ) + gamma_bytes = elf_shared_object("libgamma_renamed.so", ("libc.so.6",)) + write(workspace.staged / extension, extension_bytes) + write(build / "lib/libgamma_renamed.so", gamma_bytes) + write(build / "lib/libdelta.a", b"!\ndelta\n") + write(build / "generated/_generated.py", _GENERATED) + # Empty marker exercises RECORD size 0 and path-only attribution. + write(build / "generated/stubs/isaaccapture/py.typed", b"") + + # Model a package-manager header installed into the build tree. + write( + build / "vcpkg_installed/x64-linux/include/eigen/Core.h", + b"// Copyright (C) 2008 Gael Guennebaud \n", + ) + + write(workspace.staged / "isaaccapture/__init__.py", _AUTHORED_INIT) + write(workspace.staged / "isaaccapture/_generated.py", _GENERATED) + write(workspace.staged / "isaaccapture/py.typed", b"") + write(workspace.staged / "isaaccapture/viz/libgamma_renamed.so", gamma_bytes) + write(workspace.staged / "isaaccapture/viz/GAMMA_LICENSE", spdx_text("Zlib")) + write( + workspace.staged / "isaaccapture/sdk/native/libdemo.so", + sdk_members["libdemo.so"], + ) + # patchelf drops a NEEDED entry, so the shipped bytes differ from the archive's. + write( + workspace.staged / "isaaccapture/sdk/native/libextra.so", + elf_shared_object("libextra.so", ("libc.so.6",)), + ) + write(workspace.staged / "vendorpy/mod.py", _VENDOR_MODULE) + write(workspace.staged / "vendorpy/assets/hand.xml", _VENDOR_ASSET_STAGED) + + _file_api_reply( + root, build, (workspace.staged / extension).relative_to(build).as_posix() + ) + _git_checkout(root) + write(build / "tests/unit_tests", b"\x7fELF-not-really") + return workspace + + +def _file_api_reply(root: Path, build: Path, extension: str) -> None: + """Write a synthetic CMake file API reply.""" + targets = [ + { + "name": "ext", + "type": "MODULE_LIBRARY", + # The project's own source, compiled against beta's headers, with + # alpha's source compiled straight in. + "artifacts": [extension], + # List a package-manager header as a source to exercise build-tree exclusion. + "sources": [ + "src/cpp/module.cpp", + "build/_deps/alpha-src/alpha.cpp", + "build/vcpkg_installed/x64-linux/include/eigen/Core.h", + ], + "includes": [ + "build/_deps/beta-src/include", + "build/_deps/alpha-src", + "build/vcpkg_installed/x64-linux/include", + "src/vendor", + ], + "dependencies": [], + }, + { + "name": "gamma", + "type": "SHARED_LIBRARY", + "artifacts": ["lib/libgamma_renamed.so"], + "sources": ["build/_deps/gamma-src/gamma.cpp"], + "includes": ["build/_deps/gamma-src"], + "dependencies": ["delta"], + }, + { + "name": "delta", + "type": "STATIC_LIBRARY", + "artifacts": ["lib/libdelta.a"], + "sources": ["build/_deps/delta-src/delta.cpp"], + "includes": ["build/_deps/delta-src"], + "dependencies": [], + }, + { + # Tests link omega; nothing they produce reaches a wheel. + "name": "unit_tests", + "type": "EXECUTABLE", + "artifacts": ["tests/unit_tests"], + "sources": ["tests/cpp/test_main.cpp"], + "includes": ["build/_deps/omega-src/include"], + "dependencies": [], + }, + ] + + reply = _reply(build) + index = [] + for spec in targets: + entry = _target_entry(spec["name"]) + payload = _target_payload(**spec) + write(reply / entry["jsonFile"], json.dumps(payload, indent=1)) + index.append(entry) + + codemodel = { + "kind": "codemodel", + "version": {"major": 2, "minor": 6}, + "paths": {"source": str(root), "build": str(build)}, + "configurations": [{"name": CONFIG, "targets": index}], + } + write(reply / "codemodel-v2-fixture.json", json.dumps(codemodel, indent=1)) + write( + reply / "index-2026-01-01T00-00-00-0000.json", + json.dumps( + { + "objects": [ + { + "kind": "codemodel", + "version": {"major": 2, "minor": 6}, + "jsonFile": "codemodel-v2-fixture.json", + } + ] + }, + indent=1, + ), + ) + + +def _source(path: str) -> dict: + """A codemodel source entry; one CMake compiles names its compile group.""" + compiled = path.endswith((".c", ".cpp")) + return {"path": path, **({"compileGroupIndex": 0} if compiled else {})} + + +def _target_payload( + name: str, + type: str, # noqa: A002 - the codemodel's own key + artifacts: list[str], + sources: list[str], + includes: list[str] | None, + dependencies: list[str], +) -> dict: + """A codemodel target; one with no compile groups compiles nothing.""" + payload = { + "name": name, + "type": type, + "artifacts": [{"path": item} for item in artifacts], + "sources": [_source(item) for item in sources], + "dependencies": [{"id": dep} for dep in dependencies], + } + if includes is not None: + payload["compileGroups"] = [{"includes": [{"path": item} for item in includes]}] + return payload + + +def _target_entry(name: str) -> dict: + return {"id": name, "name": name, "jsonFile": f"target-{name}-{CONFIG}.json"} + + +@contextlib.contextmanager +def _edit_json(path: Path): + payload = json.loads(path.read_text(encoding="utf-8")) + yield payload + path.write_text(json.dumps(payload, indent=1), encoding="utf-8") + + +def _reply(build: Path) -> Path: + return build / ".cmake" / "api" / "v1" / "reply" + + +def _edit_target(build: Path, target: str): + return _edit_json(_reply(build) / f"target-{target}-{CONFIG}.json") + + +def _edit_index(build: Path): + return _edit_json(next(_reply(build).glob("index-*.json"))) + + +def link_line( + build: Path, + target: str, + fragments: tuple[str | tuple[str, str], ...], + *, + dependencies: tuple[str, ...] = (), + directory: str | None = None, + sysroot: Path | None = None, +) -> None: + """Give a fixture target the link line CMake records for a linkable target. + + A fragment is its text, filed under `libraries`, or (text, role). + `directory` is the target's build directory, which is what a relative + fragment is relative to under the Makefile generators. `sysroot` is + CMAKE_SYSROOT, as the reply records it. + """ + with _edit_target(build, target) as payload: + payload["link"] = { + "language": "CXX", + "commandFragments": [ + {"fragment": item, "role": "libraries"} + if isinstance(item, str) + else {"fragment": item[0], "role": item[1]} + for item in fragments + ], + } + if sysroot is not None: + payload["link"]["sysroot"] = {"path": str(sysroot)} + payload["dependencies"] += [{"id": item} for item in dependencies] + if directory is not None: + set_directory(build, target, directory) + + +def set_toolchains( + build: Path, + link_directories: dict[str, list[str] | None], + compilers: dict[str, str] | None = None, +) -> None: + """A toolchains reply: each language's compiler and implicit link directories. + + None for a language's directories is what CMake writes for a language the + top-level project does not enable. + """ + compilers = compilers or {} + toolchains = [] + for language, directories in link_directories.items(): + compiler: dict = { + "implicit": {} if directories is None else {"linkDirectories": directories} + } + if language in compilers: + compiler["path"] = compilers[language] + toolchains.append({"language": language, "compiler": compiler}) + write( + _reply(build) / "toolchains-v1-fixture.json", + json.dumps( + { + "kind": "toolchains", + "version": {"major": 1, "minor": 0}, + "toolchains": toolchains, + } + ), + ) + with _edit_index(build) as index: + index["objects"].append( + { + "kind": "toolchains", + "version": {"major": 1, "minor": 0}, + "jsonFile": "toolchains-v1-fixture.json", + } + ) + + +def add_include(build: Path, target: str, directory: Path) -> None: + with _edit_target(build, target) as payload: + payload["compileGroups"][0]["includes"].append({"path": str(directory)}) + + +def add_source(build: Path, target: str, path: Path) -> None: + with _edit_target(build, target) as payload: + payload["sources"].append(_source(str(path))) + + +def depfile( + build: Path, + target: str, + source: Path | str, + read: tuple[Path | str, ...] = (), + directory: str = ".", +) -> None: + """The depfile GCC leaves beside one of a target's objects under the + Makefile generators: the object, its source, every header it read, and + `-MP`'s empty rule per header. `directory` is the target's build directory. + """ + obj = f"CMakeFiles/{target}.dir/{Path(source).name}.o" + rule = " \\\n ".join([f"{obj}: {source}", *(str(item) for item in read)]) + write( + build / directory / f"{obj}.d", + rule + "\n" + "".join(f"\n{item}:\n" for item in read), + ) + + +def compiler_depend( + build: Path, target: str, rules: dict[str, tuple[str, ...]], directory: str = "." +) -> None: + """compiler_depend.make as CMake consolidates a target's depfiles: one rule + per object, paths inside the build tree relative to its top. No rules is + the placeholder CMake writes before the first build.""" + path = build / directory / "CMakeFiles" / f"{target}.dir" / "compiler_depend.make" + if not rules: + write( + path, + f"# Empty compiler generated dependencies file for {target}.\n" + "# This may be replaced when dependencies are built.\n", + ) + return + write( + path, + "# CMAKE generated file: DO NOT EDIT!\n\n" + + "\n".join( + " \\\n ".join([f"{obj}: {prerequisites[0]}", *prerequisites[1:]]) + for obj, prerequisites in rules.items() + ) + + "\n", + ) + + +def add_target( + build: Path, name: str, kind: str, sources: tuple[str, ...], dependent: str +) -> None: + """One more target, which `dependent` depends on, producing nothing shipped.""" + entry = _target_entry(name) + payload = _target_payload( + name=name, + type=kind, + artifacts=[f"tools/{name}"] if kind == "EXECUTABLE" else [], + sources=list(sources), + includes=None if kind == "UTILITY" else [], + dependencies=[], + ) + write(_reply(build) / entry["jsonFile"], json.dumps(payload)) + with _edit_json(_reply(build) / "codemodel-v2-fixture.json") as codemodel: + codemodel["configurations"][0]["targets"].append(entry) + with _edit_target(build, dependent) as target: + target["dependencies"].append({"id": name}) + + +def set_directory(build: Path, target: str, directory: str) -> None: + with _edit_target(build, target) as payload: + payload["paths"] = {"build": directory, "source": directory} + + +def set_type(build: Path, target: str, kind: str) -> None: + with _edit_target(build, target) as payload: + payload["type"] = kind + + +def set_generator(build: Path, name: str) -> None: + with _edit_index(build) as index: + index["cmake"] = {"generator": {"name": name, "multiConfig": False}} + + +def vcpkg_port( + installed: Path, + port: str, + version: str, + files: dict[str, bytes], + licence: str, + source: str | tuple[str, ...], + triplet: str = "x64-linux", +) -> None: + """A port as vcpkg installs it: its files, its install list and its SBOM. + + Shaped after what vcpkg 2026-04-08 writes for fmt 11.0.2#1. + """ + base = version.partition("#")[0] + for relative, data in files.items(): + write(installed / triplet / relative, data) + share = installed / triplet / "share" / port + write(share / "copyright", licence) + write( + share / "vcpkg.spdx.json", + json.dumps( + { + "spdxVersion": "SPDX-2.2", + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "name": port, + "SPDXID": "SPDXRef-port", + "versionInfo": version, + "downloadLocation": "git+https://github.com/Microsoft/vcpkg@07a73a7565e5de9eb42e90c16c133bdfdfebbcda", + "homepage": f"https://github.com/{port}lib/{port}", + }, + { + "name": f"{port}:{triplet}", + "SPDXID": "SPDXRef-binary", + "downloadLocation": "NONE", + }, + *( + { + "SPDXID": f"SPDXRef-resource-{index}", + "name": item, + "downloadLocation": f"git+https://github.com/{item}@{base}", + } + for index, item in enumerate( + (source,) if isinstance(source, str) else source + ) + ), + ], + } + ), + ) + listed = sorted( + {f"{triplet}/{item}" for item in files} + | { + f"{triplet}/share/{port}/copyright", + f"{triplet}/share/{port}/vcpkg.spdx.json", + } + ) + write( + installed / "vcpkg" / "info" / f"{port}_{base}_{triplet}.list", + "".join(f"{line}\n" for line in [f"{triplet}/", *listed]), + ) + + +def stub_compiler(directory: Path, libraries: Path, sysroot: Path, search: str) -> Path: + """A compiler driver that answers the questions the collector asks one. + + Its linker lists `search` as a SEARCH_DIR under the sysroot, `=` and all. + """ + linker = directory / "ld" + write( + linker, + f'#!/bin/sh\necho \'SEARCH_DIR("={search}"); SEARCH_DIR("/nonexistent")\'\n', + ) + compiler = directory / "cc" + write( + compiler, + "#!/bin/sh\n" + 'case "$1" in\n' + f' -print-search-dirs) echo "libraries: ={libraries}" ;;\n' + f' -print-prog-name=ld) echo "{linker}" ;;\n' + f' -print-sysroot) echo "{sysroot}" ;;\n' + "esac\n", + ) + for path in (linker, compiler): + path.chmod(0o755) + return compiler + + +def elf_object() -> bytes: + """A relocatable ELF, the kind a compiler writes and an archive holds.""" + image = bytearray(elf_shared_object("object.o")) + struct.pack_into(" bytes: + """One `ar` member header, its name written as given.""" + return ( + name.ljust(16) + + b"0".ljust(12) + + b"0".ljust(6) + + b"0".ljust(6) + + b"644".ljust(8) + + str(size).encode().ljust(10) + + b"`\n" + ) + + +def ar_archive(members: dict[str, bytes]) -> bytes: + """A static archive in the GNU `ar` format, holding `members`.""" + image = bytearray(b"!\n") + for name, data in members.items(): + image += ar_header(f"{name}/".encode(), len(data)) + image += data + b"\n" * (len(data) % 2) + return bytes(image) + + +# Wheel + +METADATA = "\n".join( + [ + "Metadata-Version: 2.4", + "Name: isaaccapture", + "Version: 0.4.0", + "Summary: Isaac Capture", + "Author: Example Org", + "Project-URL: Homepage, https://example.com/isaaccapture", + "License-Expression: Apache-2.0", + "License-File: LICENSE.md", + "Requires-Python: >=3.11", + # Model the cyclic dependency between the main and transition wheels. + "Requires-Dist: isaacteleop>=0.4.0", + "Requires-Dist: numpy>=1.23.0", + "Requires-Dist: pyyaml>=6.0.3", + "", + "Isaac Capture long description.", + "", + ] +) + + +AUDITWHEEL_SBOM = "auditwheel.cdx.json" +AUDITWHEEL_CDX = json.dumps( + { + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": {"component": {"type": "library", "name": "isaaccapture"}}, + "components": [], + } +).encode("utf-8") + + +def wheel_payload(workspace: Workspace) -> dict[str, bytes]: + """The wheel setuptools would produce from the staged tree.""" + payload: dict[str, bytes] = {} + for path in sorted(workspace.staged.rglob("*")): + if path.is_file(): + payload[path.relative_to(workspace.staged).as_posix()] = path.read_bytes() + + payload[f"{DIST_INFO}/METADATA"] = METADATA.encode("utf-8") + payload[f"{DIST_INFO}/WHEEL"] = ( + b"Wheel-Version: 1.0\nGenerator: setuptools\nRoot-Is-Purelib: false\n" + ) + payload[f"{DIST_INFO}/licenses/LICENSE.md"] = ( + workspace.root / "LICENSE.md" + ).read_bytes() + payload[f"{DIST_INFO}/top_level.txt"] = b"isaaccapture\nvendorpy\n" + # Include auditwheel's PEP 770 document to test coexistence. + payload[f"{DIST_INFO}/sboms/{AUDITWHEEL_SBOM}"] = AUDITWHEEL_CDX + return payload + + +def collect(workspace: Workspace, **options) -> dict: + out_dir = workspace.root / "sbom" + manifest = build_module.build( + workspace.root, workspace.build, workspace.wheel, out_dir, **options + ) + with zipfile.ZipFile(workspace.wheel) as archive: + notices = archive.read(f"{DIST_INFO}/licenses/THIRD-PARTY-NOTICES.md") + return { + "out_dir": out_dir, + "manifest": manifest, + **{ + kind: json.loads((out_dir / f"{STEM}.{kind}.json").read_text("utf-8")) + for kind in ("spdx", "licenses", "build-evidence") + }, + "md": (out_dir / f"{STEM}.licenses.md").read_text("utf-8"), + "notices": notices.decode("utf-8"), + } + + +def collect_and_bind(workspace: Workspace, **options) -> dict: + """`collect`, with the manifest written beside it and the paths tests read.""" + collected = collect(workspace, **options) + out_dir = collected["out_dir"] + manifest_path = out_dir / "manifest.json" + manifest_path.write_text( + json.dumps(collected["manifest"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return { + **collected, + "workspace": workspace, + "root": workspace.root, + "wheel": workspace.wheel, + "manifest_path": manifest_path, + "evidence_path": out_dir / f"{STEM}.build-evidence.json", + "spdx_path": out_dir / f"{STEM}.spdx.json", + } + + +def members(wheel: Path) -> dict[str, bytes]: + """A wheel's members, RECORD left out so `write_wheel` can restate it.""" + with zipfile.ZipFile(wheel) as archive: + return { + name: archive.read(name) + for name in archive.namelist() + if not name.endswith(".dist-info/RECORD") + } + + +def drop_member(wheel: Path, name: str) -> None: + """Remove one member after the fact, leaving RECORD as it was.""" + with zipfile.ZipFile(wheel) as archive: + payload = { + item: archive.read(item) for item in archive.namelist() if item != name + } + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for item in sorted(payload): + archive.writestr(item, payload[item]) + + +def replace_member(wheel: Path, name: str, data: bytes) -> None: + """Change one member after the fact, leaving RECORD as it was.""" + with zipfile.ZipFile(wheel) as archive: + payload = {item: archive.read(item) for item in archive.namelist()} + payload[name] = data + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for item in sorted(payload): + archive.writestr(item, payload[item]) + + +def _record_hash(data: bytes) -> str: + digest = hashlib.sha256(data).digest() + return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def write_wheel(path: Path, payload: dict[str, bytes]) -> Path: + """Write a wheel with a correct RECORD, the way a real build leaves one.""" + dist_info = next( + name.split("/", 1)[0] + for name in payload + if name.endswith(".dist-info/METADATA") + ) + record_name = f"{dist_info}/RECORD" + + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + for name in sorted(payload): + writer.writerow((name, _record_hash(payload[name]), str(len(payload[name])))) + writer.writerow((record_name, "", "")) + + # Fix ZIP timestamps for reproducible wheel bytes. + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name in [*sorted(payload), record_name]: + data = ( + payload[name] if name in payload else buffer.getvalue().encode("utf-8") + ) + member = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + member.external_attr = 0o644 << 16 + member.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(member, data) + return path diff --git a/scripts/sbom/tests/test_discovery.py b/scripts/sbom/tests/test_discovery.py new file mode 100644 index 0000000000..bcc8fa64ba --- /dev/null +++ b/scripts/sbom/tests/test_discovery.py @@ -0,0 +1,1452 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build provenance discovery tests.""" + +from __future__ import annotations + +import json +import os +import subprocess +import zipfile +import zlib +from pathlib import Path + +import pytest +import synth +from isaaccapture_sbom import discovery + + +def test_source_trees_are_found_without_being_listed(workspace): + components = discovery.discover_source_trees(workspace.build / "_deps") + + assert set(components) == { + "alpha", + "beta", + "delta", + "gamma", + "license-list-data", + "omega", + "vendorpy", + } + + +def test_identity_comes_off_the_remote_url(workspace): + alpha = discovery.discover_source_trees(workspace.build / "_deps")["alpha"] + + assert alpha.supplier == "Organization: example-org" + assert alpha.homepage == "https://github.com/example-org/alpha" + # With no checkout, the declared tag stands in for a commit throughout. + assert alpha.purl == "pkg:github/example-org/alpha@v1.2.3" + assert ( + alpha.download_location == "git+https://github.com/example-org/alpha.git@v1.2.3" + ) + assert alpha.version == "v1.2.3" + + +SDL_RELEASE = ( + "https://github.com/libsdl-org/SDL/releases/download/" + "release-2.32.10/SDL2-2.32.10.tar.gz" +) +SDL_HASH = "SHA256=" + "5f" * 32 + + +@pytest.mark.parametrize("hook", [False, True], ids=["shell", "under-a-hook"]) +def test_an_archive_fetched_inside_a_checkout_is_not_that_checkout( + workspace, monkeypatch, hook +): + """GIT_DIR can override -C, even for an archive nested under the project checkout.""" + deps = workspace.build / "_deps" + synth.write(deps / "sdl2-src/LICENSE.txt", synth.spdx_text("Zlib")) + synth.declare(deps, "sdl2", urls=(SDL_RELEASE,), digest=SDL_HASH) + synth.git( + workspace.root, + "remote", + "add", + "origin", + "https://github.com/example/enclosing-project.git", + ) + if hook: + monkeypatch.setenv("GIT_DIR", str(workspace.root / ".git")) + + components = discovery.discover_source_trees(deps) + sdl2 = components["sdl2"] + + assert (sdl2.name, sdl2.supplier, sdl2.homepage, sdl2.purl, sdl2.version) == ( + "SDL", + "Organization: libsdl-org", + "https://github.com/libsdl-org/SDL", + "pkg:github/libsdl-org/sdl@release-2.32.10", + "release-2.32.10", + ) + assert sdl2.download_location == SDL_RELEASE + assert f"Declared archive: {SDL_RELEASE} ({SDL_HASH})." in sdl2.source_info + # A tree declared by GIT_REPOSITORY with no .git left takes no describe either. + for component in components.values(): + assert "enclosing-project" not in repr(component) + assert "git describe" not in component.source_info + + +def test_the_checkout_itself_is_read_whatever_git_dir_says( + workspace, tmp_path, monkeypatch +): + head = discovery.git(workspace.root, "rev-parse", "HEAD") + synth.clone_of(tmp_path / "other", "https://example.invalid/other.git", "v0") + monkeypatch.setenv("GIT_DIR", str(tmp_path / "other" / ".git")) + + assert discovery.git(workspace.root, "rev-parse", "HEAD") == head + assert "VERSION" in discovery.tracked_files(workspace.root) + + +DIRECT = pytest.mark.parametrize("direct", [False, True], ids=["sub-build", "direct"]) + + +@DIRECT +@pytest.mark.parametrize( + ("urls", "digest", "purl", "version", "downloadable"), + [ + ( + ("https://downloads.example.org/foo-1.4.2.tar.gz",), + "SHA256=AB12", + "pkg:generic/foo@1.4.2?checksum=sha256:ab12" + "&download_url=https://downloads.example.org/foo-1.4.2.tar.gz", + "1.4.2", + True, + ), + ( + ( + "https://mirror-a.example.org/foo-1.4.2.tar.gz", + "https://mirror-b.example.org/foo-1.4.3.tar.gz", + ), + "SHA3_256=CD34", + "pkg:generic/foo@1.4.2?checksum=sha3-256:cd34" + "&download_url=https://mirror-a.example.org/foo-1.4.2.tar.gz", + "1.4.2", + True, + ), + ( + ("https://gitlab.com/example-org/foo/-/archive/v1.5.0/foo-v1.5.0.tar.gz",), + None, + "pkg:generic/foo@1.5.0?download_url=" + "https://gitlab.com/example-org/foo/-/archive/v1.5.0/foo-v1.5.0.tar.gz", + "1.5.0", + True, + ), + ( + ("https://downloads.sourceforge.net/project/foo/foo-2.3.tar.bz2",), + None, + "pkg:generic/foo@2.3" + "?download_url=https://downloads.sourceforge.net/project/foo/foo-2.3.tar.bz2", + "2.3", + True, + ), + ( + ("/home/builder/archives/foo-1.4.2.tar.gz",), + None, + "pkg:generic/foo@1.4.2", + "1.4.2", + False, + ), + ( + ("file:///home/builder/archives/foo.zip",), + None, + "pkg:generic/foo", + "NOASSERTION", + False, + ), + ( + ("https://github.com/example-org/foo/archive/refs/tags/v3.0.tar.gz",), + None, + "pkg:github/example-org/foo@v3.0", + "v3.0", + True, + ), + ( + ("https://github.com/example-org/foo/archive/0a1b2c3d.zip",), + None, + "pkg:github/example-org/foo@0a1b2c3d", + "0a1b2c3d", + True, + ), + ( + ( + "https://github.com/example-org/foo/releases/download/v1.0/foo-linux.zip", + ), + None, + "pkg:github/example-org/foo@v1.0", + "v1.0", + True, + ), + ( + ( + "https://github.com/example-org/foo/archive/refs/tags/" + "releases/cpp/v2.1.1.tar.gz", + ), + None, + "pkg:github/example-org/foo@releases/cpp/v2.1.1", + "releases/cpp/v2.1.1", + True, + ), + ( + ("https://codeload.github.com/example-org/foo/tar.gz/refs/tags/v4.2",), + None, + "pkg:github/example-org/foo@v4.2", + "v4.2", + True, + ), + ], +) +def test_a_declared_archive_is_named_by_its_url( + tmp_path, urls, digest, purl, version, downloadable, direct +): + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", urls=urls, digest=digest, direct=direct) + + foo = discovery.discover_source_trees(deps)["foo"] + + download = urls[0] if downloadable else "NOASSERTION" + assert (foo.purl, foo.version, foo.download_location) == (purl, version, download) + github = purl.startswith("pkg:github/") + assert foo.supplier == ("Organization: example-org" if github else "NOASSERTION") + assert "/home/builder" not in foo.source_info + + +@DIRECT +def test_a_declaration_is_matched_to_the_tree_it_names(tmp_path, direct): + """`license_list_data` fills `license-list-data-src`: the content name is not the tree.""" + deps = tmp_path / "build" / "_deps" + (deps / "foo-bar-src").mkdir(parents=True) + synth.declare( + deps, + "foo_bar", + repository="https://github.com/example-org/foo-bar.git", + tag="v5", + source="foo-bar-src", + direct=direct, + ) + + foo_bar = discovery.discover_source_trees(deps)["foo-bar"] + + assert (foo_bar.purl, foo_bar.version) == ( + "pkg:github/example-org/foo-bar@v5", + "v5", + ) + assert "Declared constraint: v5." in foo_bar.source_info + + +@pytest.mark.parametrize("stale_is_newer", [False, True], ids=["older", "newer"]) +def test_a_stamp_left_by_an_earlier_declaration_does_not_count( + tmp_path, stale_is_newer +): + """Switching method leaves the other stamp behind, and CMake rewrites a + stamp only when its contents change, so its mtime cannot decide.""" + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", repository="https://example.invalid/foo.git", tag="v1") + synth.declare(deps, "foo", urls=("https://downloads.example.org/foo-2.0.tar.gz",)) + git_stamp = next(deps.glob("foo-subbuild/**/*-gitinfo.txt")) + url_stamp = next(deps.glob("foo-subbuild/**/*-urlinfo.txt")) + stale, current = (git_stamp, url_stamp) + os.utime(stale, ns=(2, 2) if stale_is_newer else (0, 0)) + os.utime(current, ns=(1, 1)) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert (foo.version, foo.purl.split("?")[0]) == ("2.0", "pkg:generic/foo@2.0") + assert "example.invalid" not in repr(foo) + + +def test_after_a_policy_change_the_newer_layout_is_the_declaration(tmp_path): + """Sub-build stamps stay behind when CMP0168 turns NEW, and the reverse.""" + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", urls=("https://downloads.example.org/foo-1.0.tar.gz",)) + synth.declare( + deps, "foo", urls=("https://downloads.example.org/foo-2.0.tar.gz",), direct=True + ) + os.utime(next(deps.glob("foo-subbuild/**/*-urlinfo.txt")), ns=(0, 0)) + + assert discovery.discover_source_trees(deps)["foo"].version == "2.0" + + +def test_a_checkout_takes_its_git_stamp_over_a_newer_url_stamp(tmp_path): + deps = tmp_path / "build" / "_deps" + commit = synth.clone_of(deps / "foo-src", "https://git.example.org/foo.git", "v3") + synth.declare(deps, "foo", repository="https://git.example.org/foo.git", tag="v3") + synth.declare(deps, "foo", urls=("https://downloads.example.org/foo-2.0.tar.gz",)) + os.utime(next(deps.glob("foo-subbuild/**/*-gitinfo.txt")), ns=(0, 0)) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert foo.version == commit + assert "Declared constraint: v3." in foo.source_info + assert "downloads.example.org" not in repr(foo) + + +def test_git_config_in_the_environment_does_not_choose_the_remote( + tmp_path, monkeypatch +): + deps = tmp_path / "build" / "_deps" + synth.clone_of(deps / "foo-src", "https://git.example.org/foo.git", "v1") + synth.clone_of(tmp_path / "other", "https://git.example.org/other.git", "v1") + monkeypatch.setenv("GIT_CONFIG", str(tmp_path / "other" / ".git" / "config")) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert foo.homepage == "https://git.example.org/foo.git" + + +@pytest.mark.parametrize( + ("repository", "tag", "purl", "download"), + [ + ( + "https://git.example.org/foo.git", + "v1", + "pkg:generic/foo@v1", + "git+https://git.example.org/foo.git@v1", + ), + ( + "https://git.example.org/foo.git", + "v1.0+build", + "pkg:generic/foo@v1.0%2Bbuild", + "git+https://git.example.org/foo.git@v1.0+build", + ), + ( + "git@git.example.org:org/foo.git", + "v1", + "pkg:generic/foo@v1", + "git+ssh://git@git.example.org/org/foo.git@v1", + ), + ("/srv/git/foo.git", "v1", "pkg:generic/foo@v1", "NOASSERTION"), + ], +) +def test_a_declared_repository_off_github_is_named_by_its_tag( + tmp_path, repository, tag, purl, download +): + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", repository=repository, tag=tag) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert (foo.purl, foo.download_location, foo.version) == (purl, download, tag) + + +def test_a_moved_build_tree_still_reads_its_clone_script(tmp_path): + deps = tmp_path / "old" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", repository="https://example.invalid/foo.git", tag="v7") + (tmp_path / "old").rename(tmp_path / "new") + + foo = discovery.discover_source_trees(tmp_path / "new" / "_deps")["foo"] + + assert foo.version == "v7" + + +def test_credentials_in_a_declared_url_are_not_published(tmp_path): + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + (deps / "bar-src").mkdir(parents=True) + synth.declare( + deps, + "foo", + urls=("https://u:s3cret@dl.example.org/foo-1.0.tgz?token=t0ken#sig=s1g",), + ) + synth.declare(deps, "bar", repository="https://tok3n@git.example.org/bar.git") + + components = discovery.discover_source_trees(deps) + + for secret in ("s3cret", "t0ken", "s1g"): + assert secret not in repr(components["foo"]) + assert "tok3n" not in repr(components["bar"]) + assert components["foo"].download_location == "https://dl.example.org/foo-1.0.tgz" + assert components["foo"].purl.endswith( + "download_url=https://dl.example.org/foo-1.0.tgz" + ) + + +def test_configuration_from_the_environment_reaches_git(workspace, monkeypatch): + """A container's safe.directory arrives this way; it does not pick a repository.""" + monkeypatch.setenv("GIT_CONFIG_COUNT", "1") + monkeypatch.setenv("GIT_CONFIG_KEY_0", "sbom.probe") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", "kept") + + assert discovery.git(workspace.root, "config", "--get", "sbom.probe") == "kept" + + +@pytest.mark.parametrize( + ("in_another_checkout", "separate_git_dir", "through_symlink"), + [(False, False, False), (True, True, False), (False, False, True)], + ids=["standalone", "submodule-like", "symlinked"], +) +def test_a_checkout_of_its_own_answers_for_itself( + workspace, tmp_path, in_another_checkout, separate_git_dir, through_symlink +): + build = workspace.build if in_another_checkout else tmp_path / "build" + root = build / "_deps" / "epsilon-src" + synth.write(root / "LICENSE", synth.spdx_text("MIT")) + commit = synth.clone_of( + root, + "https://github.com/example-org/epsilon.git", + "v2.0.0", + git_dir=tmp_path / "modules" / "epsilon" if separate_git_dir else None, + ) + synth.declare( + build / "_deps", "epsilon", repository="https://example.invalid/x.git" + ) + if through_symlink: + (tmp_path / "link").symlink_to(build, target_is_directory=True) + build = tmp_path / "link" + + epsilon = discovery.discover_source_trees(build / "_deps")["epsilon"] + + assert epsilon.version == commit + assert epsilon.purl == f"pkg:github/example-org/epsilon@{commit}" + assert epsilon.download_location.endswith(f"/epsilon.git@{commit}") + assert "git describe: v2.0.0." in epsilon.source_info + + +def test_licenses_come_from_the_trees_own_files(workspace): + components = discovery.discover_source_trees(workspace.build / "_deps") + + assert components["alpha"].license_concluded == "MIT" + assert components["delta"].license_concluded == "BSD-3-Clause" + assert components["gamma"].license_concluded == "Zlib" + + +def test_link_graph_finds_static_and_header_only_contributions(workspace): + graph = discovery.BuildGraph(workspace.build) + extension = graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so") + + assert len(extension) == 1 + contributions = graph.contributions(extension[0]) + + assert contributions["alpha"] == {"compiled-source", "header-include"} + # beta ships headers only; an include path is the only trace it leaves. + assert contributions["beta"] == {"header-include"} + + +def test_link_graph_recurses_through_static_archives(workspace): + """delta reaches the wheel only through libdelta.a inside libgamma_renamed.so.""" + graph = discovery.BuildGraph(workspace.build) + gamma = graph.by_name("libgamma_renamed.so")[0] + + contributions = graph.contributions(gamma) + + assert "gamma" in contributions + assert "delta" in contributions + + +def test_first_party_is_decided_by_whose_sources_were_compiled(workspace): + graph = discovery.BuildGraph(workspace.build) + repo = workspace.root.resolve() + + extension = graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] + gamma = graph.by_name("libgamma_renamed.so")[0] + + assert graph.first_party(extension, repo) + assert not graph.first_party(gamma, repo) + + +def test_a_test_only_dependency_reaches_no_shipped_artifact(workspace): + graph = discovery.BuildGraph(workspace.build) + extension = graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] + gamma = graph.by_name("libgamma_renamed.so")[0] + + assert "omega" not in graph.contributions(extension) + assert "omega" not in graph.contributions(gamma) + assert "omega" in graph.contributions(graph.by_name("unit_tests")[0]) + + +def test_archive_members_are_indexed_by_content(workspace): + index = discovery.discover_archives( + workspace.root, frozenset(discovery.tracked_files(workspace.root)) + ) + + assert synth.SDK_ARCHIVE in index.archives + demo = index.by_name["libdemo.so"] + assert len(demo) == 1 + assert index.by_hash[demo[0].sha256].container == synth.SDK_ARCHIVE + + +def test_archive_version_and_license_come_from_the_archive(workspace): + index = discovery.discover_archives( + workspace.root, frozenset(discovery.tracked_files(workspace.root)) + ) + record = index.archives[synth.SDK_ARCHIVE] + + assert record["version_text"] == "1.2.0" + assert [member.path for member, _ in record["license_members"]] == ["LICENSE.txt"] + + +def test_an_archive_root_sidecar_is_not_a_license_member(tmp_path): + archive = tmp_path / "sdk.zip" + with zipfile.ZipFile(archive, "w") as handle: + handle.writestr("LICENSE.txt", synth.spdx_text("MIT")) + handle.writestr("libdemo.so", b"\0") + handle.writestr("libdemo.so.license", "SPDX-License-Identifier: MIT\n") + handle.writestr("stb.license", synth.spdx_text("MIT")) + index = discovery.ArchiveIndex() + + index.add_archive(archive, "sdk.zip") + + assert [ + member.path for member, _ in index.archives["sdk.zip"]["license_members"] + ] == ["LICENSE.txt", "stb.license"] + + +def test_a_toolkit_licence_beside_a_library_is_not_a_sidecar(tmp_path): + synth.write(tmp_path / "lib/libx.so.1", "\0") + synth.write(tmp_path / "lib/libx.so.1.license", "SPDX-License-Identifier: MIT\n") + synth.write(tmp_path / "EULA.txt", "terms\n") + + assert discovery._license_near(tmp_path / "lib/libx.so.1") == tmp_path / "EULA.txt" + + +def test_a_build_tree_without_a_file_api_reply_says_so(workspace): + import shutil + + shutil.rmtree(workspace.build / ".cmake") + + with pytest.raises(discovery.FileApiError, match="cmake_file_api"): + discovery.BuildGraph(workspace.build, synth.CONFIG) + + +def test_a_stale_codemodel_beside_the_current_one_is_not_read(workspace): + """The stale codemodel sorts later; only the index selects the current one.""" + reply = workspace.build / ".cmake" / "api" / "v1" / "reply" + stale = json.loads((reply / "codemodel-v2-fixture.json").read_text()) + stale["configurations"] = [{"name": synth.CONFIG, "targets": []}] + (reply / "codemodel-v2-stale.json").write_text(json.dumps(stale)) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + + assert graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so") + + +def test_a_configuration_the_codemodel_does_not_describe_is_refused(workspace): + with pytest.raises(discovery.FileApiError, match="Debug"): + discovery.BuildGraph(workspace.build, "Debug") + + +def test_the_graph_is_read_from_the_codemodel_not_from_generator_output(workspace): + assert not list(workspace.build.rglob("link.txt")) + assert not (workspace.build / "compile_commands.json").exists() + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + + assert graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so") + assert graph.by_name("libgamma_renamed.so") + + +def test_third_party_checked_in_here_is_still_third_party(workspace): + """Copyright metadata, not directory placement, identifies vendored code.""" + from isaaccapture_sbom import evidence as evidence_module + + space = evidence_module.discover(workspace.root, workspace.build) + vendored = { + key: component + for key, component in space.components.items() + if component.kind == "vendored-source" + } + + assert [c.name for c in vendored.values()] == [ + "Vendored source (Upstream Widgets Ltd)" + ] + assert [c.supplier for c in vendored.values()] == [ + "Organization: Upstream Widgets Ltd" + ] + component = next(iter(vendored.values())) + assert component.license_concluded == "BSL-1.0" + # Use upstream reference texts, not the repository's own license pool. + assert [item.path for item in component.evidence] == ["BSL-1.0.txt"] + assert all(item.origin == "spdx-reference" for item in component.evidence) + assert "Upstream Widgets Ltd" in component.copyright_text + + +def test_a_vendored_header_is_credited_to_the_artifact_that_includes_it(workspace): + from isaaccapture_sbom import evidence as evidence_module + + space = evidence_module.discover(workspace.root, workspace.build) + artifact = space.graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] + + contributions = space.graph.contributions(artifact, space.ownership) + + assert "vendored:upstream-widgets-ltd" in contributions + assert contributions["vendored:upstream-widgets-ltd"] == {"header-include"} + + +def test_the_codemodel_reader_works_on_a_codemodel_cmake_wrote( + live_build_dir, checkout +): + """Check real codemodel structure without depending on the current dependency set.""" + graph = discovery.BuildGraph(live_build_dir) + + assert graph.artifacts, "a configured build declares at least one artifact" + assert any( + graph.first_party(artifact, checkout) for artifact in graph.artifacts.values() + ), "this project compiles some of its own sources" + + for output, artifact in graph.artifacts.items(): + assert output.is_absolute() + assert graph.by_name(output.name), "every artifact is findable by name" + for how in graph.contributions(artifact).values(): + assert how <= {"compiled-source", "header-include"} + + # Relative link paths must resolve to this build's indexed outputs. + named = {path for node in graph._nodes.values() for path in node.linked} + assert named & set(graph.artifacts), "this project links static libraries it builds" + + +def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data): + """A second configured build contains staged source copies that must not enter the source index.""" + from isaaccapture_sbom import evidence, licensing + + licensing.load_corpus(license_data) + stray = workspace.root / "build-debug" + (stray / "isaaccapture").mkdir(parents=True) + (stray / "CMakeCache.txt").write_text("CMAKE_BUILD_TYPE:STRING=Debug\n") + copy = stray / "isaaccapture" / "__init__.py" + copy.write_text( + (workspace.root / "src/python/isaaccapture/__init__.py").read_text() + ) + + discovered = evidence.discover(workspace.root, workspace.build) + indexed = set(discovered.repo_files.by_hash.values()) + + assert "./src/python/isaaccapture/__init__.py" in indexed + assert not any(item.startswith("./build-debug/") for item in indexed) + + +def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): + """The fetch script preserves archive origin in a .source sidecar.""" + from isaaccapture_sbom import evidence, licensing + + licensing.load_corpus(license_data) + archive = workspace.root / synth.SDK_ARCHIVE + + unknown = evidence.discover(workspace.root, workspace.build) + before = unknown.components[synth.SDK_ARCHIVE] + assert before.download_location == "NOASSERTION" + assert before.supplier == "NOASSERTION" + + archive.with_name(archive.name + ".source").write_text( + "https://api.ngc.nvidia.com/v2/resources/example/sdk.tar.gz\n" + ) + known = evidence.discover(workspace.root, workspace.build) + after = known.components[synth.SDK_ARCHIVE] + + assert after.download_location.startswith("https://api.ngc.nvidia.com/") + assert after.supplier == "Organization: nvidia" + assert "fetched from https://" in after.source_info + + +def test_a_build_tree_with_no_configuration_says_so(workspace): + from isaaccapture_sbom import evidence + + cache = workspace.build / "CMakeCache.txt" + cache.write_text( + "\n".join( + line + for line in cache.read_text().splitlines() + if not line.startswith("CMAKE_BUILD_TYPE:") + ) + + "\n" + ) + + with pytest.raises(evidence.EvidenceError, match="CMAKE_BUILD_TYPE"): + evidence.discover(workspace.root, workspace.build) + + +def test_the_supplier_is_read_from_the_project_not_written_here(workspace): + + assert discovery.project_supplier(workspace.root) == "Organization: NVIDIA" + + pyproject = workspace.root / "pyproject.toml" + pyproject.write_text( + pyproject.read_text().replace( + '{ name = "NVIDIA" }', '{ name = "Someone Else" }' + ), + encoding="utf-8", + ) + + assert discovery.project_supplier(workspace.root) == "Organization: Someone Else" + + +def test_a_project_naming_several_authors_stops_rather_than_choosing(workspace): + """SPDX records one supplier; which one is not this tool's call to make.""" + + pyproject = workspace.root / "pyproject.toml" + pyproject.write_text( + pyproject.read_text().replace( + '{ name = "NVIDIA" }', + '{ name = "NVIDIA" }, { name = "Other Party" }', + ), + encoding="utf-8", + ) + + with pytest.raises(discovery.SupplierError, match="names 2 project authors"): + discovery.project_supplier(workspace.root) + + +def test_a_relative_build_dir_still_keeps_the_staged_tree_out( + workspace, license_data, monkeypatch +): + """The workflow passes a relative build directory; staging must still be excluded.""" + from isaaccapture_sbom import evidence, licensing + + licensing.load_corpus(license_data) + monkeypatch.chdir(workspace.root) + + discovered = evidence.discover(workspace.root, Path("build")) + indexed = set(discovered.build_files.by_hash.values()) + + assert indexed, "the build tree is indexed" + assert not any("/python_package/" in item for item in indexed) + + +def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( + workspace, license_data +): + """Installed dependency headers must not become one vendored component per copyright holder.""" + from isaaccapture_sbom import evidence, licensing + + licensing.load_corpus(license_data) + port = workspace.build / "vcpkg_installed/x64-linux/include/eigen/Core.h" + assert port.is_file(), "the fixture no longer installs a port into the build tree" + + discovered = evidence.discover(workspace.root, workspace.build) + holders = [key for key in discovered.components if key.startswith("vendored:")] + + assert not any("guennebaud" in key for key in holders), holders + + +def test_vendored_material_asserts_no_package_identifier(workspace, license_data): + """Copyright ownership does not establish a resolvable package identity.""" + from isaaccapture_sbom import licensing + + licensing.load_corpus(license_data) + source = workspace.root / "src" / "vendor" / "widget.hpp" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text( + "// SPDX-FileCopyrightText: Copyright 2023 Upstream Widgets Ltd\n" + "// SPDX-License-Identifier: BSL-1.0\n" + ) + components, _ = discovery.discover_vendored( + workspace.root, {source}, {"Example Org"}, tracked=() + ) + component = next(iter(components.values())) + + assert component.kind == "vendored-source" + assert component.purl == "NOASSERTION" + assert component.supplier == "Organization: Upstream Widgets Ltd" + + +def _discover(root: Path, *candidates: str): + """Discover over build inputs alone, as `discover_vendored` sees them.""" + components, _ = discovery.discover_vendored( + root, {root / item for item in candidates}, {"NVIDIA"}, tracked=() + ) + return components + + +def _vendored(root: Path, files: dict[str, str | bytes]): + """Discover over files as a checkout tracking all of them would.""" + for relative, data in files.items(): + synth.write(root / relative, data) + components, _ = discovery.discover_vendored( + root, set(), {"NVIDIA"}, tracked=sorted(files) + ) + return components + + +def _link(root: Path, link: str, target: str) -> None: + (root / link).parent.mkdir(parents=True, exist_ok=True) + (root / link).symlink_to(target) + + +def _notices(components) -> dict[str, set[str]]: + return {key: set(c.copyright_text.splitlines()) for key, c in components.items()} + + +_LICENCE_TAG = "SPDX-" + "License-Identifier: MIT" +_IGNORED_BAR = "// REUSE-IgnoreStart\n// Copyright 2020 Bar Ltd\n// REUSE-IgnoreEnd\n" +_HEADERS = { + "copyright-and-licence": f"// Copyright 2020 Bar Ltd\n// {_LICENCE_TAG}\n", + "copyright-only": "// Copyright 2020 Bar Ltd\n", + "unparsed-and-licence": f"// Copyright: 2020 Bar Ltd\n// {_LICENCE_TAG}\n", + "unparsed-only": "// Copyright: 2020 Bar Ltd\n", + "ignored-and-licence": f"{_IGNORED_BAR}// {_LICENCE_TAG}\n", + "ignored-only": _IGNORED_BAR, + "ignored-and-contributor": f"{_IGNORED_BAR}// SPDX-FileContributor: Someone\n", +} +_BAR = {"vendored:bar-ltd": {"Copyright 2020 Bar Ltd"}} +_BAR_UNPARSED = {"vendored:bar-ltd": {"Copyright: 2020 Bar Ltd"}} +_FOO = "Copyright 2020 Foo Inc" + + +_DECLARED = { + "header": { + "src/x.h": "// Copyright 2020 Foo Inc\n// SPDX-" + + "License-Identifier: MIT OR Apache-2.0\n" + }, + "sidecar": { + "src/x.h": "// Copyright 2020 Foo Inc\n", + "src/x.h.license": "SPDX-FileCopyrightText: Copyright 2020 Foo Inc\n" + "SPDX-" + "License-Identifier: MIT OR Apache-2.0\n", + }, + "REUSE.toml": { + "src/x.h": "// Copyright 2020 Foo Inc\n", + "REUSE.toml": synth.reuse_toml( + synth.annotation("src/x.h", "Foo Inc", licence="MIT OR Apache-2.0") + ), + }, +} + + +@pytest.mark.parametrize( + "files", [pytest.param(files, id=name) for name, files in _DECLARED.items()] +) +def test_a_vendored_files_terms_are_what_reuse_declares(tmp_path, files): + """All declaration forms preserve OR and supply every referenced text.""" + for relative, text in files.items(): + synth.write(tmp_path / relative, text) + + component = _discover(tmp_path, "src/x.h")["vendored:foo-inc"] + + assert component.license_concluded == "MIT OR Apache-2.0" + assert component.license_declared == "MIT OR Apache-2.0" + assert sorted(item.path for item in component.evidence) == [ + "Apache-2.0.txt", + "MIT.txt", + ] + assert all(item.origin == "spdx-reference" for item in component.evidence) + assert component.copyright_text == "Copyright 2020 Foo Inc" + + +@pytest.mark.parametrize( + ("tag", "reason"), + [ + ("", "no current SPDX-listed licence"), + ( + "// SPDX-" + "License-Identifier: LicenseRef-Foo\n", + "no current SPDX-listed licence", + ), + # MIT has a text; the LicenseRef would be defined by it in the document. + ( + "// SPDX-" + "License-Identifier: MIT AND LicenseRef-Foo\n", + "no current SPDX-listed licence with a reference text for LicenseRef-Foo", + ), + ("// SPDX-" + "License-Identifier: not ( an expression\n", "does not parse"), + # A valid tag, then one quoted in the body: REUSE reads both. + ( + "// SPDX-" + "License-Identifier: MIT\n" + 'const char *tag = "// SPDX-' + 'License-Identifier: GPL-2.0-only\\n";\n', + "does not parse", + ), + ( + "// SPDX-" + "License-Identifier: MIT AND Foo-1.0\n", + "Foo-1.0 is not an SPDX identifier", + ), + ( + "// SPDX-" + "License-Identifier: MIT AND Bar-2.0 AND Foo-1.0\n", + "Bar-2.0, Foo-1.0 are not SPDX identifiers", + ), + # `+` is SPDX's operator, so the id is known; there is just no text for it. + ( + "// SPDX-" + "License-Identifier: Apache-2.0+\n", + "no current SPDX-listed licence with a reference text", + ), + # Listed, but as a licence: the exception slot takes only exceptions. + ( + "// SPDX-" + "License-Identifier: MIT WITH Apache-2.0\n", + "Apache-2.0 is not an SPDX exception", + ), + ( + "// SPDX-" + "License-Identifier: MIT WITH LicenseRef-Foo\n", + "LicenseRef-Foo is not an SPDX exception", + ), + ( + "// SPDX-" + "License-Identifier: MIT AND rsync-linking-exception\n", + "rsync-linking-exception is an SPDX exception, not a licence", + ), + ], + ids=[ + "no-info", + "licenseref-only", + "licenseref-beside-listed", + "unparseable", + "quoted-in-body", + "unknown-id", + "unknown-ids", + "or-later", + "licence-as-exception", + "licenseref-as-exception", + "exception-as-licence", + ], +) +def test_a_file_lending_nothing_is_named_not_covered_by_its_sibling( + tmp_path, tag, reason +): + (tmp_path / "src").mkdir() + (tmp_path / "src/a.h").write_text( + "// Copyright 2020 Foo Inc\n// SPDX-" + "License-Identifier: MIT\n" + ) + (tmp_path / "src/b.h").write_text("// Copyright 2020 Foo Inc\n" + tag) + + components = _discover(tmp_path, "src/a.h", "src/b.h") + + [(path, why)] = components["vendored:foo-inc"].unlicensed + assert path == "src/b.h" + assert reason in why + + +@pytest.mark.parametrize( + "expression", + ["BSL-1.0 WITH rsync-linking-exception", "GPL-2.0+"], + ids=["exception-newer-than-grammar", "deprecated-or-later"], +) +def test_a_listed_identifier_passes_the_per_file_gate(tmp_path, expression): + """Listed in the pinned SPDX data, whatever the grammar knows: an exception + newer than it, or an id it maps through SPDX's `+` or-later operator.""" + (tmp_path / "x.h").write_text( + "// Copyright 2020 Foo Inc\n// SPDX-" + f"License-Identifier: {expression}\n" + ) + + assert _discover(tmp_path, "x.h")["vendored:foo-inc"].unlicensed == () + + +def test_a_link_without_reuse_info_takes_its_targets(tmp_path): + """A symlink's own REUSE metadata wins; without it, use the target's metadata.""" + synth.write(tmp_path / "src/real/x.h", "// Copyright 2020 Foo Inc\n") + _link(tmp_path, "src/link/x.h", "../real/x.h") + synth.write( + tmp_path / "REUSE.toml", + synth.reuse_toml(synth.annotation("src/real/x.h", "Foo Inc")), + ) + + component = _discover(tmp_path, "src/link/x.h")["vendored:foo-inc"] + + assert component.unlicensed == () + assert component.license_concluded == "MIT" + + synth.write( + tmp_path / "src/link/x.h.license", + "SPDX-FileCopyrightText: Copyright 2020 Foo Inc\n" + "SPDX-" + "License-Identifier: BSD-3-Clause\n", + ) + component = _discover(tmp_path, "src/link/x.h")["vendored:foo-inc"] + + assert component.license_concluded == "BSD-3-Clause" + + +@pytest.mark.parametrize( + "files", + [ + pytest.param( + { + "src/x.h": "#pragma once\n", + "src/x.h.license": f"SPDX-FileCopyrightText: {_FOO}\n{_LICENCE_TAG}\n", + }, + id="sidecar", + ), + pytest.param( + { + "assets/icon.svg": "\n", + "REUSE.toml": synth.reuse_toml(synth.annotation("assets/*", "Foo Inc")), + }, + id="reuse-toml-svg", + ), + pytest.param( + { + "assets/data.json": "{}\n", + "REUSE.toml": synth.reuse_toml(synth.annotation("assets/*", "Foo Inc")), + }, + id="reuse-toml-json", + ), + ], +) +def test_a_holder_declared_beside_the_file_is_read(tmp_path, files): + """A file of any type whose copyright only REUSE metadata states is still + someone else's; the sidecar and REUSE.toml are not candidates themselves.""" + component = _vendored(tmp_path, files)["vendored:foo-inc"] + + [covered] = [name for name in files if not name.endswith((".license", ".toml"))] + assert covered in component.source_info + assert component.license_concluded == "MIT" + assert component.copyright_text == _FOO + + +@pytest.mark.parametrize( + ("header", "holder", "precedence", "expected"), + [ + pytest.param( + "copyright-and-licence", + "Foo Inc", + "override", + {"vendored:foo-inc": {_FOO}}, + id="override-outranks-the-header", + ), + pytest.param( + "copyright-only", None, "override", _BAR, id="override-licence-only" + ), + pytest.param( + "copyright-and-licence", + "Foo Inc", + "closest", + _BAR, + id="closest-takes-a-full-header-first", + ), + pytest.param( + "unparsed-only", + "NVIDIA CORPORATION", + "closest", + {}, + id="closest-stands-in-for-a-file-reuse-finds-nothing-in", + ), + pytest.param( + "unparsed-and-licence", + "NVIDIA CORPORATION", + "closest", + _BAR_UNPARSED, + id="closest-beside-a-licence-tag", + ), + pytest.param( + "ignored-and-contributor", + "NVIDIA CORPORATION", + "closest", + _BAR, + id="closest-beside-a-contributor-line", + ), + *( + pytest.param( + header, + "NVIDIA CORPORATION", + "aggregate", + _BAR_UNPARSED if header.startswith("unparsed") else _BAR, + id=f"aggregate-project-{header}", + ) + for header in ( + "copyright-and-licence", + "copyright-only", + "unparsed-and-licence", + "ignored-and-licence", + "unparsed-only", + "ignored-only", + ) + ), + pytest.param( + "copyright-and-licence", + "Foo Inc", + "aggregate", + {"vendored:foo-inc": {_FOO, "Copyright 2020 Bar Ltd"}}, + id="aggregate-both-give-notices", + ), + ], +) +def test_a_header_counts_where_reuse_applies_it_beside_an_annotation( + tmp_path, header, holder, precedence, expected +): + """`override` sets the file aside; `aggregate` applies it beside the + annotation whatever REUSE parsed; `closest` applies it where REUSE took + anything from it, and stands in for it where REUSE found nothing.""" + toml = synth.reuse_toml(synth.annotation("src/x.h", holder, precedence=precedence)) + components = _vendored(tmp_path, {"src/x.h": _HEADERS[header], "REUSE.toml": toml}) + + assert _notices(components) == expected + + +def test_declared_notices_naming_the_project_keep_the_file_first_party(tmp_path): + components = _vendored( + tmp_path, + { + "src/x.h": _HEADERS["copyright-only"], + "src/x.h.license": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA " + "CORPORATION & AFFILIATES. All rights reserved.\n" + "SPDX-" + "License-Identifier: Apache-2.0\n", + }, + ) + + assert components == {} + + +def test_a_file_with_several_declared_holders_is_grouped_by_one(tmp_path): + """Grouped under the first other holder by name, whatever its case, with + every notice but the project's as its copyright.""" + components = _vendored( + tmp_path, + { + "src/x.h": "#pragma once\n", + "src/x.h.license": "SPDX-FileCopyrightText: 2020 Zed Corp\n" + "SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION\n" + "SPDX-FileCopyrightText: 2021 acme Inc\n" + _LICENCE_TAG + "\n", + }, + ) + + assert _notices(components) == { + "vendored:acme-inc": {"2020 Zed Corp", "2021 acme Inc"} + } + + +def _png_with_text(keyword: bytes, text: bytes) -> bytes: + def chunk(kind: bytes, data: bytes) -> bytes: + body = kind + data + return len(data).to_bytes(4, "big") + body + zlib.crc32(body).to_bytes(4, "big") + + header = (1).to_bytes(4, "big") * 2 + bytes([8, 0, 0, 0, 0]) + pixels = zlib.compress(b"\x00\x00") + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"tEXt", keyword + b"\x00" + text) + + chunk(b"IDAT", pixels) + + chunk(b"IEND", b"") + ) + + +def test_a_binary_file_is_not_scraped_for_a_holder(tmp_path): + """A PNG's `Copyright` text chunk is not a header.""" + png = _png_with_text(b"Copyright", b"\nCopyright 2020 Foo Inc\n") + assert discovery._COPYRIGHT_TAG.search(png.decode("utf-8", "replace")) + + assert _vendored(tmp_path, {"assets/logo.png": png}) == {} + + +def test_license_files_and_the_pool_are_not_candidates(tmp_path): + """REUSE does not cover `LICENSES/` or `LICENSE*`/`COPYING*` files, and the + holder one names is its template's or its licensor's.""" + upstream = "Copyright (c) 2020 Foo Inc\n\nPermission is hereby granted\n" + components = _vendored( + tmp_path, + { + "LICENSE": upstream, + "COPYING.txt": upstream, + "deps/foo/LICENSE.md": upstream, + "LICENSES/MIT.txt": synth.spdx_text("MIT"), + }, + ) + + assert components == {} + + +def test_a_build_input_reuse_does_not_cover_is_not_a_candidate(tmp_path): + synth.write(tmp_path / "inc/x.h", "// Copyright 2020 Foo Inc\n") + synth.write(tmp_path / "inc/x.h.license", f"SPDX-FileCopyrightText: {_FOO}\n") + synth.write(tmp_path / "inc/LICENSE", "Copyright (c) 2020 Bar Ltd\n") + + components = _discover(tmp_path, "inc/x.h", "inc/x.h.license", "inc/LICENSE") + + assert list(components) == ["vendored:foo-inc"] + assert "Files: inc/x.h." in components["vendored:foo-inc"].source_info + + +@pytest.mark.parametrize( + "relative", + ["subprojects/foo/x.h", "gen/x.h", "sub/x.h"], + ids=["meson-subproject", "vcs-ignored", "submodule"], +) +def test_a_build_input_the_project_does_not_cover_is_still_read(tmp_path, relative): + """A compiled file ships whether or not REUSE lints it; a tracked one there is + held to REUSE's coverage.""" + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + synth.write(tmp_path / ".gitignore", "gen/\n") + synth.write(tmp_path / ".gitmodules", '[submodule "sub"]\n\tpath = sub\n') + synth.write(tmp_path / relative, "// Copyright 2020 Foo Inc\n") + + assert list(_discover(tmp_path, relative)) == ["vendored:foo-inc"] + components, _ = discovery.discover_vendored( + tmp_path, set(), {"NVIDIA"}, tracked=[relative] + ) + assert components == {} + + +def test_the_files_the_build_read_are_named_first(tmp_path): + for name in ("src/a.h", "src/z.h"): + synth.write(tmp_path / name, "// Copyright 2020 Foo Inc\n") + + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "src/z.h"}, {"NVIDIA"}, tracked=["src/a.h", "src/z.h"] + ) + + assert "Files: src/z.h, src/a.h." in components["vendored:foo-inc"].source_info + + +def test_a_tracked_non_code_file_reaches_discovery(workspace, license_data): + """Through the build's own entry point, not only `discover_vendored`.""" + from isaaccapture_sbom import evidence, licensing + + licensing.load_corpus(license_data) + synth.write(workspace.root / "assets/vendor/icon.svg", "\n") + synth.write( + workspace.root / "REUSE.toml", + synth.reuse_toml(synth.annotation("assets/vendor/**", "Foo Inc")), + ) + synth.track(workspace.root) + + discovered = evidence.discover(workspace.root, workspace.build) + + assert ( + "assets/vendor/icon.svg" + in discovered.components["vendored:foo-inc"].source_info + ) + + +def test_a_copyright_string_in_a_first_party_body_names_no_holder(tmp_path): + """REUSE reports every copyright line in a file; only the head's first one + is the file's own.""" + components = _vendored( + tmp_path, + { + "src/x.py": "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA " + "CORPORATION & AFFILIATES. All rights reserved.\n" + "# SPDX-" + "License-Identifier: Apache-2.0\n\n" + 'NOTICE = """\nCopyright 2019 Someone Else\n"""\n', + }, + ) + + assert components == {} + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + pytest.param( + { + "src/x.h": _HEADERS["copyright-only"], + "src/x.h.license": _LICENCE_TAG + "\n", + }, + _BAR, + id="sidecar-licence-only", + ), + pytest.param( + {"src/x.h": _HEADERS["unparsed-and-licence"]}, + _BAR_UNPARSED, + id="line-reuse-does-not-parse", + ), + pytest.param( + {"src/x.h": _HEADERS["ignored-and-licence"]}, + _BAR, + id="inside-an-ignore-block", + ), + pytest.param( + { + "src/x.h": _HEADERS["copyright-only"], + "src/x.h.license": f"SPDX-FileCopyrightText: 2020\n{_LICENCE_TAG}\n", + }, + _BAR, + id="declared-notice-naming-nobody", + ), + ], +) +def test_without_a_declared_holder_the_head_still_decides(tmp_path, files, expected): + assert _notices(_vendored(tmp_path, files)) == expected + + +def test_the_head_is_read_in_the_encoding_it_is_written_in(tmp_path): + header = _HEADERS["copyright-and-licence"].encode("utf-16") + + assert list(_vendored(tmp_path, {"src/x.h": header})) == ["vendored:bar-ltd"] + + +@pytest.mark.parametrize("tag", ["SPDX-FileCopyrightText", "SPDX-SnippetCopyrightText"]) +def test_a_declared_notice_is_published_as_written(tmp_path, tag): + components = _vendored( + tmp_path, + { + "src/x.h": "#pragma once\n", + "src/x.h.license": f"{tag}: 2020,2021 Foo Inc\n{_LICENCE_TAG}\n", + }, + ) + + assert components["vendored:foo-inc"].copyright_text == "2020,2021 Foo Inc" + + +@pytest.mark.parametrize( + "sidecar", + [f"SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION\n{_LICENCE_TAG}\n", ""], + ids=["declaring", "empty"], +) +def test_an_aggregate_annotation_beside_a_sidecar_sets_the_file_aside( + tmp_path, sidecar +): + """REUSE reads the sidecar in the file's place, aggregate or not, and + whether or not the sidecar says anything.""" + toml = synth.annotation("src/x.h", "NVIDIA CORPORATION", precedence="aggregate") + components = _vendored( + tmp_path, + { + "src/x.h": _HEADERS["copyright-and-licence"], + "src/x.h.license": sidecar, + "REUSE.toml": synth.reuse_toml(toml), + }, + ) + + assert components == {} + + +def test_a_nested_override_sets_the_file_aside_beneath_an_aggregate(tmp_path): + """REUSE stops reading the file at any override, whatever lies above it.""" + nvidia = "NVIDIA CORPORATION" + components = _vendored( + tmp_path, + { + "src/x.h": _HEADERS["copyright-and-licence"], + "REUSE.toml": synth.reuse_toml( + synth.annotation("**", nvidia, precedence="aggregate") + ), + "src/REUSE.toml": synth.reuse_toml( + synth.annotation("**", nvidia, precedence="override") + ), + }, + ) + + assert components == {} + + +def test_a_link_takes_its_targets_holder_where_it_has_none(tmp_path): + synth.write(tmp_path / "vendor/x.h", f"// {_LICENCE_TAG}\n") + synth.write( + tmp_path / "vendor/x.h.license", "SPDX-FileCopyrightText: 2020 Foo Inc\n" + ) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + component = _discover(tmp_path, "src/x.h")["vendored:foo-inc"] + + # REUSE gives the target no licence: its sidecar sets the header aside. + assert component.license_concluded == "NOASSERTION" + assert component.unlicensed + + +def test_a_link_takes_its_own_licence_and_its_targets_holder(tmp_path): + synth.write(tmp_path / "vendor/x.h", _HEADERS["copyright-and-licence"]) + _link(tmp_path, "src/x.h", "../vendor/x.h") + synth.write( + tmp_path / "src/x.h.license", "SPDX-" + "License-Identifier: BSD-3-Clause\n" + ) + + component = _discover(tmp_path, "src/x.h")["vendored:bar-ltd"] + + assert component.license_concluded == "BSD-3-Clause" + + +@pytest.mark.parametrize( + ("sidecar", "expected"), + [ + pytest.param( + "SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION\n" + "SPDX-" + "License-Identifier: Apache-2.0\n", + {}, + id="target-sidecar-names-the-project", + ), + pytest.param( + f"SPDX-FileCopyrightText: {_FOO}\nSPDX-" + + "License-Identifier: BSD-3-Clause\n", + {"vendored:foo-inc": "BSD-3-Clause"}, + id="target-sidecar-names-another", + ), + ], +) +def test_a_link_reads_its_target_as_reuse_does(tmp_path, sidecar, expected): + """The bytes a link shares with its target are read under the target's + precedence: a header the target's sidecar sets aside decides nothing.""" + synth.write(tmp_path / "vendor/x.h", _HEADERS["copyright-and-licence"]) + synth.write(tmp_path / "vendor/x.h.license", sidecar) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + components = _discover(tmp_path, "src/x.h") + + assert {key: c.license_concluded for key, c in components.items()} == expected + + +@pytest.mark.parametrize("through", ["file", "link"]) +@pytest.mark.parametrize( + ("header", "holder", "precedence", "expected"), + [ + pytest.param( + "copyright-and-licence", + "Foo Inc", + "aggregate", + {"vendored:foo-inc": "Apache-2.0 AND MIT"}, + id="aggregate", + ), + pytest.param( + "unparsed-only", + "NVIDIA CORPORATION", + "aggregate", + {"vendored:bar-ltd": "Apache-2.0"}, + id="aggregate-unparsed", + ), + pytest.param( + "unparsed-and-licence", + "NVIDIA CORPORATION", + "closest", + {"vendored:bar-ltd": "MIT"}, + id="closest", + ), + ], +) +def test_a_link_with_its_own_annotation_reads_as_a_file_in_its_place( + tmp_path, through, header, holder, precedence, expected +): + annotation = synth.annotation( + "src/**", holder, licence="Apache-2.0", precedence=precedence + ) + synth.write(tmp_path / "REUSE.toml", synth.reuse_toml(annotation)) + if through == "link": + synth.write(tmp_path / "vendor/x.h", _HEADERS[header]) + _link(tmp_path, "src/x.h", "../vendor/x.h") + else: + synth.write(tmp_path / "src/x.h", _HEADERS[header]) + + components = _discover(tmp_path, "src/x.h") + + assert {key: c.license_concluded for key, c in components.items()} == expected + + +def test_a_tracked_link_is_not_a_candidate(tmp_path): + """REUSE skips links; only a build input is judged by its target.""" + synth.write(tmp_path / "vendor/x.h", _HEADERS["copyright-and-licence"]) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + components, _ = discovery.discover_vendored( + tmp_path, set(), {"NVIDIA"}, tracked=["src/x.h", "vendor/x.h"] + ) + + assert "Files: vendor/x.h." in components["vendored:bar-ltd"].source_info + + +def test_a_link_keeps_its_own_holder_where_its_target_has_no_info(tmp_path): + synth.write(tmp_path / "vendor/x.h", "#pragma once\n") + _link(tmp_path, "src/x.h", "../vendor/x.h") + synth.write(tmp_path / "src/x.h.license", "SPDX-FileCopyrightText: 2020 Foo Inc\n") + + assert list(_discover(tmp_path, "src/x.h")) == ["vendored:foo-inc"] + + +def test_a_link_read_as_its_target_takes_the_targets_precedence(tmp_path): + synth.write(tmp_path / "vendor/x.h", _HEADERS["unparsed-only"]) + synth.write( + tmp_path / "REUSE.toml", + synth.reuse_toml( + synth.annotation("vendor/x.h", "NVIDIA CORPORATION", precedence="aggregate") + ), + ) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + assert list(_discover(tmp_path, "src/x.h")) == ["vendored:bar-ltd"] + + +@pytest.mark.parametrize( + ("name", "head"), + [ + ("icon.svg", "\n\n"), + ("x.c", "/* Copyright 2020 Bar Ltd */\nint x;\n"), + ], + ids=["html", "c"], +) +def test_comment_markup_is_not_part_of_a_scraped_holder(tmp_path, name, head): + component = _vendored(tmp_path, {f"src/{name}": head})["vendored:bar-ltd"] + + assert component.name == "Vendored source (Bar Ltd)" + assert component.copyright_text == "Copyright 2020 Bar Ltd" diff --git a/scripts/sbom/tests/test_elf.py b/scripts/sbom/tests/test_elf.py new file mode 100644 index 0000000000..ad6941349f --- /dev/null +++ b/scripts/sbom/tests/test_elf.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ELF metadata reader tests.""" + +from __future__ import annotations + +import pytest +import synth +from isaaccapture_sbom import elf + + +def test_reads_soname_and_needed(): + image = synth.elf_shared_object("libcloudxr.so", ("libPoco.so", "libcudart.so.12")) + info = elf.read_dynamic(image) + + assert info.soname == "libcloudxr.so" + assert info.needed == ("libPoco.so", "libcudart.so.12") + assert info.machine == "x86_64" + + +def test_reads_a_library_with_no_dependencies(): + info = elf.read_dynamic(synth.elf_shared_object("libisaacteleop_mujoco.so")) + + assert info.soname == "libisaacteleop_mujoco.so" + assert info.needed == () + + +def test_rejects_non_elf_payloads(): + assert not elf.is_elf(b"# not an elf\n") + with pytest.raises(elf.NotAnElf): + elf.read_dynamic(b"PK\x03\x04 zip, not elf") + + +def test_reads_a_real_system_library(): + """Guards the reader against the synthesizer agreeing with itself.""" + import ctypes.util + + name = ctypes.util.find_library("c") + if name is None: + pytest.skip("no system libc to read") + for candidate in ( + "/lib/x86_64-linux-gnu", + "/lib/aarch64-linux-gnu", + "/usr/lib", + "/lib", + ): + path = f"{candidate}/{name}" + try: + data = open(path, "rb").read() # noqa: SIM115 + except OSError: + continue + if elf.is_elf(data): + assert elf.read_dynamic(data).soname == name + return + pytest.skip("could not locate the system libc on disk") + + +def test_verification_does_not_parse_binaries(built): + """A consumer checking a downloaded wheel should not run a binary parser + over it; scanning only reads the dynamic section when asked to.""" + from isaaccapture_sbom import wheelfile + + plain = wheelfile.scan(built["wheel"]) + analyzed = wheelfile.scan(built["wheel"], analyze_elf=True) + + assert not any(entry.is_elf for entry in plain.entries) + assert any(entry.is_elf for entry in analyzed.entries) + assert {e.name: e.sha256 for e in plain.entries} == { + e.name: e.sha256 for e in analyzed.entries + } diff --git a/scripts/sbom/tests/test_gates.py b/scripts/sbom/tests/test_gates.py new file mode 100644 index 0000000000..60478fc789 --- /dev/null +++ b/scripts/sbom/tests/test_gates.py @@ -0,0 +1,894 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The conditions that must stop a wheel from being published.""" + +from __future__ import annotations + +import json +import re +import shutil +import zipfile + +import pytest +import synth +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import cli +from isaaccapture_sbom import licensing +from isaaccapture_sbom import wheelfile +from isaaccapture_sbom import validate as validate_module + +DIST_INFO = synth.DIST_INFO +WHEEL_NAME = synth.WHEEL_NAME + + +def _rebuild(workspace, payload): + synth.write_wheel(workspace.wheel, payload) + return build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + + +def test_a_member_from_nowhere_blocks_the_build(workspace): + payload = synth.wheel_payload(workspace) + payload["isaaccapture/mystery/libmystery.so.7"] = synth.elf_shared_object( + "libmystery.so.7" + ) + + with pytest.raises(build_module.BuildError, match=r"libmystery\.so\.7"): + _rebuild(workspace, payload) + + +def test_a_component_with_no_license_text_blocks_the_build(workspace): + (workspace.build / "_deps/alpha-src/LICENSE").unlink() + + with pytest.raises(build_module.BuildError, match="alpha"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_a_root_sidecar_is_not_a_fetched_component_s_license_text(workspace): + alpha = workspace.build / "_deps/alpha-src" + (alpha / "LICENSE").unlink() + (alpha / "README.md").write_text("# alpha\n", encoding="utf-8") + (alpha / "README.md.license").write_text( + "SPDX-License-Identifier: MIT\n", encoding="utf-8" + ) + + with pytest.raises(build_module.BuildError, match=r"no license text.*alpha"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_an_empty_license_file_does_not_count_as_evidence(workspace): + # delta reaches the wheel through a static archive, so emptying its license + # leaves the wheel's contents untouched and isolates the evidence gate. + (workspace.build / "_deps/delta-src/BSD-LICENSE").write_text( + " \n", encoding="utf-8" + ) + + with pytest.raises(build_module.BuildError, match="delta"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_a_library_the_host_cannot_resolve_is_not_claimed_as_the_host_s(workspace): + """Resolving the SONAME is required evidence for attribution to the build host.""" + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libunknown-a1b2c3.so"] = synth.elf_shared_object( + "libunknown.so.1", ("libc.so.6",) + ) + synth.write_wheel(workspace.wheel, payload) + + with pytest.raises(build_module.BuildError, match="could not be traced"): + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + system_resolver=lambda soname: { + "soname": soname, + "resolved_path": None, + "package": None, + "copyright": None, + }, + ) + + +def test_the_build_host_is_the_last_resort_not_a_directory_name(workspace): + """Build provenance takes precedence even inside a repair tool's directory.""" + produced = (workspace.build / "lib/libgamma_renamed.so").read_bytes() + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libgamma_renamed-abc123.so"] = produced + staged = workspace.staged / "isaaccapture.libs" + staged.mkdir(parents=True, exist_ok=True) + (staged / "libgamma_renamed-abc123.so").write_bytes(produced) + _rebuild(workspace, payload) + + stem = WHEEL_NAME.removesuffix(".whl") + evidence = json.loads( + (workspace.root / "sbom" / f"{stem}.build-evidence.json").read_text() + ) + entry = next( + item + for item in evidence["attributions"] + if item["path"] == "isaaccapture.libs/libgamma_renamed-abc123.so" + ) + + assert entry["origin"] == "built" + assert entry["primary"] == "gamma" + + +def test_a_vendored_library_is_attributed_to_the_package_that_supplied_it( + workspace, tmp_path +): + copyright_file = tmp_path / "copyright" + copyright_file.write_text(synth.spdx_text("Zlib"), encoding="utf-8") + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libz-a1b2c3.so"] = synth.elf_shared_object( + "libz.so.1", ("libc.so.6",) + ) + synth.write_wheel(workspace.wheel, payload) + + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + system_resolver=lambda soname: { + "soname": soname, + "resolved_path": "/usr/lib/libz.so.1.3", + "package": "zlib1g", + # The package name identifies the software, not its supplier. + "supplier": "Organization: Ubuntu Developers (ubuntu-devel@example.com)", + "copyright": str(copyright_file), + }, + ) + + report = json.loads( + ( + workspace.root / "sbom" / f"{WHEEL_NAME.removesuffix('.whl')}.licenses.json" + ).read_text() + ) + vendored = next( + item for item in report["components"] if item["component"] == "system:libz.so.1" + ) + assert vendored["license_concluded"] == "Zlib" + assert vendored["supplier"] == ( + "Organization: Ubuntu Developers (ubuntu-devel@example.com)" + ) + + +def test_editing_a_file_after_the_fact_fails_verification(built): + synth.replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") + + failures = validate_module.check(built["wheel"]) + + assert any("SHA256 mismatch" in failure for failure in failures) + assert any("RECORD digest mismatch" in failure for failure in failures) + + +def test_adding_a_file_after_the_fact_fails_verification(built): + synth.replace_member(built["wheel"], "isaaccapture/surprise.py", b"# added later\n") + + failures = validate_module.check(built["wheel"]) + + assert any("not covered by the SBOM" in failure for failure in failures) + + +def test_mutating_the_wheel_breaks_its_manifest_binding(built): + synth.replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") + + failures = validate_module.check( + built["wheel"], + build_evidence=built["evidence_path"], + manifest=built["manifest_path"], + ) + + assert any("modified after its digest was bound" in failure for failure in failures) + + +def test_an_empty_member_verifies(built): + """RECORD states size 0 for py.typed and namespace __init__.py.""" + with zipfile.ZipFile(built["wheel"]) as archive: + empty = [item.filename for item in archive.infolist() if item.file_size == 0] + + assert empty, "fixture no longer carries an empty member" + assert validate_module.check(built["wheel"]) == [] + + +def test_another_tools_document_beside_ours_is_kept(built): + """auditwheel ships `sboms/auditwheel.cdx.json`; PEP 770 shares that directory.""" + with zipfile.ZipFile(built["wheel"]) as archive: + names = set(archive.namelist()) + + assert f"{DIST_INFO}/sboms/{synth.AUDITWHEEL_SBOM}" in names + assert validate_module.check(built["wheel"]) == [] + + +def test_our_own_document_going_missing_is_rejected(built): + synth.drop_member(built["wheel"], f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json") + + failures = validate_module.check(built["wheel"]) + + assert any("embedded SBOM" in failure for failure in failures) + + +def test_one_wheel_described_by_several_builds_merges_to_one_record(tmp_path, built): + """Matrix entries may produce identical universal wheels; merge their records.""" + first = tmp_path / "a.json" + second = tmp_path / "b.json" + for path in (first, second): + path.write_text(json.dumps(built["manifest"]), encoding="utf-8") + + merged = build_module.merge_manifests([first, second]) + names = [item["filename"] for item in merged["wheels"]] + + assert names == sorted(set(names)), "a repeated record was published twice" + + +def test_two_wheels_wearing_one_filename_are_refused(tmp_path, built): + first = tmp_path / "a.json" + second = tmp_path / "b.json" + first.write_text(json.dumps(built["manifest"]), encoding="utf-8") + other = json.loads(json.dumps(built["manifest"])) + other["wheels"][0]["sha256"] = "0" * 64 # a different wheel, same filename + second.write_text(json.dumps(other), encoding="utf-8") + + with pytest.raises(build_module.BuildError, match="different contents"): + build_module.merge_manifests([first, second]) + + +def _release_layout(built, tmp_path): + """The shape publish-sbom-evidence assembles from the matrix artifacts.""" + wheels = tmp_path / "release" / "wheels" + evidence = tmp_path / "release" / "evidence" + wheels.mkdir(parents=True) + evidence.mkdir(parents=True) + (wheels / built["wheel"].name).write_bytes(built["wheel"].read_bytes()) + for sidecar in built["out_dir"].glob("*"): + if sidecar.name != "manifest.json": + (evidence / sidecar.name).write_bytes(sidecar.read_bytes()) + merged = tmp_path / "release" / "isaaccapture-sbom-manifest.json" + merged.write_text( + json.dumps(built["manifest"], indent=2, sort_keys=True), encoding="utf-8" + ) + return merged, wheels, evidence + + +def test_a_complete_release_evidence_set_verifies(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + + assert validate_module.check_set(merged, wheels, evidence) == [] + assert _check_set_cli(merged, wheels, evidence) == 0 + + +def test_a_missing_sidecar_fails_the_release_set(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + next(evidence.glob("*.licenses.md")).unlink() + + failures = validate_module.check_set(merged, wheels, evidence) + + assert any("is missing" in failure for failure in failures) + + +def test_a_missing_wheel_fails_the_release_set(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + (wheels / built["wheel"].name).unlink() + + failures = validate_module.check_set(merged, wheels, evidence) + + assert any("is missing from" in failure for failure in failures) + + +def _check_set_cli(merged, wheels, evidence): + return cli.main( + [ + "check-set", + "--manifest", + str(merged), + "--wheel-dir", + str(wheels), + "--evidence-dir", + str(evidence), + ] + ) + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + ({}, "no 'wheels' list"), + ([], "not a JSON object"), + ({"wheels": []}, "lists no wheels"), + ({"wheels": "x"}, "no 'wheels' list"), + ({"wheels": ["x"]}, "entry is not an object"), + ({"wheels": [{}]}, "entry names no wheel"), + ({"wheels": [{"filename": 1}]}, "entry names no wheel"), + ({"wheels": [{"filename": "../x.whl"}]}, "not a bare wheel file name"), + ({"wheels": [{"filename": "/tmp/x.whl"}]}, "not a bare wheel file name"), + ({"wheels": [{"filename": "a\\b.whl"}]}, "not a bare wheel file name"), + ({"wheels": [{"filename": "C:x.whl"}]}, "not a bare wheel file name"), + ], +) +def test_manifest_records_refuses(tmp_path, manifest, message): + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + validate_module.ManifestError, match=f"{re.escape(str(path))}: .*{message}" + ): + validate_module.manifest_records(path) + + +def test_manifest_records_names_the_file_it_cannot_parse(tmp_path): + path = tmp_path / "manifest.json" + path.write_text("{", encoding="utf-8") + + with pytest.raises( + validate_module.ManifestError, match=rf"{re.escape(str(path))}: .*not JSON" + ): + validate_module.manifest_records(path) + + +def test_check_set_refuses_sidecars_that_are_not_named_files(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + manifest = json.loads(merged.read_text(encoding="utf-8")) + manifest["wheels"][0]["sidecars"] = list(manifest["wheels"][0]["sidecars"]) + merged.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + validate_module.ManifestError, match="sidecars must each name a file" + ): + validate_module.check_set(merged, wheels, evidence) + + +def _manifest_of(tmp_path, records): + path = tmp_path / "manifest.json" + path.write_text(json.dumps({"wheels": records}), encoding="utf-8") + return path + + +def _record(filename="x-1.0-py3-none-any.whl", sidecar="x.spdx.json"): + return { + "filename": filename, + "sha256": "0" * 64, + "size": 0, + "sbom_in_wheel": "x-1.0.dist-info/sboms/x.spdx.json", + "sidecars": {"spdx": {"filename": sidecar, "sha256": "0" * 64}}, + } + + +@pytest.mark.parametrize("sidecar", ["../x.spdx.json", "/tmp/x.spdx.json", ".."]) +def test_check_set_refuses_a_sidecar_that_is_not_a_bare_name(tmp_path, sidecar): + manifest = _manifest_of(tmp_path, [_record(sidecar=sidecar)]) + + with pytest.raises(validate_module.ManifestError, match="not a bare file name"): + validate_module.check_set(manifest, tmp_path, tmp_path) + + +def test_check_set_refuses_a_sidecar_with_no_digest(tmp_path): + record = _record() + del record["sidecars"]["spdx"]["sha256"] + manifest = _manifest_of(tmp_path, [record]) + + with pytest.raises( + validate_module.ManifestError, match="sidecars must each name a file" + ): + validate_module.check_set(manifest, tmp_path, tmp_path) + + +def test_check_set_refuses_a_wheel_advertised_twice(tmp_path): + manifest = _manifest_of(tmp_path, [_record(), _record()]) + + with pytest.raises(validate_module.ManifestError, match="advertised 2 times"): + validate_module.check_set(manifest, tmp_path, tmp_path) + + +def _empty_manifest(tmp_path): + path = tmp_path / "manifest.json" + path.write_text(json.dumps({"wheels": []}), encoding="utf-8") + return path + + +def test_check_set_refuses_a_manifest_listing_no_wheel(tmp_path, capsys): + manifest = _empty_manifest(tmp_path) + + with pytest.raises(validate_module.ManifestError, match="lists no wheels"): + validate_module.check_set(manifest, tmp_path, tmp_path) + assert _check_set_cli(manifest, tmp_path, tmp_path) == 2 + assert "lists no wheels" in capsys.readouterr().err + + +def _tiny_wheel(tmp_path): + return synth.write_wheel( + tmp_path / "x-1.0-py3-none-any.whl", + { + "x-1.0.dist-info/METADATA": b"Metadata-Version: 2.4\nName: x\nVersion: 1.0\n", + "x-1.0.dist-info/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\n", + }, + ) + + +def test_check_refuses_a_manifest_listing_no_wheel(tmp_path): + with pytest.raises(validate_module.ManifestError, match="lists no wheels"): + validate_module.check(_tiny_wheel(tmp_path), manifest=_empty_manifest(tmp_path)) + + +@pytest.mark.parametrize( + ("records", "message"), + [ + ([_record(sidecar="../x.spdx.json")], "not a bare file name"), + ([_record(), _record()], "advertised 2 times"), + ], +) +def test_check_refuses_a_malformed_record_for_its_wheel(tmp_path, records, message): + manifest = _manifest_of(tmp_path, records) + + with pytest.raises(validate_module.ManifestError, match=message): + validate_module.check(_tiny_wheel(tmp_path), manifest=manifest) + + +def test_merge_manifests_refuses_a_manifest_listing_no_wheel(tmp_path): + good = tmp_path / "a" / "manifest.json" + good.parent.mkdir() + good.write_text( + json.dumps({"wheels": [{"filename": "x-1.0-py3-none-any.whl"}]}), + encoding="utf-8", + ) + (tmp_path / "b").mkdir() + empty = _empty_manifest(tmp_path / "b") + + with pytest.raises( + validate_module.ManifestError, match=f"{re.escape(str(empty))}: " + ): + build_module.merge_manifests([good, empty]) + output = str(tmp_path / "m.json") + assert cli.main(["merge-manifests", "--output", output, str(empty)]) == 2 + + +def test_a_wheel_the_manifest_does_not_list_fails_the_release_set( + built, tmp_path, capsys +): + merged, wheels, evidence = _release_layout(built, tmp_path) + stray = wheels / "isaacteleop-0.4.0-py3-none-any.whl" + stray.write_bytes(built["wheel"].read_bytes()) + + failures = validate_module.check_set(merged, wheels, evidence) + + assert failures == [ + f"{stray.name} is in {wheels}, and {merged.name} does not list it" + ] + assert _check_set_cli(merged, wheels, evidence) == 1 + assert "does not list it" in capsys.readouterr().err + + +def test_missing_spdx_license_data_stops_the_build(workspace): + shutil.rmtree(workspace.build / "_deps/license-list-data-src") + + with pytest.raises(licensing.CorpusError, match="SPDX license list data not found"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_a_malformed_reuse_tag_does_not_reach_the_document(workspace): + """Reject malformed tags before emitting a document that fails SPDX conformance.""" + payload = synth.wheel_payload(workspace) + good = "# SPDX-" + "License-Identifier: mit or Apache-2.0\n" + bad = "# SPDX-" + "License-Identifier: !!! not an expression\n" + for name, text in (("tagged_ok.py", good), ("tagged_bad.py", bad)): + payload[f"isaaccapture/{name}"] = text.encode() + (workspace.root / "src/python/isaaccapture" / name).write_text(text) + (workspace.staged / "isaaccapture" / name).write_text(text) + synth.track(workspace.root) + _rebuild(workspace, payload) + + with zipfile.ZipFile(workspace.wheel) as archive: + spdx = json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + concluded = { + item["fileName"].removeprefix("./"): item["licenseConcluded"] + for item in spdx["files"] + } + + assert concluded["isaaccapture/tagged_ok.py"] == "MIT OR Apache-2.0" + assert concluded["isaaccapture/tagged_bad.py"] == "NOASSERTION" + + +def test_building_twice_on_the_same_wheel_is_refused(built): + with pytest.raises(build_module.BuildError, match="already carries"): + build_module.build( + built["workspace"].root, + built["workspace"].build, + built["wheel"], + built["out_dir"], + ) + + +def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data): + """Patched libraries can lose hash identity while retaining a shared filename. + A contradictory build ID must prevent attribution by name. + """ + + licensing.load_corpus(license_data) + payload = synth.wheel_payload(workspace) + # Match the SDK basename but contradict its build ID and content hash. + payload["isaaccapture/other_sdk/libdemo.so"] = synth.elf_shared_object( + "libdemo.so", + ("libc.so.6", "libm.so.6"), + build_id="2222222222222222222222222222222222222222", + ) + synth.write_wheel(workspace.wheel, payload) + + with pytest.raises(build_module.BuildError, match="could not be traced"): + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + ) + + +def test_an_unreadable_archive_says_so(workspace, license_data): + """Only fetched archives are indexed, so an unreadable one indicates a broken input.""" + from isaaccapture_sbom import discovery + + licensing.load_corpus(license_data) + (workspace.root / synth.SDK_ARCHIVE).write_bytes(b"not an archive at all") + + with pytest.raises(discovery.ArchiveError, match="cannot be read"): + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + ) + + +def test_bytes_more_than_one_component_holds_name_no_single_source( + workspace, license_data +): + """Identical license bytes can belong to several projects. + Do not assign ownership based on index order. + """ + + licensing.load_corpus(license_data) + shared = synth.spdx_text("Apache-2.0").encode() + for key in ("alpha", "beta"): + (workspace.build / "_deps" / f"{key}-src").mkdir(parents=True, exist_ok=True) + (workspace.build / "_deps" / f"{key}-src" / "LICENSE").write_bytes(shared) + + payload = synth.wheel_payload(workspace) + payload["isaaccapture/viz/SOMEDEP_LICENSE"] = shared + synth.write_wheel(workspace.wheel, payload) + + build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + with zipfile.ZipFile(workspace.wheel) as archive: + spdx = json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + entry = next( + item for item in spdx["files"] if item["fileName"].endswith("SOMEDEP_LICENSE") + ) + + assert "shared-content" in entry["comment"] + assert "more than one component" in entry["comment"] + assert "alpha" in entry["comment"] and "beta" in entry["comment"] + + +def test_the_excluded_list_cannot_exempt_a_smuggled_file(built): + """The checked document controls its exclusion list; trusting it would let an editor + exempt arbitrary added files from coverage checks. + """ + with zipfile.ZipFile(built["wheel"]) as archive: + payload = {item: archive.read(item) for item in archive.namelist()} + # write_wheel regenerates RECORD; remove it here to avoid testing duplicate members. + payload.pop(f"{DIST_INFO}/RECORD", None) + + payload["isaaccapture/_backdoor.py"] = b"import os\n" + document = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + spdx = json.loads(payload[document]) + for package in spdx["packages"]: + if package["SPDXID"] == "SPDXRef-Package-wheel": + package["packageVerificationCode"][ + "packageVerificationCodeExcludedFiles" + ].append("./isaaccapture/_backdoor.py") + payload[document] = json.dumps(spdx, indent=2, sort_keys=True).encode() + b"\n" + synth.write_wheel(built["wheel"], payload) + + failures = validate_module.check(built["wheel"]) + + assert any("excluded-files list must be exactly" in item for item in failures) + assert any("not covered by the SBOM" in item for item in failures) + + +def test_two_members_under_one_name_are_refused(built): + """Name-based reads select one duplicate; extractors may select another. + No single hash can describe both members. + """ + with zipfile.ZipFile(built["wheel"]) as archive: + names = [item for item in archive.namelist() if not item.endswith("/")] + payload = {item: archive.read(item) for item in names} + + smuggled = built["wheel"].with_name("duplicated.whl") + with zipfile.ZipFile(smuggled, "w", zipfile.ZIP_DEFLATED) as archive: + for name in names: + if name == "isaaccapture/__init__.py": + archive.writestr(name, b"import os\n") + archive.writestr(name, payload[name]) + + with pytest.raises(wheelfile.WheelError, match="more than one member"): + validate_module.check(smuggled) + + +def test_one_discovery_serves_every_wheel_without_bleeding_between_them( + workspace, tmp_path +): + """Host-library discoveries belong to each wheel's evidence. + Mutating shared discovery would leak components into subsequent wheels. + """ + from isaaccapture_sbom import evidence as evidence_module + + copyright_file = tmp_path / "copyright" + copyright_file.write_text(synth.spdx_text("Zlib"), encoding="utf-8") + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libz-a1b2c3.so"] = synth.elf_shared_object( + "libz.so.1", ("libc.so.6",) + ) + synth.write_wheel(workspace.wheel, payload) + + plain = workspace.wheel.parent / "plain-1.0-py3-none-any.whl" + synth.write_wheel(plain, synth.wheel_payload(workspace)) + + licensing.load_corpus(workspace.build / "_deps" / "license-list-data-src" / "json") + discovery = evidence_module.discover(workspace.root, workspace.build) + resolver = lambda soname: { # noqa: E731 + "soname": soname, + "resolved_path": "/usr/lib/libz.so.1.3", + "package": "zlib1g", + "supplier": "Organization: Ubuntu Developers (ubuntu-devel@example.com)", + "copyright": str(copyright_file), + } + for wheel in (workspace.wheel, plain): + build_module.build( + workspace.root, + workspace.build, + wheel, + workspace.root / "sbom", + system_resolver=resolver, + discovery=discovery, + ) + + def components(wheel_name): + stem = wheel_name.removesuffix(".whl") + sidecar = workspace.root / "sbom" / f"{stem}.build-evidence.json" + return json.loads(sidecar.read_text())["components"] + + assert "system:libz.so.1" in components(WHEEL_NAME), ( + "the wheel's own evidence must account for what its members needed" + ) + assert "system:libz.so.1" not in components(plain.name), ( + "a wheel that ships no such library must not inherit it" + ) + assert validate_module.check(workspace.wheel) == [] + + +def test_a_third_partys_licence_does_not_discharge_this_one(workspace, license_data): + """A third-party license with the same basename cannot satisfy the requirement + to package the distribution's own terms. + """ + + licensing.load_corpus(license_data) + assert build_module._own_license_texts( + "Apache-2.0", {"third-party/openxr-sdk/LICENSES/Apache-2.0.txt"} + ), "a third party's copy must not count as this distribution's" + assert not build_module._own_license_texts( + "Apache-2.0", {"LICENSES/Apache-2.0.txt"} + ), "the distribution's own copy counts" + assert build_module._declared_without_text( + "Apache-2.0", {"third-party/openxr-sdk/licenses/apache-2.0.txt"} + ) == ["Apache-2.0"] + + +@pytest.mark.parametrize("kind", [*validate_module.CONTENT_SIDECARS, "build_evidence"]) +def test_builds_of_one_wheel_may_differ_only_in_build_evidence(tmp_path, built, kind): + """Every entry builds a `py3-none-any` wheel. Its evidence describes each + build, so it may differ and the first entry's is bound; what is derived + from the wheel alone may not.""" + first = tmp_path / "a.json" + second = tmp_path / "b.json" + first.write_text(json.dumps(built["manifest"]), encoding="utf-8") + other = json.loads(json.dumps(built["manifest"])) + other["wheels"][0]["sidecars"][kind]["sha256"] = "1" * 64 + second.write_text(json.dumps(other), encoding="utf-8") + + if kind == "build_evidence": + merged = build_module.merge_manifests([first, second]) + assert merged["wheels"] == built["manifest"]["wheels"] + else: + with pytest.raises( + build_module.BuildError, match=rf"described it .*\({kind}\)" + ): + build_module.merge_manifests([first, second]) + + +def _two_builds_of_one_wheel(built, tmp_path): + """Two entries' sidecar directories whose build evidence differs.""" + entries = [] + for name, note in (("a", ""), ("b", "\n")): + entry = tmp_path / "sidecars" / name + shutil.copytree(built["out_dir"], entry) + evidence = entry / built["evidence_path"].name + evidence.write_bytes(evidence.read_bytes() + note.encode()) + manifest = json.loads((entry / "manifest.json").read_text(encoding="utf-8")) + manifest["wheels"][0]["sidecars"]["build_evidence"]["sha256"] = ( + build_module.sha256_file(evidence) + ) + (entry / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + entries.append(entry) + return entries + + +def test_a_merged_release_holds_the_evidence_it_binds_and_verifies(built, tmp_path): + first, second = _two_builds_of_one_wheel(built, tmp_path) + release = tmp_path / "release" + wheels = tmp_path / "wheels" + wheels.mkdir() + shutil.copy(built["wheel"], wheels) + + # Named out of order: the first in path order is bound, and copied. + assert _merge_cli([second, first], release) == 0 + + manifest = release / "isaaccapture-sbom-manifest.json" + bound = json.loads(manifest.read_text(encoding="utf-8"))["wheels"][0]["sidecars"] + assert {item.name for item in release.iterdir()} == { + manifest.name, + *(item["filename"] for item in bound.values()), + } + evidence = release / bound["build_evidence"]["filename"] + assert evidence.read_bytes() == (first / evidence.name).read_bytes() + assert validate_module.check_set(manifest, wheels, release) == [] + + +def test_evidence_the_manifest_does_not_bind_fails_the_release_set(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + stray = evidence / f"b.{built['evidence_path'].name}" + stray.write_bytes(built["evidence_path"].read_bytes()) + + failures = validate_module.check_set(merged, wheels, evidence) + + assert failures == [ + f"{stray.name} is in {evidence}, and {merged.name} does not bind it" + ] + + +def test_a_file_named_like_a_manifest_kept_elsewhere_is_stray(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + stray = evidence / merged.name + stray.write_text("{}", encoding="utf-8") + + assert validate_module.check_set(merged, wheels, evidence) == [ + f"{stray.name} is in {evidence}, and {merged.name} does not bind it" + ] + + +def test_merging_into_the_directory_holding_the_evidence(built, tmp_path): + entry = tmp_path / "entry" + shutil.copytree(built["out_dir"], entry) + + assert _merge_cli([entry], entry) == 0 + + +def test_a_release_checked_from_one_directory_verifies(built, tmp_path): + """The documented recipe: wheels, evidence and manifest side by side.""" + merged, wheels, evidence = _release_layout(built, tmp_path) + shutil.copy(next(wheels.iterdir()), evidence) + shutil.copy(merged, evidence) + + manifest = evidence / merged.name + assert validate_module.check_set(manifest, evidence, evidence) == [] + + stray = evidence / "isaacteleop-0.4.0-py3-none-any.whl" + shutil.copy(built["wheel"], stray) + assert validate_module.check_set(manifest, evidence, evidence) == [ + f"{stray.name} is in {evidence}, and {manifest.name} does not list it" + ] + + +def _merge_cli(entries, release): + return cli.main( + [ + "merge-manifests", + "--output", + str(release / "isaaccapture-sbom-manifest.json"), + "--evidence-out", + str(release), + *(str(entry / "manifest.json") for entry in entries), + ] + ) + + +def test_a_missing_bound_sidecar_copies_nothing(built, tmp_path, capsys): + first, second = _two_builds_of_one_wheel(built, tmp_path) + (first / built["spdx_path"].name).unlink() + release = tmp_path / "release" + + assert _merge_cli([first, second], release) == 2 + assert "sidecar spdx is missing" in capsys.readouterr().err + assert not release.exists() + + +def test_merge_manifests_refuses_malformed_sidecars(tmp_path): + record = _record() + record["sidecars"] = list(record["sidecars"]) + + with pytest.raises( + validate_module.ManifestError, match="sidecars must each name a file" + ): + build_module.merge_manifests([_manifest_of(tmp_path, [record])]) + + +def test_reuse_toml_declares_the_terms_of_an_untagged_vendored_file(workspace): + """Model an untagged vendored header such as Chromium's key table.""" + synth.write( + workspace.root / "src/vendor/keys/include/keys/table.h", synth.UNTAGGED_HEADER + ) + synth.write( + workspace.root / "REUSE.toml", + synth.reuse_toml( + synth.annotation( + "src/vendor/keys/**", + "The Keys Authors", + licence="BSD-3-Clause", + year=2013, + ) + ), + ) + synth.track(workspace.root) + _rebuild(workspace, synth.wheel_payload(workspace)) + + stem = WHEEL_NAME.removesuffix(".whl") + evidence = json.loads( + (workspace.root / "sbom" / f"{stem}.build-evidence.json").read_text() + ) + component = evidence["components"][synth.UNTAGGED_KEY] + assert component["in_this_wheel"] + assert component["license_concluded"] == "BSD-3-Clause" + assert [item["path"] for item in component["evidence"]] == ["BSD-3-Clause.txt"] + + +@pytest.mark.parametrize( + ("relative", "text", "key"), + [ + pytest.param( + "src/vendor/keys/include/keys/table.h", + synth.UNTAGGED_HEADER, + synth.UNTAGGED_KEY, + id="no-sibling", + ), + pytest.param( + "src/vendor/widgets_extra.h", + "// Copyright 2024, Upstream Widgets Ltd.\n", + "vendored:upstream-widgets-ltd", + id="no-info", + ), + pytest.param( + "src/vendor/widgets_extra.h", + "// Copyright 2024, Upstream Widgets Ltd.\n" + "// SPDX-" + "License-Identifier: LicenseRef-Widgets\n", + "vendored:upstream-widgets-ltd", + id="licenseref-only", + ), + ], +) +def test_each_vendored_file_needs_a_listed_licence_of_its_own( + workspace, relative, text, key +): + """A listed license on upstream_helper.h cannot cover other files that declare + no license or a LicenseRef without text. + """ + synth.write(workspace.root / relative, text) + synth.track(workspace.root) + + with pytest.raises(build_module.BuildError) as raised: + _rebuild(workspace, synth.wheel_payload(workspace)) + + message = str(raised.value) + assert "fails the per-file gate" in message + assert f"{relative} ({key}): no current SPDX-listed licence" in message + assert "upstream_helper.h" not in message diff --git a/scripts/sbom/tests/test_licensing.py b/scripts/sbom/tests/test_licensing.py new file mode 100644 index 0000000000..ca09c845fb --- /dev/null +++ b/scripts/sbom/tests/test_licensing.py @@ -0,0 +1,898 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""License text identification and declaration tests.""" + +from __future__ import annotations + +import math +from dataclasses import replace + +import pytest +import synth +from isaaccapture_sbom import discovery, document, licensing, report + + +def test_corpus_loads(): + version, entries = licensing.corpus() + + assert version.startswith("3.") + assert len(entries) > 400 + assert {"MIT", "Apache-2.0", "BSD-3-Clause", "Zlib", "Qhull"} <= set(entries) + + +def test_deprecated_identifiers_are_not_offered(): + _, entries = licensing.corpus() + + assert "GPL-2.0" not in entries, "superseded by GPL-2.0-only/-or-later" + assert "GPL-2.0-only" in entries + + +def test_an_extended_license_is_not_mistaken_for_the_one_it_contains(): + """ECL-2.0 contains Apache-2.0; CC-BY-NC-4.0 contains CC-BY-4.0. + + Containment alone cannot distinguish the extended license from its shorter base. + """ + apache = [ + license_id + for license_id, _, _ in licensing.identify(synth.spdx_text("Apache-2.0")) + ] + creative = [ + license_id + for license_id, _, _ in licensing.identify(synth.spdx_text("CC-BY-4.0")) + ] + + assert apache == ["Apache-2.0"] + assert creative == ["CC-BY-4.0"] + + +def test_the_extended_license_still_wins_when_it_is_the_real_one(): + """A longer match must not win solely because it is longer.""" + matches = [ + license_id + for license_id, _, _ in licensing.identify(synth.spdx_text("ECL-2.0")) + ] + + assert matches == ["ECL-2.0"] + + +def test_identifies_a_canonical_text(): + matches = licensing.identify(synth.spdx_text("MIT")) + + assert [license_id for license_id, _, _ in matches] == ["MIT"] + + +def test_identifies_a_text_wrapped_in_a_copyright_line(): + text = "Copyright (c) 2019 Some Person\n\n" + synth.spdx_text("Zlib") + + assert [license_id for license_id, _, _ in licensing.identify(text)] == ["Zlib"] + + +def test_a_longer_license_subsumes_the_shorter_one_it_contains(): + matches = licensing.identify(synth.spdx_text("BSD-3-Clause")) + + assert [license_id for license_id, _, _ in matches] == ["BSD-3-Clause"] + + +def test_a_dual_licensed_file_reports_both(): + text = synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0") + + identified = {license_id for license_id, _, _ in licensing.identify(text)} + + assert identified == {"Apache-2.0", "CC-BY-4.0"} + + +def _varied_prose(word_count: int) -> str: + """Filler whose wording does not repeat, the way real agreements read.""" + alphabet = "abcdefghijklmnopqrstuvwxyz" + return " ".join( + "term" + + alphabet[index % 26] + + alphabet[(index // 26) % 26] + + alphabet[(index // 676) % 26] + for index in range(word_count) + ) + + +def test_a_license_quoted_in_an_appendix_is_not_the_document_s_license(): + agreement = ( + "PROPRIETARY AGREEMENT\n\n" + + _varied_prose(6000) + + "\n\nAPPENDIX: third-party notices\n\n" + + synth.spdx_text("BSL-1.0") + ) + + assert licensing.identify(agreement) == [] + + +def test_a_real_license_survives_surrounding_packaging_boilerplate(): + wrapped = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" + "Upstream-Name: thing\n\n" + + synth.spdx_text("Zlib") + + "\n\n" + + _varied_prose(200) + ) + + assert [license_id for license_id, _, _ in licensing.identify(wrapped)] == ["Zlib"] + + +def _grant( + key: str, name: str, text: str, kind: str = "grant" +) -> licensing.LicenseEvidence: + return licensing.evidence_for(key, "component-file", f"{key}/{name}", text, kind) + + +def test_unmatched_grant_becomes_a_license_ref_not_a_guess(): + evidence = [_grant("demo", "LICENSE", synth.PROPRIETARY_LICENSE)] + + concluded, _ = licensing.expression(evidence, "demo") + + assert concluded == "LicenseRef-demo" + + +def test_a_matched_grant_does_not_hide_an_unmatched_one(): + mit = _grant("demo", "LICENSE", synth.spdx_text("MIT")) + eula = _grant("demo", "LICENSE-EULA.txt", synth.PROPRIETARY_LICENSE) + + assert licensing.expression([mit, eula], "demo") == ( + "LicenseRef-demo AND MIT", + "LicenseRef-demo AND MIT", + ) + assert licensing.carried_texts([mit, eula]) == [eula] + assert licensing.verbatim_terms([mit, eula]) == synth.PROPRIETARY_LICENSE + assert licensing.why_unnamed([mit, eula]) == ( + "matched no text in the SPDX reference corpus" + ) + + +_GPL_TIE = ( + "matches the reference text of GPL-3.0-only or GPL-3.0-or-later, and the " + "text alone does not say which applies" +) +_NEWER = "newer than the SPDX grammar this document ships with" + + +def test_a_text_two_ids_share_is_carried_rather_than_named_as_both(): + gpl = _grant("gpl", "COPYING", synth.spdx_text("GPL-3.0-only")) + mit = _grant("gpl", "LICENSE-MIT", synth.spdx_text("MIT")) + eula = _grant("gpl", "EULA", synth.PROPRIETARY_LICENSE) + + assert licensing.expression([gpl], "gpl")[0] == "LicenseRef-gpl" + assert licensing.expression([gpl, mit], "gpl")[0] == "LicenseRef-gpl AND MIT" + assert licensing.carried_texts([gpl, mit]) == [gpl] + assert licensing.why_unnamed([gpl, mit]) == _GPL_TIE + assert licensing.why_unnamed([eula, gpl]) == ( + f"matched no text in the SPDX reference corpus; {_GPL_TIE}" + ) + + +@pytest.mark.parametrize("license_id", ["MPL-2.0", "OFL-1.1"]) +def test_a_tie_with_a_plain_id_names_the_plain_id(license_id): + text = _grant("font", "LICENSE", synth.spdx_text(license_id)) + assert len(licensing.alternatives(text.matches)[0]) > 1, "not a tie any more" + + assert licensing.expression([text], "font")[0] == license_id + assert text.identified == license_id + + +def test_a_tie_the_plain_id_names_is_not_given_as_a_reason(): + both = _grant( + "c", + "COPYING", + synth.spdx_text("MPL-2.0") + "\n\n" + synth.spdx_text("LGPL-2.1-only"), + ) + + assert licensing.expression([both], "c")[0] == "LicenseRef-c AND MPL-2.0" + assert licensing.why_unnamed([both]) == ( + "matches the reference text of LGPL-2.1-only or LGPL-2.1-or-later, and " + "the text alone does not say which applies" + ) + + +def test_a_tied_text_says_how_little_of_it_the_reference_text_is(): + padded = _grant( + "c", "COPYING", synth.spdx_text("LGPL-2.1-only") + _varied_prose(3000) + ) + share = licensing._read_grant(padded).explained + # Counted once it explains too little; counted per id it would not. + assert licensing._EXPLAINED_THRESHOLD / 2 < share < licensing._EXPLAINED_THRESHOLD + + assert licensing.why_unnamed([padded]) == ( + "matches the reference text of LGPL-2.1-only or LGPL-2.1-or-later, and " + "the text alone does not say which applies; what matched in the SPDX " + f"reference corpus accounts for only {math.floor(share * 100)}% of c/COPYING, " + "its copyright notices aside" + ) + + +def _share(text: licensing.LicenseEvidence) -> str: + return f"{math.floor(licensing._read_grant(text).explained * 100)}%" + + +def test_a_mostly_unmatched_grant_is_named_and_carried(): + whole = _grant("demo", "LICENSE", synth.spdx_text("MIT")) + padded = _grant("demo", "COPYING", synth.spdx_text("MIT") + _varied_prose(300)) + zlib = _grant("demo", "COPYING.zlib", synth.spdx_text("Zlib") + _varied_prose(250)) + + assert licensing.expression([whole], "demo")[0] == "MIT" + assert licensing.expression([padded, whole], "demo")[0] == ( + "LicenseRef-demo AND MIT" + ) + assert licensing.carried_texts([padded, whole]) == [padded] + assert licensing.why_unnamed([padded, zlib]) == ( + "what matched in the SPDX reference corpus accounts for only " + f"{_share(padded)} of demo/COPYING, its copyright notices aside; what matched " + f"in the SPDX reference corpus accounts for only {_share(zlib)} of " + "demo/COPYING.zlib, its copyright notices aside" + ) + + +_FMT_EXCEPTION = ( + "As an exception, if, as a result of your compiling your source code, portions " + "of this Software are embedded into a machine-executable object form of such " + "source code, you may redistribute such embedded portions in such object form " + "without including the above copyright and permission notices." +) + + +@pytest.mark.parametrize( + "shape, share, concluded", + [ + # fmt's own shape: named at 0.75, carried at 0.80. + ("exception", (0.75, 0.80), "LicenseRef-demo AND MIT"), + # Two thirds of it: carried at 0.85, named at 0.80. + ("part-exception", (0.80, 0.85), "MIT"), + # A holder list is not terms, however long. + ("holders", (0.99, 1.01), "MIT"), + ], +) +def test_how_much_of_a_grant_must_match_for_its_ids_to_say_it_all( + shape, share, concluded +): + mit = synth.spdx_text("MIT") + text = { + "exception": f"{mit}\n\n--- Optional exception to the license ---\n\n" + + _FMT_EXCEPTION, + "part-exception": mit + "\n\n" + " ".join(_FMT_EXCEPTION.split()[:34]), + "holders": "\n".join( + f"Copyright (c) 2015 Holder Number{index} " + for index in range(8) + ) + + "\n\n" + + mit, + }[shape] + grant = _grant("demo", "LICENSE", text) + assert share[0] < licensing._read_grant(grant).explained < share[1] + + assert licensing.expression([grant], "demo")[0] == concluded + + +@pytest.fixture +def mit_is_newer(monkeypatch): + """MIT as an id the grammar this document ships with cannot express.""" + expressible = licensing.expressible + monkeypatch.setattr( + licensing, "expressible", lambda key: key != "MIT" and expressible(key) + ) + + +def _reference(key: str, license_id: str) -> licensing.LicenseEvidence: + return licensing.evidence_for( + key, + licensing.SPDX_REFERENCE, + f"{license_id}.txt", + synth.spdx_text(license_id), + "grant", + ) + + +@pytest.mark.parametrize( + "texts, concluded, carried, reason", + [ + ( + ["MIT", "Zlib"], + "LicenseRef-demo AND Zlib", + [0], + f"matched MIT, {_NEWER}", + ), + ( + ["MIT", None], + "LicenseRef-demo", + [0, 1], + f"matched no text in the SPDX reference corpus; matched MIT, {_NEWER}", + ), + ( + ["MIT", "GPL-3.0-only"], + "LicenseRef-demo", + [0, 1], + f"{_GPL_TIE}; matched MIT, {_NEWER}", + ), + # A reference text is declared, not unmatched, and its id is not + # restated as matched. + ( + ["reference", "MIT", "Zlib"], + "LicenseRef-demo AND Zlib", + [0, 1], + f"declares MIT, {_NEWER}", + ), + ], + ids=["beside-named", "beside-unmatched", "beside-tie", "declared"], +) +def test_an_id_the_grammar_cannot_name_carries_only_its_texts( + mit_is_newer, texts, concluded, carried, reason +): + evidence = [ + _reference("demo", "MIT") + if source == "reference" + else _grant("demo", f"LICENSE-{index}", synth.PROPRIETARY_LICENSE) + if source is None + else _grant("demo", f"LICENSE-{index}", synth.spdx_text(source)) + for index, source in enumerate(texts) + ] + + assert licensing.expression(evidence, "demo")[0] == concluded + assert licensing.carried_texts(evidence) == [evidence[i] for i in carried] + assert licensing.why_unnamed(evidence) == reason + + +def test_a_declaration_the_grammar_cannot_name_travels_whole(mit_is_newer): + """An OR cannot be kept in part, so every text it declares is carried.""" + declared, evidence = licensing.declared_terms("vendored:x", {"MIT OR Zlib"}) + assert declared == "LicenseRef-vendored-x" + assert [item.identified for item in licensing.carried_texts(evidence)] == [ + "MIT", + "Zlib", + ] + declared, evidence = licensing.declared_terms("vendored:y", {"Zlib OR BSL-1.0"}) + assert declared == "Zlib OR BSL-1.0" + assert licensing.carried_texts(evidence) == [] + + +@pytest.mark.parametrize( + "carried, total, subject", + [ + (1, 1, "the shipped grant is"), + (2, 2, "its 2 shipped grants are"), + (1, 2, "1 of its 2 shipped grants is"), + (2, 3, "2 of its 3 shipped grants are"), + ], +) +def test_the_report_says_how_many_grants_are_carried(carried, total, subject): + assert report._carried_subject(carried, total) == subject + + +def test_the_report_says_whether_a_carried_grant_is_named_too(mit_is_newer): + padded = _grant("demo", "COPYING", synth.spdx_text("Zlib") + _varied_prose(250)) + eula = _grant("demo", "EULA", synth.PROPRIETARY_LICENSE) + _, evidence = licensing.declared_terms("vendored:x", {"MIT OR Zlib"}) + + assert report._carried_reason([padded]) == ( + "the shipped grant is carried verbatim as well as named: what matched in " + f"the SPDX reference corpus accounts for only {_share(padded)} of " + "demo/COPYING, its copyright notices aside" + ) + assert report._carried_reason([padded, eula]).startswith( + "its 2 shipped grants are carried verbatim, 1 of them as well as named: " + ) + # A declaration carried whole names none of its ids. + assert report._carried_reason(evidence) == ( + "its 2 shipped grants are carried verbatim rather than named: declares " + f"MIT, {_NEWER}" + ) + + +def test_grants_concluding_nothing_are_not_blamed_on_a_pool(monkeypatch): + """Grants conclude nothing where the SPDX grammar rejects their ids or a + reference text names none.""" + mit = _grant("demo", "LICENSE", synth.spdx_text("MIT")) + pool = _grant("demo", "LICENSES/MIT.txt", mit.text, kind="pool") + unnamed = replace(mit, origin=licensing.SPDX_REFERENCE, identified=None) + reason = "its grants (demo/LICENSE) yield no SPDX expression" + + assert licensing.expression([unnamed, pool], "demo")[0] == "NOASSERTION" + assert licensing.why_unresolved([unnamed, pool]) == reason + + def rejected(*_args, **_kwargs): + raise licensing.ExpressionError("rejected") + + monkeypatch.setattr(licensing, "combine_expressions", rejected) + + assert licensing.expression([mit, pool], "demo")[0] == "NOASSERTION" + assert licensing.why_unresolved([mit, pool]) == reason + assert licensing.why_unresolved([pool]) == ( + "only REUSE pool texts ship (MIT); no grant states which of them are " + "the component's terms" + ) + + +def test_no_grant_at_all_is_noassertion(): + assert licensing.expression([], "demo") == ("NOASSERTION", "NOASSERTION") + + +def test_finds_a_prefixed_license_file(tmp_path): + (tmp_path / "BSD-LICENSE").write_text( + synth.spdx_text("BSD-3-Clause"), encoding="utf-8" + ) + + evidence = licensing.discover_in_tree("ccd", tmp_path, "ccd-src") + + assert [item.identified for item in evidence] == ["BSD-3-Clause"] + + +def test_a_reuse_pool_does_not_become_the_project_license(tmp_path): + """LICENSES/ holds texts for individual files, not the project's own grant.""" + (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") + pool = tmp_path / "LICENSES" + pool.mkdir() + (pool / "MIT.txt").write_text(synth.spdx_text("MIT"), encoding="utf-8") + (pool / "Zlib.txt").write_text(synth.spdx_text("Zlib"), encoding="utf-8") + + evidence = licensing.discover_in_tree("openxr", tmp_path, "openxr-src") + concluded, _ = licensing.expression(evidence, "openxr") + + assert concluded == "Apache-2.0" + assert sum(1 for item in evidence if item.kind == "pool") == 2, ( + "pool texts still ship" + ) + + +def test_falls_back_to_a_readme_section_when_no_license_file_exists(tmp_path): + (tmp_path / "README.md").write_text( + "# Thing\n\nSome prose.\n\n## Licence\nPublic domain or MIT License.\n", + encoding="utf-8", + ) + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert [item.origin for item in evidence] == ["readme-section"] + assert licensing.expression(evidence, "thing")[0] == "LicenseRef-thing" + + +def test_notices_are_packaged_but_do_not_set_the_expression(tmp_path): + (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") + (tmp_path / "NOTICE").write_text( + "This product includes software from X.\n", encoding="utf-8" + ) + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert {item.kind for item in evidence} == {"grant", "notice"} + assert licensing.expression(evidence, "thing")[0] == "Apache-2.0" + + +def test_shipping_a_license_file_is_how_authors_declare_one(tmp_path): + """Declared is not an echo of the conclusion: it is read from the same file, + because shipping that file is the declaration.""" + (tmp_path / "LICENSE").write_text(synth.spdx_text("MIT"), encoding="utf-8") + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert licensing.expression(evidence, "thing") == ("MIT", "MIT") + + +def test_a_component_with_no_grant_declares_nothing(tmp_path): + (tmp_path / "README.md").write_text("nothing to see", encoding="utf-8") + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert licensing.expression(evidence, "thing") == ("NOASSERTION", "NOASSERTION") + + +def test_a_tag_on_a_documentation_file_is_not_the_package_declaration(tmp_path): + """OpenXR's COPYING.adoc is tagged CC-BY-4.0 for its prose. + + That tag does not license the SDK; unmatched prose travels as a LicenseRef. + """ + (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") + # REUSE-IgnoreStart + (tmp_path / "COPYING").write_text( + "SPDX-License-Identifier: CC-BY-4.0\n\nThis project uses several licences.\n", + encoding="utf-8", + ) + # REUSE-IgnoreEnd + + concluded, declared = licensing.expression( + licensing.discover_in_tree("thing", tmp_path, "thing-src"), "thing" + ) + + assert concluded == "Apache-2.0 AND LicenseRef-thing" + assert "CC-BY-4.0" not in declared + + +@pytest.mark.parametrize( + "files, concluded", + [ + ({"LICENSE": ("Apache-2.0", "CC-BY-4.0")}, "Apache-2.0 AND CC-BY-4.0"), + ( + {"LICENSE-MIT": ("MIT",), "LICENSE-APACHE": ("Apache-2.0",)}, + "Apache-2.0 AND MIT", + ), + ], +) +def test_two_identified_licenses_combine(files, concluded): + """Combined expression objects reject truth-value testing. + + Exercise the combiner with multiple licenses to catch accidental bool checks. + """ + evidence = [ + _grant("dual", name, "\n\n".join(map(synth.spdx_text, ids))) + for name, ids in files.items() + ] + + assert licensing.expression(evidence, "dual")[0] == concluded + assert licensing.verbatim_terms(evidence) == "\n\n".join( + item.text for item in evidence + ) + + +def test_a_bare_license_file_covers_its_directory(tmp_path): + synth.write(tmp_path / "LICENSE", synth.spdx_text("MIT")) + synth.write(tmp_path / "third/.license", "SPDX-FileCopyrightText: 2020 A\n") + # Named like a sidecar, with no file beside it to annotate. + synth.write(tmp_path / "third/stb.license", synth.spdx_text("MIT")) + + nested = [ + item + for item in licensing.discover_in_tree("thing", tmp_path, "thing-src") + if item.kind == "nested" + ] + + assert [(item.path, item.scope) for item in nested] == [ + ("thing-src/third/.license", "thing-src/third"), + ("thing-src/third/stb.license", "thing-src/third"), + ] + + +def test_a_text_carrying_two_licences_lists_both_not_a_choice(): + dual = _grant( + "dual", + "LICENSE", + synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0"), + ) + + assert dual.identified == "Apache-2.0 AND CC-BY-4.0" + component = discovery.Component( + key="dual", + name="dual", + kind="source-tree", + supplier="NOASSERTION", + homepage="NOASSERTION", + purl="pkg:generic/dual", + version="1", + download_location="NOASSERTION", + source_info="", + evidence=(dual,), + ) + rendered = document._attribution_texts(component, synth.DIST_INFO)[0] + assert " identified as Apache-2.0 (" in rendered + assert "; CC-BY-4.0 (" in rendered + assert "reference text of" not in rendered + + +def test_a_choice_is_never_joined_to_a_licence_with_and(): + gpl = [("GPL-3.0-only", 1.0, 0.6), ("GPL-3.0-or-later", 1.0, 0.6)] + alternatives = "matches the reference text of GPL-3.0-only or GPL-3.0-or-later" + + assert licensing.matched_prose(gpl) == alternatives + assert ( + licensing.matched_prose([("Apache-2.0", 1.0, 0.2), *gpl, ("MIT", 1.0, 0.1)]) + == f"Apache-2.0 AND MIT; {alternatives}" + ) + + +def test_an_index_of_licenses_is_not_a_grant(tmp_path): + """A cone-mode sparse checkout keeps root files, so a repository that + merely catalogues licences can present names that look like grants.""" + (tmp_path / "licenses.md").write_text("| MIT | Apache-2.0 |\n", encoding="utf-8") + (tmp_path / "accessingLicenses.md").write_text( + "how to fetch them\n", encoding="utf-8" + ) + + assert licensing.discover_in_tree("catalogue", tmp_path, "catalogue-src") == [] + + +@pytest.mark.parametrize( + ("name", "beside", "expected"), + [ + ("README.md.license", {"README.md"}, None), + ("LICENSE.license", {"LICENSE"}, None), + ("stb.license", set(), "grant"), + (".license", {""}, "grant"), + ], +) +def test_a_license_suffix_is_a_sidecar_only_beside_what_it_annotates( + name, beside, expected +): + assert licensing.classify(name, beside.__contains__) == expected + + +def test_a_root_sidecar_is_not_a_fetched_tree_s_grant(tmp_path): + (tmp_path / "README.md").write_text("# Thing\n\nprose\n", encoding="utf-8") + (tmp_path / "README.md.license").write_text( + "SPDX-License-Identifier: MIT\n\n## License\nMIT\n", encoding="utf-8" + ) + + assert licensing.discover_in_tree("thing", tmp_path, "thing-src") == [] + + +def test_root_license_named_files_are_kept_unless_they_annotate_one(tmp_path): + synth.write(tmp_path / "LICENSE", synth.spdx_text("MIT")) + synth.write(tmp_path / "stb.license", synth.spdx_text("MIT")) + synth.write(tmp_path / ".license", synth.spdx_text("MIT")) + synth.write(tmp_path / "mit", "data\n") + synth.write(tmp_path / "mit.license", "SPDX-License-Identifier: MIT\n") + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert [(item.path, item.kind) for item in evidence] == [ + ("thing-src/.license", "grant"), + ("thing-src/LICENSE", "grant"), + ("thing-src/stb.license", "grant"), + ] + + +def test_an_orphan_license_file_alone_is_a_fetched_tree_s_grant(tmp_path): + synth.write(tmp_path / "stb.license", synth.spdx_text("MIT")) + + (item,) = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert (item.kind, item.identified) == ("grant", "MIT") + + +def test_a_project_title_is_not_a_license_section(tmp_path): + (tmp_path / "README.md").write_text( + "# License List Data\n\nGenerated data.\n", encoding="utf-8" + ) + + assert licensing.discover_in_tree("catalogue", tmp_path, "catalogue-src") == [] + + +def test_a_real_license_section_is_still_found(tmp_path): + (tmp_path / "README.md").write_text( + "# Thing\n\nprose\n\n## Licence\nPublic domain or MIT License.\n", + encoding="utf-8", + ) + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert [item.origin for item in evidence] == ["readme-section"] + + +def test_a_notice_another_already_covers_is_dropped(): + folded = licensing.fold_notices( + [ + "Copyright (c) 2025 Acme Ltd.", + "Copyright (c) 2025-2026 Acme Ltd.", + "Copyright (c) 2026 Acme Ltd.", + ] + ) + + assert folded == ["Copyright (c) 2025-2026 Acme Ltd."] + + +def test_folding_never_claims_a_year_no_file_claims(): + """Merging ranges would assert 2022; neither notice does.""" + assert licensing.fold_notices( + ["Copyright (c) 2021 Acme Ltd.", "Copyright (c) 2023 Acme Ltd."] + ) == ["Copyright (c) 2021 Acme Ltd.", "Copyright (c) 2023 Acme Ltd."] + + +def test_folding_does_not_merge_different_holders(): + folded = licensing.fold_notices( + ["Copyright (c) 2026 Acme Ltd.", "Copyright (c) 2026 Other Corp."] + ) + + assert len(folded) == 2 + + +def test_a_template_is_not_a_notice(): + assert licensing.read_notices("Copyright [yyyy] [name of copyright owner]") == [] + assert licensing.read_notices("Copyright (c) ") == [] + assert licensing.read_notices( + "MIT\n\nCopyright (c) 2016 Wenzel Jakob\n\nPermission" + ) == ["Copyright (c) 2016 Wenzel Jakob"] + + +def test_a_debian_copyright_field_names_its_holder_either_way(license_data): + """Debian Copyright fields can omit years and contain prose or None. + + The colon in Copyright: prevents matching it as an ordinary notice. + """ + licensing.load_corpus(license_data) + + same_line = "Copyright: Poul-Henning Kamp \nLicense: Beerware\n" + indented = "Copyright:\n Colin Plumb\n Todd C. Miller\nLicense: public-domain\n" + + assert licensing.read_notices(same_line) == [ + "Poul-Henning Kamp " + ] + assert licensing.read_notices(indented) == ["Colin Plumb", "Todd C. Miller"] + assert licensing.read_notices("Copyright: None\nLicense: MIT\n") == [] + assert ( + licensing.read_notices( + "Copyright: This code is derived from software contributed by X\n" + ) + == [] + ) + + +def test_every_holder_in_a_text_is_read_not_just_the_first(license_data): + """Notices can span lines or follow long lead-ins; every holder must survive.""" + licensing.load_corpus(license_data) + + assert licensing.read_notices( + "Copyright (c) 2002-2006 Marcus Geelnard\n\n" + "Copyright (c) 2006-2019 Camilla Lowy\n" + ) == [ + "Copyright (c) 2002-2006 Marcus Geelnard", + "Copyright (c) 2006-2019 Camilla Lowy", + ] + # The holder on the line below, with no comma to join them. + assert licensing.read_notices( + "Copyright © 1980, 1982, 1986, 1989-1994\n" + " The Regents of the University of California.\n" + ) == [ + "Copyright © 1980, 1982, 1986, 1989-1994 The Regents of the University " + "of California." + ] + # A lead-in longer than a name, and a clause broken by the end of its line. + assert licensing.read_notices( + "Box collision code (engine_collision_box.c) is Copyright 2016 S Kolev.\n" + ) == ["Copyright 2016 S Kolev."] + assert licensing.read_notices( + "2010), this software is Copyright (c) 2007-2010 by B Lepilleur, and is\n" + "released under the terms of the MIT License.\n" + ) == ["Copyright (c) 2007-2010 by B Lepilleur"] + + +def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): + """Preserve OR through matching and roll-up; rebuilding from identifiers would turn + a choice into simultaneous obligations. + """ + + licensing.load_corpus(license_data) + source = tmp_path / "vendored.h" + source.write_text( + "// SPDX-FileCopyrightText: Copyright 2020 Upstream Widgets Ltd\n" + "// SPDX-License-Identifier: MIT OR GPL-2.0-or-later\n" + ) + components, _ = discovery.discover_vendored( + tmp_path, {source}, {"NVIDIA"}, tracked=() + ) + component = next(iter(components.values())) + + assert component.license_concluded == "MIT OR GPL-2.0-or-later" + assert component.license_declared == "MIT OR GPL-2.0-or-later" + assert licensing.operands_of("MIT OR GPL-2.0-or-later") == [ + "(MIT OR GPL-2.0-or-later)" + ] + assert licensing.operands_of("Apache-2.0 AND CC-BY-4.0") == [ + "Apache-2.0", + "CC-BY-4.0", + ] + + +def test_an_exception_is_not_a_licence_needing_its_own_text(license_data): + """A WITH exception modifies a license; it is not a separate license operand + requiring its own packaged text in this gate. + """ + licensing.load_corpus(license_data) + + assert licensing.is_exception("LLVM-exception") + assert licensing.is_exception("Classpath-exception-2.0") + assert not licensing.is_exception("Apache-2.0") + + +@pytest.mark.parametrize("content", [None, "{not json", '{"licenses": []}']) +def test_the_exception_list_is_required(tmp_path, content): + """Without it, an exception newer than the grammar reads as an unknown id.""" + synth._stage_license_data(tmp_path) + exceptions = tmp_path / "exceptions.json" + if content is None: + exceptions.unlink() + else: + exceptions.write_text(content) + + with pytest.raises(licensing.CorpusError, match=r"exceptions\.json"): + licensing.load_corpus(tmp_path) + + +def test_a_machine_readable_copyright_field_is_read_as_debian_writes_it(): + """Field names ignore case; `None` (or `n/a`) states no holder, and what + follows it in the field is commentary. A notice wrapped over lines stays + one paragraph, its lines in order.""" + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" + "Files: a\ncopyright: 2001 Lower Case Holder\n .\n 2002 Second Holder\n" + "License: MIT\n\n" + "Files: b\nCopyright: None\n Copied from elsewhere\nLicense: public-domain\n\n" + "Files: c\nCopyright: n/a\nLicense: MIT\n\n" + "Files: d\nCopyright:\n Copyright 1989\n The Regents. All rights reserved.\n" + "License: BSD-3-clause\n" + ) + + assert licensing.dep5_copyright(text) == [ + "2001 Lower Case Holder", + "2002 Second Holder", + "Copyright 1989\nThe Regents. All rights reserved.", + ] + + +def test_the_debian_packaging_stanza_names_no_holder_of_the_library(): + """`Files: debian/*` credits whoever packaged it; `*` and the header count.""" + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" + "Copyright: 2000 Header Holder\n\n" + "Files: *\nCopyright: 2001 Upstream Holder\nLicense: MIT\n\n" + "Files: debian/*\n debian/patches/fix.patch\n" + "Copyright: 2020 Debian Packager\nLicense: MIT\n\n" + "Files:\n debian/*\nCopyright: 2020 Second Packager\nLicense: MIT\n\n" + "Files: debian ./debian/control\nCopyright: 2020 Third Packager\n" + "License: MIT\n\n" + "Files: debian/rules src/mixed.c\nCopyright: 2021 Mixed Stanza Holder\n" + "License: MIT\n" + ) + + assert licensing.dep5_copyright(text) == [ + "2000 Header Holder", + "2001 Upstream Holder", + "2021 Mixed Stanza Holder", + ] + + +def test_a_field_paragraph_is_split_into_the_notices_it_states(): + """A wrapped line continues the preceding notice unless it starts a new one.""" + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" + "Files: *\nCopyright: Copyright © 1980, 1982\n" + " The Regents of the University. All rights reserved.\n" + " 2004-2024 Guillem Jover\n 2012 Guillem Jover\n .\n" + " Some code is derived from software contributed to Berkeley by\n" + " Somebody, as part of Summer of Code\n 2005 program.\n" + "License: BSD-3-clause\n\n" + "Files: md5.c\nCopyright:\n Colin Plumb\n Todd C. Miller\nLicense: MIT\n" + ) + + notices = licensing.dep5_copyright(text) + + assert notices == [ + "Copyright © 1980, 1982\nThe Regents of the University. All rights reserved.", + "2004-2024 Guillem Jover", + "2012 Guillem Jover", + "Some code is derived from software contributed to Berkeley by\n" + "Somebody, as part of Summer of Code\n2005 program.", + "Colin Plumb", + "Todd C. Miller", + ] + assert "2012 Guillem Jover" not in licensing.fold_notices(notices) + + +@pytest.mark.parametrize( + ("field", "expected"), + [ + # A year list wrapping onto the holder's line is one notice. + ( + "Copyright: © 1994, 1995,\n 1996 Free Software Foundation, Inc.\n", + ["© 1994, 1995,\n1996 Free Software Foundation, Inc."], + ), + # A holder whose name reads like prose does not swallow the next one. + ( + "Copyright: 1999-2022 Python Software Foundation\n" + " 1992-1993 Jean-loup Gailly\n", + ["1999-2022 Python Software Foundation", "1992-1993 Jean-loup Gailly"], + ), + # A line ending in a comma runs on into a lower-case one. + ( + "Copyright:\n Christian Tismer ,\n contributors\n", + ["Christian Tismer ,\ncontributors"], + ), + ], +) +def test_a_notice_is_not_cut_or_merged_at_a_line_break(field, expected): + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" + f"Files: *\n{field}License: MIT\n" + ) + + assert licensing.dep5_copyright(text) == expected diff --git a/scripts/sbom/tests/test_nested_terms.py b/scripts/sbom/tests/test_nested_terms.py new file mode 100644 index 0000000000..0eb546f7e3 --- /dev/null +++ b/scripts/sbom/tests/test_nested_terms.py @@ -0,0 +1,578 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Scope nested license terms to the files contributed to the wheel.""" + +from __future__ import annotations + +import json +import logging +from types import SimpleNamespace + +import pytest +import synth +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import discovery +from isaaccapture_sbom import evidence as evidence_module +from isaaccapture_sbom import inventory as inventory_module +from isaaccapture_sbom import licensing +from isaaccapture_sbom import report + +BUNDLED = "Copyright (c) 2019 Bundled Authors\n\n" + "Permission is hereby granted." +EXTENSION = "isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so" + + +@pytest.fixture +def alpha(workspace): + return workspace.build / "_deps" / "alpha-src" + + +@pytest.fixture +def beta(workspace): + return workspace.build / "_deps" / "beta-src" + + +def _deps(workspace): + return workspace.build / "_deps" + + +def _compiled(workspace): + """The sources the fixture's `ext` target compiles, as a depfile names them.""" + return ( + (workspace.root / "src/cpp/module.cpp").resolve(), + (_deps(workspace) / "alpha-src/alpha.cpp").resolve(), + ) + + +def _depfiles(workspace, read=(), extra=()) -> None: + """A depfile per object `ext` compiles; the first also read `read`.""" + first, *rest = (*_compiled(workspace), *(item.resolve() for item in extra)) + synth.depfile(workspace.build, "ext", first, tuple(read)) + for source in rest: + synth.depfile(workspace.build, "ext", source) + + +def _kept(workspace, built=None) -> dict[str, dict | None]: + """Each nested text the build found, with what kept it, or None if dropped.""" + built = built or synth.collect(workspace) + found = { + item["path"]: item.get("reached_by") + for component in built["build-evidence"]["components"].values() + for item in component["evidence"] + if item["kind"] == "nested" + } + for root in sorted(_deps(workspace).glob("*-src")): + for item in licensing.discover_in_tree(root.name, root, root.name): + if item.kind == "nested": + found.setdefault(item.path, None) + return found + + +def _package(built, name): + return next(item for item in built["spdx"]["packages"] if item["name"] == name) + + +def test_a_compiled_in_library_keeps_its_terms_and_holder(workspace, alpha): + synth.write(alpha / "third/bundled.cpp", "// bundled\n") + synth.write(alpha / "third/LICENSE", BUNDLED) + synth.write(alpha / "test/LICENSE", "Copyright (c) 2018 Test Authors\n") + synth.add_source(workspace.build, "ext", alpha / "third/bundled.cpp") + _depfiles(workspace, extra=(alpha / "third/bundled.cpp",)) + + built = synth.collect(workspace) + kept = _kept(workspace, built) + + assert kept["alpha-src/third/LICENSE"]["how"] == "compiled-source" + assert kept["alpha-src/third/LICENSE"]["path"] == "alpha-src/third/bundled.cpp" + assert kept["alpha-src/test/LICENSE"] is None + copyright_text = _package(built, "alpha")["copyrightText"] + assert "Bundled Authors" in copyright_text + assert "Test Authors" not in copyright_text + + +def test_bundled_gpl_text_names_no_steward_and_reads_as_alternatives(workspace, alpha): + # As upstreams ship it: `(C)` and `http://`, where the list has `©` and `https://`. + gpl = ( + synth.spdx_text("GPL-3.0-only") + .replace("©", "(C)") + .replace("https://fsf.org/", "http://fsf.org/") + ) + synth.write(alpha / "third/bundled.cpp", "// bundled\n") + synth.write(alpha / "third/LICENSE-gpl3.txt", gpl) + synth.add_source(workspace.build, "ext", alpha / "third/bundled.cpp") + _depfiles(workspace, extra=(alpha / "third/bundled.cpp",)) + + built = synth.collect(workspace) + + alternatives = "matches the reference text of GPL-3.0-only or GPL-3.0-or-later" + package = _package(built, "alpha") + assert "Free Software Foundation" not in package["copyrightText"] + assert package["licenseConcluded"] == "MIT" + texts = [item for item in package["attributionTexts"] if "LICENSE-gpl3" in item] + assert "matches the reference text of GPL-3.0-only (" in texts[0] + assert texts[0].endswith(report.NESTED_ROLE) + row = next(line for line in built["md"].splitlines() if "| alpha |" in line) + assert f"→ {alternatives} ({report.NESTED_ROLE})" in row + assert "GPL-3.0-only AND" not in built["md"] + json.dumps(built["spdx"]) + assert f"LICENSE-gpl3.txt` — {report.NESTED_ROLE}" in built["notices"] + + +def test_a_header_the_depfile_says_was_read_keeps_its_terms(workspace, beta): + synth.write(beta / "include/beta/fmt/format.h", "#pragma once\n") + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.write(beta / "include/beta/unused/LICENSE", BUNDLED) + synth.write(beta / "docs/LICENSE", BUNDLED) + _depfiles(workspace, read=(beta / "include/beta/fmt/format.h",)) + + kept = _kept(workspace) + + assert kept["beta-src/include/beta/fmt/LICENSE"] == { + "how": "header-read", + "path": "beta-src/include/beta/fmt/format.h", + "member": EXTENSION, + } + assert kept["beta-src/include/beta/unused/LICENSE"] is None + assert kept["beta-src/docs/LICENSE"] is None + + +def test_a_depfile_overrules_the_include_directory(workspace, beta): + synth.write(beta / "include/COPYING", BUNDLED) + _depfiles(workspace) + + assert _kept(workspace)["beta-src/include/COPYING"] is None + + +def _no_record(workspace) -> None: + """No dependency output at all.""" + + +def _one_record_short(workspace) -> None: + """module.cpp includes alpha.cpp, which has no object record of its own.""" + module, alpha = _compiled(workspace) + synth.depfile(workspace.build, "ext", module, (alpha,)) + + +def _placeholder(workspace) -> None: + """The compiler_depend.make CMake writes before the first build.""" + synth.compiler_depend(workspace.build, "ext", {}) + + +@pytest.mark.parametrize("records", [_no_record, _one_record_short, _placeholder]) +def test_without_a_record_per_object_include_directories_stand_in( + workspace, beta, records +): + synth.write(beta / "include/COPYING", BUNDLED) + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.write(beta / "docs/LICENSE", BUNDLED) + records(workspace) + + kept = _kept(workspace) + + assert kept["beta-src/include/COPYING"]["how"] == "include-dir" + assert kept["beta-src/include/COPYING"]["path"] == "beta-src/include" + assert kept["beta-src/include/beta/fmt/LICENSE"]["how"] == "include-dir" + assert kept["beta-src/docs/LICENSE"] is None + + +def test_a_stale_depfile_is_ignored_and_a_precompiled_header_counts(workspace, beta): + for name in ("stale", "pch", "unity"): + synth.write(beta / f"{name}/{name}.h", "#pragma once\n") + synth.write(beta / f"{name}/LICENSE", BUNDLED) + _depfiles(workspace) + # A source the target no longer compiles, and a header CMake generated into + # the objects directory to precompile. + synth.depfile( + workspace.build, + "ext", + (workspace.root / "src/cpp/removed.cpp").resolve(), + ((beta / "stale/stale.h").resolve(),), + ) + synth.depfile( + workspace.build, + "ext", + (workspace.build / "CMakeFiles/ext.dir/cmake_pch.hxx").resolve(), + ((beta / "pch/pch.h").resolve(),), + ) + # A unity source left from a build the codemodel no longer describes. + synth.depfile( + workspace.build, + "ext", + (workspace.build / "CMakeFiles/ext.dir/Unity/unity_0_cxx.cxx").resolve(), + ((beta / "unity/unity.h").resolve(),), + ) + + kept = _kept(workspace) + + assert kept["beta-src/stale/LICENSE"] is None + assert kept["beta-src/unity/LICENSE"] is None + assert kept["beta-src/pch/LICENSE"]["how"] == "header-read" + + +def test_an_include_directory_holding_the_whole_tree_keeps_its_terms(workspace, beta): + synth.write(beta / "vendor/LICENSE", BUNDLED) + # `-I _deps` credits every fetched tree, so each needs its own terms. + synth.write(_deps(workspace) / "license-list-data-src/LICENSE", BUNDLED) + synth.add_include(workspace.build, "ext", workspace.build / "_deps") + + assert _kept(workspace)["beta-src/vendor/LICENSE"] == { + "how": "include-dir", + "path": "beta-src", + "member": EXTENSION, + } + + +def test_without_dependency_output_listed_sources_stand_in(workspace, beta): + synth.write(beta / "src/impl.h", "#pragma once\n") + synth.write(beta / "src/LICENSE", BUNDLED) + synth.add_source(workspace.build, "ext", beta / "src/impl.h") + + assert _kept(workspace)["beta-src/src/LICENSE"]["how"] == "listed-source" + + +def _depfiles_in_sub(workspace, monkeypatch) -> None: + """Depfiles name paths from where make ran the compiler.""" + module, alpha = _compiled(workspace) + header = "../_deps/beta-src/include/beta/fmt/format.h" + synth.depfile(workspace.build, "ext", module, (header,), directory="sub") + synth.depfile(workspace.build, "ext", alpha, directory="sub") + + +def _compiler_depend_in_sub(workspace, monkeypatch) -> None: + """compiler_depend.make names paths inside the build from its top.""" + module, _ = _compiled(workspace) + objects = "sub/CMakeFiles/ext.dir" + header = "_deps/beta-src/include/beta/fmt/format.h" + synth.compiler_depend( + workspace.build, + "ext", + { + f"{objects}/module.cpp.o": (str(module), header), + f"{objects}/alpha.cpp.o": ("_deps/alpha-src/alpha.cpp",), + }, + directory="sub", + ) + + +def _ninja_log_in_sub(workspace, monkeypatch) -> None: + """Ninja's deps log names paths from the top of the build.""" + module, _ = _compiled(workspace) + objects = "sub/CMakeFiles/ext.dir" + _ninja( + workspace, + monkeypatch, + f"{objects}/module.cpp.o: #deps 2, deps mtime 1 (VALID)\n" + f" {module}\n _deps/beta-src/include/beta/fmt/format.h\n\n" + f"{objects}/alpha.cpp.o: #deps 1, deps mtime 1 (VALID)\n" + " _deps/alpha-src/alpha.cpp\n\n", + ) + + +@pytest.mark.parametrize( + "records", [_depfiles_in_sub, _compiler_depend_in_sub, _ninja_log_in_sub] +) +def test_relative_paths_in_dependency_output_resolve_as_written( + workspace, beta, records, monkeypatch +): + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.set_directory(workspace.build, "ext", "sub") + records(workspace, monkeypatch) + + assert _kept(workspace)["beta-src/include/beta/fmt/LICENSE"]["how"] == "header-read" + + +def test_a_linked_archive_keeps_every_nested_text_of_its_component(workspace, beta): + synth.write(beta / "lib/libbeta.a", synth.ar_archive({"b.o": synth.elf_object()})) + synth.write(beta / "vendor/LICENSE", BUNDLED) + synth.link_line(workspace.build, "ext", (str(beta / "lib/libbeta.a"),)) + _depfiles(workspace) + + assert _kept(workspace)["beta-src/vendor/LICENSE"] == { + "how": "prebuilt-library", + "path": "beta-src", + "member": EXTENSION, + } + + +def test_an_archive_owned_outside_the_component_trees_keeps_them_too( + workspace, beta, tmp_path +): + prebuilt = tmp_path / "prebuilt" + synth.write(prebuilt / "libbeta.a", synth.ar_archive({"b.o": synth.elf_object()})) + synth.write(beta / "vendor/LICENSE", BUNDLED) + synth.link_line(workspace.build, "ext", (str(prebuilt / "libbeta.a"),)) + _depfiles(workspace) + space = evidence_module.discover(workspace.root, workspace.build) + space.ownership.register_root(prebuilt, "beta") + + evidence = synth.collect(workspace, discovery=space)["build-evidence"] + + nested = next( + item + for item in evidence["components"]["beta"]["evidence"] + if item["path"] == "beta-src/vendor/LICENSE" + ) + assert nested["reached_by"]["how"] == "prebuilt-library" + + +def test_a_header_read_from_the_build_tree_is_its_source_file(workspace, beta): + built = _deps(workspace) / "beta-build" + synth.write(beta / "include/beta/fmt/format.h", "// fmt\n") + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.write(beta / "gen/LICENSE", BUNDLED) + # A copy staged elsewhere, and a header generated where its tree would hold it. + synth.write(built / "staged/format.h", "// fmt\n") + synth.write(built / "gen/config.h", "#define GENERATED 1\n") + _depfiles(workspace, read=(built / "staged/format.h", built / "gen/config.h")) + + kept = _kept(workspace) + + assert kept["beta-src/include/beta/fmt/LICENSE"]["path"] == ( + "beta-src/include/beta/fmt/format.h" + ) + assert kept["beta-src/gen/LICENSE"]["path"] == "beta-src/gen/config.h" + + +def test_a_component_path_outside_its_trees_is_not_reach(workspace, alpha, tmp_path): + synth.write(alpha / "fmt/LICENSE", BUNDLED) + elsewhere = tmp_path / "other" / "_deps" / "alpha-src" / "fmt" / "format.h" + _depfiles(workspace, read=(elsewhere,)) + + assert _kept(workspace)["alpha-src/fmt/LICENSE"] is None + + +def test_a_build_tree_file_keeps_its_place_and_names_only_its_own_sources( + workspace, alpha, beta, tmp_path +): + deps = _deps(workspace) + synth.write(beta / "include/beta/fmt/format.h", "// fmt\n") + synth.write(alpha / "other/other.h", "// other\n") + synth.write(deps / "beta-build/gen/format.h", "// fmt\n") + synth.write(deps / "beta-build/gen/other.h", "// other\n") + synth.write(beta / "a/dup.h", "// dup\n") + synth.write(beta / "b/dup.h", "// dup\n") + # Empty, like many files of any tree: the bytes say nothing about origin. + synth.write(beta / "empty/blank.h", "") + synth.write(deps / "beta-build/gen/blank.h", "") + # A source tree reached through a link has its `-build` sibling beside the link. + real = tmp_path / "elsewhere" / "linked" + real.mkdir(parents=True) + (deps / "linked-src").symlink_to(real) + synth.write(deps / "linked-build/gen/config.h", "#define X 1\n") + space = evidence_module.discover(workspace.root, workspace.build) + resolver = inventory_module.Resolver(space) + + def shown(key, path): + return sorted(resolver._displayed({key: {path.resolve(): "header-read"}})[key]) + + assert shown("beta", deps / "beta-build/gen/format.h") == [ + "beta-src/gen/format.h", + "beta-src/include/beta/fmt/format.h", + ] + assert shown("beta", deps / "beta-build/gen/other.h") == ["beta-src/gen/other.h"] + assert shown("beta", deps / "beta-build/gen/blank.h") == ["beta-src/gen/blank.h"] + assert shown("beta", beta / "a/dup.h") == ["beta-src/a/dup.h"] + assert shown("linked", deps / "linked-build/gen/config.h") == [ + "linked-src/gen/config.h" + ] + + +def test_reach_is_attributed_with_the_ownership_asked_about(workspace): + config = _deps(workspace) / "beta-build/gen/config.h" + synth.write(config, "#define X 1\n") + _depfiles(workspace, read=(config,)) + space = evidence_module.discover(workspace.root, workspace.build) + artifact = space.graph.by_name(EXTENSION.rsplit("/", 1)[-1])[0] + + # Only the ownership index knows a `-build` tree's owner. + assert "beta" not in space.graph.reached(artifact) + assert "beta" in space.graph.reached(artifact, space.ownership) + + +def test_a_build_tool_or_custom_target_is_no_reach(workspace, beta): + synth.write(beta / "codegen/LICENSE", BUNDLED) + synth.write(beta / "scripts/LICENSE", BUNDLED) + synth.add_target( + workspace.build, + "beta_codegen", + "EXECUTABLE", + (str(beta / "codegen/main.cpp"),), + "ext", + ) + synth.add_target( + workspace.build, + "beta_generate", + "UTILITY", + (str(beta / "scripts/generate.py"),), + "ext", + ) + _depfiles(workspace) + + kept = _kept(workspace) + + assert kept["beta-src/codegen/LICENSE"] is None + assert kept["beta-src/scripts/LICENSE"] is None + + +def test_a_reuse_sidecar_covers_only_its_file(workspace, alpha): + synth.write(alpha / "third/bundled.cpp", "// bundled\n") + synth.write(alpha / "third/data.txt", "data\n") + synth.write(alpha / "third/data.txt.license", "SPDX-FileCopyrightText: 2020 A\n") + synth.write(alpha / "third/used.h", "#pragma once\n") + synth.write(alpha / "third/used.h.license", "SPDX-FileCopyrightText: 2020 B\n") + synth.add_source(workspace.build, "ext", alpha / "third/bundled.cpp") + _depfiles( + workspace, read=(alpha / "third/used.h",), extra=(alpha / "third/bundled.cpp",) + ) + + kept = _kept(workspace) + + assert kept["alpha-src/third/data.txt.license"] is None + assert kept["alpha-src/third/used.h.license"] == { + "how": "header-read", + "path": "alpha-src/third/used.h", + "member": EXTENSION, + } + + +def test_copied_and_derived_members_keep_the_terms_beside_them(workspace): + vendorpy = _deps(workspace) / "vendorpy-src" + synth.write(vendorpy / "pkg/LICENSE", BUNDLED) + synth.write(vendorpy / "pkg/assets/NOTICE", BUNDLED) + synth.write(vendorpy / "docs/LICENSE", BUNDLED) + + kept = _kept(workspace) + + assert kept["vendorpy-src/pkg/LICENSE"] == { + "how": "copied-file", + "path": "vendorpy-src/pkg/mod.py", + "member": "vendorpy/mod.py", + } + assert kept["vendorpy-src/pkg/assets/NOTICE"]["how"] == "derived-file" + assert kept["vendorpy-src/docs/LICENSE"] is None + + +def _ninja(workspace, monkeypatch, log: str) -> None: + synth.set_generator(workspace.build, "Ninja") + asked = discovery._ask + monkeypatch.setattr( + discovery, + "_ask", + lambda *command, **options: ( + log if command[0] == "ninja" else asked(*command, **options) + ), + ) + + +def test_ninja_deps_log_is_read_for_what_a_target_read( + workspace, monkeypatch, caplog, beta +): + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + header = (beta / "include/beta/fmt/format.h").resolve() + _ninja( + workspace, + monkeypatch, + "".join( + f"CMakeFiles/ext.dir/{source.name}.o: #deps 2, deps mtime 1 (VALID)\n" + f" {source}\n {header}\n\n" + for source in _compiled(workspace) + ) + # An output whose depfile listed nothing, as a precompiled header's can. + + "CMakeFiles/ext.dir/cmake_pch.hxx.gch: #deps 0, deps mtime 1 (VALID)\n\n", + ) + + with caplog.at_level(logging.WARNING, logger="isaaccapture.sbom"): + kept = _kept(workspace) + + assert kept["beta-src/include/beta/fmt/LICENSE"]["how"] == "header-read" + assert not [item for item in caplog.records if "ninja -t deps" in item.message] + + +def test_ninja_records_count_only_for_their_own_objects_directory( + workspace, monkeypatch, beta +): + synth.write(beta / "other/LICENSE", BUNDLED) + other = (beta / "other/other.h").resolve() + module, _ = _compiled(workspace) + _ninja( + workspace, + monkeypatch, + "".join( + f"CMakeFiles/ext.dir/{source.name}.o: #deps 1, deps mtime 1 (VALID)\n" + f" {source}\n\n" + for source in _compiled(workspace) + ) + # Another target compiling the same source with other headers. + + "sub/CMakeFiles/other.dir/module.cpp.o: #deps 2, deps mtime 1 (VALID)\n" + f" {module}\n {other}\n\n", + ) + + assert _kept(workspace)["beta-src/other/LICENSE"] is None + + +def test_without_ninja_the_fallback_is_announced_once( + workspace, monkeypatch, caplog, beta +): + synth.write(beta / "include/COPYING", BUNDLED) + _ninja(workspace, monkeypatch, "") + + with caplog.at_level(logging.WARNING, logger="isaaccapture.sbom"): + kept = _kept(workspace) + + assert kept["beta-src/include/COPYING"]["how"] == "include-dir" + assert ( + len([item for item in caplog.records if "ninja -t deps" in item.message]) == 1 + ) + + +def test_depfile_rules_are_read_as_make_reads_them(): + text = ( + "# CMAKE generated file: DO NOT EDIT!\n" + "obj.o: /src/a\\ b.cpp \\\n /inc/x$$y.h \\\n /inc/z.h\n" + "\n/inc/z.h:\n" + ) + + assert discovery._make_rules(text) == [["/src/a b.cpp", "/inc/x$y.h", "/inc/z.h"]] + + +def test_registering_an_owner_forgets_what_was_asked_before(tmp_path): + ownership = discovery.Ownership() + source = tmp_path / "tree" / "a.h" + vendored = tmp_path / "vendor" / "b.h" + assert ownership.owner_of(source) is None + ownership.register_root(tmp_path / "tree", "tree") + assert ownership.owner_of(source) == "tree" + + assert ownership.owner_of(vendored) is None + ownership.register_file(vendored, "vendored:b") + assert ownership.owner_of(vendored) == "vendored:b" + + +def test_a_steward_line_is_recognised_however_its_copy_spells_it(): + notice = "Copyright (C) 2007 Free Software Foundation, Inc. " + + assert licensing.notice_is_the_licence_authors(notice, ["GPL-3.0-only"]) + + +def test_the_evidence_names_the_first_member_that_reached_a_text(workspace, beta): + synth.write(beta / "vendor/LICENSE", BUNDLED) + component = evidence_module.discover(workspace.root, workspace.build).components[ + "beta" + ] + read = {"beta": {"beta-src/vendor/v.h": "header-read"}} + inventory = SimpleNamespace( + attributions={ + member: inventory_module.Attribution( + path=member, origin="built", detail="", read=read + ) + for member in ("z/late.so", "a/early.so") + } + ) + + settled = build_module._with_reached_nested(inventory, {"beta": component}) + + kept = next( + item + for item in settled["beta"].evidence + if item.path == "beta-src/vendor/LICENSE" + ) + assert kept.reached_by.member == "a/early.so" diff --git a/scripts/sbom/tests/test_spdx_conformance.py b/scripts/sbom/tests/test_spdx_conformance.py new file mode 100644 index 0000000000..ce0befbc81 --- /dev/null +++ b/scripts/sbom/tests/test_spdx_conformance.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate generated documents with the third-party SPDX validator.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("spdx_tools", reason="spdx-tools is a dev dependency of this suite") + +from spdx_tools.spdx.parser.parse_anything import parse_file # noqa: E402 +from spdx_tools.spdx.validation.document_validator import ( # noqa: E402 + validate_full_spdx_document, +) + + +def test_emitted_document_is_valid_spdx_2_3(built): + document = parse_file(str(built["spdx_path"])) + + messages = validate_full_spdx_document(document) + + assert [message.validation_message for message in messages] == [] + + +def test_relationships_survive_the_round_trip(built): + document = parse_file(str(built["spdx_path"])) + + kinds = { + relationship.relationship_type.name for relationship in document.relationships + } + + assert {"DESCRIBES", "STATIC_LINK", "CONTAINS", "DEPENDS_ON"} <= kinds + assert {"GENERATED_FROM", "COPY_OF"} & kinds + + +def test_an_identifier_newer_than_the_grammar_still_validates(workspace): + """A vendored file may name an exception the pinned list has and the + grammar consumers validate with does not; it travels as a LicenseRef with + its texts, and the report lists it as an evidence gap.""" + import synth + + synth.write( + workspace.root / "src/vendor/rsync_helper.h", + "// Copyright 2024 Rsync Folk\n// SPDX-" + + "License-Identifier: BSL-1.0 WITH rsync-linking-exception\n", + ) + synth.track(workspace.root) + synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) + collected = synth.collect(workspace) + + document = parse_file(str(collected["out_dir"] / f"{synth.STEM}.spdx.json")) + assert validate_full_spdx_document(document) == [] + + report = collected["licenses"] + component = next( + item + for item in report["components"] + if item["component"] == "vendored:rsync-folk" + ) + assert component["license_concluded"] == "LicenseRef-vendored-rsync-folk" + gaps = { + item["component"]: item + for item in report["components_with_unidentified_license"] + } + assert "rsync-linking-exception" in gaps["vendored:rsync-folk"]["reason"] + assert "rsync-linking-exception.txt" in gaps["vendored:rsync-folk"]["evidence"] diff --git a/scripts/sbom/tests/test_static_archives.py b/scripts/sbom/tests/test_static_archives.py new file mode 100644 index 0000000000..42735edd27 --- /dev/null +++ b/scripts/sbom/tests/test_static_archives.py @@ -0,0 +1,1276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Static link inputs, linker resolution, and package ownership tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import synth +from packageurl import PackageURL +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import discovery +from isaaccapture_sbom import document as document_module +from isaaccapture_sbom import evidence as evidence_module + +EXTENSION = "_ext.cpython-312-x86_64-linux-gnu.so" +ONE_OBJECT = synth.ar_archive({"runtime.o": b"\x7fELF object"}) +# An agreement that quotes the notices of what it bundles, as the CUDA EULA does. +EULA = ( + synth.PROPRIETARY_LICENSE + + "\nThird-party notices\n\nCopyright (c) 2011 Upstream Compression Ltd.\n" +) + + +def _no_package(monkeypatch) -> None: + """The build machine's package manager owns nothing outside its own trees.""" + monkeypatch.setattr( + discovery, "_identify_host_package", lambda library, origin: origin + ) + + +def _packaged(monkeypatch, package, copyright, **fields) -> None: + """dpkg names `package` as the owner of whatever is asked about. + + `package` and `copyright` may be callables of the library's path. + """ + + def owned(library, origin): + def value(item): + return item(library) if callable(item) else item + + return { + **origin, + **fields, + "package": value(package), + "copyright": str(value(copyright)), + } + + monkeypatch.setattr(discovery, "_identify_host_package", owned) + + +def _row(collected: dict, key: str) -> dict: + return next( + item for item in collected["licenses"]["components"] if item["component"] == key + ) + + +def _library_dir(root, *names) -> None: + """A directory holding each `lib` as both a shared library and an archive.""" + for name in names: + synth.write(root / f"lib{name}.so", synth.elf_shared_object(f"lib{name}.so")) + synth.write(root / f"lib{name}.a", ONE_OBJECT) + + +def _inputs(workspace, *fragments, **link) -> dict[str, discovery.LinkInput]: + synth.link_line(workspace.build, "ext", fragments, **link) + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + linked = graph.static_inputs(graph.by_name(EXTENSION)[0]) + return {item.named: item for item in linked} + + +def _package(row: dict, spdx: dict) -> dict: + package_id = document_module.component_package_id(row["component"]) + return next(item for item in spdx["packages"] if item["SPDXID"] == package_id) + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + (synth.ar_archive({}), 0), + (ONE_OBJECT, 1), + (synth.ar_archive({"a.o": b"a", "b.o": b"bb"}), 2), + # A GNU symbol table is an index, not a member. + (b"!\n" + synth.ar_header(b"/", 4) + b"\0\0\0\0" + ONE_OBJECT[8:], 1), + # BSD names a long member, the symbol table among them, inside its data. + ( + b"!\n" + + synth.ar_header(b"#1/20", 24) + + b"__.SYMDEF SORTED\0\0\0\0" + + b"\0\0\0\0" + + ONE_OBJECT[8:], + 1, + ), + # A thin archive stores no member data, only headers. + ( + b"!\n" + + synth.ar_header(b"a.o/", 100) + + synth.ar_header(b"b.o/", 200), + 2, + ), + (b"/* GNU ld script */\nGROUP ( libm.so.6 )\n", None), + ], +) +def test_archive_members_are_counted_from_the_ar_headers(tmp_path, data, expected): + path = tmp_path / "lib.a" + path.write_bytes(data) + + assert discovery.archive_members(path) == expected + + +@pytest.mark.parametrize( + ("generator", "directory", "fragment"), + [ + (None, None, "lib/libdelta.a"), + ("Ninja", "src/ext", "lib/libdelta.a"), + ("Unix Makefiles", "src/ext", "../../lib/libdelta.a"), + ], +) +def test_an_archive_a_target_here_built_maps_to_that_target( + workspace, generator, directory, fragment +): + """Not a declared dependency of ext; its link line is the only evidence.""" + if generator: + synth.set_generator(workspace.build, generator) + synth.link_line(workspace.build, "ext", (fragment,), directory=directory) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + extension = graph.by_name(EXTENSION)[0] + + assert "compiled-source" in graph.contributions(extension)["delta"] + assert graph.static_inputs(extension) == {} + + +def test_an_archive_through_a_static_library_is_on_the_consumers_link_line( + workspace, tmp_path +): + """CMake gives a static library no link line: what it requires is flattened + onto each consumer's, beside the static library's own archive.""" + runtime = tmp_path / "host" / "libruntime.a" + synth.write(runtime, ONE_OBJECT) + synth.link_line( + workspace.build, + "ext", + ("lib/libdelta.a", str(runtime)), + dependencies=("delta",), + ) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + extension = graph.by_name(EXTENSION)[0] + + assert "delta" in graph.contributions(extension) + assert graph.static_inputs(extension) == { + discovery.LinkInput(str(runtime), runtime, "archive", members=1): {"ext"} + } + + +def test_an_archive_a_dependency_links_is_followed_like_its_sources( + workspace, tmp_path +): + """ext -> delta (static) -> gamma (shared), whose link line names it.""" + runtime = tmp_path / "host" / "libruntime.a" + synth.write(runtime, ONE_OBJECT) + synth.link_line(workspace.build, "ext", (), dependencies=("delta",)) + synth.link_line(workspace.build, "delta", (), dependencies=("gamma",)) + synth.link_line(workspace.build, "gamma", (str(runtime),)) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + extension = graph.by_name(EXTENSION)[0] + + assert list(graph.static_inputs(extension).values()) == [{"gamma"}] + + +def test_static_code_is_found_in_every_form_a_link_line_takes(workspace, tmp_path): + host = tmp_path / "host dir" + synth.write(host / "libspaced.a", ONE_OBJECT) + synth.write(host / "libwhole.a", ONE_OBJECT) + synth.write(host / "libflagged.a", ONE_OBJECT) + synth.write(host / "libversioned.a.1", ONE_OBJECT) + synth.write(host / "prebuilt.o", synth.elf_object()) + synth.write(host / "libshared.so", synth.elf_shared_object("libshared.so")) + synth.write(host / "exports.map", "{ global: *; };\n") + + inputs = _inputs( + workspace, + f'"{host}/libspaced.a"', + f'"-Wl,--whole-archive,{host}/libwhole.a,--no-whole-archive"', + (f'"{host}/libflagged.a"', "flags"), + f'"{host}/libversioned.a.1"', + f'"{host}/prebuilt.o"', + f'"{host}/libshared.so"', + (f'"-Wl,--version-script,{host}/exports.map"', "flags"), + ("-fuse-ld=gold", "flags"), + ) + + assert {name: item.kind for name, item in inputs.items()} == { + f"{host}/libspaced.a": "archive", + f"{host}/libwhole.a": "archive", + f"{host}/libflagged.a": "archive", + f"{host}/libversioned.a.1": "archive", + f"{host}/prebuilt.o": "object", + } + + +def test_a_library_name_resolves_the_way_ld_resolves_it(workspace, tmp_path): + searched = tmp_path / "searched" + implicit = tmp_path / "implicit" + _library_dir(searched, "both", "forced") + synth.write(searched / "libstaticonly.a", ONE_OBJECT) + synth.write(implicit / "libcompiler.a", ONE_OBJECT) + synth.set_toolchains(workspace.build, {"CXX": [str(implicit)]}) + + inputs = _inputs( + workspace, + (f"-L{searched}", "libraryPath"), + "-lboth", + "-lstaticonly", + "-lcompiler", + "-l:libboth.a", + "-lmissing", + "-l:libmissing.a", + "-Wl,-Bstatic", + "-lforced", + ) + + # -Bstatic selects an archive; the earlier shared library belongs to DT_NEEDED. + assert {name: (item.kind, item.path) for name, item in inputs.items()} == { + "-lforced": ("archive", searched / "libforced.a"), + "-lstaticonly": ("archive", searched / "libstaticonly.a"), + "-lcompiler": ("archive", implicit / "libcompiler.a"), + "-l:libboth.a": ("archive", searched / "libboth.a"), + "-lmissing": ("unresolved", None), + "-l:libmissing.a": ("unresolved", None), + } + + +def test_every_spelling_of_a_search_resolves_alike(workspace, tmp_path): + joined, separated = tmp_path / "joined", tmp_path / "separated" + _library_dir(joined, "eqsign", "spaced", "linkerspaced") + _library_dir(separated, "fromsep") + synth.write(joined / "libarchive.a.1", ONE_OBJECT) + synth.write(joined / "libshared.so.1", synth.elf_shared_object("libshared.so.1")) + synth.write(tmp_path / "xl" / "libxl.a", ONE_OBJECT) + + inputs = _inputs( + workspace, + f"-Wl,--library-path={joined}", + f"-Wl,-L,{separated}", + "-Wl,-Bstatic", + "-Wl,--library=eqsign", + "-l spaced", + "-Wl,-l,linkerspaced", + "-lfromsep", + "-Wl,-Bdynamic", + "-l:libarchive.a.1", + "-l:libshared.so.1", + f"-Xlinker {tmp_path}/xl/libxl.a", + ) + + assert {name: item.path for name, item in inputs.items()} == { + "-leqsign": joined / "libeqsign.a", + "-lspaced": joined / "libspaced.a", + "-llinkerspaced": joined / "liblinkerspaced.a", + "-lfromsep": separated / "libfromsep.a", + # Explicit names are classified by content, regardless of suffix. + "-l:libarchive.a.1": joined / "libarchive.a.1", + f"{tmp_path}/xl/libxl.a": tmp_path / "xl" / "libxl.a", + } + + +@pytest.mark.parametrize( + ("fragments", "expected"), + [ + # -Bdynamic resets; push-state saves and pop-state restores. + ( + ( + "-Wl,-Bstatic", + "-Wl,-Bdynamic", + "-lreset", + "-Wl,--push-state,-Bstatic,-lpushed,--pop-state", + "-lafter", + ), + {"-lpushed": "libpushed.a"}, + ), + # Pushed while static: the pop restores static mode, not the default. + ( + ( + "-Wl,-Bstatic", + "-Wl,--push-state,-Bdynamic,-lpushdyn,--pop-state", + "-lpopped", + ), + {"-lpopped": "libpopped.a"}, + ), + # The driver's -static covers the whole line, wherever it is written. + (("-lafter", ("-static", "flags")), {"-lafter": "libafter.a"}), + ], +) +def test_static_mode_follows_ld_and_the_driver( + workspace, tmp_path, fragments, expected +): + directory = tmp_path / "libs" + _library_dir(directory, "reset", "pushed", "after", "pushdyn", "popped") + + inputs = _inputs(workspace, f"-L{directory}", *fragments) + + assert {name: item.path for name, item in inputs.items()} == { + name: directory / file for name, file in expected.items() + } + + +def test_an_option_value_is_not_an_input(workspace, tmp_path): + synth.write(tmp_path / "libvalue.a", ONE_OBJECT) + + inputs = _inputs( + workspace, + (f"-o {tmp_path}/libvalue.a", "flags"), + f"-Wl,--dependency-file,{tmp_path}/deps.d", + f"-Wl,--require-defined,{tmp_path}/libvalue.a", + (f"-Wl,-rpath,{tmp_path}", "flags"), + ) + + assert inputs == {} + + +def test_an_unknown_path_names_itself_and_the_option_it_may_belong_to(workspace): + (item,) = _inputs(workspace, "-Wl,--some-new-option,mystery-value").values() + + assert item.kind == "unresolved" + assert item.named == "mystery-value" + assert "linker option" in item.reason + + +def test_the_compiler_and_linker_are_asked_where_the_reply_is_silent( + workspace, tmp_path +): + """C enabled only below the top level gets no implicit directories from + CMake, so the compiler is asked, and then its linker's SEARCH_DIRs.""" + from_compiler = tmp_path / "compiler" + from_linker = tmp_path / "sysroot" / "stub-only" / "lib" + synth.write(from_compiler / "libfromcompiler.a", ONE_OBJECT) + synth.write(from_linker / "libfromlinker.a", ONE_OBJECT) + compiler = synth.stub_compiler( + tmp_path / "bin", from_compiler, tmp_path / "sysroot", "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + inputs = _inputs(workspace, "-lfromcompiler", "-lfromlinker") + + assert {name: item.path for name, item in inputs.items()} == { + "-lfromcompiler": from_compiler / "libfromcompiler.a", + "-lfromlinker": from_linker / "libfromlinker.a", + } + + +def test_a_search_directory_under_the_sysroot_is_read_there(workspace, tmp_path): + """`-L=/dir` names `/dir`, not a directory called `=`.""" + sysroot = tmp_path / "sysroot" + synth.write(sysroot / "sub" / "libinroot.a", ONE_OBJECT) + synth.write(tmp_path / "compiler" / "libinroot.a", ONE_OBJECT) + compiler = synth.stub_compiler( + tmp_path / "bin", tmp_path / "compiler", sysroot, "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + (item,) = _inputs(workspace, "-L=/sub", "-linroot").values() + + assert item.path == sysroot / "sub" / "libinroot.a" + + +def test_a_script_in_the_sysroot_names_files_in_the_sysroot(workspace, tmp_path): + """libc.so in a sysroot names `/usr/lib/...`, which ld reads under it.""" + sysroot = tmp_path / "sysroot" + synth.write(sysroot / "lib" / "libroot_nonshared.a", ONE_OBJECT) + synth.write(sysroot / "lib" / "libprefixed.a", ONE_OBJECT) + synth.write( + sysroot / "lib" / "libroot.so", + "/* GNU ld script */\nGROUP ( /lib/libroot_nonshared.a =/lib/libprefixed.a )\n", + ) + compiler = synth.stub_compiler( + tmp_path / "bin", tmp_path / "compiler", sysroot, "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + synth.link_line(workspace.build, "ext", (f"-L{sysroot}/lib", "-lroot")) + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + inputs = graph.static_inputs(graph.by_name(EXTENSION)[0]) + + assert sorted(item.path for item in inputs) == [ + sysroot / "lib" / "libprefixed.a", + sysroot / "lib" / "libroot_nonshared.a", + ] + + +def test_cmake_sysroot_overrides_the_compilers_own(workspace, tmp_path): + """CMAKE_SYSROOT reaches the linker as --sysroot, for `-L=` and for ld's + `=` search directories alike.""" + compiler_root, cmake_root = tmp_path / "compiler-root", tmp_path / "cmake-root" + for root in (compiler_root, cmake_root): + synth.write(root / "sub" / "libfoo.a", ONE_OBJECT) + synth.write(root / "stub-only" / "lib" / "libfromld.a", ONE_OBJECT) + compiler = synth.stub_compiler( + tmp_path / "bin", tmp_path / "compiler", compiler_root, "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + inputs = _inputs(workspace, "-L=/sub", "-lfoo", "-lfromld", sysroot=cmake_root) + + assert {name: item.path for name, item in inputs.items()} == { + "-lfoo": cmake_root / "sub" / "libfoo.a", + "-lfromld": cmake_root / "stub-only" / "lib" / "libfromld.a", + } + + +@pytest.mark.parametrize( + ("fragments", "expected"), + [ + (("-Wl,-Bstatic", "-l:libgrp.so"), ["libz.a"]), + (("-Wl,-Bstatic", "{d}/libgrp.so"), ["libz.a"]), + (("-Wl,-Bstatic", "-l:libouter.so"), ["libz.a"]), + (("-lgrp",), []), + ], +) +def test_static_mode_applies_inside_a_linker_script( + workspace, tmp_path, fragments, expected +): + _library_dir(tmp_path, "z") + synth.write(tmp_path / "libgrp.so", "/* GNU ld script */\nGROUP ( -lz )\n") + synth.write(tmp_path / "libouter.so", "/* GNU ld script */\nINPUT ( libgrp.so )\n") + + inputs = _inputs( + workspace, f"-L{tmp_path}", *(item.format(d=tmp_path) for item in fragments) + ) + + assert [item.path.name for item in inputs.values()] == expected + + +def test_no_compiler_on_record_says_to_reconfigure(workspace): + cache = workspace.build / "CMakeCache.txt" + cache.write_text( + "".join( + line + for line in cache.read_text().splitlines(keepends=True) + if "_COMPILER:" not in line + ) + ) + + (item,) = _inputs(workspace, "-lanything").values() + + assert "reconfigure" in item.reason + + +def test_a_linker_script_is_followed_to_what_it_names(workspace, tmp_path): + """libc.so names libc_nonshared.a beside libc.so.6; glibc's libm.a is a + script naming libm-.a.""" + directory = tmp_path / "libs" + synth.write(directory / "libscripted.so.6", synth.elf_shared_object("x.so.6")) + synth.write(directory / "libscripted_nonshared.a", ONE_OBJECT) + synth.write(directory / "libinner.a", ONE_OBJECT) + synth.write( + directory / "libscripted.so", + "/* GNU ld script */\nOUTPUT_FORMAT(elf64-x86-64)\n" + f"GROUP ( {directory}/libscripted.so.6 {directory}/libscripted_nonshared.a" + f" AS_NEEDED ( {directory}/libscripted.so.6 ) )\nINPUT(-linner)\n", + ) + synth.write(directory / "libm-9.99.a", ONE_OBJECT) + synth.write(directory / "libmath.a", "/* GNU ld script */\nGROUP ( libm-9.99.a )\n") + # ld looks beside the script first, then where the link runs, then -L. + elsewhere = tmp_path / "elsewhere" + synth.write(directory / "libbeside.a", ONE_OBJECT) + synth.write(elsewhere / "libbeside.a", ONE_OBJECT) + synth.write(workspace.build / "libbeside.a", ONE_OBJECT) + synth.write(workspace.build / "libcwd.a", ONE_OBJECT) + synth.write( + elsewhere / "libnear.so", + "/* GNU ld script */\nGROUP ( libbeside.a libcwd.a )\n", + ) + + synth.link_line( + workspace.build, + "ext", + (f"-L{directory}", "-lscripted", "-l:libmath.a", f"{elsewhere}/libnear.so"), + ) + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + inputs = graph.static_inputs(graph.by_name(EXTENSION)[0]) + + assert sorted((item.named, item.path) for item in inputs) == sorted( + [ + (f"{elsewhere}/libnear.so", elsewhere / "libbeside.a"), + (f"{elsewhere}/libnear.so", (workspace.build / "libcwd.a").resolve()), + ("-l:libmath.a", directory / "libm-9.99.a"), + ("-lscripted", directory / "libinner.a"), + ("-lscripted", directory / "libscripted_nonshared.a"), + ] + ) + + +def test_a_script_naming_itself_says_so(workspace, tmp_path): + synth.write(tmp_path / "libself.so", "/* GNU ld script */\nGROUP ( libself.so )\n") + + inputs = _inputs(workspace, f"-L{tmp_path}", "-lself") + + assert [item.reason for item in inputs.values()] == [ + "linker scripts nest too deeply here" + ] + + +@pytest.mark.parametrize("kind", ["EXECUTABLE", "MODULE_LIBRARY"]) +def test_a_build_order_dependency_links_nothing_into_the_member(workspace, kind): + """Build-order dependencies contribute no linked bytes, including their transitive libraries.""" + synth.set_type(workspace.build, "unit_tests", kind) + synth.link_line(workspace.build, "gamma", ("-l:libviagamma.a",)) + synth.link_line( + workspace.build, "unit_tests", ("-l:libnothing.a",), dependencies=("gamma",) + ) + synth.link_line(workspace.build, "ext", (), dependencies=("unit_tests",)) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + + assert graph.static_inputs(graph.by_name(EXTENSION)[0]) == {} + assert { + item.named for item in graph.static_inputs(graph.by_name("unit_tests")[0]) + } == {"-l:libnothing.a", "-l:libviagamma.a"} + + +def test_a_fragment_that_does_not_parse_is_unresolved_not_dropped(workspace): + inputs = _inputs(workspace, '"/opt/lib/libunbalanced.a') + + (item,) = inputs.values() + assert item.kind == "unresolved" + assert "does not parse" in item.reason + + +def _link(workspace, *fragments) -> None: + synth.link_line(workspace.build, "ext", fragments) + + +def test_a_packaged_archive_is_inventoried_through_its_package( + workspace, tmp_path, monkeypatch +): + archive = tmp_path / "host" / "usr" / "lib" / "libsquash.a" + synth.write(archive, ONE_OBJECT) + copyright_file = tmp_path / "host" / "copyright" + synth.write(copyright_file, synth.spdx_text("Zlib")) + _packaged( + monkeypatch, + "libsquash-dev", + copyright_file, + supplier="Organization: Example Distro (devel@example.com)", + version="1.0-1", + apt_origin="Ubuntu", + apt_origin_from="installed version", + ) + _link(workspace, str(archive)) + + collected = synth.collect(workspace) + + row = _row(collected, "system:libsquash.a") + assert row["license_concluded"] == "Zlib" + assert row["how_it_reaches_the_wheel"] == ["prebuilt-library"] + assert row["purl"] == "pkg:deb/ubuntu/libsquash-dev@1.0-1" + assert row["source_info"].startswith(f"Statically linked from {archive}.") + evidence = collected["build-evidence"] + record = evidence["static_inputs"][str(archive)] + assert record["package"] == "libsquash-dev" + assert record["apt_origin"] == "Ubuntu" + assert record["apt_origin_from"] == "installed version" + assert record["linked_by"] == ["ext"] + linked = next( + item for item in evidence["link_artifacts"] if item["target"] == "ext" + ) + assert linked["static_inputs"] == [str(archive)] + + +# `apt-cache policy` as apt prints it. NVIDIA's CUDA repository is a flat one, +# `deb / /`, listed with an empty distribution. +UBUNTU_SOURCE = "http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages" +NVIDIA_SOURCE = ( + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64" + " Packages" +) +SECURITY_SOURCE = "http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages" +FLAT_SOURCE = "https://repo.example.com/apt ./ Packages" +# A repository with no Release file states no Origin. +BARE_SOURCE = "file:/srv/debs ./ Packages" +# libnvidia-container's Origin is a URL. +URL_ORIGIN = "https://nvidia.github.io/libnvidia-container/stable" +URL_SOURCE = f"{URL_ORIGIN}/deb/amd64 Packages" +# A mirror spelling Ubuntu's Origin in lower case. +MIRROR_SOURCE = "http://mirror.example.org/ubuntu jammy-updates/main amd64 Packages" +HOST_SOURCE = "https://packages.example.com/apt stable/main amd64 Packages" +APT_SOURCES = f"""Package files: + 100 /var/lib/dpkg/status + release a=now + 600 {NVIDIA_SOURCE} + release o=NVIDIA,l=NVIDIA CUDA,c= + origin developer.download.nvidia.com + 500 {UBUNTU_SOURCE} + release v=22.04,o=Ubuntu,a=jammy-updates,n=jammy,l=Ubuntu,c=main,b=amd64 + origin archive.ubuntu.com + 500 {FLAT_SOURCE} + release o=Example,c= + origin repo.example.com + 500 {SECURITY_SOURCE} + release v=22.04,o=Ubuntu,a=jammy-security,n=jammy,l=Ubuntu,c=main,b=amd64 + origin security.ubuntu.com + 500 {BARE_SOURCE} + 500 {URL_SOURCE} + release o={URL_ORIGIN},l=NVIDIA CORPORATION ,c= + origin nvidia.github.io + 500 {MIRROR_SOURCE} + release o=ubuntu,a=jammy-updates,c=main,b=amd64 + origin mirror.example.org + 500 {HOST_SOURCE} + release o=packages.example.com,a=stable,c=main,b=amd64 + origin packages.example.com +Pinned packages: +""" + + +def _version_table(*versions: tuple) -> str: + """`(version, installed, sources[, note])` per version, as apt lists them.""" + lines = ["libsquash-dev:", " Installed: 1.0-1", " Version table:"] + for version, installed, sources, *note in versions: + lines.append( + " ".join([f" {'***' if installed else ' '} {version} 500", *note]) + ) + lines += [f" 500 {source}" for source in sources] + return "\n".join(lines) + "\n" + + +STATUS = "/var/lib/dpkg/status" +LIBRARY = "/usr/lib/libsquash.a" + + +def _host_package( + monkeypatch, + policy, + owned=f"libsquash-dev:amd64: {LIBRARY}\n", + qualified="libsquash-dev:amd64", + sources=None, +): + """What `_identify_host_package` makes of dpkg answering `owned`, and what it + asked. dpkg-query -W and apt answer only for `qualified`.""" + answers = { + ("dpkg-query", "-S", LIBRARY): owned, + ("dpkg-query", "-W", "-f=${Version}", qualified): "1.0-1", + ("apt-cache", "policy"): APT_SOURCES if sources is None else sources, + ("apt-cache", "policy", qualified): policy, + } + asked = [] + + def ask(*command, env=None): + asked.append(command) + return answers.get(command, "") + + monkeypatch.setattr(discovery, "_ask", ask) + return discovery._identify_host_package(Path(LIBRARY), {}), asked + + +@pytest.mark.parametrize( + ("policy", "namespace", "decided"), + [ + pytest.param( + _version_table( + ("1.0-1", True, [UBUNTU_SOURCE, STATUS]), + ("0.9-1", False, [FLAT_SOURCE]), + ), + "ubuntu", + "installed version", + id="installed-version-decides-not-another", + ), + pytest.param( + _version_table(("1.0-1", True, [NVIDIA_SOURCE, STATUS])), + "nvidia", + "installed version", + id="nvidia", + ), + pytest.param( + _version_table(("1.0-1", True, [FLAT_SOURCE])), + "example", + "installed version", + id="flat", + ), + pytest.param( + _version_table(("1.0-1", True, [UBUNTU_SOURCE, FLAT_SOURCE, STATUS])), + None, + None, + id="disagreeing", + ), + pytest.param( + _version_table( + ("1.0-1", True, [BARE_SOURCE, STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE]), + ), + None, + None, + id="no-origin", + ), + # As for libc6 in an ubuntu:22.04 image. + pytest.param( + _version_table( + ("1.1-1", False, [UBUNTU_SOURCE, SECURITY_SOURCE]), + ("1.0-1", True, [STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE]), + ), + "ubuntu", + "other versions", + id="installed-version-unlisted-other-versions-decide", + ), + pytest.param( + _version_table( + ("1.1-1", False, [UBUNTU_SOURCE]), + ("1.0-1", True, [STATUS]), + ("0.9-1", False, [FLAT_SOURCE]), + ), + None, + None, + id="installed-version-unlisted-others-disagree", + ), + pytest.param( + _version_table(("1.0-1", True, [STATUS])), None, None, id="local-deb" + ), + pytest.param("", None, None, id="apt-knows-nothing"), + # A phased update carries a note after its priority, translated. + pytest.param( + _version_table( + ("1.0-1", True, [UBUNTU_SOURCE, STATUS], "(phased 20%)"), + ("0.9-1", False, [FLAT_SOURCE]), + ), + "ubuntu", + "installed version", + id="phased-installed", + ), + pytest.param( + _version_table( + ("1.0-1", True, [NVIDIA_SOURCE, STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE], "(phased 20%)"), + ), + "nvidia", + "installed version", + id="phased-after-installed", + ), + pytest.param( + _version_table( + ("1.0-1", True, [NVIDIA_SOURCE, STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE], "(gestaffelt 20%)"), + ), + "nvidia", + "installed version", + id="phased-german", + ), + pytest.param( + _version_table(("1.0-1", True, [UBUNTU_SOURCE, MIRROR_SOURCE, STATUS])), + "ubuntu", + "installed version", + id="origins-agree-ignoring-case", + ), + pytest.param( + _version_table(("1.0-1", True, [URL_SOURCE, STATUS])), + None, + "installed version", + id="url-origin-is-no-namespace", + ), + pytest.param( + _version_table(("1.0-1", True, [HOST_SOURCE, STATUS])), + None, + "installed version", + id="host-origin-is-no-namespace", + ), + ], +) +def test_a_packages_namespace_is_the_archive_that_publishes_it( + monkeypatch, policy, namespace, decided +): + origin, _ = _host_package(monkeypatch, policy) + + purl = discovery.system_component("libsquash.a", origin).purl + + # pkg:deb requires its vendor namespace; without one, nothing is claimed. + expected = f"pkg:deb/{namespace}/" if namespace else "pkg:generic/" + assert purl == f"{expected}libsquash-dev@1.0-1" + parsed = PackageURL.from_string(purl) + assert (parsed.type, parsed.namespace, parsed.name, parsed.version) == ( + "deb" if namespace else "generic", + namespace, + "libsquash-dev", + "1.0-1", + ) + assert origin["apt_origin_from"] == decided + + +@pytest.mark.parametrize( + ("owned", "qualified"), + [ + pytest.param( + f"libsquash-dev, libsquash-extra:amd64: {LIBRARY}\n", + "libsquash-dev", + id="several-owners-the-first-as-printed", + ), + pytest.param( + f"libsquash-dev:i386: {LIBRARY}\n", + "libsquash-dev:i386", + id="foreign-architecture", + ), + ], +) +def test_dpkg_and_apt_are_asked_about_the_owner_dpkg_names( + monkeypatch, owned, qualified +): + origin, asked = _host_package( + monkeypatch, + _version_table(("1.0-1", True, [UBUNTU_SOURCE])), + owned=owned, + qualified=qualified, + ) + + assert origin["package"] == "libsquash-dev" + assert origin["version"] == "1.0-1" + assert origin["apt_origin"] == "Ubuntu" + assert { + ("dpkg-query", "-W", "-f=${Version}", qualified), + ("dpkg-query", "-W", "-f=${Maintainer}", qualified), + ("apt-cache", "policy", qualified), + } <= set(asked) + + +@pytest.mark.parametrize( + "owned", + [ + pytest.param( + f"diversion by squash-gl from: {LIBRARY}\n" + f"diversion by squash-gl to: {LIBRARY}.distrib\n", + id="diverted-unowned", + ), + pytest.param( + f"diversion by squash-gl from: {LIBRARY}\n" + f"diversion by squash-gl to: {LIBRARY}.distrib\n" + f"libsquash-dev:amd64: {LIBRARY}\n", + id="diverted-the-file-may-not-be-the-owners", + ), + pytest.param( + f"diversion by squash-gl to: {LIBRARY}\n", + id="only-a-diversion-names-the-path", + ), + ], +) +def test_a_diversion_line_names_no_owner(monkeypatch, owned): + origin, _ = _host_package(monkeypatch, "", owned=owned) + + assert origin.get("package") is None + purl = discovery.system_component("libsquash.a", origin).purl + assert purl == "pkg:generic/libsquash.a" + + +def test_apt_sources_are_read_whatever_the_language(monkeypatch): + """Headings are translated; a pinned package may be named `release`.""" + sources = ( + APT_SOURCES.replace("Package files:", "Fichiers du paquet :").replace( + "Pinned packages:", "Paquets épinglés :" + ) + + " release -> 2.0-1 avec la priorité 990\n" + + " 123 -> 1.0 avec la priorité 1001\n" + ) + _host_package(monkeypatch, "", sources=sources) + + releases = discovery._apt_releases() + + assert set(releases) == { + STATUS, + NVIDIA_SOURCE, + UBUNTU_SOURCE, + FLAT_SOURCE, + SECURITY_SOURCE, + BARE_SOURCE, + URL_SOURCE, + MIRROR_SOURCE, + HOST_SOURCE, + } + assert releases[BARE_SOURCE] == {} + assert releases[NVIDIA_SOURCE]["o"] == "NVIDIA" + assert releases[URL_SOURCE]["o"] == URL_ORIGIN + + +# A machine-readable copyright file whose licence paragraph quotes a notice. +DEP5 = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" + "Upstream-Name: squash\n\n" + "Files: *\nCopyright: 2001-2020 Squash Authors\n" + " 2019 Example Corp\nLicense: Squash\n\n" + "License: Squash\n Permission is granted to use this software freely.\n" + " Copyright (c) 1995 Someone Quoted In The Text\n" +) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # Not machine-readable: the notices it quotes belong to what it bundles. + (EULA, "NOASSERTION"), + (DEP5, "2001-2020 Squash Authors\n2019 Example Corp"), + # Free-form, with a `Copyright:` heading at the start of a line. + ( + "This package was debianized by someone.\n\n" + "Copyright: 2001 Quoted Upstream\n", + "NOASSERTION", + ), + ], +) +def test_a_system_component_names_only_the_holders_its_package_declares( + workspace, tmp_path, monkeypatch, text, expected +): + archive = tmp_path / "host" / "libsquash.a" + synth.write(archive, ONE_OBJECT) + copyright_file = tmp_path / "host" / "copyright" + synth.write(copyright_file, text) + _packaged(monkeypatch, "libsquash-dev", copyright_file) + _link(workspace, str(archive)) + + collected = synth.collect(workspace) + + row = _row(collected, "system:libsquash.a") + assert _package(row, collected["spdx"])["copyrightText"] == expected + + +def test_a_cuda_archive_is_a_component_under_the_toolkits_terms( + workspace, tmp_path, monkeypatch +): + toolkit = tmp_path / "cuda-99.9" + archive = toolkit / "targets" / "x86_64-linux" / "lib" / "libcudart_static.a" + synth.write(archive, ONE_OBJECT) + synth.write(toolkit / "EULA.txt", EULA) + _no_package(monkeypatch) + monkeypatch.setattr(discovery, "_toolkit_roots", lambda: [toolkit.resolve()]) + _link(workspace, str(archive)) + + collected = synth.collect(workspace) + + key = "system:libcudart_static.a" + row = _row(collected, key) + assert row["how_it_reaches_the_wheel"] == ["prebuilt-library"] + assert [item["path"] for item in row["evidence"]] == [ + str((toolkit / "EULA.txt").resolve()) + ] + assert row["license_concluded"].startswith("LicenseRef-") + assert _package(row, collected["spdx"])["copyrightText"] == "NOASSERTION" + + member = next( + item + for item in collected["build-evidence"]["attributions"] + if item["path"].endswith(EXTENSION) + ) + assert member["components"][key] == ["prebuilt-library"] + assert { + "spdxElementId": document_module.WHEEL_PACKAGE_ID, + "relatedSpdxElement": document_module.component_package_id(key), + "relationshipType": "STATIC_LINK", + } in collected["spdx"]["relationships"] + + +def test_two_archives_sharing_a_name_are_two_components( + workspace, tmp_path, monkeypatch +): + first = tmp_path / "one" / "libsame.a" + second = tmp_path / "two" / "libsame.a" + for archive in (first, second): + synth.write(archive, ONE_OBJECT) + synth.write(archive.parent / "copyright", synth.spdx_text("Zlib")) + _packaged( + monkeypatch, + lambda library: f"libsame-{library.parent.name}", + lambda library: library.parent / "copyright", + ) + _link(workspace, str(first), str(second)) + + collected = synth.collect(workspace) + + records = collected["build-evidence"]["static_inputs"] + keys = {records[str(first)]["component"], records[str(second)]["component"]} + assert len(keys) == 2 + assert keys <= {item["component"] for item in collected["licenses"]["components"]} + + +def test_an_archive_with_no_members_is_recorded_not_inventoried( + workspace, tmp_path, monkeypatch +): + archive = tmp_path / "host" / "librt.a" + synth.write(archive, synth.ar_archive({})) + _no_package(monkeypatch) + _link(workspace, str(archive)) + + collected = synth.collect(workspace) + + record = collected["build-evidence"]["static_inputs"][str(archive)] + assert record["members"] == 0 + assert record["component"] is None + assert "no members" in record["outcome"] + assert not any( + item["component"].endswith("librt.a") + for item in collected["licenses"]["components"] + ) + + +def _tracked_archive(workspace, monkeypatch, holder=None): + """A tracked archive linked into ext, held by `holder` through REUSE.toml.""" + relative = "prebuilt/libtracked.a" + synth.write(workspace.root / relative, ONE_OBJECT) + if holder: + synth.write( + workspace.root / "REUSE.toml", + synth.reuse_toml(synth.annotation(relative, holder)), + ) + synth.track(workspace.root) + _no_package(monkeypatch) + _link(workspace, str(workspace.root / relative)) + return (workspace.root / relative).resolve() + + +def test_a_tracked_archive_is_credited_to_the_holder_reuse_names( + workspace, monkeypatch +): + _tracked_archive(workspace, monkeypatch, "Upstream Archive Co") + + collected = synth.collect(workspace) + + row = _row(collected, "vendored:upstream-archive-co") + assert "prebuilt-library" in row["how_it_reaches_the_wheel"] + + +def test_a_tracked_archive_naming_no_other_holder_is_the_projects( + workspace, monkeypatch +): + archive = _tracked_archive(workspace, monkeypatch) + + collected = synth.collect(workspace) + + record = collected["build-evidence"]["static_inputs"][str(archive)] + assert record["component"] is None + assert "held by this project" in record["outcome"] + + +@pytest.mark.parametrize("where", ["host", "build", "unresolved"]) +def test_an_input_nothing_explains_stops_the_build( + workspace, tmp_path, monkeypatch, where +): + _no_package(monkeypatch) + monkeypatch.setattr(discovery, "_toolkit_roots", list) + if where == "unresolved": + fragment = "-l:libmystery.a" + else: + root = workspace.build / "prebuilt" if where == "build" else tmp_path / "opt" + synth.write(root / "libmystery.a", ONE_OBJECT) + fragment = str((root / "libmystery.a").resolve()) + _link(workspace, fragment) + + with pytest.raises( + build_module.BuildError, match=r"libmystery\.a \(linked by ext\)" + ): + synth.collect(workspace) + + +FMT_LICENCE_HEAD = ( + "Copyright (c) 2012 - present, Victor Zverovich and {fmt} contributors\n\n" +) + + +def _fmt_port(installed) -> None: + synth.vcpkg_port( + installed, + "fmt", + "11.0.2#1", + { + "lib/libfmt.a": ONE_OBJECT, + "debug/lib/libfmtd.a": ONE_OBJECT, + "include/fmt/core.h": b"#pragma once\n", + }, + FMT_LICENCE_HEAD + synth.spdx_text("MIT"), + "fmtlib/fmt", + ) + + +@pytest.mark.parametrize("outside_build", [False, True]) +def test_a_vcpkg_archive_is_credited_to_the_port_that_installed_it( + workspace, tmp_path, monkeypatch, outside_build +): + """spdlog built against vcpkg's fmt puts libfmt.a on every consumer's line.""" + installed = ( + tmp_path / "elsewhere" / "vcpkg_installed" + if outside_build + else workspace.build / "vcpkg_installed" + ) + if outside_build: + cache = workspace.build / "CMakeCache.txt" + cache.write_text(cache.read_text() + f"VCPKG_INSTALLED_DIR:PATH={installed}\n") + _fmt_port(installed) + _no_package(monkeypatch) + _link(workspace, str(installed / "x64-linux" / "lib" / "libfmt.a")) + + collected = synth.collect(workspace) + + row = _row(collected, "vcpkg:fmt") + assert row["kind"] == "vcpkg-port" + assert row["version"] == "11.0.2" + assert row["license_concluded"] == "MIT" + assert row["purl"] == "pkg:github/fmtlib/fmt@11.0.2" + assert row["supplier"] == "Organization: fmtlib" + assert "prebuilt-library" in row["how_it_reaches_the_wheel"] + assert "vcpkg port fmt 11.0.2#1 for x64-linux" in row["source_info"] + package = _package(row, collected["spdx"]) + assert "Victor Zverovich" in package["copyrightText"] + assert { + "spdxElementId": document_module.WHEEL_PACKAGE_ID, + "relatedSpdxElement": package["SPDXID"], + "relationshipType": "STATIC_LINK", + } in collected["spdx"]["relationships"] + + +@pytest.mark.parametrize("target", ["x64-linux", "arm64-linux"]) +def test_a_port_installed_for_two_triplets_owns_both_installs(workspace, target): + """A port the build needs as a host tool and as a library has a list per + triplet. Both installs are its files; the component describes the target's.""" + installed = workspace.build / "vcpkg_installed" + triplets = ("arm64-linux", "x64-linux") + for triplet in triplets: + synth.vcpkg_port( + installed, + "fmt", + "11.0.2#1", + {"lib/libfmt.a": ONE_OBJECT}, + synth.spdx_text("MIT"), + "fmtlib/fmt", + triplet=triplet, + ) + + components, owned = discovery.discover_vcpkg(installed, target) + + assert {installed / triplet / "lib" / "libfmt.a" for triplet in triplets} <= set( + owned["vcpkg:fmt"] + ) + assert components["vcpkg:fmt"].source_info.startswith( + f"Built and installed by vcpkg port fmt 11.0.2#1 for {target}." + ) + + +def test_the_cache_names_the_triplet_a_port_is_described_from(workspace): + installed = workspace.build / "vcpkg_installed" + for triplet in ("arm64-linux", "x64-linux"): + synth.vcpkg_port( + installed, + "fmt", + "11.0.2#1", + {"lib/libfmt.a": ONE_OBJECT}, + synth.spdx_text("MIT"), + "fmtlib/fmt", + triplet=triplet, + ) + cache = workspace.build / "CMakeCache.txt" + cache.write_text(cache.read_text() + "VCPKG_TARGET_TRIPLET:STRING=x64-linux\n") + + space = evidence_module.discover(workspace.root, workspace.build) + + assert "for x64-linux." in space.components["vcpkg:fmt"].source_info + + +def test_a_file_no_port_lists_is_still_unexplained(workspace, monkeypatch): + installed = workspace.build / "vcpkg_installed" + _fmt_port(installed) + stray = installed / "x64-linux" / "lib" / "libstray.a" + synth.write(stray, ONE_OBJECT) + _no_package(monkeypatch) + _link(workspace, str(stray)) + + with pytest.raises(build_module.BuildError, match=r"libstray\.a \(linked by ext\)"): + synth.collect(workspace) + + +def test_an_unlinked_port_is_recorded_as_a_vcpkg_port_not_in_the_wheel(workspace): + synth.vcpkg_port( + workspace.build / "vcpkg_installed", + "zlib", + "1.3.1", + {"lib/libz.a": ONE_OBJECT}, + synth.spdx_text("Zlib"), + "madler/zlib", + ) + + collected = synth.collect(workspace) + + zlib = collected["build-evidence"]["components"]["vcpkg:zlib"] + assert (zlib["kind"], zlib["in_this_wheel"]) == ("vcpkg-port", False) + + +def _eigen_port(installed) -> None: + synth.vcpkg_port( + installed, + "eigen3", + "3.4.0#5", + {"include/eigen3/Eigen/Core": b"#pragma once\n"}, + synth.spdx_text("MIT"), + ("libeigen/eigen", "libeigen/eigen-patches"), + ) + + +def test_an_include_directory_one_port_owns_credits_that_port(workspace): + """The fixture's extension also searches the shared include root.""" + installed = workspace.build / "vcpkg_installed" + _fmt_port(installed) + _eigen_port(installed) + synth.add_include( + workspace.build, "ext", installed / "x64-linux" / "include" / "eigen3" + ) + + space = evidence_module.discover(workspace.root, workspace.build) + graph = space.graph + contributions = graph.contributions(graph.by_name(EXTENSION)[0], space.ownership) + + assert contributions["vcpkg:eigen3"] == {"header-include"} + assert "vcpkg:fmt" not in contributions + + +def test_a_port_with_several_sources_names_no_upstream(workspace): + _eigen_port(workspace.build / "vcpkg_installed") + + collected = synth.collect(workspace) + + row = _row(collected, "vcpkg:eigen3") + assert row["version"] == "3.4.0" + assert row["purl"] == "pkg:generic/eigen3@3.4.0" + assert row["supplier"] == "NOASSERTION" + assert "vcpkg port eigen3 3.4.0#5" in row["source_info"] + + +def test_a_symlinked_script_is_read_beside_the_link(workspace, tmp_path): + """ld resolves a script's relative names beside the name it was given.""" + target, links = tmp_path / "target", tmp_path / "links" + synth.write(target / "libreal.so", "/* GNU ld script */\nGROUP ( libmine.a )\n") + synth.write(target / "libmine.a", ONE_OBJECT) + synth.write(links / "libmine.a", ONE_OBJECT) + (links / "libsym.so").symlink_to(target / "libreal.so") + + (item,) = _inputs(workspace, f"-L{links}", "-lsym").values() + + assert item.path == links / "libmine.a" + + +def test_ownership_sees_a_file_registered_after_a_lookup(tmp_path): + ownership = discovery.Ownership() + include = tmp_path / "include" + ownership.register_file(include / "a" / "a.h", "port-a", by_directory=False) + assert ownership.owners_under(include) == {"port-a"} + + ownership.register_file(include / "b" / "b.h", "port-b", by_directory=False) + ownership.register_file(include / "c.h", "fetched") + + assert ownership.owners_under(include) == {"fetched"} + assert ownership.owners_under(include / "b") == {"port-b"} + + +def test_an_empty_archive_credits_no_one_however_it_is_held(workspace): + installed = workspace.build / "vcpkg_installed" + synth.vcpkg_port( + installed, + "stubs", + "1.0", + {"lib/libstub.a": synth.ar_archive({})}, + synth.spdx_text("MIT"), + "example/stubs", + ) + _link(workspace, str(installed / "x64-linux" / "lib" / "libstub.a")) + + space = evidence_module.discover(workspace.root, workspace.build) + contributions = space.graph.contributions( + space.graph.by_name(EXTENSION)[0], space.ownership + ) + + assert "vcpkg:stubs" not in contributions diff --git a/scripts/sbom/tests/test_wheel_sbom.py b/scripts/sbom/tests/test_wheel_sbom.py new file mode 100644 index 0000000000..a52658fd0f --- /dev/null +++ b/scripts/sbom/tests/test_wheel_sbom.py @@ -0,0 +1,726 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End-to-end wheel evidence and verification tests.""" + +from __future__ import annotations + +import json +import re +import zipfile +from pathlib import Path + +import pytest +import synth +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import document as document_module +from isaaccapture_sbom import inventory as inventory_module +from isaaccapture_sbom import licensing +from isaaccapture_sbom import report as report_module +from isaaccapture_sbom import validate as validate_module +from isaaccapture_sbom import wheelfile + +DIST_INFO = synth.DIST_INFO +WHEEL_NAME = synth.WHEEL_NAME + + +def _names(wheel): + with zipfile.ZipFile(wheel) as archive: + return set(archive.namelist()) + + +def _embedded(wheel): + with zipfile.ZipFile(wheel) as archive: + return json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + + +def _wheel_package(spdx: dict) -> dict: + return next( + item for item in spdx["packages"] if item["SPDXID"] == "SPDXRef-Package-wheel" + ) + + +def _release_failures(built) -> list[str]: + return validate_module.check( + built["wheel"], + build_evidence=built["evidence_path"], + manifest=built["manifest_path"], + ) + + +def test_wheel_verifies_against_its_own_sbom(built): + assert validate_module.check(built["wheel"]) == [] + + +def test_release_validation_passes(built): + assert _release_failures(built) == [] + + +def test_sbom_lands_in_dist_info_sboms_beside_any_other_tools(built): + sboms = {name for name in _names(built["wheel"]) if f"{DIST_INFO}/sboms/" in name} + + # PEP 770 shares the directory: auditwheel wrote its document before us. + assert sboms == { + f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json", + f"{DIST_INFO}/sboms/{synth.AUDITWHEEL_SBOM}", + } + + +def test_every_wheel_file_is_covered(built): + spdx = _embedded(built["wheel"]) + documented = {item["fileName"].removeprefix("./") for item in spdx["files"]} + excluded = { + item.removeprefix("./") + for package in spdx["packages"] + for item in package.get("packageVerificationCode", {}).get( + "packageVerificationCodeExcludedFiles", [] + ) + } + + assert _names(built["wheel"]) - documented == excluded + + +def test_every_member_records_where_it_came_from(built): + evidence = json.loads(built["evidence_path"].read_text(encoding="utf-8")) + origins = {item["path"]: item["origin"] for item in evidence["attributions"]} + + assert origins["isaaccapture/__init__.py"] == "repo-source" + assert origins["isaaccapture/_generated.py"] == "generated" + assert origins["isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so"] == "built" + assert origins["isaaccapture/viz/libgamma_renamed.so"] == "built" + assert origins["isaaccapture/viz/GAMMA_LICENSE"] == "copied" + assert origins["isaaccapture/sdk/native/libdemo.so"] == "archive-copy" + assert origins["vendorpy/mod.py"] == "copied" + + +def test_post_processed_files_are_recorded_as_derived_not_as_copies(built): + evidence = json.loads(built["evidence_path"].read_text(encoding="utf-8")) + origins = {item["path"]: item for item in evidence["attributions"]} + + patched = origins["isaaccapture/sdk/native/libextra.so"] + assert patched["origin"] == "archive-derived" + assert "modified copy of" in patched["detail"] + + rewritten = origins["vendorpy/assets/hand.xml"] + assert rewritten["origin"] == "derived" + + +def test_components_are_discovered_not_declared(built): + spdx = _embedded(built["wheel"]) + names = {package["name"] for package in spdx["packages"]} + + assert {"alpha", "beta", "gamma", "delta", "vendorpy"} <= names + assert "demosdk-1.2.0-linux-amd64" in names + assert "omega" not in names, "a test-only dependency is not in the wheel" + + +def test_a_dependency_reaching_no_wheel_file_is_recorded_by_the_build(built): + components = built["build-evidence"]["components"] + + assert components["omega"]["in_this_wheel"] is False, "a test-only dependency" + # Fetched reference data belongs in build evidence, even though no wheel ships it. + assert components["license-list-data"]["in_this_wheel"] is False + assert "omega" not in json.dumps(built["licenses"]) + assert "omega" not in built["md"] + + +def test_where_the_license_list_was_read_is_left_to_the_build_evidence(built): + evidence = built["build-evidence"]["license_data"] + report = built["licenses"]["license_data"] + + assert "source" in evidence and "source" not in report + assert report["commit"] == evidence["commit"] + + +def test_licenses_are_packaged_and_declared(built): + names = _names(built["wheel"]) + third_party = { + name for name in names if f"{DIST_INFO}/licenses/third-party/" in name + } + + assert f"{DIST_INFO}/licenses/THIRD-PARTY-NOTICES.md" in names + assert any("/alpha/" in name for name in third_party) + assert any("demosdk" in name for name in third_party) + + raw, metadata = wheelfile.read_metadata(built["wheel"], DIST_INFO) + declared = set(metadata.get("license_files") or []) + # PEP 639 paths resolve relative to .dist-info/licenses/. + prefix = f"{DIST_INFO}/licenses/" + assert {name.removeprefix(prefix) for name in third_party} <= declared + for value in declared: + assert f"{prefix}{value}" in names, ( + f"License-File {value!r} resolves to nothing" + ) + assert "LICENSE.md" in declared, ( + "the project's own license must survive the rewrite" + ) + + # Avoid RFC 822 folding: simpler metadata readers may not unfold paths. + assert not [ + line for line in raw.decode().splitlines() if line.startswith((" ", "\t")) + ] + + +def test_an_unidentified_grant_travels_verbatim_in_the_document(built): + spdx = _embedded(built["wheel"]) + extracted = {item["licenseId"]: item for item in spdx["hasExtractedLicensingInfos"]} + + # Use the component path; archive basenames can collide. + ref = next(item for item in extracted if "demosdk" in item) + assert "deps" in ref and "vendor" in ref, ref + assert "DEMO CORP" in extracted[ref]["extractedText"] + + +@pytest.fixture +def eula_beside_mit(workspace): + """alpha ships its MIT LICENSE and an EULA that matches nothing.""" + eula = synth.PROPRIETARY_LICENSE.replace("Demo SDK", "Alpha runtime") + (workspace.build / "_deps" / "alpha-src" / "LICENSE-EULA.txt").write_text( + eula, encoding="utf-8" + ) + synth.track(workspace.root) + return {**synth.collect_and_bind(workspace), "eula": eula} + + +def test_an_unmatched_grant_beside_a_matched_one_is_carried_and_reported( + eula_beside_mit, +): + built = eula_beside_mit + package = next( + item for item in built["spdx"]["packages"] if item["name"] == "alpha" + ) + assert package["licenseConcluded"] == "LicenseRef-alpha AND MIT" + extracted = { + item["licenseId"]: item for item in built["spdx"]["hasExtractedLicensingInfos"] + } + assert extracted["LicenseRef-alpha"]["extractedText"] == built["eula"] + assert extracted["LicenseRef-alpha"]["comment"] == ( + "Matched no text in the SPDX reference corpus; reproduced here verbatim " + "from alpha-src/LICENSE-EULA.txt." + ) + unidentified = { + item["component"]: item + for item in built["licenses"]["components_with_unidentified_license"] + } + assert unidentified["alpha"] == { + "component": "alpha", + "reason": "1 of its 2 shipped grants is carried verbatim rather than " + "named: matched no text in the SPDX reference corpus", + "evidence": ["alpha-src/LICENSE-EULA.txt"], + } + assert _release_failures(built) == [] + + +def test_a_component_shipping_only_a_reuse_pool_is_a_reported_gap(workspace): + """A REUSE pool passes the gate without concluding a license; report that gap.""" + source = workspace.build / "_deps" / "alpha-src" + (source / "LICENSES").mkdir() + (source / "LICENSE").rename(source / "LICENSES" / "MIT.txt") + synth.track(workspace.root) + built = synth.collect_and_bind(workspace) + + report = built["licenses"] + alpha = next(item for item in report["components"] if item["component"] == "alpha") + assert alpha["license_concluded"] == "NOASSERTION" + listing = { + key: [item for item in report[key] if item["component"] == "alpha"] + for key in report_module.GAP_LISTS + } + assert [key for key, found in listing.items() if found] == [ + "components_with_unresolved_license" + ] + [entry] = listing["components_with_unresolved_license"] + assert entry["evidence"] == ["alpha-src/LICENSES/MIT.txt"] + assert entry["reason"].startswith("only REUSE pool texts ship (MIT)") + [record] = built["manifest"]["wheels"] + assert record["components_with_unresolved_license"] == [entry] + assert _wheel_package(built["spdx"])["licenseConcluded"] == "NOASSERTION" + assert _release_failures(built) == [] + + +def _report_of(**gaps) -> dict: + """A report payload with one identified component and the given gap lists.""" + row = { + "name": "alpha", + "kind": "source-tree", + "version": "1", + "license_concluded": "MIT", + "how_it_reaches_the_wheel": [], + "evidence": [], + } + return { + "wheel": WHEEL_NAME, + "generated_from_commit": None, + "spdx_license_list_version": None, + "components": [row], + "externally_supplied_runtime": [], + "consumer_requirements": [], + **{key: gaps.get(key, []) for key in report_module.GAP_LISTS}, + } + + +def test_an_identified_set_reports_no_gaps(): + rendered = report_module.report_markdown(_report_of()) + + assert rendered.endswith( + "## License evidence gaps\n\n" + "None; every redistributed component ships an identified license.\n" + ) + + +@pytest.mark.parametrize("key", list(report_module.GAP_LISTS)) +def test_any_gap_list_withholds_the_all_identified_summary(key): + gap = {"component": "alpha", "reason": "why"} + rendered = report_module.report_markdown(_report_of(**{key: [gap]})) + + assert f"{report_module.GAP_LISTS[key]}\n\n- `alpha` — why\n" in rendered + assert "None; every redistributed component" not in rendered + + +def _rewrite_alpha(built, expression: str) -> list[str]: + """Rewrite alpha's expressions in the embedded document; what check says.""" + member = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + payload = synth.members(built["wheel"]) + spdx = json.loads(payload[member]) + for package in spdx["packages"]: + if package["name"] == "alpha": + package["licenseConcluded"] = package["licenseDeclared"] = expression + payload[member] = json.dumps(spdx).encode() + synth.write_wheel(built["wheel"], payload) + return validate_module.check(built["wheel"], build_evidence=built["evidence_path"]) + + +def _disagreement(expression: str) -> list[str]: + recorded = "LicenseRef-alpha AND MIT" + return [ + f"alpha: the document says {field} {expression!r}, the build recorded " + f"{recorded!r}" + for field in ("licenseConcluded", "licenseDeclared") + ] + + +@pytest.mark.parametrize( + "rewritten", ["LicenseRef-alpha", "Apache-2.0 AND LicenseRef-alpha"] +) +def test_the_named_half_of_a_mixed_conclusion_is_checked(eula_beside_mit, rewritten): + assert _rewrite_alpha(eula_beside_mit, rewritten) == _disagreement(rewritten) + + +def test_another_component_s_licence_ref_is_not_accepted(eula_beside_mit): + """Swapped, alpha's EULA would be published as the SDK's terms; every + extracted text would still belong to some component.""" + other = next( + item["licenseId"] + for item in eula_beside_mit["spdx"]["hasExtractedLicensingInfos"] + if "demosdk" in item["licenseId"] + ) + rewritten = f"{other} AND MIT" + + assert _rewrite_alpha(eula_beside_mit, rewritten) == _disagreement(rewritten) + + +def test_reuse_tags_on_shipped_files_are_reported_per_file(built): + spdx = _embedded(built["wheel"]) + tagged = { + item["fileName"].removeprefix("./"): item.get("licenseConcluded") + for item in spdx["files"] + } + + assert tagged["vendorpy/mod.py"] == "Apache-2.0" + assert tagged["isaaccapture/__init__.py"] == "Apache-2.0" + + +def test_record_is_consistent_after_the_rewrite(built): + record = wheelfile.read_record(built["wheel"], DIST_INFO) + with zipfile.ZipFile(built["wheel"]) as archive: + members = set(archive.namelist()) + for name, (digest, size) in record.items(): + if name == f"{DIST_INFO}/RECORD": + continue + data = archive.read(name) + assert digest == wheelfile.record_hash(data), name + assert size == str(len(data)), name + assert set(record) == members + + +def test_manifest_binds_the_final_wheel_digest(built): + entry = built["manifest"]["wheels"][0] + + assert entry["filename"] == WHEEL_NAME + assert entry["sha256"] == build_module.sha256_file(built["wheel"]) + assert entry["sbom_in_wheel"] == f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + + +def test_published_copy_matches_the_embedded_one(built): + with zipfile.ZipFile(built["wheel"]) as archive: + embedded = archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json") + + assert built["spdx_path"].read_bytes() == embedded + + +def test_external_runtime_dependencies_are_not_claimed_as_contents(built): + spdx = _embedded(built["wheel"]) + external = { + package["name"] + for package in spdx["packages"] + if package["SPDXID"].startswith("SPDXRef-Package-external-") + } + + assert {"libc.so.6", "libstdc++.so.6"} <= external + assert "libdemo.so" not in external, "a library inside the wheel is not external" + + +def test_consumer_requirements_come_from_wheel_metadata(built): + spdx = _embedded(built["wheel"]) + comments = " ".join( + package.get("comment", "") + for package in spdx["packages"] + if package["SPDXID"].startswith("SPDXRef-Package-pypi-") + ) + + assert "numpy>=1.23.0" in comments + assert "pyyaml>=6.0.3" in comments + + +def test_build_is_reproducible_under_source_date_epoch(tmp_path, monkeypatch): + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + + digests = [] + for run in ("a", "b"): + space = synth.create(tmp_path / run) + synth.write_wheel(space.wheel, synth.wheel_payload(space)) + build_module.build(space.root, space.build, space.wheel, space.root / "sbom") + digests.append(build_module.sha256_file(space.wheel)) + + assert digests[0] == digests[1] + + +def test_source_date_epoch_normalizes_timestamps_the_input_wheel_carried( + workspace, monkeypatch +): + """Normalize copied members too; otherwise reproducibility depends on the input wheel + already having reproducible timestamps. + """ + import zipfile as zf + + stamped = synth.wheel_payload(workspace) + with zf.ZipFile(workspace.wheel, "w", zf.ZIP_DEFLATED) as archive: + for name in sorted(stamped): + archive.writestr( + zf.ZipInfo(name, date_time=(2021, 6, 5, 4, 3, 2)), stamped[name] + ) + archive.writestr(zf.ZipInfo("x", date_time=(2021, 6, 5, 4, 3, 2)), b"") + synth.write_wheel(workspace.wheel, stamped) + + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + + with zf.ZipFile(workspace.wheel) as archive: + stamps = {info.date_time for info in archive.infolist()} + assert len(stamps) == 1, f"members carry differing timestamps: {stamps}" + + +@pytest.fixture +def shared_text(workspace, license_data): + """alpha and beta ship the demo SDK's proprietary terms byte for byte.""" + + licensing.load_corpus(license_data) + for key in ("alpha", "beta"): + root = workspace.build / "_deps" / f"{key}-src" + root.mkdir(parents=True, exist_ok=True) + (root / "LICENSE").write_text(synth.PROPRIETARY_LICENSE, encoding="utf-8") + synth.track(workspace.root) + + payload = synth.wheel_payload(workspace) + for key in ("alpha", "beta"): + payload[f"isaaccapture/{key}_notice.txt"] = synth.PROPRIETARY_LICENSE.encode() + synth.write_wheel(workspace.wheel, payload) + return synth.collect_and_bind(workspace) + + +def test_components_sharing_a_licence_text_share_one_entry(shared_text): + """Identical terms shared by several components need only one extracted text entry.""" + spdx = _embedded(shared_text["wheel"]) + bodies = [item["extractedText"] for item in spdx["hasExtractedLicensingInfos"]] + + assert len(bodies) == len(set(bodies)), "a text is carried more than once" + used = { + ref + for package in spdx["packages"] + for field in ("licenseConcluded", "licenseDeclared") + for ref in re.findall(r"LicenseRef-[\w.-]+", package.get(field, "")) + } + defined = {item["licenseId"] for item in spdx["hasExtractedLicensingInfos"]} + assert used <= defined, f"undefined after sharing: {sorted(used - defined)}" + + +def test_a_shared_id_is_held_to_the_text_it_names(shared_text): + """gamma's Zlib text is some component's terms, so only the id catches it.""" + member = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + payload = synth.members(shared_text["wheel"]) + spdx = json.loads(payload[member]) + entry = next( + item + for item in spdx["hasExtractedLicensingInfos"] + if item["licenseId"].startswith("LicenseRef-shared-text-") + ) + entry["extractedText"] = synth.spdx_text("Zlib") + payload[member] = json.dumps(spdx).encode() + synth.write_wheel(shared_text["wheel"], payload) + + failures = validate_module.check( + shared_text["wheel"], build_evidence=shared_text["evidence_path"] + ) + + assert failures == [ + f"{entry['licenseId']}: the text it carries is not the one its id names" + ] + + +def test_a_requirement_this_run_also_built_is_not_an_unknown( + workspace, license_data, tmp_path +): + """Use sibling metadata for dependencies built in the same run. + + Digest-bound references cannot represent cycles, and a universal wheel must not + pin a platform-specific sibling from one matrix entry. + """ + from isaaccapture_sbom import cli + + synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) + companion = workspace.wheel.parent / "isaacteleop-0.4.0-py3-none-any.whl" + synth.write_wheel( + companion, + { + "isaacteleop-0.4.0.dist-info/METADATA": "\n".join( + [ + "Metadata-Version: 2.4", + "Name: isaacteleop", + "Version: 0.4.0", + "Author: Example Org", + "License-Expression: Apache-2.0", + "Requires-Dist: isaaccapture>=0.4.0", + "", + ] + ).encode(), + "isaacteleop-0.4.0.dist-info/WHEEL": ( + b"Wheel-Version: 1.0\nGenerator: setuptools\nRoot-Is-Purelib: true\n" + ), + }, + ) + + out = workspace.root / "sbom" + assert ( + cli.main( + [ + "--repo-root", + str(workspace.root), + "build", + "--build-dir", + str(workspace.build), + "--license-data", + str(license_data), + "--out-dir", + str(out), + "--wheel", + str(companion), + "--wheel", + str(workspace.wheel), + ] + ) + == 0 + ) + + def sibling_row(document, name): + spdx = json.loads((out / document).read_text()) + row = next( + package + for package in spdx["packages"] + if package["name"] == name + and package["SPDXID"].startswith("SPDXRef-Package-pypi-") + ) + return spdx, row + + forward, isaaccapture = sibling_row( + "isaacteleop-0.4.0-py3-none-any.spdx.json", "isaaccapture" + ) + back, isaacteleop = sibling_row( + f"{WHEEL_NAME.removesuffix('.whl')}.spdx.json", "isaacteleop" + ) + for spdx, row in ((forward, isaaccapture), (back, isaacteleop)): + assert "externalDocumentRefs" not in spdx + assert row["versionInfo"] != "NOASSERTION" + assert row["licenseDeclared"] == "Apache-2.0" + assert row["supplier"] != "NOASSERTION" + assert "Built by this same build" in row["comment"] + + assert validate_module.check(companion) == [] + + +def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_data): + """A metadata-only wheel has no source notices, and a reference license names no holder. + Use distribution metadata to identify its licensor. + """ + thin = workspace.wheel.parent / "thin-0.4.0-py3-none-any.whl" + synth.write_wheel( + thin, + { + "thin-0.4.0.dist-info/METADATA": "\n".join( + [ + "Metadata-Version: 2.4", + "Name: thin", + "Version: 0.4.0", + "Author: Example Org", + "License-Expression: Apache-2.0", + "", + ] + ).encode(), + "thin-0.4.0.dist-info/WHEEL": ( + b"Wheel-Version: 1.0\nGenerator: setuptools\nRoot-Is-Purelib: true\n" + ), + }, + ) + out = workspace.root / "sbom" + build_module.build(workspace.root, workspace.build, thin, out, license_data) + + with zipfile.ZipFile(thin) as archive: + spdx = json.loads( + archive.read( + "thin-0.4.0.dist-info/sboms/thin-0.4.0-py3-none-any.whl.spdx.json" + ) + ) + notices = archive.read( + "thin-0.4.0.dist-info/licenses/THIRD-PARTY-NOTICES.md" + ).decode() + members = set(archive.namelist()) + + assert _wheel_package(spdx)["copyrightText"] != "NOASSERTION" + assert "Copyright" in notices + assert "Example Org" in notices + # The SPDX spelling of a supplier is not prose. + assert "distributed by Organization:" not in notices + assert "thin-0.4.0.dist-info/licenses/LICENSES/Apache-2.0.txt" in members + + +_IDSTRING = re.compile(r"SPDXRef-[A-Za-z0-9.-]+") + + +def _file_ids_of(names: list[str], version: str = "0.4.0") -> dict[str, str]: + """Each file's id in a document over a wheel holding only these members.""" + entries = tuple( + wheelfile.Entry(name=name, size=0, sha256="0" * 64, sha1="0" * 40) + for name in names + ) + spdx = document_module.build_document( + { + "supplier": "Organization: Example Org", + "homepage": "https://example.invalid", + "license": "Apache-2.0", + "document_namespace": "https://example.invalid/spdx", + }, + {"license_list_version": "3.27"}, + wheelfile.WheelInfo( + path=Path(f"isaaccapture-{version}-py3-none-any.whl"), + distribution="isaaccapture", + version=version, + dist_info=f"isaaccapture-{version}.dist-info", + entries=entries, + ), + inventory_module.Inventory( + attributions={}, + components_present={}, + component_roles={}, + external_runtime={}, + requires_dist=[], + ), + [], + ) + return { + item["fileName"].removeprefix("./"): item["SPDXID"] for item in spdx["files"] + } + + +def test_a_file_id_is_its_sanitised_path_and_a_digest_of_it(built): + spdx = _embedded(built["wheel"]) + + for item in spdx["files"]: + assert item["SPDXID"] == document_module.file_id( + item["fileName"].removeprefix("./"), DIST_INFO + ) + # Pinned: a change to what is hashed renames every file in every document. + assert ( + document_module.file_id( + "isaaccapture/cloudxr/oob_teleop_progress.py", DIST_INFO + ) + == "SPDXRef-File-isaaccapture-cloudxr-oob-teleop-progress.py-f80fee5e" + ) + assert ( + document_module.file_id(f"{DIST_INFO}/METADATA", DIST_INFO) + == "SPDXRef-File-dist-info-METADATA-ad68becb" + ) + + +def test_adding_a_file_renumbers_no_other_file(): + before = _file_ids_of(["a/one.py", "b/three.py", "c/four.so"]) + after = _file_ids_of(["a/one.py", "a/two.py", "b/three.py", "c/four.so"]) + + assert {name: after[name] for name in before} == before + + +@pytest.mark.parametrize(("old", "new"), [("1.7+local", "1.8"), ("1!1.7", "1!1.8")]) +def test_a_version_bump_renames_no_file(old, new): + def members(version): + return [ + "isaaccapture/cloudxr/oob_teleop_progress.py", + f"isaaccapture-{version}.dist-info/METADATA", + f"isaaccapture-{version}.dist-info/licenses/third-party/fmt/LICENSE", + f"isaaccapture-{version}.data/scripts/isaaccapture-tool", + ] + + before = _file_ids_of(members(old), version=old) + after = _file_ids_of(members(new), version=new) + + assert list(before.values()) == list(after.values()) + + +@pytest.mark.parametrize( + "path", + [ + f"sub/{DIST_INFO}/METADATA", + f"{DIST_INFO}X/f", + f"{DIST_INFO.removesuffix('.dist-info')}.dataX/f", + ], +) +def test_only_the_versioned_directories_are_read_as_fixed(path): + assert document_module.file_id(path, DIST_INFO).startswith( + f"SPDXRef-File-{licensing.spdx_safe(path)}-" + ) + + +def test_paths_that_sanitise_alike_get_distinct_ids(): + ids = _file_ids_of(["pkg/a_b.py", "pkg/a-b.py", "pkg/a b.py", "pkg/a__b.py"]) + + assert len(set(ids.values())) == len(ids) + + +def test_a_path_with_no_safe_character_still_gets_a_valid_id(): + assert _IDSTRING.fullmatch(document_module.file_id("データ/名前", DIST_INFO)) + + +def test_a_top_level_dist_info_is_not_the_versioned_one(): + ids = _file_ids_of(["dist-info/METADATA", "isaaccapture-0.4.0.dist-info/METADATA"]) + + assert len(set(ids.values())) == 2 + + +def test_two_files_one_id_stops_the_build(monkeypatch): + monkeypatch.setattr(document_module, "file_id", lambda path, dist_info: "x") + + with pytest.raises( + ValueError, + match=r"isaaccapture-0\.4\.0-py3-none-any\.whl: a\.py and b\.py " + r"would both be x", + ): + _file_ids_of(["a.py", "b.py"]) diff --git a/src/compat/pyproject.toml b/src/compat/pyproject.toml index 1c373fac26..25f6433c6b 100644 --- a/src/compat/pyproject.toml +++ b/src/compat/pyproject.toml @@ -11,8 +11,14 @@ dynamic = ["version"] description = "Transition package for the rename to isaaccapture" requires-python = ">=3.11" license = "Apache-2.0" +authors = [ + { name = "NVIDIA" }, +] dependencies = ["isaaccapture>=1.6.dev0,<1.9"] +[project.urls] +Homepage = "https://github.com/NVIDIA/IsaacCapture" + [project.optional-dependencies] dev = ["isaaccapture[dev]>=1.6.dev0,<1.9"] ui = ["isaaccapture[ui]>=1.6.dev0,<1.9"] diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index 7008bd42e7..b758b2bcfc 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -23,6 +23,8 @@ If work under **`src/core/`** went wrong—**user** correction, **pre-commit/CI* - **All C++ diagnostics go through `isaaccapture::Logger`** (`log_bridge/cpp/inc/log_bridge/logger.hpp`), never `std::cout`/`std::cerr`/`printf`, and the logger name is always `isaaccapture..`. The env-var contract and wire format are shared with the Python half, so a change to either must land in both — see [`log_bridge/AGENTS.md`](log_bridge/AGENTS.md) before touching the logging machinery, and the repo root [`AGENTS.md`](../../AGENTS.md) "Logging" section for the rules that apply to every call site. - **Run `clang-format` on touched C++ before pushing** — CI rejects unformatted C++ and pre-commit does not catch it. See the formatting instructions in the repo root [`AGENTS.md`](../../AGENTS.md) ("Pre-commit — match CI before you stop") for the exact commands; the source of truth lives there, not here. - **Authored `.py` under `src/python/isaaccapture/` carries no build-system registration.** The `CONFIGURE_DEPENDS` glob in ``src/python/CMakeLists.txt`` stages every `.py` there and `[tool.setuptools.packages.find]` discovers the packages, so a subpackage is a directory with an `__init__.py` and nothing else — no packages list, no `DEPENDS` entry. Targets that *produce* files in the staging tree (pybind ``.so``/``.pyd``, ``isaacteleop_python`` copies, ``stage_generated_tracker_exports``, ``viz_py``, vendored runtimes, …) must be **dependencies of** ``python_package`` (``DEPENDS`` / ``add_dependencies(python_package …)`` in ``src/core/python/CMakeLists.txt``), not the other way around — otherwise Windows ninja races the wheel build and setuptools fails with ``error: package directory 'isaaccapture\\' does not exist``. **``python_package`` waits for ``viz_py`` when ``BUILD_VIZ=ON``** (``.pyd`` link race). **Central stubgen** in ``src/core/python/CMakeLists.txt`` (not a ``viz_py`` POST_BUILD) DEPENDS on ``python_package``, so ``stage_generated_tracker_exports`` must stay on ``python_package``'s DEPENDS — otherwise stubbing ``isaaccapture.viz._viz`` (or any module that loads ``isaaccapture/__init__.py``) fails on the missing export. **For ``pip install -e .`` (scikit-build-core), ``_generated_tracker_exports.py`` must also be ``install(FILES ...)`` under ``isaacteleop_wheel``** — the directory install excludes ``*.py`` so authored sources stay redirect-resolved, but that module is not under ``src/python/``. +- **The staging tree (`build/python_package/$`) is a copy destination, never the only place a file exists.** A build step that writes straight into it leaves a wheel member with no build-tree origin, and the SBOM collector cannot explain it — it deliberately does not index staging, because indexing it would let the wheel explain itself. Generate into `build/generated//` and copy in, the way `stage_generated_tracker_exports` and the stub target both do. + - **Editing a `.fbs` under `schema/fbs/` can make every recording already on disk unreadable.** Recorded MCAP embeds the schema it was written under; `schema_tests` compares the current schemas against `schema/golden/` and `McapTrackerViewers` re-checks at replay time. Read [`schema/AGENTS.md`](schema/AGENTS.md) and [`schema/README.md`](schema/README.md) before changing one. - **Schema-based tracker sources exist only in a configured build tree.** They are generated at CMake *configure* time from `deviceio_trackers/trackers.toml` into `${CMAKE_BINARY_DIR}/generated/trackers/`, so grepping `src/` for a tracker such as `Se3Tracker` finds the manifest entry and the generator, not a `.cpp`. Searching, debugging, and clangd all need `cmake -B build` to have run first (already true for the `flatc` output). See [`deviceio_trackers/AGENTS.md`](deviceio_trackers/AGENTS.md) and [`codegen/AGENTS.md`](codegen/AGENTS.md) before adding or editing a tracker. - **`OpenXRSession` waits for the headset by default** (`wait_for_system = true`). `xrGetSystem` answers `XR_ERROR_FORM_FACTOR_UNAVAILABLE` (-35) until a system exists, and the retry loop is unbounded, holds the GIL for the whole wait (the pybind constructor releases nothing), and defers `Ctrl-C` until a headset turns up. Anything that constructs a session where no client will ever connect — CI, a headless smoke test — must impose its own timeout; `deps/cloudxr/docker-compose.test.yaml` wraps the CloudXR GPU tests in one for that reason. diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index 35ffc430ed..dea5a3baff 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -12,6 +12,34 @@ if(NOT EXISTS "${ROBOTIC_GROUNDING_STAGED_DIR}/__init__.py") endif() message(STATUS "[grounding] bundling robotic_grounding/ from ${ROBOTIC_GROUNDING_STAGED_DIR}") +# Share the root pyproject.toml's license-file list across both wheel builds. +set(_root_pyproject "${CMAKE_SOURCE_DIR}/pyproject.toml") +set(_license_files_script "${CMAKE_CURRENT_SOURCE_DIR}/license_files.py") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${_root_pyproject}" "${_license_files_script}") +execute_process( + COMMAND "${Python3_EXECUTABLE}" "${_license_files_script}" "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE _license_files_out + ERROR_VARIABLE _license_files_err + RESULT_VARIABLE _license_files_rc + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(NOT _license_files_rc EQUAL 0) + message(FATAL_ERROR "Cannot read license-files from ${_root_pyproject}: ${_license_files_err}") +endif() +string(REPLACE "\n" ";" _license_files_out "${_license_files_out}") +list(POP_FRONT _license_files_out ISAACCAPTURE_LICENSE_FILES) +list(REMOVE_DUPLICATES _license_files_out) + +set(_license_staging_commands) +foreach(_license_file IN LISTS _license_files_out) + get_filename_component(_license_dir "${_license_file}" DIRECTORY) + list(APPEND _license_staging_commands + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/${_license_dir}" + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/${_license_file}" "${CMAKE_BINARY_DIR}/python_package/$/${_license_file}" + ) +endforeach() + # Generate pyproject.toml from template with ISAAC_TELEOP_PYPROJECT_VERSION configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml.in" @@ -42,8 +70,10 @@ add_custom_target(python_package ALL COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/pyproject.toml" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" "${CMAKE_BINARY_DIR}/python_package/$/" - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/LICENSE.md" "${CMAKE_BINARY_DIR}/python_package/$/LICENSE.md" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/README.md" "${CMAKE_BINARY_DIR}/python_package/$/README.md" + # Remove stale license texts dropped from the list since the last build. + COMMAND ${CMAKE_COMMAND} -E rm -rf "${CMAKE_BINARY_DIR}/python_package/$/LICENSES" + ${_license_staging_commands} COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-dev.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-ui.txt" "${CMAKE_BINARY_DIR}/python_package/$/" @@ -71,6 +101,8 @@ endif() set(STUBGEN_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/generate_stubs.py") set(STUBGEN_PYPROJECT "${CMAKE_CURRENT_SOURCE_DIR}/stubgen_pyproject.toml") set(STUBGEN_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python_package/$") +# Generate outside staging so the SBOM can trace each stub to a build-tree origin. +set(STUBGEN_OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated/stubs/$") set(PY_TYPED_MARKER "${STUBGEN_PACKAGE_DIR}/isaaccapture/py.typed") set(STUBGEN_MODULES @@ -102,7 +134,7 @@ foreach(_module IN LISTS STUBGEN_MODULES) list(APPEND STUBGEN_COMMANDS COMMAND ${CMAKE_COMMAND} -E env --unset=VIRTUAL_ENV uv run --project "${CMAKE_BINARY_DIR}/stubgen" --python ${ISAAC_TELEOP_PYTHON_VERSION} - python "${STUBGEN_SCRIPT}" ${_module} "${STUBGEN_PACKAGE_DIR}" + python "${STUBGEN_SCRIPT}" ${_module} "${STUBGEN_PACKAGE_DIR}" "${STUBGEN_OUTPUT_DIR}" ) endforeach() @@ -115,10 +147,14 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy "${STUBGEN_PYPROJECT}" "${CMAKE_BINARY_DIR}/stubgen/pyproject.toml" # Generate stubs for each pybind11 module using the stubgen pyproject.toml + COMMAND ${CMAKE_COMMAND} -E make_directory "${STUBGEN_OUTPUT_DIR}/isaaccapture" ${STUBGEN_COMMANDS} # Create py.typed marker file (PEP 561) - also serves as build stamp - COMMAND ${CMAKE_COMMAND} -E touch "${PY_TYPED_MARKER}" + COMMAND ${CMAKE_COMMAND} -E touch "${STUBGEN_OUTPUT_DIR}/isaaccapture/py.typed" + + # Copy last so PY_TYPED_MARKER stamps a complete staging tree. + COMMAND ${CMAKE_COMMAND} -E copy_directory "${STUBGEN_OUTPUT_DIR}" "${STUBGEN_PACKAGE_DIR}" DEPENDS python_package "${STUBGEN_SCRIPT}" "${STUBGEN_PYPROJECT}" $ $ diff --git a/src/core/python/generate_stubs.py b/src/core/python/generate_stubs.py index 676c06a0ed..80f0f25384 100644 --- a/src/core/python/generate_stubs.py +++ b/src/core/python/generate_stubs.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Generate Python stub file (.pyi) for a single Isaac Teleop pybind11 module. @@ -6,10 +6,10 @@ This script uses pybind11-stubgen's Python API to generate type stubs for IDE intellisense. Usage: - python generate_stubs.py + python generate_stubs.py Example: - python generate_stubs.py isaaccapture.deviceio._deviceio /path/to/python_package + python generate_stubs.py isaaccapture.deviceio._deviceio build/python_package/Release build/generated/stubs """ import sys @@ -27,18 +27,13 @@ def _err(message: str) -> None: print(message, file=sys.stderr) -def generate_stub(module_name: str, package_dir: Path) -> bool: - """Generate stub file for a single pybind11 module. +def generate_stub(module_name: str, import_dir: Path, output_dir: Path) -> bool: + """Generate stubs for a fully qualified module imported from the staged package. - Args: - module_name: Fully qualified module name (e.g., "isaaccapture.deviceio._deviceio") - package_dir: Path to the python_package directory containing isaaccapture. - - Returns: - True if successful, False otherwise. + Keep output_dir outside staging to preserve a traceable build-tree origin. + Return whether generation succeeded. """ - # Add package_dir to sys.path so pybind11-stubgen can import the module - sys.path.insert(0, str(package_dir)) + sys.path.insert(0, str(import_dir)) try: from pybind11_stubgen import ( @@ -59,7 +54,7 @@ def generate_stub(module_name: str, package_dir: Path) -> bool: # Configure stubgen using the proper API args = CLIArgs( module_name=module_name, - output_dir=str(package_dir), + output_dir=str(output_dir), root_suffix="", ignore_invalid_expressions=None, ignore_invalid_identifiers=None, @@ -111,21 +106,24 @@ def generate_stub(module_name: str, package_dir: Path) -> bool: def main() -> int: """Main entry point.""" - if len(sys.argv) != 3: - _err(f"Usage: {sys.argv[0]} ") + if len(sys.argv) != 4: + _err(f"Usage: {sys.argv[0]} ") _err( - f"Example: {sys.argv[0]} isaaccapture.deviceio._deviceio /path/to/python_package" + f"Example: {sys.argv[0]} isaaccapture.deviceio._deviceio " + "build/python_package/Release build/generated/stubs" ) return 1 module_name = sys.argv[1] - package_dir = Path(sys.argv[2]).resolve() + import_dir = Path(sys.argv[2]).resolve() + output_dir = Path(sys.argv[3]).resolve() - if not package_dir.exists(): - _err(f"Error: Package directory does not exist: {package_dir}") + if not import_dir.exists(): + _err(f"Error: Import directory does not exist: {import_dir}") return 1 + output_dir.mkdir(parents=True, exist_ok=True) - if generate_stub(module_name, package_dir): + if generate_stub(module_name, import_dir, output_dir): return 0 else: _err("Stub generation failed.") diff --git a/src/core/python/license_files.py b/src/core/python/license_files.py new file mode 100644 index 0000000000..8a0b9f43a2 --- /dev/null +++ b/src/core/python/license_files.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Print the root pyproject.toml's project.license-files for CMake. + +The first line is a JSON/TOML array for pyproject.toml.in; each line +after it is one file those patterns match, relative to the repository root. +""" + +import json +import sys +import tomllib +from pathlib import Path + + +def main(root: Path) -> int: + with (root / "pyproject.toml").open("rb") as handle: + patterns = tomllib.load(handle)["project"]["license-files"] + print(json.dumps(patterns)) + for pattern in patterns: + # PEP 639: a pattern that matches nothing is an error, not an empty set. + matches = sorted(path for path in root.glob(pattern) if path.is_file()) + if not matches: + print( + f"project.license-files: {pattern!r} matches no file", file=sys.stderr + ) + return 1 + for path in matches: + print(path.relative_to(root).as_posix()) + return 0 + + +if __name__ == "__main__": + sys.exit(main(Path(sys.argv[1]))) diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index fff1897dca..e9ced4db91 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -9,6 +9,7 @@ # See: numpy.org/devdocs/dev/depending_on_numpy.html [build-system] +# setuptools>=77 implements PEP 639 license metadata and file placement. requires = ["setuptools>=77", "wheel", "numpy>=2.0"] build-backend = "setuptools.build_meta" @@ -19,7 +20,8 @@ description = "Isaac Teleop - Teleoperation with Device I/O" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" -license-files = ["LICENSE.md"] +# The root pyproject.toml's list, staged beside this file by python_package. +license-files = @ISAACCAPTURE_LICENSE_FILES@ authors = [ {name = "NVIDIA"} ] diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 98c75f7bc3..e777b9c8f5 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -75,3 +75,4 @@ Shared C++ fixtures (not executables) live under `tests/cpp/viz/support/` - `src/core/codegen/test_*.py` — unittest, co-located with the generator. - `examples/oxr/python/test_*.py` — standalone scripts for CloudXR GPU CI. - ROS2 Docker integration helpers under `examples/teleop_ros2/.../integration_tests/`. +- `scripts/sbom/tests/` — the SBOM collector's tests, run from its own uv project. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 59e4ea5dc6..84bd3a5aa1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ # IsaacTeleop tests (pytest + Catch2/CTest) # ============================================================================== # -# Production libraries live under src/ and examples/; all pytest and Catch2 +# Production libraries live under src/ and examples/; pytest and Catch2 # suites live here under tests/python/ and tests/cpp/. if(BUILD_TESTING) diff --git a/tests/python/packaging/test_wheel_contents.py b/tests/python/packaging/test_wheel_contents.py index 33c8c0674f..0d0b93a12a 100644 --- a/tests/python/packaging/test_wheel_contents.py +++ b/tests/python/packaging/test_wheel_contents.py @@ -11,6 +11,7 @@ import os import sys +import tomllib import zipfile from email.parser import BytesParser from pathlib import Path @@ -136,6 +137,33 @@ def test_capture_license_ships_in_distribution_metadata() -> None: assert not any("isaaccapture_egopost/" in name for name in archive.namelist()) +def test_wheel_ships_exactly_the_root_license_files() -> None: + root = repo_root() + with (root / "pyproject.toml").open("rb") as handle: + patterns = tomllib.load(handle)["project"]["license-files"] + expected = { + path.relative_to(root).as_posix() + for pattern in patterns + for path in root.glob(pattern) + if path.is_file() + } + with zipfile.ZipFile(_wheel()) as archive: + metadata_path = next( + name for name in archive.namelist() if name.endswith(".dist-info/METADATA") + ) + metadata = BytesParser().parsebytes(archive.read(metadata_path)) + # Third-party texts the SBOM collector appends live under other names. + own = { + name + for name in metadata.get_all("License-File", []) + if name == "LICENSE.md" or name.startswith("LICENSES/") + } + assert own == expected + licenses_dir = metadata_path.removesuffix("METADATA") + "licenses/" + for name in own: + assert archive.read(licenses_dir + name) == (root / name).read_bytes() + + @pytest.mark.skipif( not sys.platform.startswith("linux"), reason="the plugin is Linux only" )